From 94ffda577c9f2cae2bd2460b9fa849d488d118ea Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Thu, 21 Apr 2022 09:50:58 +0800 Subject: [PATCH 01/66] [Eager]Fix SetDeviceId in eager_final_state_api from python_c_gen.py (#42025) --- .../final_state_generator/python_c_gen.py | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/eager/auto_code_generator/final_state_generator/python_c_gen.py b/paddle/fluid/eager/auto_code_generator/final_state_generator/python_c_gen.py index e2bb4104551e3..7ca5fc833ea8d 100644 --- a/paddle/fluid/eager/auto_code_generator/final_state_generator/python_c_gen.py +++ b/paddle/fluid/eager/auto_code_generator/final_state_generator/python_c_gen.py @@ -100,6 +100,9 @@ def FindParsingFunctionFromAttributeType(atype): {} tstate = PyEval_SaveThread(); + + // Set Device ID +{} auto out = {}({}); @@ -118,6 +121,19 @@ def FindParsingFunctionFromAttributeType(atype): """ +FUNCTION_SET_DEVICE_TEMPLATE = \ +""" + {} + if (paddle::platform::is_gpu_place(place)) {{ +#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) + phi::backends::gpu::SetDeviceId(place.device); + VLOG(1) <<"CurrentDeviceId: " << phi::backends::gpu::GetCurrentDeviceId() << " from " << (int)place.device; +#else + PADDLE_THROW(paddle::platform::errors::PreconditionNotMet( + "PaddlePaddle should compile with GPU if use CUDAPlace.")); +#endif + }} +""" FUNCTION_NAME_TEMPLATE = \ "{}{}{}" @@ -293,14 +309,23 @@ def GeneratePythonCFunction(self): "false") parse_attributes_str = "" + expected_place_str = "auto place = egr::Controller::Instance().GetExpectedPlace();\n" # Generate Python-C Attributes Parsing Logic for name, atype, _, pos in orig_forward_attrs_list: parsing_function_name = FindParsingFunctionFromAttributeType(atype) + # Used input argument place if specified from Python frontend. + if len(expected_place_str + ) != 0 and parsing_function_name == "CastPyArg2Place": + expected_place_str = "" + assert name == "place", "Only support 'place' as template argument name in FUNCTION_SET_DEVICE_TEMPLATE." + parse_attributes_str += PARSE_PYTHON_C_ARGS_TEMPLATE.format( name, pos, atype, name, parsing_function_name, name, forward_api_name, pos) + set_device_str = FUNCTION_SET_DEVICE_TEMPLATE.format(expected_place_str) + # Generate Dygraph Function Call Logic num_args = len(forward_inputs_position_map.keys()) + len( orig_forward_attrs_list) @@ -326,8 +351,8 @@ def GeneratePythonCFunction(self): "pythonc_record_event", forward_api_name, "pybind_imperative_func") self.python_c_function_str = PYTHON_C_FUNCTION_TEMPLATE.format( forward_api_name, pythonc_record_event_str, forward_api_name, - get_eager_tensor_str, parse_attributes_str, fwd_function_name, - dygraph_function_call_str, return_str) + get_eager_tensor_str, parse_attributes_str, set_device_str, + fwd_function_name, dygraph_function_call_str, return_str) # Set prefix of forward_api_name to avoid conflicts prefix = self.namespace.strip("::") @@ -361,8 +386,9 @@ def GeneratePythonCFunction(self): self.python_c_function_str += PYTHON_C_FUNCTION_TEMPLATE.format( inplaced_forward_api_name, pythonc_record_event_str, inplaced_forward_api_name, get_eager_tensor_str, - parse_attributes_str, inplaced_fwd_function_name, - dygraph_function_call_str, return_str) + parse_attributes_str, set_device_str, + inplaced_fwd_function_name, dygraph_function_call_str, + return_str) # Generate Python-C Function Registration self.python_c_function_reg_str += "\n," + PYTHON_C_FUNCTION_REG_TEMPLATE.format( From f5ac996165e46e36c0d4c4b54e384c1c7c54b386 Mon Sep 17 00:00:00 2001 From: David Nicolas <37790151+liyongchao911@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:27:43 +0800 Subject: [PATCH 02/66] fix api math equation dispaly issue; test=document_fix (#42058) --- python/paddle/tensor/math.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py index ad65a22dfae92..7e0b2e5424dad 100644 --- a/python/paddle/tensor/math.py +++ b/python/paddle/tensor/math.py @@ -94,7 +94,7 @@ def log(x, name=None): .. math:: - Out = \\ln(x) + Out = \ln(x) Args: x (Tensor): Input Tensor. Must be one of the following types: float32, float64. @@ -213,7 +213,7 @@ def stanh(x, scale_a=0.67, scale_b=1.7159, name=None): .. math:: - out = b * \\frac{e^{a * x} - e^{-a * x}}{e^{a * x} + e^{-a * x}} + out = b * \frac{e^{a * x} - e^{-a * x}}{e^{a * x} + e^{-a * x}} Parameters: x (Tensor): The input Tensor with data type float32, float64. @@ -1848,7 +1848,7 @@ def logsumexp(x, axis=None, keepdim=False, name=None): This OP calculates the log of the sum of exponentials of ``x`` along ``axis`` . .. math:: - logsumexp(x) = \\log\\sum exp(x) + logsumexp(x) = \log\sum exp(x) Args: x (Tensor): The input Tensor with data type float32 or float64, which @@ -2417,7 +2417,7 @@ def log1p(x, name=None): Calculates the natural log of the given input tensor, element-wise. .. math:: - Out = \\ln(x+1) + Out = \ln(x+1) Args: x (Tensor): Input Tensor. Must be one of the following types: float32, float64. @@ -2455,7 +2455,7 @@ def log2(x, name=None): .. math:: - Out = \\log_2x + Out = \log_2x Args: x (Tensor): Input tensor must be one of the following types: float32, float64. @@ -2507,7 +2507,7 @@ def log10(x, name=None): .. math:: - Out = \\log_10_x + Out = \log_10_x Args: x (Tensor): Input tensor must be one of the following types: float32, float64. @@ -3289,7 +3289,7 @@ def tanh(x, name=None): Tanh Activation Operator. .. math:: - out = \\frac{e^{x} - e^{-x}}{e^{x} + e^{-x}} + out = \frac{e^{x} - e^{-x}}{e^{x} + e^{-x}} Args: x (Tensor): Input of Tanh operator, an N-D Tensor, with data type float32, float64 or float16. From 1bf2eeab421c779f889573b172543e49570ac2cc Mon Sep 17 00:00:00 2001 From: pangyoki Date: Thu, 21 Apr 2022 10:29:26 +0800 Subject: [PATCH 03/66] add _grad_name and _grad_value for eager tensor (#41990) * add _grad_name and _grad_value for eager tensor * fix paddle_enforce * fix paddle_enforce 2 * fix grad_name * _grad_value return lodtensor rather than tensor * fix --- paddle/fluid/pybind/eager_method.cc | 44 +++++++++++++++++++ .../fluid/tests/unittests/test_var_base.py | 13 ++++++ 2 files changed, 57 insertions(+) diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc index ca757fac9a6e3..13fba2baa1d6c 100644 --- a/paddle/fluid/pybind/eager_method.cc +++ b/paddle/fluid/pybind/eager_method.cc @@ -1492,6 +1492,46 @@ static PyObject* tensor__offset(TensorObject* self, PyObject* args, EAGER_CATCH_AND_THROW_RETURN_NULL } +static PyObject* tensor__grad_name(TensorObject* self, PyObject* args, + PyObject* kwargs) { + EAGER_TRY + paddle::experimental::Tensor* grad = + egr::EagerUtils::mutable_grad(self->tensor); + PADDLE_ENFORCE_EQ(grad != nullptr, true, + platform::errors::InvalidArgument( + "Detected NULL grad. Please check if you have manually " + "cleared the grad inside autograd_meta")); + return ToPyObject(grad->name()); + EAGER_CATCH_AND_THROW_RETURN_NULL +} + +static PyObject* tensor__grad_value(TensorObject* self, PyObject* args, + PyObject* kwargs) { + EAGER_TRY + paddle::experimental::Tensor* grad = + egr::EagerUtils::mutable_grad(self->tensor); + PADDLE_ENFORCE_EQ(grad != nullptr, true, + platform::errors::InvalidArgument( + "Detected NULL grad. Please check if you have manually " + "cleared the grad inside autograd_meta")); + + if (!grad->defined()) { + Py_IncRef(Py_None); + return Py_None; + } + if (grad->is_dense_tensor()) { + auto* grad_tensor = + static_cast(grad->impl().get()); + return ToPyObject(grad_tensor); + } else { + PADDLE_THROW(paddle::platform::errors::Fatal( + "this method is only supported for DenseTensor")); + Py_IncRef(Py_None); + return Py_None; + } + EAGER_CATCH_AND_THROW_RETURN_NULL +} + #if defined(PADDLE_WITH_CUDA) static PyObject* tensor_method__uva(TensorObject* self, PyObject* args, PyObject* kwargs) { @@ -1633,6 +1673,10 @@ PyMethodDef variable_methods[] = { METH_VARARGS | METH_KEYWORDS, NULL}, {"_offset", (PyCFunction)(void (*)(void))tensor__offset, METH_VARARGS | METH_KEYWORDS, NULL}, + {"_grad_name", (PyCFunction)(void (*)(void))tensor__grad_name, + METH_VARARGS | METH_KEYWORDS, NULL}, + {"_grad_value", (PyCFunction)(void (*)(void))tensor__grad_value, + METH_VARARGS | METH_KEYWORDS, NULL}, #if defined(PADDLE_WITH_CUDA) {"_tensor_uva", (PyCFunction)(void (*)(void))tensor_method__uva, METH_VARARGS | METH_KEYWORDS, NULL}, diff --git a/python/paddle/fluid/tests/unittests/test_var_base.py b/python/paddle/fluid/tests/unittests/test_var_base.py index 2729aabf604bb..e6e608bea23f4 100644 --- a/python/paddle/fluid/tests/unittests/test_var_base.py +++ b/python/paddle/fluid/tests/unittests/test_var_base.py @@ -1743,5 +1743,18 @@ def test_copy_gradient_from(self): self.func_test_copy_gradient_from() +class TestEagerTensorGradNameValue(unittest.TestCase): + def test_eager_tensor_grad_name_value(self): + with _test_eager_guard(): + a_np = np.array([2, 3]).astype('float32') + a = paddle.to_tensor(a_np) + a.stop_gradient = False + b = a**2 + self.assertEqual(a._grad_value(), None) + b.backward() + self.assertEqual('eager_tmp' in a._grad_name(), True) + self.assertNotEqual(a._grad_value(), None) + + if __name__ == '__main__': unittest.main() From 3da8066a05c2eadc172e0669cbb6bcdbc7f8d057 Mon Sep 17 00:00:00 2001 From: Weilong Wu Date: Thu, 21 Apr 2022 10:32:25 +0800 Subject: [PATCH 04/66] [Eager] Support numpy.narray as input for eager expand (#42043) --- paddle/fluid/pybind/eager_utils.cc | 3 ++- .../tests/unittests/test_expand_v2_op.py | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc index ec391a7fa64a8..9719963d51da0 100644 --- a/paddle/fluid/pybind/eager_utils.cc +++ b/paddle/fluid/pybind/eager_utils.cc @@ -1101,7 +1101,8 @@ paddle::experimental::IntArray CastPyArg2IntArray(PyObject* obj, // obj could be: int, float, bool, paddle.Tensor PyTypeObject* type = obj->ob_type; auto type_name = std::string(type->tp_name); - if (type_name == "list" || type_name == "tuple") { + if (type_name == "list" || type_name == "tuple" || + type_name == "numpy.ndarray") { std::vector value = CastPyArg2Ints(obj, op_type, arg_pos); return paddle::experimental::IntArray(value); diff --git a/python/paddle/fluid/tests/unittests/test_expand_v2_op.py b/python/paddle/fluid/tests/unittests/test_expand_v2_op.py index 592a635ddcccc..4932ea8a1b5c9 100644 --- a/python/paddle/fluid/tests/unittests/test_expand_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_expand_v2_op.py @@ -20,6 +20,7 @@ import paddle.fluid as fluid from paddle.fluid import compiler, Program, program_guard import paddle +from paddle.fluid.framework import _test_eager_guard # Situation 1: shape is a list(without tensor) @@ -243,6 +244,30 @@ def test_shape_with_var(self): self.assertListEqual(list(out.shape), [-1, -1, -1]) +# Test python Dygraph API +class TestExpandV2DygraphAPI(unittest.TestCase): + def test_expand_times_is_tensor(self): + with paddle.fluid.dygraph.guard(): + with _test_eager_guard(): + paddle.seed(1) + a = paddle.rand([2, 5]) + egr_expand_1 = paddle.expand(a, shape=[2, 5]) + np_array = np.array([2, 5]) + egr_expand_2 = paddle.expand(a, shape=np_array) + + paddle.seed(1) + a = paddle.rand([2, 5]) + expand_1 = paddle.expand(a, shape=[2, 5]) + np_array = np.array([2, 5]) + expand_2 = paddle.expand(a, shape=np_array) + + self.assertTrue( + np.array_equal(egr_expand_1.numpy(), egr_expand_2.numpy())) + self.assertTrue(np.array_equal(expand_1.numpy(), expand_2.numpy())) + self.assertTrue( + np.array_equal(expand_1.numpy(), egr_expand_1.numpy())) + + if __name__ == "__main__": paddle.enable_static() unittest.main() From f2f1de7baaa6c9f20a7a8a66d3da85b7a48005e2 Mon Sep 17 00:00:00 2001 From: Ruibiao Chen Date: Thu, 21 Apr 2022 11:12:39 +0800 Subject: [PATCH 05/66] Support cinn_launch op in standalone executor (#42046) * Support cinn_launch OP in standalone executor * Remove some redundant code --- .../framework/new_executor/interpretercore.cc | 11 ++++++++++- .../new_executor/interpretercore_util.cc | 17 ++++++++++++++--- .../framework/new_executor/new_executor_defs.cc | 10 ++++++++++ .../framework/new_executor/new_executor_defs.h | 4 ++++ 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/framework/new_executor/interpretercore.cc b/paddle/fluid/framework/new_executor/interpretercore.cc index 74310a6046c7d..a4fcf0773f623 100644 --- a/paddle/fluid/framework/new_executor/interpretercore.cc +++ b/paddle/fluid/framework/new_executor/interpretercore.cc @@ -428,8 +428,17 @@ void InterpreterCore::BuildAndCacheInstructionCtx(Instruction* instr_node) { } outs_map.emplace(var_name_item.first, std::move(out_vars)); } + // set runtime_ctx and infershape_ctx_ - instr_node->ResetContext(ins_map, outs_map); + if (instr_node->OpBase()->Type() == "cinn_launch") { // OP use scope in + // kernel + Scope* local_scope = create_local_scope_ + ? global_scope_->GetMutableLocalScope() + : global_scope_->GetMutableScope(); + instr_node->ResetContextWithScope(ins_map, outs_map, *local_scope); + } else { + instr_node->ResetContext(ins_map, outs_map); + } } void InterpreterCore::BuildSkipShareLoDInfo() { diff --git a/paddle/fluid/framework/new_executor/interpretercore_util.cc b/paddle/fluid/framework/new_executor/interpretercore_util.cc index ed813c78bc368..afddcb580b9d8 100644 --- a/paddle/fluid/framework/new_executor/interpretercore_util.cc +++ b/paddle/fluid/framework/new_executor/interpretercore_util.cc @@ -392,8 +392,19 @@ void build_op_func_list(const platform::Place& place, platform::DeviceContextPool::Instance(); auto* dev_ctx = pool.Get(place); Scope scope; + Scope* runtime_scope = &scope; + // NOTE(Ruibiao): We do not encourage directly using scope in OP kernel. + // But some OPs do have such behavior (e.g., cinn_launch OP). Here special + // treatment for them. + if (op_with_kernel->Type() == "cinn_launch") { + VLOG(6) << "OP(" << op_with_kernel->Type() << ") use scope in kernel, " + "so pass a real scope to " + "ExecutionContext"; + runtime_scope = local_scope; + } + auto expected_kernel_key = op_with_kernel->GetExpectedKernelType( - ExecutionContext(*op, scope, *dev_ctx, runtime_context)); + ExecutionContext(*op, *runtime_scope, *dev_ctx, runtime_context)); op_with_kernel->ResetKernelType(new OpKernelType(expected_kernel_key)); // change device by the device_guard() @@ -441,8 +452,8 @@ void build_op_func_list(const platform::Place& place, op_with_kernel->Info().infer_shape_(&infer_shape_ctx); } - auto exec_ctx = - ExecutionContext(*op_with_kernel, scope, *dev_ctx, runtime_context); + auto exec_ctx = ExecutionContext(*op_with_kernel, *runtime_scope, + *dev_ctx, runtime_context); auto run_phi_kernel = false; if (phi::KernelFactory::Instance().HasCompatiblePhiKernel( diff --git a/paddle/fluid/framework/new_executor/new_executor_defs.cc b/paddle/fluid/framework/new_executor/new_executor_defs.cc index 86d534b0b4edd..3c2395d4320a1 100644 --- a/paddle/fluid/framework/new_executor/new_executor_defs.cc +++ b/paddle/fluid/framework/new_executor/new_executor_defs.cc @@ -755,6 +755,16 @@ void Instruction::ResetContext(const VariableValueMap& in_vars, new ExecutionContext(*OpBase(), scope_, dev_ctx_, *runtime_ctx_.get())); } +void Instruction::ResetContextWithScope(const VariableValueMap& in_vars, + const VariableValueMap& out_vars, + const framework::Scope& scope) { + runtime_ctx_.reset(new RuntimeContext(in_vars, out_vars)); + infershape_ctx_.reset( + new InterpretercoreInferShapeContext(*OpBase(), *runtime_ctx_.get())); + execution_ctx_.reset( + new ExecutionContext(*OpBase(), scope, dev_ctx_, *runtime_ctx_.get())); +} + std::shared_ptr Instruction::InnerRuntimeContext() const { return runtime_ctx_; } diff --git a/paddle/fluid/framework/new_executor/new_executor_defs.h b/paddle/fluid/framework/new_executor/new_executor_defs.h index 6a1e46e359242..28b9f6f0130f5 100644 --- a/paddle/fluid/framework/new_executor/new_executor_defs.h +++ b/paddle/fluid/framework/new_executor/new_executor_defs.h @@ -347,6 +347,10 @@ class Instruction { void ResetContext(const VariableValueMap& in_vars, const VariableValueMap& out_vars); + void ResetContextWithScope(const VariableValueMap& in_vars, + const VariableValueMap& out_vars, + const framework::Scope& scope); + std::shared_ptr InnerRuntimeContext() const; std::shared_ptr InnerInferShapeContext() From 0d28ee29066ba46b48198d62605f6b61fcf92719 Mon Sep 17 00:00:00 2001 From: Wilber Date: Thu, 21 Apr 2022 11:17:08 +0800 Subject: [PATCH 06/66] infer add io stream. (#42031) * infer add io stream. * add macro --- cmake/external/lite.cmake | 2 +- .../fluid/inference/api/analysis_predictor.cc | 18 +++ .../fluid/inference/api/analysis_predictor.h | 4 + .../inference/api/details/zero_copy_tensor.cc | 133 ++++++++++++++++++ paddle/fluid/inference/api/paddle_api.h | 12 ++ paddle/fluid/inference/api/paddle_tensor.h | 5 + 6 files changed, 173 insertions(+), 1 deletion(-) diff --git a/cmake/external/lite.cmake b/cmake/external/lite.cmake index f1d206dd5e199..0031757467f37 100644 --- a/cmake/external/lite.cmake +++ b/cmake/external/lite.cmake @@ -50,7 +50,7 @@ if (NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR) set(LITE_INSTALL_DIR ${THIRD_PARTY_PATH}/install/lite) if(NOT LITE_GIT_TAG) - set(LITE_GIT_TAG 4ab64daecc11fbf74fffdc6a4733f388472e7d5d) + set(LITE_GIT_TAG 81ef66554099800c143a0feff6e0a491b3b0d12e) endif() if(NOT CUDA_ARCH_NAME) diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 7ec3271c66573..015f4471a0246 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -1931,11 +1931,29 @@ bool InternalUtils::RunWithExternalStream(paddle_infer::Predictor *p, #endif return false; } + void InternalUtils::UpdateConfigInterleaved(paddle_infer::Config *c, bool with_interleaved) { #ifdef PADDLE_WITH_CUDA c->trt_with_interleaved_ = with_interleaved; #endif } + +void InternalUtils::SyncStream(paddle_infer::Predictor *p) { +#ifdef PADDLE_WITH_CUDA + auto *pred = dynamic_cast(p->predictor_.get()); + paddle::platform::DeviceContextPool &pool = + paddle::platform::DeviceContextPool::Instance(); + auto *dev_ctx = reinterpret_cast( + pool.Get(pred->place_)); + cudaStreamSynchronize(dev_ctx->stream()); +#endif +} +void InternalUtils::SyncStream(cudaStream_t stream) { +#ifdef PADDLE_WITH_CUDA + cudaStreamSynchronize(stream); +#endif +} + } // namespace experimental } // namespace paddle_infer diff --git a/paddle/fluid/inference/api/analysis_predictor.h b/paddle/fluid/inference/api/analysis_predictor.h index d9992f3fbef9d..e96526730fdea 100644 --- a/paddle/fluid/inference/api/analysis_predictor.h +++ b/paddle/fluid/inference/api/analysis_predictor.h @@ -38,6 +38,9 @@ namespace paddle_infer { using float16 = paddle::platform::float16; +namespace experimental { +class InternalUtils; +}; } /// /// \file analysis_predictor.h @@ -492,6 +495,7 @@ class AnalysisPredictor : public PaddlePredictor { std::shared_ptr fleet_exe_; std::shared_ptr task_node_; #endif + friend class paddle_infer::experimental::InternalUtils; }; } // namespace paddle diff --git a/paddle/fluid/inference/api/details/zero_copy_tensor.cc b/paddle/fluid/inference/api/details/zero_copy_tensor.cc index 0f26a1076a68c..7461724afb4dd 100644 --- a/paddle/fluid/inference/api/details/zero_copy_tensor.cc +++ b/paddle/fluid/inference/api/details/zero_copy_tensor.cc @@ -714,4 +714,137 @@ template void Tensor::ORTCopyToCpu(int8_t *data) const; template void Tensor::ORTCopyToCpu(float16 *data) const; #endif +namespace experimental { +template +void InternalUtils::CopyFromCpuWithIoStream(paddle_infer::Tensor *t, + const T *data, + cudaStream_t stream) { + if (t->tensor_ == nullptr) { + PADDLE_ENFORCE_EQ( + t->name_.empty(), false, + paddle::platform::errors::PreconditionNotMet( + "Need to SetName first, so that the corresponding tensor can " + "be retrieved.")); + auto *scope = static_cast(t->scope_); + auto *var = scope->FindVar(t->name_); + PADDLE_ENFORCE_NOT_NULL( + var, paddle::platform::errors::PreconditionNotMet( + "No tensor called [%s] in the runtime scope", t->name_)); + auto *tensor = var->GetMutable(); + t->tensor_ = tensor; + } + + auto *tensor = static_cast(t->tensor_); + PADDLE_ENFORCE_GE(tensor->numel(), 0, + paddle::platform::errors::PreconditionNotMet( + "You should call Tensor::Reshape(const " + "std::vector &shape)" + "function before copying data from cpu.")); + size_t ele_size = tensor->numel() * sizeof(T); + if (t->place_ == PlaceType::kCPU) { + auto *t_data = tensor->mutable_data(paddle::platform::CPUPlace()); + std::memcpy(static_cast(t_data), data, ele_size); + } else if (t->place_ == PlaceType::kGPU) { +#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) + paddle::platform::CUDAPlace gpu_place(t->device_); + auto *t_data = tensor->mutable_data(gpu_place); + paddle::memory::Copy(gpu_place, static_cast(t_data), + paddle::platform::CPUPlace(), data, ele_size, stream); +#else + PADDLE_THROW(paddle::platform::errors::Unavailable( + "Can not create tensor with CUDA place because paddle is not compiled " + "with CUDA.")); +#endif + } else { + PADDLE_THROW(paddle::platform::errors::InvalidArgument( + "CopyFromCpuWithIoStream only supports CPU and GPU now.")); + } +} + +template +void InternalUtils::CopyToCpuWithIoStream(paddle_infer::Tensor *t, T *data, + cudaStream_t stream) { + if (t->tensor_ == nullptr) { + PADDLE_ENFORCE_EQ( + t->name_.empty(), false, + paddle::platform::errors::PreconditionNotMet( + "Need to SetName first, so that the corresponding tensor can " + "be retrieved.")); + auto *scope = static_cast(t->scope_); + auto *var = scope->FindVar(t->name_); + PADDLE_ENFORCE_NOT_NULL( + var, paddle::platform::errors::PreconditionNotMet( + "No tensor called [%s] in the runtime scope", t->name_)); + auto *tensor = var->GetMutable(); + t->tensor_ = tensor; + } + + auto *tensor = static_cast(t->tensor_); + auto ele_num = tensor->numel(); + auto *t_data = tensor->data(); + auto t_place = tensor->place(); + + paddle::framework::Tensor out; + auto mem_allocation = + std::make_shared( + static_cast(data), ele_num * sizeof(T), + paddle::platform::CPUPlace()); + out.ResetHolder(mem_allocation); + + if (paddle::platform::is_cpu_place(t_place)) { +#ifdef PADDLE_WITH_MKLDNN + if (tensor->layout() == paddle::framework::DataLayout::kMKLDNN) + paddle::framework::innerTransDataLayoutFromMKLDNN( + tensor->layout(), paddle::platform::MKLDNNDeviceContext::tls() + .get_cur_paddle_data_layout(), + *tensor, &out, paddle::platform::CPUPlace(), true); + else + std::memcpy(static_cast(data), t_data, ele_num * sizeof(T)); +#else + std::memcpy(static_cast(data), t_data, ele_num * sizeof(T)); +#endif + } else if (t->place_ == PlaceType::kGPU) { +#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) + paddle::memory::Copy(paddle::platform::CPUPlace(), + static_cast(data), t_place, t_data, + ele_num * sizeof(T), stream); +#else + PADDLE_THROW(paddle::platform::errors::Unavailable( + "Can not create tensor with CUDA place because paddle is not compiled " + "with CUDA.")); +#endif + } else { + PADDLE_THROW(paddle::platform::errors::InvalidArgument( + "CopyToCpuWithIoStream only supports CPU and GPU now.")); + } +} + +template void InternalUtils::CopyFromCpuWithIoStream( + paddle_infer::Tensor *t, const float *data, cudaStream_t stream); +template void InternalUtils::CopyFromCpuWithIoStream( + paddle_infer::Tensor *t, const int64_t *data, cudaStream_t stream); +template void InternalUtils::CopyFromCpuWithIoStream( + paddle_infer::Tensor *t, const int32_t *data, cudaStream_t stream); +template void InternalUtils::CopyFromCpuWithIoStream( + paddle_infer::Tensor *t, const uint8_t *data, cudaStream_t stream); +template void InternalUtils::CopyFromCpuWithIoStream( + paddle_infer::Tensor *t, const int8_t *data, cudaStream_t stream); +template void InternalUtils::CopyFromCpuWithIoStream( + paddle_infer::Tensor *t, const float16 *data, cudaStream_t stream); + +template void InternalUtils::CopyToCpuWithIoStream( + paddle_infer::Tensor *t, float *data, cudaStream_t stream); +template void InternalUtils::CopyToCpuWithIoStream( + paddle_infer::Tensor *t, int64_t *data, cudaStream_t stream); +template void InternalUtils::CopyToCpuWithIoStream( + paddle_infer::Tensor *t, int32_t *data, cudaStream_t stream); +template void InternalUtils::CopyToCpuWithIoStream( + paddle_infer::Tensor *t, uint8_t *data, cudaStream_t stream); +template void InternalUtils::CopyToCpuWithIoStream( + paddle_infer::Tensor *t, int8_t *data, cudaStream_t stream); +template void InternalUtils::CopyToCpuWithIoStream( + paddle_infer::Tensor *t, float16 *data, cudaStream_t stream); + +} // namespace experimental + } // namespace paddle_infer diff --git a/paddle/fluid/inference/api/paddle_api.h b/paddle/fluid/inference/api/paddle_api.h index 0f8f9e0a975ba..dc9f7debe5f2f 100644 --- a/paddle/fluid/inference/api/paddle_api.h +++ b/paddle/fluid/inference/api/paddle_api.h @@ -420,8 +420,10 @@ using hipStream_t = struct ihipStream_t*; namespace paddle_infer { class Predictor; +class Tensor; using Config = paddle::AnalysisConfig; namespace experimental { +// Unstable interface, may be modified or deleted in the future. class PD_INFER_DECL InternalUtils { public: // Note: Can only be used under thread_local semantics. @@ -429,8 +431,18 @@ class PD_INFER_DECL InternalUtils { cudaStream_t stream); static bool RunWithExternalStream(paddle_infer::Predictor* pred, hipStream_t stream); + static void UpdateConfigInterleaved(paddle_infer::Config* c, bool with_interleaved); + + static void SyncStream(paddle_infer::Predictor* pred); + static void SyncStream(cudaStream_t stream); + template + static void CopyFromCpuWithIoStream(paddle_infer::Tensor* t, const T* data, + cudaStream_t stream); + template + static void CopyToCpuWithIoStream(paddle_infer::Tensor* t, T* data, + cudaStream_t stream); }; } // namespace experimental } // namespace paddle_infer diff --git a/paddle/fluid/inference/api/paddle_tensor.h b/paddle/fluid/inference/api/paddle_tensor.h index 2afe2d32e2f60..6f99ed6e25a28 100644 --- a/paddle/fluid/inference/api/paddle_tensor.h +++ b/paddle/fluid/inference/api/paddle_tensor.h @@ -39,6 +39,10 @@ namespace contrib { class TensorUtils; } +namespace experimental { +class InternalUtils; +}; + /// \brief Paddle data type. enum DataType { FLOAT32, @@ -198,6 +202,7 @@ class PD_INFER_DECL Tensor { #endif friend class paddle_infer::contrib::TensorUtils; + friend class paddle_infer::experimental::InternalUtils; #if defined(PADDLE_WITH_TESTING) && defined(PADDLE_WITH_INFERENCE_API_TEST) friend class paddle_infer::InferApiTesterUtils; #endif From 83d6e315596d648ec6403cde625b6bf98b2740bd Mon Sep 17 00:00:00 2001 From: Zhen Wang Date: Wed, 20 Apr 2022 20:26:06 -0700 Subject: [PATCH 07/66] Move pass optimizations into CINN. (#42047) * Move pass optimizations into CINN. * Update the commit id of used cinn codes. --- cmake/external/cinn.cmake | 2 +- .../framework/paddle2cinn/cinn_compiler.cc | 33 ++++++------------- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/cmake/external/cinn.cmake b/cmake/external/cinn.cmake index 1ca029b3add4c..004bf353d34e8 100644 --- a/cmake/external/cinn.cmake +++ b/cmake/external/cinn.cmake @@ -26,7 +26,7 @@ add_definitions(-w) ###################################### include(ExternalProject) set(CINN_PREFIX_DIR ${THIRD_PARTY_PATH}/CINN) -set(CINN_GIT_TAG 08d7680dd91dfaa65787969050eb8f1143654f10) +set(CINN_GIT_TAG eedb801ca39bfc6b9621bc76c24a0bf98cb8404b) set(CINN_OPTIONAL_ARGS -DPY_VERSION=${PY_VERSION} -DWITH_CUDA=${WITH_GPU} -DWITH_CUDNN=${WITH_GPU} diff --git a/paddle/fluid/framework/paddle2cinn/cinn_compiler.cc b/paddle/fluid/framework/paddle2cinn/cinn_compiler.cc index 67393c288df86..51dca93c7c7f0 100644 --- a/paddle/fluid/framework/paddle2cinn/cinn_compiler.cc +++ b/paddle/fluid/framework/paddle2cinn/cinn_compiler.cc @@ -25,14 +25,10 @@ #include "cinn/auto_schedule/tuning.h" #include "cinn/common/target.h" #include "cinn/common/type.h" -#include "cinn/frontend/decomposer/use_decomposer.h" -#include "cinn/frontend/pass/use_program_pass.h" -#include "cinn/frontend/program_pass.h" +#include "cinn/frontend/optimize.h" #include "cinn/frontend/syntax.h" #include "cinn/hlir/framework/graph.h" #include "cinn/hlir/framework/graph_compiler.h" -#include "cinn/hlir/framework/pass.h" -#include "cinn/hlir/pass/use_pass.h" #include "gflags/gflags.h" #include "paddle/fluid/framework/framework.pb.h" #include "paddle/fluid/framework/ir/graph.h" @@ -58,13 +54,11 @@ namespace paddle2cinn { using ir::Graph; using ir::Node; using inference::analysis::Dot; -using ::cinn::common::Target; -using ::cinn::common::Float; -using ::cinn::hlir::framework::GraphCompiler; using ::cinn::auto_schedule::AutoTuner; +using ::cinn::common::Target; +using ::cinn::frontend::Optimize; using ::cinn::hlir::framework::BuildScope; -using ::cinn::frontend::ProgramPass; -using ::cinn::hlir::framework::ApplyPass; +using ::cinn::hlir::framework::GraphCompiler; CinnCompiler* CinnCompiler::GetInstance() { static CinnCompiler instance; @@ -75,7 +69,7 @@ const CinnCompiledObject& CinnCompiler::Compile( const Graph& graph, const std::map& input_tensors, const Target& target, void* stream) { - VLOG(1) << "-- The graph to be compiled is:\n" << VizGraph(graph); + VLOG(4) << "-- The graph to be compiled is:\n" << VizGraph(graph); CinnCacheKeyByAddress cur_key_by_address(graph, input_tensors, target.arch_str()); CinnCacheKeyByStructure cur_key_by_struct; @@ -258,22 +252,15 @@ std::unique_ptr CinnCompiler::CompileGraph( CinnGraphSymbolization symbol{compiled_num, graph, target, input_tensors}; auto frontend_program = symbol(); auto fetch_ids = symbol.GetFetchIds(); - ProgramPass::Apply(&frontend_program, fetch_ids, target, {"Decomposer"}); - ::cinn::frontend::ApplyPass(&frontend_program, fetch_ids, "RemoveIdentity"); - ::cinn::frontend::ApplyPass(&frontend_program, fetch_ids, "TransposeFolding"); - ProgramPass::Apply(&frontend_program, fetch_ids, target, {"GemmRewriter"}); + VLOG(4) << "All fetch var ids in CINN: " + << string::join_strings(fetch_ids, ','); - auto cinn_graph = std::make_shared<::cinn::hlir::framework::Graph>( - frontend_program, target); - VLOG(1) << "-- The " << compiled_num << "-th compilation (" + auto cinn_graph = Optimize(&frontend_program, fetch_ids, target); + VLOG(4) << "-- The " << compiled_num << "-th compilation (" << target.arch_str() << "), and its related graph:\n" << cinn_graph->Visualize(); - ApplyPass(cinn_graph.get(), "OpFusion"); - auto scope = BuildScope(target, cinn_graph); - - VLOG(4) << "All fetch var ids in CINN: " - << string::join_strings(fetch_ids, ','); + auto scope = BuildScope(target, cinn_graph); auto graph_compiler = std::make_unique(target, scope, cinn_graph); GraphCompiler::CompileOptions options; From 2f28399794014d9dd0d37dcb21537e3e03bfe92a Mon Sep 17 00:00:00 2001 From: Sing_chan <51314274+betterpig@users.noreply.github.com> Date: Thu, 21 Apr 2022 11:42:30 +0800 Subject: [PATCH 08/66] block kernel_signature in windows (#42033) --- paddle/fluid/pybind/CMakeLists.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index b0ebe5026b5d4..9c509bbd2c455 100755 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -181,10 +181,9 @@ if(WITH_PYTHON) target_link_libraries(op_function_generator ${OP_FUNCTION_GENERETOR_DEPS}) add_executable(eager_op_function_generator eager_op_function_generator.cc) target_link_libraries(eager_op_function_generator ${OP_FUNCTION_GENERETOR_DEPS}) - add_executable(kernel_signature_generator kernel_signature_generator.cc) - target_link_libraries(kernel_signature_generator ${OP_FUNCTION_GENERETOR_DEPS}) - if(WIN32) - target_link_libraries(kernel_signature_generator shlwapi.lib) + if(NOT WIN32) + add_executable(kernel_signature_generator kernel_signature_generator.cc) + target_link_libraries(kernel_signature_generator ${OP_FUNCTION_GENERETOR_DEPS}) endif() get_property (os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES) From c3b0b68005ddec4f6c8ea0cb3deff6732bd34f1a Mon Sep 17 00:00:00 2001 From: JingZhuangzhuang <75348594+JZZ-NOTE@users.noreply.github.com> Date: Thu, 21 Apr 2022 13:35:35 +0800 Subject: [PATCH 09/66] update ampere sm (#42023) * update ampere sm * update ampere sm * update ampere sm --- cmake/cuda.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmake/cuda.cmake b/cmake/cuda.cmake index 75f4f19244494..4894d615c2a35 100644 --- a/cmake/cuda.cmake +++ b/cmake/cuda.cmake @@ -132,7 +132,11 @@ function(select_nvcc_arch_flags out_variable) elseif(${CUDA_ARCH_NAME} STREQUAL "Turing") set(cuda_arch_bin "75") elseif(${CUDA_ARCH_NAME} STREQUAL "Ampere") - set(cuda_arch_bin "80") + if (${CMAKE_CUDA_COMPILER_VERSION} LESS 11.1) # CUDA 11.0 + set(cuda_arch_bin "80") + elseif (${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0) # CUDA 11.1+ + set(cuda_arch_bin "80 86") + endif() elseif(${CUDA_ARCH_NAME} STREQUAL "All") set(cuda_arch_bin ${paddle_known_gpu_archs}) elseif(${CUDA_ARCH_NAME} STREQUAL "Auto") From 9774f9650e6f2f39c0972d968c945c365e75c769 Mon Sep 17 00:00:00 2001 From: Zhangjingyu06 <92561254+Zhangjingyu06@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:16:27 +0800 Subject: [PATCH 10/66] modify batch_norm and batch_norm_grad. *test=kunlun (#41976) --- paddle/fluid/operators/batch_norm_op_xpu.cc | 176 +++++++++++--------- 1 file changed, 101 insertions(+), 75 deletions(-) diff --git a/paddle/fluid/operators/batch_norm_op_xpu.cc b/paddle/fluid/operators/batch_norm_op_xpu.cc index d6826e8710e85..da138fb482e5a 100644 --- a/paddle/fluid/operators/batch_norm_op_xpu.cc +++ b/paddle/fluid/operators/batch_norm_op_xpu.cc @@ -1,5 +1,4 @@ /* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -38,15 +37,25 @@ class BatchNormXPUKernel : public framework::OpKernel { bool global_stats = test_mode || use_global_stats; const auto &data_layout_str = ctx.Attr("data_layout"); const auto data_layout = framework::StringToDataLayout(data_layout_str); + PADDLE_ENFORCE_EQ(data_layout_str == "NCHW" || data_layout_str == "NHWC", + true, + platform::errors::InvalidArgument( + "The 'data_layout' attribute must be NCHW or NHWC. " + "But recevived 'data_layout' is [%s].", + data_layout_str)); + const auto *x = ctx.Input("X"); const auto &x_dims = x->dims(); - int temp = x_dims[3]; - temp = (x_dims.size() != 4) ? 1 : temp; - bool is_nchw = (data_layout == DataLayout::kNCHW); - const int N = x_dims[0]; - const int C = is_nchw ? x_dims[1] : temp; - const int H = is_nchw ? x_dims[2] : x_dims[1]; - const int W = is_nchw ? temp : x_dims[2]; + PADDLE_ENFORCE_EQ( + x_dims.size() >= 2 && x_dims.size() <= 5, true, + platform::errors::InvalidArgument( + "The size of input's dimensions should be between 2 and 5" + "But received: the size of input's dimensions is [%d]", + x_dims.size())); + + int N, C, H, W, D; + ExtractNCWHD(x_dims, data_layout, &N, &C, &H, &W, &D); + const auto *scale = ctx.Input("Scale"); const auto *bias = ctx.Input("Bias"); const auto *x_data = x->data(); @@ -67,6 +76,7 @@ class BatchNormXPUKernel : public framework::OpKernel { saved_variance->mutable_data(ctx.GetPlace()); auto &dev_ctx = ctx.template device_context(); + bool is_nchw = data_layout_str == "NCHW"; if (!global_stats) { auto *mean_out_data = mean_out->data(); @@ -83,35 +93,29 @@ class BatchNormXPUKernel : public framework::OpKernel { &mom_cpu); momentum = mom_tensor->data()[0]; } - if (C == 1) { - int r = xpu::batch_norm(dev_ctx.x_context(), x_data, y_data, N, 1, H, - W, epsilon, momentum, scale_data, bias_data, - saved_mean_data, saved_variance_data, - mean_out_data, variance_out_data, true); - PADDLE_ENFORCE_EQ( - r, xpu::Error_t::SUCCESS, - platform::errors::External( - "The batch_norm XPU API return wrong value[%d %s]", r, - XPUAPIErrorMsg[r])); - } else { - int r = xpu::batch_norm(dev_ctx.x_context(), x_data, y_data, N, C, H, - W, epsilon, momentum, scale_data, bias_data, - saved_mean_data, saved_variance_data, - mean_out_data, variance_out_data, is_nchw); - PADDLE_ENFORCE_EQ( - r, xpu::Error_t::SUCCESS, - platform::errors::External( - "The batch_norm XPU API return wrong value[%d %s]", r, - XPUAPIErrorMsg[r])); - } + + int r = xpu::batch_norm(dev_ctx.x_context(), x_data, y_data, N, C, H, + W, epsilon, momentum, scale_data, bias_data, + saved_mean_data, saved_variance_data, + mean_out_data, variance_out_data, is_nchw); + PADDLE_ENFORCE_EQ(r, xpu::Error_t::SUCCESS, + platform::errors::External( + "The batch_norm XPU API return wrong value[%d %s]", + r, XPUAPIErrorMsg[r])); } else { + PADDLE_ENFORCE_EQ( + data_layout_str == "NCHW", true, + platform::errors::InvalidArgument( + "The batch_norm_infer 'data_layout' attribute must be NCHW. " + "But recevived 'data_layout' is [%s].", + data_layout_str)); const auto *mean = ctx.Input("Mean"); const auto *variance = ctx.Input("Variance"); const auto *mean_data = mean->data(); const auto *variance_data = variance->data(); int r = xpu::batch_norm_infer(dev_ctx.x_context(), x_data, y_data, N, C, H, W, epsilon, scale_data, bias_data, - mean_data, variance_data, true); + mean_data, variance_data, is_nchw); PADDLE_ENFORCE_EQ( r, xpu::Error_t::SUCCESS, platform::errors::External( @@ -172,6 +176,13 @@ class BatchNormGradXPUKernel : public framework::OpKernel { const float epsilon = ctx.Attr("epsilon"); const auto data_layout = framework::StringToDataLayout(data_layout_str); + PADDLE_ENFORCE_EQ(data_layout_str == "NCHW" || data_layout_str == "NHWC", + true, + platform::errors::InvalidArgument( + "The 'data_layout' attribute must be NCHW or NHWC. " + "But recevived 'data_layout' is [%s].", + data_layout_str)); + auto *d_x = ctx.Output(framework::GradVarName("X")); auto *d_scale = ctx.Output(framework::GradVarName("Scale")); auto *d_bias = ctx.Output(framework::GradVarName("Bias")); @@ -204,13 +215,15 @@ class BatchNormGradXPUKernel : public framework::OpKernel { } const auto &x_dims = x->dims(); - int temp = x_dims[3]; - temp = (x_dims.size() != 4) ? 1 : temp; - bool is_nchw = (data_layout == DataLayout::kNCHW); - const int N = x_dims[0]; - const int C = is_nchw ? x_dims[1] : temp; - const int H = is_nchw ? x_dims[2] : x_dims[1]; - const int W = is_nchw ? temp : x_dims[2]; + PADDLE_ENFORCE_EQ( + x_dims.size() >= 2 && x_dims.size() <= 5, true, + platform::errors::InvalidArgument( + "The size of input's dimensions should be between 2 and 5" + "But received: the size of input's dimensions is [%d]", + x_dims.size())); + + int N, C, H, W, D; + ExtractNCWHD(x_dims, data_layout, &N, &C, &H, &W, &D); const auto *x_data = x->data(); const auto *d_y_data = d_y->data(); @@ -235,42 +248,45 @@ class BatchNormGradXPUKernel : public framework::OpKernel { "the size of scale's dimensions is [%d], the dimensions of scale " "is [%s].", scale->dims().size(), scale->dims())); + PADDLE_ENFORCE_EQ( + scale->dims()[0], C, + platform::errors::InvalidArgument( + "The first dimension of scale must equal to Channels[%d]. But " + "received: the first dimension of scale is [%d]", + C, scale->dims()[0])); auto &dev_ctx = ctx.template device_context(); xpu::ctx_guard RAII_GUARD(dev_ctx.x_context()); - const T *mean_data = nullptr; - const T *inv_var_data = nullptr; + const auto *batch_mean = ctx.Input("SavedMean"); + const auto *batch_inv_std = ctx.Input("SavedVariance"); + const auto *global_mean = ctx.Input("Mean"); + const auto *global_var = ctx.Input("Variance"); // TODO(guozibin): hadle the situation case of N * H * W = 1 - if (!use_global_stats) { - const auto *saved_mean = ctx.Input("SavedMean"); - // SavedVariance have been reverted in forward operator - const auto *saved_inv_variance = ctx.Input("SavedVariance"); - mean_data = saved_mean->data(); - inv_var_data = saved_inv_variance->data(); - } else { - const auto *running_mean = ctx.Input("Mean"); - const auto *running_variance = ctx.Input("Variance"); - mean_data = running_mean->data(); - inv_var_data = running_variance->data(); - float *running_inv_var_data = - RAII_GUARD.alloc_l3_or_gm(running_variance->numel()); - float *epsilon_data = RAII_GUARD.alloc_l3_or_gm(1); - int r1 = calculate_inv_var(dev_ctx.x_context(), inv_var_data, epsilon, C, - epsilon_data, running_inv_var_data); - PADDLE_ENFORCE_EQ(r1, XPU_SUCCESS, platform::errors::External( - "XPU API(batch_norm_grad " - "calculate_inv_var function) " - "return wrong value[%d %s]", - r1, XPUAPIErrorMsg[r1])); - inv_var_data = running_inv_var_data; - } if (is_inplace) { + float *global_inv_std_data = nullptr; + if (use_global_stats) { + global_inv_std_data = + RAII_GUARD.alloc_l3_or_gm(global_var->numel()); + float *epsilon_data = RAII_GUARD.alloc_l3_or_gm(1); + int r1 = + calculate_inv_var(dev_ctx.x_context(), global_var->data(), + epsilon, C, epsilon_data, global_inv_std_data); + PADDLE_ENFORCE_EQ(r1, XPU_SUCCESS, platform::errors::External( + "XPU API(batch_norm_grad " + "calculate_inv_var function) " + "return wrong value[%d %s]", + r1, XPUAPIErrorMsg[r1])); + } auto px = *x; + auto *inv_std_data = + use_global_stats ? global_inv_std_data : batch_inv_std->data(); + auto mean_data = use_global_stats ? global_mean->data() + : batch_mean->data(); int r2 = calculate_inv_BN_Y( dev_ctx.x_context(), px.mutable_data(ctx.GetPlace()), - scale->data(), bias->data(), mean_data, inv_var_data, N, + scale->data(), bias->data(), mean_data, inv_std_data, N, C, H * W, x->data()); PADDLE_ENFORCE_EQ(r2, XPU_SUCCESS, platform::errors::External( "XPU API(batch_norm_grad " @@ -278,19 +294,29 @@ class BatchNormGradXPUKernel : public framework::OpKernel { "return wrong value[%d %s]", r2, XPUAPIErrorMsg[r2])); } - if (!d_x) { - d_x_data = RAII_GUARD.alloc_l3_or_gm(x->numel()); - } - if (!d_scale) { - d_scale_data = RAII_GUARD.alloc_l3_or_gm(C); - } - if (!d_bias_data) { - d_bias_data = RAII_GUARD.alloc_l3_or_gm(C); - } - int r3 = xpu::batch_norm_grad( - dev_ctx.x_context(), x_data, d_y_data, d_x_data, N, C, H, W, scale_data, - mean_data, inv_var_data, d_scale_data, d_bias_data, is_nchw); + int r3; + bool is_nchw = data_layout_str == "NCHW"; + if (use_global_stats) { + r3 = xpu::batch_norm_grad( + dev_ctx.x_context(), x_data, d_y_data, d_x_data, N, C, H, W, + scale_data, nullptr, nullptr, d_scale_data, d_bias_data, is_nchw, + global_mean->data(), global_var->data(), epsilon); + } else { + if (!d_x) { + d_x_data = RAII_GUARD.alloc_l3_or_gm(x->numel()); + } + if (!d_scale) { + d_scale_data = RAII_GUARD.alloc_l3_or_gm(C); + } + if (!d_bias_data) { + d_bias_data = RAII_GUARD.alloc_l3_or_gm(C); + } + r3 = xpu::batch_norm_grad( + dev_ctx.x_context(), x_data, d_y_data, d_x_data, N, C, H, W, + scale_data, batch_mean->data(), batch_inv_std->data(), + d_scale_data, d_bias_data, is_nchw); + } PADDLE_ENFORCE_EQ(r3, XPU_SUCCESS, platform::errors::External( "XPU API(batch_norm_grad) return " "wrong value[%d %s]", From 7003dcaa2f5814da8584d5cf3b9b1a97cffdc8f2 Mon Sep 17 00:00:00 2001 From: sneaxiy <32832641+sneaxiy@users.noreply.github.com> Date: Thu, 21 Apr 2022 14:21:41 +0800 Subject: [PATCH 11/66] Support FP16 argmax/argmin kernel (#42038) * support int16 argmax kernel * add fp16 test --- paddle/phi/kernels/gpu/arg_min_max_kernel.cu | 2 ++ .../tests/unittests/test_arg_min_max_op.py | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/paddle/phi/kernels/gpu/arg_min_max_kernel.cu b/paddle/phi/kernels/gpu/arg_min_max_kernel.cu index 6feee512cc9f4..385ddb5e521a2 100644 --- a/paddle/phi/kernels/gpu/arg_min_max_kernel.cu +++ b/paddle/phi/kernels/gpu/arg_min_max_kernel.cu @@ -259,6 +259,7 @@ PD_REGISTER_KERNEL(arg_min, GPU, ALL_LAYOUT, phi::ArgMinKernel, + phi::dtype::float16, float, double, int32_t, @@ -270,6 +271,7 @@ PD_REGISTER_KERNEL(arg_max, GPU, ALL_LAYOUT, phi::ArgMaxKernel, + phi::dtype::float16, float, double, int32_t, diff --git a/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py b/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py index c11fb3d1e28aa..cbcb4af926951 100644 --- a/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py @@ -68,6 +68,26 @@ def initTestCase(self): self.axis = 0 +@unittest.skipIf(not paddle.is_compiled_with_cuda(), + "FP16 test runs only on GPU") +class TestCase0FP16(BaseTestCase): + def initTestCase(self): + self.op_type = 'arg_max' + self.dims = (3, 4, 5) + self.dtype = np.float16 + self.axis = 0 + + +@unittest.skipIf(not paddle.is_compiled_with_cuda(), + "FP16 test runs only on GPU") +class TestCase1FP16(BaseTestCase): + def initTestCase(self): + self.op_type = 'arg_min' + self.dims = (3, 4) + self.dtype = np.float16 + self.axis = 1 + + class TestCase2_1(BaseTestCase): def initTestCase(self): self.op_type = 'arg_max' @@ -202,4 +222,5 @@ def setUp(self): if __name__ == '__main__': + paddle.enable_static() unittest.main() From 9b1107e445d1846d97ff2408a39c3820f63f1efa Mon Sep 17 00:00:00 2001 From: Jiabin Yang <360788950@qq.com> Date: Thu, 21 Apr 2022 15:47:34 +0800 Subject: [PATCH 12/66] [Eager] Watch all eager files for now (#42039) * watch all eager files for now * support eager watch files --- tools/check_file_diff_approvals.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/check_file_diff_approvals.sh b/tools/check_file_diff_approvals.sh index ce67912eb2266..b0800a9cd845e 100644 --- a/tools/check_file_diff_approvals.sh +++ b/tools/check_file_diff_approvals.sh @@ -59,6 +59,16 @@ API_FILES=("CMakeLists.txt" "paddle/scripts/paddle_build.bat" "tools/windows/run_unittests.sh" "tools/parallel_UT_rule.py" + "python/paddle/fluid/dygraph/layers.py" + "paddle/fluid/eager/grad_node_info.h" + "paddle/fluid/eager/grad_node_info.cc" + "paddle/fluid/eager/grad_tensor_holder.h" + "paddle/fluid/eager/grad_tensor_holder.cc" + "paddle/fluid/eager/tensor_wrapper.h" + "paddle/fluid/eager/autograd_meta.cc" + "paddle/fluid/eager/autograd_meta.h" + "paddle/fluid/eager/backward.cc" + "paddle/fluid/eager/backward.h" ) approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000` @@ -178,6 +188,9 @@ for API_FILE in ${API_FILES[*]}; do elif [ "${API_FILE}" == "python/paddle/fluid/parallel_executor.py" ]; then echo_line="You must have one RD (Xreki,luotao1,zhhsplendid) approval for ${API_FILE}, which manages the underlying code for PaddlePaddle.\n" check_approval 1 12538138 6836917 7913861 + elif [ "${API_FILE}" == "python/paddle/fluid/dygraph/layers.py" ] || [ "${API_FILE}" == "paddle/fluid/eager/grad_node_info.h" ] || [ "${API_FILE}" == "paddle/fluid/eager/grad_node_info.cc" ] || [ "${API_FILE}" == "paddle/fluid/eager/grad_tensor_holder.h" ] || [ "${API_FILE}" == "paddle/fluid/eager/grad_tensor_holder.cc" ] || [ "${API_FILE}" == "paddle/fluid/eager/tensor_wrapper.h" ] || [ "${API_FILE}" == "paddle/fluid/eager/autograd_meta.cc"] || [ "${API_FILE}" == "paddle/fluid/eager/autograd_meta.h"] || [ "${API_FILE}" == "paddle/fluid/eager/backward.cc"] || [ "${API_FILE}" == "paddle/fluid/eager/backward.h"]; then + echo_line="You must have one RD (JiabinYang,chenwhql,phlrain) approval for ${API_FILE}, which manages the underlying code for PaddlePaddle.\n" + check_approval JiabinYang chenwhql phlrain else echo_line="You must have one RD (XiaoguangHu01,chenwhql,zhiqiu,Xreki,luotao1,qili93) approval for ${API_FILE}, which manages the underlying code for fluid.\n" check_approval 1 46782768 12538138 6836917 22561442 6888866 16605440 From 5c73822396151bd0c419bdef95ca1b8cc0482e02 Mon Sep 17 00:00:00 2001 From: 0x45f <23097963+0x45f@users.noreply.github.com> Date: Thu, 21 Apr 2022 16:08:30 +0800 Subject: [PATCH 13/66] Remove wrong check_variable_and_dtype in matrix_rank (#42062) --- python/paddle/tensor/linalg.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py index 6006a9dec0cbb..2c1732ad62848 100644 --- a/python/paddle/tensor/linalg.py +++ b/python/paddle/tensor/linalg.py @@ -1441,7 +1441,6 @@ def matrix_rank(x, tol=None, hermitian=False, name=None): if tol is None: attrs['use_default_tol'] = True elif isinstance(tol, Variable): - check_variable_and_dtype(tol, 'tol', ['float32'], 'matrix_rank') attrs['use_default_tol'] = False if tol.dtype != x.dtype: inputs['TolTensor'] = cast(tol, x.dtype) From db468d7dea23403f1bdd83223cc258bbd142e4d7 Mon Sep 17 00:00:00 2001 From: jakpiase Date: Thu, 21 Apr 2022 10:48:04 +0200 Subject: [PATCH 14/66] oneDNN md-in-tensor 2nd batch of changes (#41997) --- .../operators/mkldnn/activation_mkldnn_op.cc | 10 ++-- .../operators/mkldnn/batch_norm_mkldnn_op.cc | 51 +++------------- .../fluid/operators/mkldnn/clip_mkldnn_op.cc | 6 +- .../operators/mkldnn/concat_mkldnn_op.cc | 17 ++---- .../operators/mkldnn/expand_v2_mkldnn_op.cc | 15 +++-- .../mkldnn/gaussian_random_mkldnn_op.cc | 11 +++- .../operators/mkldnn/interpolate_mkldnn_op.cc | 20 +++---- .../operators/mkldnn/layer_norm_mkldnn_op.cc | 23 ++++---- .../operators/mkldnn/log_softmax_mkldnn_op.cc | 9 +-- .../fluid/operators/mkldnn/lrn_mkldnn_op.cc | 21 ++----- .../fluid/operators/mkldnn/pool_mkldnn_op.cc | 59 ++++--------------- .../fluid/operators/mkldnn/prelu_mkldnn_op.cc | 29 ++++----- .../fluid/operators/mkldnn/scale_mkldnn_op.cc | 3 +- .../fluid/operators/mkldnn/shape_mkldnn_op.cc | 12 ++-- .../mkldnn/shuffle_channel_mkldnn_op.cc | 8 +-- .../operators/mkldnn/softmax_mkldnn_op.cc | 27 +++------ .../operators/mkldnn/softplus_mkldnn_op.h | 13 ++-- .../mkldnn/test_expand_v2_mkldnn_op.py | 31 +++++----- 18 files changed, 121 insertions(+), 244 deletions(-) diff --git a/paddle/fluid/operators/mkldnn/activation_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/activation_mkldnn_op.cc index bdd868c1e262a..ecee094de346e 100644 --- a/paddle/fluid/operators/mkldnn/activation_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/activation_mkldnn_op.cc @@ -107,8 +107,7 @@ void eltwise_forward(const framework::ExecutionContext &ctx, astream, {{DNNL_ARG_FROM, *src_memory_p}, {DNNL_ARG_TO, *dst_memory_p}}); astream.wait(); - out->set_layout(DataLayout::kMKLDNN); - out->set_format(GetMKLDNNFormat(*dst_memory_p)); + out->set_mem_desc(dst_memory_p->get_desc()); } template @@ -136,8 +135,7 @@ void eltwise_grad(const framework::ExecutionContext &ctx, {DNNL_ARG_DIFF_SRC, *diff_src_memory_p}}); astream.wait(); - dx->set_layout(DataLayout::kMKLDNN); - dx->set_format(GetMKLDNNFormat(*diff_src_memory_p)); + dx->set_mem_desc(diff_src_memory_p->get_desc()); } template @@ -165,8 +163,7 @@ void eltwise_grad_use_out(const framework::ExecutionContext &ctx, {DNNL_ARG_DIFF_SRC, *diff_src_memory_p}}); astream.wait(); - dx->set_layout(DataLayout::kMKLDNN); - dx->set_format(GetMKLDNNFormat(*diff_src_memory_p)); + dx->set_mem_desc(diff_src_memory_p->get_desc()); } template @@ -347,6 +344,7 @@ namespace ops = paddle::operators; FOR_EACH_MKLDNN_KERNEL_FUNCTOR(REGISTER_ACTIVATION_MKLDNN_KERNEL); +// round eltwise primitive doesn't support BF16, nor does it support grad REGISTER_ACTIVATION_MKLDNN_KERNEL_FWD_ONLY(round, RoundMKLDNNFunctor); namespace ops = paddle::operators; diff --git a/paddle/fluid/operators/mkldnn/batch_norm_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/batch_norm_mkldnn_op.cc index 900d3e54c7971..3abdb905401c1 100644 --- a/paddle/fluid/operators/mkldnn/batch_norm_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/batch_norm_mkldnn_op.cc @@ -54,17 +54,6 @@ class BatchNormMKLDNNHandler : public platform::MKLDNNHandlerNoCachingT< std::vector DataLayout_error_msg = {"kNHWC", "kNCHW", "kAnyLayout", "kMKLDNN"}; - PADDLE_ENFORCE_EQ( - x->layout(), DataLayout::kMKLDNN, - platform::errors::InvalidArgument( - "Wrong layout set for X tensor. Expected layout is `kMKLDNN`, " - "But received %s.", - DataLayout_error_msg[static_cast(DataLayout::kMKLDNN)])); - PADDLE_ENFORCE_NE( - x->format(), MKLDNNMemoryFormat::undef, - platform::errors::InvalidArgument("Wrong format set for X tensor")); - - auto src_tz = phi::vectorize(x->dims()); // Flags are added by bitwise OR operation auto flags = dnnl::normalization_flags::use_scale_shift; // 001 @@ -73,14 +62,10 @@ class BatchNormMKLDNNHandler : public platform::MKLDNNHandlerNoCachingT< if (fuse_with_relu && test_mode) flags |= dnnl::normalization_flags::fuse_norm_relu; // 100 - auto md = dnnl::memory::desc( - src_tz, platform::MKLDNNGetDataType(), - platform::MKLDNNFormatForSize(src_tz.size(), x->format())); - this->AcquireForwardPrimitiveDescriptor( global_stats == true ? dnnl::prop_kind::forward_scoring : dnnl::prop_kind::forward_training, - md, epsilon, flags); + x->mem_desc(), epsilon, flags); } BatchNormMKLDNNHandler(const paddle::framework::ExecutionContext &ctx, @@ -89,14 +74,6 @@ class BatchNormMKLDNNHandler : public platform::MKLDNNHandlerNoCachingT< : platform::MKLDNNHandlerNoCachingT( mkldnn_engine, ctx.GetPlace()) { - PADDLE_ENFORCE_EQ(out_grad->layout(), DataLayout::kMKLDNN, - platform::errors::InvalidArgument( - "Wrong layout set for Input out_grad tensor")); - PADDLE_ENFORCE_NE(out_grad->format(), MKLDNNMemoryFormat::undef, - platform::errors::InvalidArgument( - "Wrong format set for Input out_grad tensor")); - - auto src_tz = phi::vectorize(in_x->dims()); auto scale_tz = phi::vectorize(scale->dims()); PADDLE_ENFORCE_EQ( scale_tz.size(), 1, @@ -104,26 +81,14 @@ class BatchNormMKLDNNHandler : public platform::MKLDNNHandlerNoCachingT< "Dims of scale tensor must be 1, but received scale's size is %d", scale_tz.size())); - MKLDNNMemoryFormat diff_fmt = - platform::MKLDNNFormatForSize(src_tz.size(), out_grad->format()); - - MKLDNNMemoryFormat src_fmt = - platform::MKLDNNFormatForSize(src_tz.size(), in_x->format()); - - auto dims = phi::vectorize(in_x->dims()); - auto diff_dst_md = - dnnl::memory::desc(dims, platform::MKLDNNGetDataType(), diff_fmt); - auto src_md = - dnnl::memory::desc(dims, platform::MKLDNNGetDataType(), src_fmt); - const float epsilon = ctx.Attr("epsilon"); this->AcquireForwardPrimitiveDescriptor( - dnnl::prop_kind::forward_training, src_md, epsilon, + dnnl::prop_kind::forward_training, in_x->mem_desc(), epsilon, dnnl::normalization_flags::use_scale_shift); this->AcquireBackwardPrimitiveDescriptor( - dnnl::prop_kind::backward, diff_dst_md, src_md, epsilon, - dnnl::normalization_flags::use_scale_shift); + dnnl::prop_kind::backward, out_grad->mem_desc(), in_x->mem_desc(), + epsilon, dnnl::normalization_flags::use_scale_shift); } std::shared_ptr AcquireScaleShiftMemory(const Tensor *scale, @@ -227,8 +192,7 @@ class BatchNormMKLDNNOpKernel : public paddle::framework::OpKernel { variance_memory = handler.AcquireVarianceMemory(batch_variance); } - y->set_layout(DataLayout::kMKLDNN); - y->set_format(platform::GetMKLDNNFormat(*dst_memory)); + y->set_mem_desc(dst_memory->get_desc()); auto &astream = platform::MKLDNNDeviceContext::tls().get_stream(); batch_norm_p->execute(astream, {{DNNL_ARG_SRC, *src_memory}, @@ -322,9 +286,8 @@ class BatchNormMKLDNNGradOpKernel : public paddle::framework::OpKernel { std::copy(std::next(it, C), std::end(diff_scaleshift_data), diff_shift_data); - // set layout/format of output tensors - diff_x->set_layout(DataLayout::kMKLDNN); - diff_x->set_format(platform::GetMKLDNNFormat(*diff_src_memory)); + // set memory descriptor of out tensor + diff_x->set_mem_desc(diff_src_memory->get_desc()); } }; } // namespace operators diff --git a/paddle/fluid/operators/mkldnn/clip_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/clip_mkldnn_op.cc index 83ccd80e171b9..bfa7db82bd148 100644 --- a/paddle/fluid/operators/mkldnn/clip_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/clip_mkldnn_op.cc @@ -46,8 +46,7 @@ class ClipMKLDNNKernel : public paddle::framework::OpKernel { {DNNL_ARG_TO, *dst_memory_p}}); astream.wait(); - out->set_layout(paddle::framework::DataLayout::kMKLDNN); - out->set_format(paddle::platform::GetMKLDNNFormat(*dst_memory_p)); + out->set_mem_desc(dst_memory_p->get_desc()); } }; @@ -83,8 +82,7 @@ class ClipGradMKLDNNKernel : public paddle::framework::OpKernel { {DNNL_ARG_DIFF_SRC, *diff_src_memory_p}}); astream.wait(); - dx->set_layout(paddle::framework::DataLayout::kMKLDNN); - dx->set_format(paddle::platform::GetMKLDNNFormat(*diff_dst_memory_p)); + dx->set_mem_desc(diff_dst_memory_p->get_desc()); } }; diff --git a/paddle/fluid/operators/mkldnn/concat_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/concat_mkldnn_op.cc index 4b8e5f0334ff6..5095fa067193a 100644 --- a/paddle/fluid/operators/mkldnn/concat_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/concat_mkldnn_op.cc @@ -68,8 +68,7 @@ class ConcatMKLDNNHandler // Create memory descriptors for each of inputs for (size_t i = 0; i < inputs.size(); ++i) { - const auto dims = phi::vectorize(inputs[i]->dims()); - srcs_md.emplace_back(memory::desc(dims, dt, inputs[i]->format())); + srcs_md.push_back(inputs[i]->mem_desc()); } auto dst_dims = phi::vectorize(output->dims()); @@ -99,9 +98,6 @@ static void EnforceLayouts(const std::vector inputs) { PADDLE_ENFORCE_EQ( input->layout(), DataLayout::kMKLDNN, platform::errors::InvalidArgument("Wrong layout set for Input tensor")); - PADDLE_ENFORCE_NE( - input->format(), MKLDNNMemoryFormat::undef, - platform::errors::InvalidArgument("Wrong format set for Input tensor")); } } @@ -147,8 +143,7 @@ class ConcatMKLDNNOpKernel : public paddle::framework::OpKernel { concat_p->execute(astream, args); astream.wait(); - output->set_layout(DataLayout::kMKLDNN); - output->set_format(platform::GetMKLDNNFormat(*dst_mem)); + output->set_mem_desc(dst_mem->get_desc()); } }; @@ -192,7 +187,7 @@ class ConcatGradMKLDNNOpKernel : public paddle::framework::OpKernel { dout_vec_dims, framework::TransToProtoVarType(dout->dtype()), dout_type, onednn_engine); auto reorder_src_memory_p = reorder_handler.AcquireSrcMemory( - dout->format(), platform::to_void_cast(dout->data())); + dout->mem_desc(), platform::to_void_cast(dout->data())); for (size_t i = 0; i < dx.size(); ++i) { if (out_var_names[i] != framework::kEmptyVarName && @@ -202,7 +197,8 @@ class ConcatGradMKLDNNOpKernel : public paddle::framework::OpKernel { dx_vec_dims, offset, reorder_src_memory_p); auto reorder_dst_memory_p = reorder_handler.AcquireDstMemory( - dx[i], dx_vec_dims, dout->format(), ctx.GetPlace()); + dx[i], dx_vec_dims, + platform::GetPlainMKLDNNFormat(dx_vec_dims.size()), ctx.GetPlace()); auto reorder_p = reorder_handler.AcquireReorder(reorder_dst_memory_p, slice_mem_p); @@ -210,8 +206,7 @@ class ConcatGradMKLDNNOpKernel : public paddle::framework::OpKernel { offset[axis] += dx[i]->dims()[axis]; - dx[i]->set_layout(framework::DataLayout::kMKLDNN); - dx[i]->set_format(platform::GetMKLDNNFormat(*reorder_dst_memory_p)); + dx[i]->set_mem_desc(reorder_dst_memory_p->get_desc()); } } astream.wait(); diff --git a/paddle/fluid/operators/mkldnn/expand_v2_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/expand_v2_mkldnn_op.cc index 7a81e90e455d3..05d6bae5f719a 100644 --- a/paddle/fluid/operators/mkldnn/expand_v2_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/expand_v2_mkldnn_op.cc @@ -115,10 +115,11 @@ class ExpandGradMKLDNNKernel : public paddle::framework::OpKernel { dout_type, onednn_engine); auto reorder_src_memory_p = reorder_handler.AcquireSrcMemory( - dout->format(), paddle::platform::to_void_cast(dout->data())); + dout->mem_desc(), paddle::platform::to_void_cast(dout->data())); - auto reorder_dst_memory_p = - reorder_handler.AcquireDstMemory(dx, dout->format(), ctx.GetPlace()); + auto reorder_dst_memory_p = reorder_handler.AcquireDstMemory( + dx, paddle::platform::GetPlainMKLDNNFormat(dx_vec_dims.size()), + ctx.GetPlace()); auto reorder_p = reorder_handler.AcquireReorder(reorder_src_memory_p, reorder_dst_memory_p); @@ -126,9 +127,7 @@ class ExpandGradMKLDNNKernel : public paddle::framework::OpKernel { reorder_p->execute(astream, *reorder_src_memory_p, *reorder_dst_memory_p); astream.wait(); - dx->set_layout(paddle::framework::DataLayout::kMKLDNN); - dx->set_format( - paddle::platform::GetMKLDNNFormat(reorder_dst_memory_p->get_desc())); + dx->set_mem_desc(reorder_dst_memory_p->get_desc()); } else { paddle::platform::ReductionMKLDNNHandler handler( dnnl::algorithm::reduction_sum, 0.0f, 0.0f, onednn_engine, @@ -145,8 +144,8 @@ class ExpandGradMKLDNNKernel : public paddle::framework::OpKernel { reduction_p->execute(astream, reduction_args); astream.wait(); dx->set_layout(paddle::framework::DataLayout::kMKLDNN); - dx->set_format(paddle::platform::GetMKLDNNFormat( - dst_memory_p->get_desc().reshape(vectorize(dx->dims())))); + dx->set_mem_desc( + dst_memory_p->get_desc().reshape(vectorize(dx->dims()))); } } }; diff --git a/paddle/fluid/operators/mkldnn/gaussian_random_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/gaussian_random_mkldnn_op.cc index de999035fa5d8..1a122503c0f3c 100644 --- a/paddle/fluid/operators/mkldnn/gaussian_random_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/gaussian_random_mkldnn_op.cc @@ -16,7 +16,7 @@ limitations under the License. */ #include "paddle/fluid/framework/generator.h" #include "paddle/fluid/operators/fill_constant_op.h" -#include "paddle/fluid/platform/mkldnn_helper.h" +#include "paddle/fluid/platform/mkldnn_reuse.h" namespace paddle { namespace operators { @@ -42,8 +42,13 @@ class GaussianMKLDNNKernel : public paddle::framework::OpKernel { data[i] = dist(*engine); } - tensor->set_layout(DataLayout::kMKLDNN); - tensor->set_format(platform::GetPlainMKLDNNFormat(tensor->dims().size())); + dnnl::memory::desc out_mem_desc( + phi::vectorize(tensor->dims()), + framework::ToMKLDNNDataType( + framework::TransToProtoVarType(tensor->dtype())), + platform::GetPlainMKLDNNFormat(tensor->dims().size())); + + tensor->set_mem_desc(out_mem_desc); } }; } // namespace operators diff --git a/paddle/fluid/operators/mkldnn/interpolate_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/interpolate_mkldnn_op.cc index 04b90d2f1f380..37d6c07290312 100644 --- a/paddle/fluid/operators/mkldnn/interpolate_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/interpolate_mkldnn_op.cc @@ -34,17 +34,14 @@ class InterpolateMKLDNNHandler public: InterpolateMKLDNNHandler(const dnnl::algorithm algo, const dnnl::engine engine, platform::Place cpu_place, - const Tensor* x, Tensor* z) + const Tensor* x, Tensor* out) : platform::MKLDNNHandlerNoCachingT( engine, cpu_place) { - const auto src_x_tz = phi::vectorize(x->dims()); - const auto dst_tz = phi::vectorize(z->dims()); - const auto src_md = dnnl::memory::desc( - src_x_tz, platform::MKLDNNGetDataType(), x->format()); + const auto dst_tz = phi::vectorize(out->dims()); const auto dst_md = memory::desc(dst_tz, platform::MKLDNNGetDataType(), MKLDNNMemoryFormat::any); this->AcquireForwardPrimitiveDescriptor(dnnl::prop_kind::forward_inference, - algo, src_md, dst_md); + algo, x->mem_desc(), dst_md); } }; @@ -133,7 +130,7 @@ class InterpolateMKLDNNKernel : public framework::OpKernel { const auto& mkldnn_engine = dev_ctx.GetEngine(); const auto* x = ctx.Input("X"); - auto* z = ctx.Output("Out"); + auto* out = ctx.Output("Out"); const auto interp_method = ctx.Attr("interp_method"); const dnnl::algorithm algo = (interp_method == "nearest") @@ -142,13 +139,13 @@ class InterpolateMKLDNNKernel : public framework::OpKernel { const auto out_dims_vec = ComputeOutputShape(ctx); framework::DDim dim_out = phi::make_ddim(out_dims_vec); - z->Resize(dim_out); + out->Resize(dim_out); InterpolateMKLDNNHandler handler(algo, mkldnn_engine, ctx.GetPlace(), x, - z); + out); auto src_memory_p = handler.AcquireSrcMemory(x); - auto dst_memory_p = handler.AcquireDstMemory(z); + auto dst_memory_p = handler.AcquireDstMemory(out); auto resampling_prim = handler.AcquireForwardPrimitive(); const std::unordered_map args = { @@ -158,8 +155,7 @@ class InterpolateMKLDNNKernel : public framework::OpKernel { resampling_prim->execute(astream, args); astream.wait(); - z->set_layout(DataLayout::kMKLDNN); - z->set_format(platform::GetMKLDNNFormat(*dst_memory_p)); + out->set_mem_desc(dst_memory_p->get_desc()); } }; diff --git a/paddle/fluid/operators/mkldnn/layer_norm_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/layer_norm_mkldnn_op.cc index 2e82b47e8da1c..8f98a0b9fbee8 100644 --- a/paddle/fluid/operators/mkldnn/layer_norm_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/layer_norm_mkldnn_op.cc @@ -25,22 +25,21 @@ class LayerNormMKLDNNHandler : public platform::MKLDNNHandlerNoCachingT< public: LayerNormMKLDNNHandler(const std::vector& dims, const float& epsilon, const dnnl::normalization_flags& flags, - const bool& is_test, const MKLDNNMemoryFormat fmt, + const bool& is_test, const Tensor* x, const dnnl::engine engine, platform::Place cpu_place) : platform::MKLDNNHandlerNoCachingT( engine, cpu_place) { - auto md = dnnl::memory::desc(dims, platform::MKLDNNGetDataType(), fmt); if (!is_test) { // TODO(grygielski) Delete forcing stats_md after DNNL 1.2 is introduced auto stats_md = dnnl::memory::desc( {begin(dims), end(dims) - 1}, platform::MKLDNNGetDataType(), - platform::MKLDNNFormatForSize(dims.size() - 1, - MKLDNNMemoryFormat::nchw)); + platform::GetPlainMKLDNNFormat(dims.size() - 1)); this->AcquireForwardPrimitiveDescriptor(dnnl::prop_kind::forward_training, - md, stats_md, epsilon, flags); + x->mem_desc(), stats_md, epsilon, + flags); } else { this->AcquireForwardPrimitiveDescriptor( - dnnl::prop_kind::forward_inference, md, epsilon, flags); + dnnl::prop_kind::forward_inference, x->mem_desc(), epsilon, flags); } } @@ -83,7 +82,7 @@ class LayerNormMKLDNNOpKernel : public paddle::framework::OpKernel { auto* x = ctx.Input("X"); auto* scale = ctx.Input("Scale"); auto* bias = ctx.Input("Bias"); - auto* y = ctx.Output("Y"); + auto* out = ctx.Output("Y"); const float epsilon = ctx.Attr("epsilon"); const auto begin_norm_axis = ctx.Attr("begin_norm_axis"); @@ -107,12 +106,11 @@ class LayerNormMKLDNNOpKernel : public paddle::framework::OpKernel { flags |= dnnl::normalization_flags::use_scale_shift; } - LayerNormMKLDNNHandler handler(src_tz, epsilon, flags, is_test, - x->format(), mkldnn_engine, - ctx.GetPlace()); + LayerNormMKLDNNHandler handler(src_tz, epsilon, flags, is_test, x, + mkldnn_engine, ctx.GetPlace()); auto src_memory = handler.AcquireSrcMemory(x); - auto dst_memory = handler.AcquireDstMemory(y); + auto dst_memory = handler.AcquireDstMemory(out); auto layer_norm_p = handler.AcquireForwardPrimitive(); @@ -140,8 +138,7 @@ class LayerNormMKLDNNOpKernel : public paddle::framework::OpKernel { layer_norm_p->execute(astream, args); astream.wait(); - y->set_layout(phi::DataLayout::kMKLDNN); - y->set_format(platform::GetMKLDNNFormat(*dst_memory)); + out->set_mem_desc(dst_memory->get_desc()); } }; diff --git a/paddle/fluid/operators/mkldnn/log_softmax_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/log_softmax_mkldnn_op.cc index 626d3ef40b166..a4d768e84d7d9 100644 --- a/paddle/fluid/operators/mkldnn/log_softmax_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/log_softmax_mkldnn_op.cc @@ -28,12 +28,8 @@ class LogSoftmaxMKLDNNHandler const int axis) : platform::MKLDNNHandlerNoCachingT( mkldnn_engine, cpu_place) { - const auto logsoftmax_tz = phi::vectorize(x->dims()); - const auto md = dnnl::memory::desc( - logsoftmax_tz, platform::MKLDNNGetDataType(), x->format()); - this->AcquireForwardPrimitiveDescriptor(dnnl::prop_kind::forward_inference, - md, axis); + x->mem_desc(), axis); } }; @@ -63,8 +59,7 @@ class LogSoftmaxMKLDNNKernel : public framework::OpKernel { {DNNL_ARG_DST, *dst_memory_p}}); astream.wait(); - out->set_layout(framework::DataLayout::kMKLDNN); - out->set_format(x->format()); + out->set_mem_desc(dst_memory_p->get_desc()); } }; } // namespace operators diff --git a/paddle/fluid/operators/mkldnn/lrn_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/lrn_mkldnn_op.cc index 849dba8538f49..d3a36555c389a 100644 --- a/paddle/fluid/operators/mkldnn/lrn_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/lrn_mkldnn_op.cc @@ -44,15 +44,11 @@ class LRNMKLDNNHandler const float k = ctx.Attr("k"); bool is_test = ctx.Attr("is_test"); - auto dims = phi::vectorize(input->dims()); - - auto src_md = dnnl::memory::desc(dims, platform::MKLDNNGetDataType(), - input->format()); - this->AcquireForwardPrimitiveDescriptor( is_test ? dnnl::prop_kind::forward_inference : dnnl::prop_kind::forward_training, - dnnl::algorithm::lrn_across_channels, src_md, n, alpha, beta, k); + dnnl::algorithm::lrn_across_channels, input->mem_desc(), n, alpha, beta, + k); } LRNMKLDNNHandler(const framework::ExecutionContext& ctx, @@ -72,20 +68,13 @@ class LRNMKLDNNHandler const float beta = ctx.Attr("beta"); const float k = ctx.Attr("k"); - auto dims = phi::vectorize(in_x->dims()); - - auto src_md = dnnl::memory::desc(dims, platform::MKLDNNGetDataType(), - in_x->format()); - auto diff_md = dnnl::memory::desc(dims, platform::MKLDNNGetDataType(), - out_grad->format()); - this->AcquireForwardPrimitiveDescriptor( dnnl::prop_kind::forward_training, dnnl::algorithm::lrn_across_channels, - src_md, n, alpha, beta, k); + in_x->mem_desc(), n, alpha, beta, k); this->AcquireBackwardPrimitiveDescriptor( - dnnl::algorithm::lrn_across_channels, src_md, diff_md, n, alpha, beta, - k); + dnnl::algorithm::lrn_across_channels, in_x->mem_desc(), + out_grad->mem_desc(), n, alpha, beta, k); } std::shared_ptr AcquireWorkspaceMemory(Tensor* workspace) { diff --git a/paddle/fluid/operators/mkldnn/pool_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/pool_mkldnn_op.cc index 1078b451c55ba..77763531c8296 100644 --- a/paddle/fluid/operators/mkldnn/pool_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/pool_mkldnn_op.cc @@ -41,13 +41,6 @@ class PoolingMKLDNNHandler : platform::MKLDNNHandlerNoCachingT( mkldnn_engine, ctx.GetPlace()) { - PADDLE_ENFORCE_EQ(input->layout(), DataLayout::kMKLDNN, - platform::errors::InvalidArgument( - "Wrong layout set for Input tensor.")); - PADDLE_ENFORCE_NE(input->format(), MKLDNNMemoryFormat::undef, - platform::errors::InvalidArgument( - "Wrong format set for Input tensor.")); - const std::string pooling_type = ctx.Attr("pooling_type"); std::vector ksize_temp = ctx.Attr>("ksize"); @@ -91,29 +84,18 @@ class PoolingMKLDNNHandler phi::funcs::UpdatePadding(&paddings, global_pooling, 0, padding_algorithm, data_dims, strides, ksize); - const auto src_tz = phi::vectorize(input->dims()); - const auto dst_tz = phi::vectorize(output->dims()); - const auto is_test = ctx.Attr("is_test"); + const bool ceil_mode = ctx.Attr("ceil_mode"); + const auto exclude_padding = ctx.Attr("exclusive"); + auto mkldnn_paddings = platform::ToMkldnnPadding(paddings); const auto dt = framework::ToMKLDNNDataType( framework::TransToProtoVarType(input->dtype())); - - const auto exclude_padding = ctx.Attr("exclusive"); - - const auto src_md = dnnl::memory::desc(src_tz, dt, input->format()); - /* create memory descriptor for pooling without specified format - * ('any') which lets a primitive (pooling in this case) choose - * the memory format preferred for best performance - */ - + const auto src_tz = phi::vectorize(input->dims()); + const auto dst_tz = phi::vectorize(output->dims()); const auto dst_md = platform::MKLDNNMemDesc(dst_tz, dt, MKLDNNMemoryFormat::any); - auto mkldnn_paddings = platform::ToMkldnnPadding(paddings); - - const bool ceil_mode = ctx.Attr("ceil_mode"); - if (ceil_mode) { CorrectOutputSize(src_tz, dst_tz, ksize, paddings, strides, mkldnn_paddings[1]); @@ -128,7 +110,8 @@ class PoolingMKLDNNHandler ? dnnl::algorithm::pooling_max : (exclude_padding ? dnnl::algorithm::pooling_avg_exclude_padding : dnnl::algorithm::pooling_avg_include_padding), - src_md, dst_md, strides, ksize, mkldnn_paddings[0], mkldnn_paddings[1]); + input->mem_desc(), dst_md, strides, ksize, mkldnn_paddings[0], + mkldnn_paddings[1]); } PoolingMKLDNNHandler(const paddle::framework::ExecutionContext& ctx, @@ -138,20 +121,6 @@ class PoolingMKLDNNHandler : platform::MKLDNNHandlerNoCachingT( mkldnn_engine, ctx.GetPlace()) { - PADDLE_ENFORCE_EQ( - in_x->layout(), DataLayout::kMKLDNN, - platform::errors::InvalidArgument("Wrong layout set for Input tensor")); - PADDLE_ENFORCE_NE( - in_x->format(), MKLDNNMemoryFormat::undef, - platform::errors::InvalidArgument("Wrong format set for Input tensor")); - - PADDLE_ENFORCE_EQ(out_grad->layout(), DataLayout::kMKLDNN, - platform::errors::InvalidArgument( - "Wrong layout set for Input output_grad tensor")); - PADDLE_ENFORCE_NE(out_grad->format(), MKLDNNMemoryFormat::undef, - platform::errors::InvalidArgument( - "Wrong format set for Input output_grad tensor")); - PADDLE_ENFORCE_EQ( ctx.Attr("is_test"), false, platform::errors::InvalidArgument( @@ -187,10 +156,7 @@ class PoolingMKLDNNHandler const auto dt = framework::ToMKLDNNDataType( framework::TransToProtoVarType(in_x->dtype())); - auto src_md = dnnl::memory::desc(src_tz, dt, in_x->format()); auto dst_md = dnnl::memory::desc(diff_dst_tz, dt, MKLDNNMemoryFormat::any); - auto diff_dst_md = dnnl::memory::desc( - diff_dst_tz, platform::MKLDNNGetDataType(), out_grad->format()); auto diff_src_md = dnnl::memory::desc( diff_src_tz, platform::MKLDNNGetDataType(), MKLDNNMemoryFormat::any); @@ -211,14 +177,15 @@ class PoolingMKLDNNHandler ? dnnl::algorithm::pooling_max : (exclude_padding ? dnnl::algorithm::pooling_avg_exclude_padding : dnnl::algorithm::pooling_avg_include_padding), - src_md, dst_md, strides, ksize, mkldnn_paddings[0], mkldnn_paddings[1]); + in_x->mem_desc(), dst_md, strides, ksize, mkldnn_paddings[0], + mkldnn_paddings[1]); this->AcquireBackwardPrimitiveDescriptor( pooling_type == "max" ? dnnl::algorithm::pooling_max : (exclude_padding ? dnnl::algorithm::pooling_avg_exclude_padding : dnnl::algorithm::pooling_avg_include_padding), - diff_src_md, diff_dst_md, strides, ksize, mkldnn_paddings[0], + diff_src_md, out_grad->mem_desc(), strides, ksize, mkldnn_paddings[0], mkldnn_paddings[1]); } @@ -327,8 +294,7 @@ class PoolMKLDNNOpKernel : public paddle::framework::OpKernel { } astream.wait(); - output->set_layout(DataLayout::kMKLDNN); - output->set_format(platform::GetMKLDNNFormat(*dst_memory)); + output->set_mem_desc(dst_memory->get_desc()); } }; @@ -369,8 +335,7 @@ class PoolMKLDNNGradOpKernel : public paddle::framework::OpKernel { } astream.wait(); - in_x_grad->set_layout(DataLayout::kMKLDNN); - in_x_grad->set_format(platform::GetMKLDNNFormat(*diff_src_memory)); + in_x_grad->set_mem_desc(diff_src_memory->get_desc()); } // Compute() }; diff --git a/paddle/fluid/operators/mkldnn/prelu_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/prelu_mkldnn_op.cc index 86ecb01c89af7..e459f8b8e1cf8 100644 --- a/paddle/fluid/operators/mkldnn/prelu_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/prelu_mkldnn_op.cc @@ -41,9 +41,6 @@ class PReluMKLDNNHandler platform::CreateKey(dev_ctx, phi::vectorize(x->dims()), uniq_name)) { if (unlikely(!this->isCached())) { - auto x_md = memory::desc(phi::vectorize(x->dims()), - MKLDNNGetDataType(), x->format()); - auto weights_dims = phi::vectorize(weights->dims()); // weights must have same size as X only for "element" case @@ -59,30 +56,28 @@ class PReluMKLDNNHandler memory::format_tag::any); this->AcquireForwardPrimitiveDescriptor(dnnl::prop_kind::forward_training, - x_md, weights_md); + x->mem_desc(), weights_md); if (!is_test) - this->AcquireBackwardPrimitiveDescriptor(x_md, weights_md, x_md, - weights_md); + this->AcquireBackwardPrimitiveDescriptor(x->mem_desc(), weights_md, + x->mem_desc(), weights_md); } } std::shared_ptr AcquireWeightsMemoryPossiblyWithReorder( - const Tensor* input, const bool is_test) { - const T* input_data = input->data(); + const Tensor* weights, const bool is_test) { + const T* weights_data = weights->data(); // if weights are 1D, every format tag is correct, so we accept // format_tag::any's output and no reorder is needed - if (input->dims().size() == 1) { + if (weights->dims().size() == 1) { return this->AcquireMemoryFromPrimitive(this->fwd_pd_->weights_desc(), - to_void_cast(input_data), + to_void_cast(weights_data), "@alpha_mem_p"); } - auto user_weights_md = memory::desc( - phi::vectorize(input->dims()), MKLDNNGetDataType(), input->format()); return this->AcquireMemoryWithReorder( - user_weights_md, this->fwd_pd_->weights_desc(), - to_void_cast(input_data), "@alpha_mem_p", is_test); + weights->mem_desc(), this->fwd_pd_->weights_desc(), + to_void_cast(weights_data), "@alpha_mem_p", is_test); } std::shared_ptr AcquireDiffWeightsMemory(Tensor* output) { @@ -128,8 +123,7 @@ class PReluMKLDNNKernel : public framework::OpKernel { {DNNL_ARG_DST, *dst_memory_p}}); astream.wait(); - out->set_layout(framework::DataLayout::kMKLDNN); - out->set_format(GetMKLDNNFormat(*dst_memory_p)); + out->set_mem_desc(dst_memory_p->get_desc()); } }; @@ -174,8 +168,7 @@ class PReluGradMKLDNNKernel : public framework::OpKernel { {DNNL_ARG_DIFF_WEIGHTS, *diff_weights_memory_p}}); astream.wait(); - dx->set_layout(framework::DataLayout::kMKLDNN); - dx->set_format(GetMKLDNNFormat(*diff_src_memory_p)); + dx->set_mem_desc(diff_src_memory_p->get_desc()); } }; } // namespace operators diff --git a/paddle/fluid/operators/mkldnn/scale_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/scale_mkldnn_op.cc index d7b4574fb0dc8..6139b3c9be22b 100644 --- a/paddle/fluid/operators/mkldnn/scale_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/scale_mkldnn_op.cc @@ -54,8 +54,7 @@ class ScaleMKLDNNKernel : public framework::OpKernel { {DNNL_ARG_TO, *dst_memory_p}}); astream.wait(); - out->set_layout(framework::DataLayout::kMKLDNN); - out->set_format(platform::GetMKLDNNFormat(*dst_memory_p)); + out->set_mem_desc(dst_memory_p->get_desc()); } }; } // namespace operators diff --git a/paddle/fluid/operators/mkldnn/shape_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/shape_mkldnn_op.cc index a3b764b0e1c46..f04c73ec0b249 100644 --- a/paddle/fluid/operators/mkldnn/shape_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/shape_mkldnn_op.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/framework/op_registry.h" -#include "paddle/fluid/platform/mkldnn_helper.h" +#include "paddle/fluid/platform/mkldnn_reuse.h" namespace paddle { namespace operators { @@ -40,9 +40,13 @@ class ShapeMKLDNNKernel : public framework::OpKernel { out_data[i] = in_dims[i]; } - auto* out = ctx.Output("Out"); - out->set_layout(framework::DataLayout::kMKLDNN); - out->set_format(platform::GetPlainMKLDNNFormat(out->dims().size())); + dnnl::memory::desc out_mem_desc( + phi::vectorize(out_t->dims()), + framework::ToMKLDNNDataType( + framework::TransToProtoVarType(out_t->dtype())), + platform::GetPlainMKLDNNFormat(out_t->dims().size())); + + out_t->set_mem_desc(out_mem_desc); } }; } // namespace operators diff --git a/paddle/fluid/operators/mkldnn/shuffle_channel_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/shuffle_channel_mkldnn_op.cc index 408de57bf946d..79b0692748dcf 100644 --- a/paddle/fluid/operators/mkldnn/shuffle_channel_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/shuffle_channel_mkldnn_op.cc @@ -29,11 +29,8 @@ class ShuffleChannelMKLDNNHandler : platform::MKLDNNHandlerNoCachingT(engine, cpu_place) { static constexpr int channel_axis = 1; - const auto md = dnnl::memory::desc(phi::vectorize(x->dims()), - MKLDNNGetDataType(), x->format()); - this->AcquireForwardPrimitiveDescriptor(dnnl::prop_kind::forward_training, - md, channel_axis, group); + x->mem_desc(), channel_axis, group); } }; @@ -64,8 +61,7 @@ class ShuffleChannelMKLDNNKernel : public framework::OpKernel { {DNNL_ARG_DST, *dst_memory_p}}); astream.wait(); - out->set_layout(framework::DataLayout::kMKLDNN); - out->set_format(x->format()); + out->set_mem_desc(dst_memory_p->get_desc()); } }; } // namespace operators diff --git a/paddle/fluid/operators/mkldnn/softmax_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/softmax_mkldnn_op.cc index a0e50aa297851..ef5d95dca3f63 100644 --- a/paddle/fluid/operators/mkldnn/softmax_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/softmax_mkldnn_op.cc @@ -47,12 +47,8 @@ class SoftmaxMKLDNNHandler platform::errors::InvalidArgument( "The shape of input and output tensor must be identical.")); - auto softmax_tz = phi::vectorize(input->dims()); - auto md = memory::desc(softmax_tz, platform::MKLDNNGetDataType(), - input->format()); - - this->AcquireForwardPrimitiveDescriptor(prop_kind::forward_scoring, md, - axis); + this->AcquireForwardPrimitiveDescriptor(prop_kind::forward_scoring, + input->mem_desc(), axis); } SoftmaxMKLDNNHandler(const framework::ExecutionContext& ctx, @@ -73,17 +69,11 @@ class SoftmaxMKLDNNHandler auto dims = out_grad->dims(); // input and output share the same shape const int axis = phi::funcs::CanonicalAxis(ctx.Attr("axis"), dims.size()); - auto softmax_tz = phi::vectorize(dims); - - auto data_softmax_md = MKLDNNMemDesc( - softmax_tz, platform::MKLDNNGetDataType(), out->format()); - auto diff_softmax_md = MKLDNNMemDesc( - softmax_tz, platform::MKLDNNGetDataType(), out_grad->format()); this->AcquireForwardPrimitiveDescriptor(prop_kind::forward_scoring, - data_softmax_md, axis); - this->AcquireBackwardPrimitiveDescriptor(diff_softmax_md, data_softmax_md, - axis); + out->mem_desc(), axis); + this->AcquireBackwardPrimitiveDescriptor(out_grad->mem_desc(), + out->mem_desc(), axis); } }; @@ -128,9 +118,7 @@ class SoftmaxMKLDNNKernel : public paddle::framework::OpKernel { }); } - output->set_layout(framework::DataLayout::kMKLDNN); - // Softmax output format is the same as input one - output->set_format(input->format()); + output->set_mem_desc(softmax_dst_memory_p->get_desc()); } }; @@ -162,8 +150,7 @@ class SoftmaxMKLDNNGradKernel : public paddle::framework::OpKernel { {DNNL_ARG_DIFF_SRC, *diff_src_memory_p}}); astream.wait(); - in_x_grad->set_layout(framework::DataLayout::kMKLDNN); - in_x_grad->set_format(platform::GetMKLDNNFormat(*diff_src_memory_p)); + in_x_grad->set_mem_desc(diff_src_memory_p->get_desc()); } }; } // namespace operators diff --git a/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h b/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h index 143038e738ec6..b6111e99b683f 100644 --- a/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h +++ b/paddle/fluid/operators/mkldnn/softplus_mkldnn_op.h @@ -29,12 +29,11 @@ class SoftplusMKLDNNHandler : platform::MKLDNNHandlerNoCachingT(engine, ctx.GetPlace()) { auto x_tz = phi::vectorize(x->dims()); - auto x_md = - dnnl::memory::desc(x_tz, platform::MKLDNNGetDataType(), x->format()); auto beta_tz = std::vector(x_tz.size(), 1); - auto beta_md = dnnl::memory::desc(beta_tz, platform::MKLDNNGetDataType(), - x->format()); + auto beta_md = + dnnl::memory::desc(beta_tz, platform::MKLDNNGetDataType(), + platform::GetPlainMKLDNNFormat(x_tz.size())); dnnl::post_ops post_ops; post_ops.append_eltwise(1.0f, dnnl::algorithm::eltwise_soft_relu, 0.0f, @@ -50,7 +49,8 @@ class SoftplusMKLDNNHandler attrs.set_post_ops(post_ops); this->AcquireForwardPrimitiveDescriptor(attrs, dnnl::algorithm::binary_mul, - x_md, beta_md, x_md); + x->mem_desc(), beta_md, + x->mem_desc()); } std::shared_ptr AcquireBetaMemory(const float* beta) { @@ -129,8 +129,7 @@ void custom_softplus_eltwise_forward(const framework::ExecutionContext& ctx) { binary_p->execute(astream, args); astream.wait(); - out->set_layout(framework::DataLayout::kMKLDNN); - out->set_format(platform::GetMKLDNNFormat(*dst_memory_p)); + out->set_mem_desc(dst_memory_p->get_desc()); } } // namespace operators } // namespace paddle diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_expand_v2_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_expand_v2_mkldnn_op.py index b814eaed62b26..6229b7f559b16 100644 --- a/python/paddle/fluid/tests/unittests/mkldnn/test_expand_v2_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_expand_v2_mkldnn_op.py @@ -28,18 +28,22 @@ def setUp(self): self.op_type = "expand_v2" self.init_data() self.x = np.random.random(self.ori_shape).astype("float32") - self.set_inputs() self.attrs = {'shape': self.shape, 'use_mkldnn': True} + self.set_inputs() + self.set_additional_inputs() output = np.tile(self.x, self.expand_times) self.outputs = {'Out': output} def set_inputs(self): self.inputs = {'X': self.x} + def set_additional_inputs(self): + pass + def init_data(self): - self.ori_shape = [1, 140] - self.shape = [12, 140] - self.expand_times = [12, 1] + self.ori_shape = [1, 1, 1, 140] + self.shape = [2, 3, 4, 140] + self.expand_times = [2, 3, 4, 1] def test_check_output(self): self.check_output_with_place(core.CPUPlace()) @@ -74,7 +78,7 @@ def init_data(self): self.ori_shape = [100, 1] self.expand_times = [1, 2] self.expand_shape = [100, 2] - self.shape = [-1, -1] + self.shape = [100, 2] def calc_expand_shapes_tensor(self): self.expand_shapes_tensor = [] @@ -82,12 +86,9 @@ def calc_expand_shapes_tensor(self): self.expand_shapes_tensor.append(("x" + str(index), np.ones( (1)).astype('int32') * ele)) - def set_inputs(self): + def set_additional_inputs(self): self.calc_expand_shapes_tensor() - self.inputs = { - 'X': self.x, - 'expand_shapes_tensor': self.expand_shapes_tensor - } + self.inputs['expand_shapes_tensor'] = self.expand_shapes_tensor class TestExpandV2ExpandShapesTensor2OneDNNOp( @@ -104,13 +105,10 @@ def init_data(self): self.ori_shape = [100] self.expand_times = [2, 1] self.expand_shape = [2, 100] - self.shape = [-1, -1] + self.shape = [2, 100] - def set_inputs(self): - self.inputs = { - 'X': self.x, - 'Shape': np.array(self.expand_shape).astype("int32") - } + def set_additional_inputs(self): + self.inputs['Shape'] = np.array(self.expand_shape).astype("int32") # BF16 TESTS @@ -118,6 +116,7 @@ def create_expand_v2_bf16_test_class(parent): @OpTestTool.skip_if_not_cpu_bf16() class TestExpandV2BF16OneDNNOp(parent): def set_inputs(self): + self.attrs['mkldnn_data_type'] = 'bfloat16' self.inputs = {"X": convert_float_to_uint16(self.x)} def calculate_grads(self): From 920d44dfe1b0e9954e1c06b110b792f5eba21f94 Mon Sep 17 00:00:00 2001 From: Asthestarsfalll <72954905+Asthestarsfalll@users.noreply.github.com> Date: Thu, 21 Apr 2022 16:52:25 +0800 Subject: [PATCH 15/66] =?UTF-8?q?=E3=80=90PaddlePaddle=20Hackathon=202?= =?UTF-8?q?=E3=80=9123=E3=80=81=E4=B8=BA=20Paddle=20=E6=96=B0=E5=A2=9E=20S?= =?UTF-8?q?oftmax2D=20=E7=BB=84=E7=BD=91API=20(#40910)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Hackathon 23 * fix bug * fix pylint error * try * fix CI-Coverage * update and add more unittest * update --- .../fluid/tests/unittests/test_softmax2d.py | 111 ++++++++++++++++++ python/paddle/nn/__init__.py | 2 + python/paddle/nn/layer/__init__.py | 1 + python/paddle/nn/layer/activation.py | 52 ++++++++ 4 files changed, 166 insertions(+) create mode 100644 python/paddle/fluid/tests/unittests/test_softmax2d.py diff --git a/python/paddle/fluid/tests/unittests/test_softmax2d.py b/python/paddle/fluid/tests/unittests/test_softmax2d.py new file mode 100644 index 0000000000000..4879e9a0efbf0 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_softmax2d.py @@ -0,0 +1,111 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import numpy as np +import paddle +import paddle.fluid as fluid +import paddle.fluid.core as core +from test_softmax_op import ref_softmax + + +class TestSoftmax2DAPI(unittest.TestCase): + def setUp(self): + self.shape = [2, 6, 5, 4] + self.x_np = np.random.uniform(-1, 1, self.shape).astype('float64') + self.axis = -3 + self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + else paddle.CPUPlace() + + def test_static_api(self): + paddle.enable_static() + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.fluid.data('X', self.x_np.shape, self.x_np.dtype) + m = paddle.nn.Softmax2D() + out = m(x) + exe = paddle.static.Executor(self.place) + res = exe.run(feed={'X': self.x_np}, fetch_list=[out]) + out_ref = ref_softmax(self.x_np, self.axis) + self.assertTrue(np.allclose(out_ref, res)) + + def test_dygraph_api(self): + paddle.disable_static(self.place) + x = paddle.to_tensor(self.x_np) + m = paddle.nn.Softmax2D() + out = m(x) + out_ref = ref_softmax(self.x_np, self.axis) + self.assertTrue(np.allclose(out_ref, out.numpy())) + paddle.enable_static() + + +class TestSoftmax2DShape(TestSoftmax2DAPI): + def setUp(self): + self.shape = [2, 6, 4] + self.x_np = np.random.uniform(-1, 1, self.shape).astype('float64') + self.axis = -3 + self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + else paddle.CPUPlace() + + +class TestSoftmax2DFloat32(TestSoftmax2DAPI): + def setUp(self): + self.shape = [2, 3, 4] + self.x_np = np.random.uniform(-1, 1, self.shape).astype('float32') + self.axis = -3 + self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + else paddle.CPUPlace() + + +class TestSoftmax2DCPU(TestSoftmax2DAPI): + def setUp(self): + self.shape = [2, 6, 4] + self.x_np = np.random.uniform(-1, 1, self.shape).astype('float64') + self.axis = -3 + self.place = paddle.CPUPlace() + + +class TestSoftmax2DRepr(unittest.TestCase): + def setUp(self): + self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + else paddle.CPUPlace() + + def test_extra_repr(self): + paddle.disable_static(self.place) + m = paddle.nn.Softmax2D(name='test') + self.assertTrue(m.extra_repr() == 'name=test') + paddle.enable_static() + + +class TestSoftmax2DError(unittest.TestCase): + def setUp(self): + self.place = paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ + else paddle.CPUPlace() + + def test_static_error(self): + paddle.enable_static() + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.fluid.data('X', [5, 5], 'float32') + m = paddle.nn.Softmax2D() + self.assertRaises(AssertionError, m, x) + + def test_dygraph_error(self): + paddle.disable_static(self.place) + x_np = np.random.randn(2, 3, 4, 2, 3) + x = paddle.to_tensor(x_np, dtype='float64') + m = paddle.nn.Softmax2D() + self.assertRaises(AssertionError, m, x) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py index b83a900059bf4..b4824eff007d6 100644 --- a/python/paddle/nn/__init__.py +++ b/python/paddle/nn/__init__.py @@ -41,6 +41,7 @@ from .layer.activation import Hardsigmoid # noqa: F401 from .layer.activation import LogSigmoid # noqa: F401 from .layer.activation import Softmax # noqa: F401 +from .layer.activation import Softmax2D # noqa: F401 from .layer.activation import Softplus # noqa: F401 from .layer.activation import Softshrink # noqa: F401 from .layer.activation import Softsign # noqa: F401 @@ -260,6 +261,7 @@ def weight_norm(*args): 'AdaptiveMaxPool1D', 'TransformerEncoder', 'Softmax', + 'Softmax2D', 'ParameterList', 'Conv2D', 'Softshrink', diff --git a/python/paddle/nn/layer/__init__.py b/python/paddle/nn/layer/__init__.py index 2b50508065605..7dd18f1fefd65 100644 --- a/python/paddle/nn/layer/__init__.py +++ b/python/paddle/nn/layer/__init__.py @@ -26,6 +26,7 @@ from .activation import Sigmoid # noqa: F401 from .activation import Softmax # noqa: F401 from .activation import LogSoftmax # noqa: F401 +from .activation import Softmax2D # noqa: F401 from .common import Bilinear # noqa: F401 from .common import Pad1D # noqa: F401 from .common import Pad2D # noqa: F401 diff --git a/python/paddle/nn/layer/activation.py b/python/paddle/nn/layer/activation.py index 400585c431830..cd82fe12fff6b 100644 --- a/python/paddle/nn/layer/activation.py +++ b/python/paddle/nn/layer/activation.py @@ -1338,3 +1338,55 @@ def forward(self, x): def extra_repr(self): name_str = ', name={}'.format(self._name) if self._name else '' return 'groups={}, axis={}{}'.format(self._groups, self._axis, name_str) + + +class Softmax2D(Layer): + r""" + Softmax2D Activation. + Given a Tensor with shape (B, C, H, W) or (C, H, W), it will apply Softmax to each location (C, h_i, w_j). + The sum of result in each location (C, H_i, W_j) will be one. + + Shape: + - Input: :math:`(B, C, H, W)` or :math:`(C, H, W)` + - Output: :math:`(B, C, H, W)` or :math:`(C, H, W)`(same as input) + + Return: + A Tensor of the same shape and dtype as input with value in range [0, 1]. + + Examples: + .. code-block:: python + + import paddle + + x = paddle.rand([1, 2, 3, 4]) + # [[[[0.42496058 0.1172187 0.14664008 0.8151267 ] + # [0.24430142 0.42052492 0.60372984 0.79307914] + # [0.4539401 0.90458065 0.10235776 0.62009853]] + + # [[0.11731581 0.16053623 0.05667042 0.91876775] + # [0.9413854 0.30770817 0.6788164 0.9543593 ] + # [0.4145064 0.75909156 0.11598814 0.73599935]]]] + m = paddle.nn.Softmax2D() + out = m(x) + # [[[[0.5763103 0.48917228 0.5224772 0.4741129 ] + # [0.3324591 0.5281743 0.48123717 0.45976716] + # [0.5098571 0.5363083 0.49659243 0.4710572 ]] + + # [[0.42368975 0.51082766 0.47752273 0.5258871 ] + # [0.66754097 0.47182566 0.5187628 0.5402329 ] + # [0.49014282 0.46369177 0.50340754 0.5289428 ]]]] + """ + + def __init__(self, name=None): + super(Softmax2D, self).__init__() + self._dtype = None + self._name = name + + def forward(self, x): + assert x.ndim == 3 or x.ndim == 4, "Softmax2D requires a 3D or 4D tensor as input. Received: {}D.".format( + x.ndim) + return F.softmax(x, axis=-3, dtype=self._dtype, name=self._name) + + def extra_repr(self): + name_str = 'name={}'.format(self._name) if self._name else '' + return name_str From 9db6c7628c2374cf3fd628521c5ac8efdb9bf3af Mon Sep 17 00:00:00 2001 From: liutiexing <74819124+liutiexing@users.noreply.github.com> Date: Thu, 21 Apr 2022 18:20:05 +0800 Subject: [PATCH 16/66] WorkQueue supports always_spinning option (#42029) * WorkQueue supports always_spinning option * update * update --- .../new_executor/interpretercore_util.h | 3 +++ .../workqueue/nonblocking_threadpool.h | 15 +++++++++++--- .../new_executor/workqueue/workqueue.cc | 12 ++++++++--- .../new_executor/workqueue/workqueue.h | 9 +++++++-- .../new_executor/workqueue/workqueue_test.cc | 20 +++++++++++++++++++ 5 files changed, 51 insertions(+), 8 deletions(-) diff --git a/paddle/fluid/framework/new_executor/interpretercore_util.h b/paddle/fluid/framework/new_executor/interpretercore_util.h index 56683330ee6cb..60ac3702f4b3c 100644 --- a/paddle/fluid/framework/new_executor/interpretercore_util.h +++ b/paddle/fluid/framework/new_executor/interpretercore_util.h @@ -63,6 +63,7 @@ class AsyncWorkQueue { group_options.emplace_back(/*name*/ "HostTasks", /*num_threads*/ host_num_threads, /*allow_spinning*/ true, + /*always_spinning*/ false, /*track_task*/ false, /*detached*/ true, /*events_waiter*/ waiter); @@ -70,6 +71,7 @@ class AsyncWorkQueue { group_options.emplace_back(/*name*/ "DeviceKernelLaunch", /*num_threads*/ deivce_num_threads, /*allow_spinning*/ true, + /*always_spinning*/ true, /*track_task*/ false, /*detached*/ true, /*events_waiter*/ waiter); @@ -77,6 +79,7 @@ class AsyncWorkQueue { group_options.emplace_back(/*name*/ "Prepare", /*num_threads*/ 1, /*allow_spinning*/ true, + /*always_spinning*/ false, /*track_task*/ false, /*detached*/ true, /*events_waiter*/ waiter); diff --git a/paddle/fluid/framework/new_executor/workqueue/nonblocking_threadpool.h b/paddle/fluid/framework/new_executor/workqueue/nonblocking_threadpool.h index a599bc41f678e..559eb6a7490cd 100644 --- a/paddle/fluid/framework/new_executor/workqueue/nonblocking_threadpool.h +++ b/paddle/fluid/framework/new_executor/workqueue/nonblocking_threadpool.h @@ -29,13 +29,13 @@ class ThreadPoolTempl { typedef RunQueue Queue; ThreadPoolTempl(const std::string& name, int num_threads, bool allow_spinning, - Environment env = Environment()) + bool always_spinning, Environment env = Environment()) : env_(env), allow_spinning_(allow_spinning), + always_spinning_(always_spinning), global_steal_partition_(EncodePartition(0, num_threads_)), blocked_(0), num_tasks_(0), - spinning_(0), done_(false), cancelled_(false), ec_(num_threads), @@ -236,11 +236,11 @@ class ThreadPoolTempl { Environment env_; const bool allow_spinning_; + const bool always_spinning_; std::vector> all_coprimes_; unsigned global_steal_partition_; std::atomic blocked_; std::atomic num_tasks_; - std::atomic spinning_; std::atomic done_; std::atomic cancelled_; EventCount ec_; @@ -417,6 +417,15 @@ class ThreadPoolTempl { ec_.Notify(true); return false; } + + // Cancel wait if always_spinning_ + if (always_spinning_) { + ec_.CancelWait(); + blocked_--; + return true; + } + + // Wait for work platform::RecordEvent record("WaitForWork", platform::TracerEventType::UserDefined, 10); ec_.CommitWait(waiter); diff --git a/paddle/fluid/framework/new_executor/workqueue/workqueue.cc b/paddle/fluid/framework/new_executor/workqueue/workqueue.cc index b8dfcad187ca0..0f0de8ef9b05d 100644 --- a/paddle/fluid/framework/new_executor/workqueue/workqueue.cc +++ b/paddle/fluid/framework/new_executor/workqueue/workqueue.cc @@ -21,6 +21,10 @@ void WorkQueueOptions::Validate() const { name.find('_'), std::string::npos, platform::errors::InvalidArgument( "WorkQueueOptions.name shouldn't contain an underline")); + PADDLE_ENFORCE_EQ( + allow_spinning == false && always_spinning == true, false, + platform::errors::InvalidArgument("WorkQueueOptions.allow_spinning must " + "be true when always_spinning is set")); } namespace { @@ -40,7 +44,8 @@ class WorkQueueImpl : public WorkQueue { options.events_waiter->RegisterEvent(kQueueDestructEvent); } queue_ = new NonblockingThreadPool(options_.name, options_.num_threads, - options_.allow_spinning); + options_.allow_spinning, + options_.always_spinning); } virtual ~WorkQueueImpl() { @@ -127,8 +132,9 @@ WorkQueueGroupImpl::WorkQueueGroupImpl( destruct_notifier_ = options.events_waiter->RegisterEvent(kQueueDestructEvent); } - queues_[idx] = new (&queues_storage_[idx]) NonblockingThreadPool( - options.name, options.num_threads, options.allow_spinning); + queues_[idx] = new (&queues_storage_[idx]) + NonblockingThreadPool(options.name, options.num_threads, + options.allow_spinning, options.always_spinning); } } diff --git a/paddle/fluid/framework/new_executor/workqueue/workqueue.h b/paddle/fluid/framework/new_executor/workqueue/workqueue.h index 0101461658d00..e9c658e3b9dc6 100644 --- a/paddle/fluid/framework/new_executor/workqueue/workqueue.h +++ b/paddle/fluid/framework/new_executor/workqueue/workqueue.h @@ -64,11 +64,12 @@ struct WorkQueueOptions { } WorkQueueOptions(const std::string& name, size_t num_threads, - bool allow_spinning, bool track_task, bool detached, - EventsWaiter* waiter) + bool allow_spinning, bool always_spinning, bool track_task, + bool detached, EventsWaiter* waiter) : name(name), num_threads(num_threads), allow_spinning(allow_spinning), + always_spinning(always_spinning), track_task(track_task), detached(detached), events_waiter(waiter) { @@ -80,7 +81,11 @@ struct WorkQueueOptions { std::string name; size_t num_threads; + // Worker threads will spin for a while if this flag is set. bool allow_spinning; + // Worker threads will never sleep if this flag is set. + // Better performance vs. higher CPU utilization. + bool always_spinning{false}; // If you need to blocking the calling thread to wait "queue empty", set // track_task = true and set events_waiter. EventsWaiter::WaitEvent will // block the calling thread until any of events (including "queue empty") diff --git a/paddle/fluid/framework/new_executor/workqueue/workqueue_test.cc b/paddle/fluid/framework/new_executor/workqueue/workqueue_test.cc index d8e09fb6baefe..857eaead5b658 100644 --- a/paddle/fluid/framework/new_executor/workqueue/workqueue_test.cc +++ b/paddle/fluid/framework/new_executor/workqueue/workqueue_test.cc @@ -48,6 +48,7 @@ TEST(WorkQueue, TestSingleThreadedWorkQueue) { EventsWaiter events_waiter; WorkQueueOptions options(/*name*/ "SingleThreadedWorkQueueForTesting", /*num_threads*/ 1, /*allow_spinning*/ true, + /*always_spinning*/ true, /*track_task*/ true, /*detached*/ true, &events_waiter); auto work_queue = CreateSingleThreadedWorkQueue(options); @@ -69,6 +70,15 @@ TEST(WorkQueue, TestSingleThreadedWorkQueue) { EXPECT_EQ(finished.load(), true); EXPECT_EQ(counter.load(), kLoopNum); EXPECT_EQ(handle.get(), 1234); + work_queue.reset(); + // Test default_options with no spinning + WorkQueueOptions default_options("SingleThreadedWorkQueueForTesting", + /*num_threads*/ 1, + /*allow_spinning*/ false, + /*track_task*/ false); + work_queue = CreateSingleThreadedWorkQueue(default_options); + handle = work_queue->AddAwaitableTask([]() { return 5678; }); + EXPECT_EQ(handle.get(), 5678); } TEST(WorkQueue, TestMultiThreadedWorkQueue) { @@ -85,6 +95,7 @@ TEST(WorkQueue, TestMultiThreadedWorkQueue) { EventsWaiter events_waiter; WorkQueueOptions options(/*name*/ "MultiThreadedWorkQueueForTesting", /*num_threads*/ 10, /*allow_spinning*/ true, + /*always_spinning*/ true, /*track_task*/ true, /*detached*/ false, &events_waiter); auto work_queue = CreateMultiThreadedWorkQueue(options); @@ -115,6 +126,13 @@ TEST(WorkQueue, TestMultiThreadedWorkQueue) { }); work_queue.reset(); waiter_thread.join(); + // Forever spin unittest + WorkQueueOptions default_options("MultiThreadedWorkQueueForTesting", + /*num_threads*/ 10, /*allow_spinning*/ false, + /*track_task*/ false); + work_queue = CreateMultiThreadedWorkQueue(default_options); + auto handle = work_queue->AddAwaitableTask([]() { return 5678; }); + EXPECT_EQ(handle.get(), 5678); } TEST(WorkQueue, TestWorkQueueGroup) { @@ -130,10 +148,12 @@ TEST(WorkQueue, TestWorkQueueGroup) { EventsWaiter events_waiter; WorkQueueOptions sq_options(/*name*/ "SingleThreadedWorkQueueForTesting", /*num_threads*/ 1, /*allow_spinning*/ true, + /*always_spinning*/ true, /*track_task*/ true, /*detached*/ false, &events_waiter); WorkQueueOptions mq_options(/*name*/ "MultiThreadedWorkQueueForTesting", /*num_threads*/ 10, /*allow_spinning*/ true, + /*always_spinning*/ true, /*track_task*/ true, /*detached*/ false, &events_waiter); auto queue_group = CreateWorkQueueGroup({sq_options, mq_options}); From fb87df663aef5ff2b808da3ae3fff7cd5762ba12 Mon Sep 17 00:00:00 2001 From: RichardWooSJTU <37864677+RichardWooSJTU@users.noreply.github.com> Date: Thu, 21 Apr 2022 19:07:55 +0800 Subject: [PATCH 17/66] Fix nms op docs (#41792) * fix nms op doc missing default value --- python/paddle/vision/ops.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/python/paddle/vision/ops.py b/python/paddle/vision/ops.py index 8fa51df9ac10d..2d60fd4561480 100644 --- a/python/paddle/vision/ops.py +++ b/python/paddle/vision/ops.py @@ -1399,26 +1399,27 @@ def nms(boxes, IoU = \frac{intersection\_area(box1, box2)}{union\_area(box1, box2)} If scores are provided, input boxes will be sorted by their scores firstly. + If category_idxs and categories are provided, NMS will be performed with a batched style, which means NMS will be applied to each category respectively and results of each category will be concated and sorted by scores. + If K is provided, only the first k elements will be returned. Otherwise, all box indices sorted by scores will be returned. Args: boxes(Tensor): The input boxes data to be computed, it's a 2D-Tensor with - the shape of [num_boxes, 4] and boxes should be sorted by their - confidence scores. The data type is float32 or float64. + the shape of [num_boxes, 4]. The data type is float32 or float64. Given as [[x1, y1, x2, y2], …], (x1, y1) is the top left coordinates, and (x2, y2) is the bottom right coordinates. Their relation should be ``0 <= x1 < x2 && 0 <= y1 < y2``. - iou_threshold(float32): IoU threshold for determine overlapping boxes. Default value: 0.3. + iou_threshold(float32, optional): IoU threshold for determine overlapping boxes. Default value: 0.3. scores(Tensor, optional): Scores corresponding to boxes, it's a 1D-Tensor with - shape of [num_boxes]. The data type is float32 or float64. + shape of [num_boxes]. The data type is float32 or float64. Default: None. category_idxs(Tensor, optional): Category indices corresponding to boxes. - it's a 1D-Tensor with shape of [num_boxes]. The data type is int64. - categories(List, optional): A list of unique id of all categories. The data type is int64. + it's a 1D-Tensor with shape of [num_boxes]. The data type is int64. Default: None. + categories(List, optional): A list of unique id of all categories. The data type is int64. Default: None. top_k(int64, optional): The top K boxes who has higher score and kept by NMS preds to - consider. top_k should be smaller equal than num_boxes. + consider. top_k should be smaller equal than num_boxes. Default: None. Returns: Tensor: 1D-Tensor with the shape of [num_boxes]. Indices of boxes kept by NMS. From ec995c594d5a787ca2ce42b94131e552b2be6c4e Mon Sep 17 00:00:00 2001 From: Aganlengzi Date: Thu, 21 Apr 2022 19:36:35 +0800 Subject: [PATCH 18/66] [CustomDevice] fix macro (#42073) * [CustomDevice] fix macro * fix --- paddle/phi/backends/device_ext.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/paddle/phi/backends/device_ext.h b/paddle/phi/backends/device_ext.h index 6315fe15afdf1..749d8d323b62d 100644 --- a/paddle/phi/backends/device_ext.h +++ b/paddle/phi/backends/device_ext.h @@ -523,14 +523,14 @@ struct CustomRuntimeParams { char reserved[32]; }; -#define PADDLE_CUSTOM_RUNTIME_CHECK_VERSION(params) \ - if ((params)->size != sizeof(DevicePluginParams) && \ - (params)->interface->size != sizeof(C_DeviceInterface)) { \ - return; \ - } \ - (params)->version.major = PADDLE_DEVICE_PLUGIN_MAJOR_VERSION; \ - (params)->version.minor = PADDLE_DEVICE_PLUGIN_MINOR_VERSION; \ - (params)->version.patch = PADDLE_DEVICE_PLUGIN_PATCH_VERSION; +#define PADDLE_CUSTOM_RUNTIME_CHECK_VERSION(params) \ + if ((params)->size != sizeof(CustomRuntimeParams) && \ + (params)->interface->size != sizeof(C_DeviceInterface)) { \ + return; \ + } \ + (params)->version.major = PADDLE_CUSTOM_RUNTIME_MAJOR_VERSION; \ + (params)->version.minor = PADDLE_CUSTOM_RUNTIME_MINOR_VERSION; \ + (params)->version.patch = PADDLE_CUSTOM_RUNTIME_PATCH_VERSION; // Plugin implement it and fill CustomRuntimeParams void InitPlugin(CustomRuntimeParams*); From 6becabaa49278803be23d9bb097b9133c1940c02 Mon Sep 17 00:00:00 2001 From: zmxdream Date: Thu, 21 Apr 2022 20:58:35 +0800 Subject: [PATCH 19/66] [XPUPS]add hashtable interface (#41987) * add hashtable interface. test=develop * update. test=develop * update. test=develop * fix. test=develop * fix optimizer config for xpups. test=develop * fix. test=develop * fix. test=develop --- .../framework/fleet/heter_ps/hashtable.h | 37 ++++- .../fleet/heter_ps/hashtable_kernel.kps | 150 +++++++++--------- .../framework/fleet/heter_ps/heter_comm.h | 6 + .../framework/fleet/heter_ps/heter_comm_inl.h | 18 +++ .../framework/fleet/heter_ps/heter_ps.cu | 10 ++ .../fluid/framework/fleet/heter_ps/heter_ps.h | 29 ++-- .../framework/fleet/heter_ps/heter_ps_base.h | 15 +- .../framework/fleet/heter_ps/optimizer_conf.h | 49 +++--- .../fluid/framework/fleet/ps_gpu_wrapper.kps | 90 +++-------- 9 files changed, 218 insertions(+), 186 deletions(-) diff --git a/paddle/fluid/framework/fleet/heter_ps/hashtable.h b/paddle/fluid/framework/fleet/heter_ps/hashtable.h index b821ccecf0a29..b860ea5d39cb5 100644 --- a/paddle/fluid/framework/fleet/heter_ps/hashtable.h +++ b/paddle/fluid/framework/fleet/heter_ps/hashtable.h @@ -41,6 +41,10 @@ limitations under the License. */ #include "xpu/kernel/simd.h" #endif +#if defined(PADDLE_WITH_XPU_KP) +#include "paddle/fluid/framework/fleet/heter_ps/optimizer_conf.h" +#endif + namespace paddle { namespace framework { @@ -56,11 +60,10 @@ class TableContainer capacity, ValType()) {} }; #elif defined(PADDLE_WITH_XPU_KP) - template class XPUCacheArray { public: - explicit XPUCacheArray(size_t capacity) : capacity_(capacity), size_(0) { + explicit XPUCacheArray(long long capacity) : capacity_(capacity), size_(0) { xpu_malloc(reinterpret_cast(&keys), capacity_ * sizeof(KeyType)); xpu_malloc(reinterpret_cast(&vals), capacity_ * sizeof(ValType)); } @@ -71,8 +74,27 @@ class XPUCacheArray { } void print() {} - // ValType* find(const KeyType& key) { return NULL; } - // bool insert(const KeyType& key, const ValType& val) { return true; } + +#if defined(__xpu__) + __device__ ValType* find(const KeyType& key) { + for (int i = 0; i < size_; i++) { + if (keys[i] == key) return &vals[i]; + } + return NULL; + } + __device__ bool insert(const KeyType& key, const ValType& val) { + // # NOTE(zhangminxu): we set the capacity larger than the feasign number of + // one batch + if (size_ == capacity_) { + return false; + } else { + keys[size_] = key; + vals[size_] = val; + size_++; + return true; + } + } +#endif int prefetch(const int dev_id, XPUStream stream = NULL) { return 0; } size_t size() { return size_; } @@ -110,6 +132,11 @@ class HashTable { void show(); +#if defined(PADDLE_WITH_XPU_KP) + void set_sparse_sgd(const OptimizerConfig& optimizer_config); + void set_embedx_sgd(const OptimizerConfig& optimizer_config); +#endif + template void dump_to_cpu(int devid, StreamType stream); @@ -151,6 +178,8 @@ class HashTable { TableContainer* container_; #elif defined(PADDLE_WITH_XPU_KP) XPUCacheArray* container_; + OptimizerConfig* xpu_optimizer_config_; + OptimizerConfig cpu_optimizer_config_; #endif int BLOCK_SIZE_{256}; float LOAD_FACTOR{0.75f}; diff --git a/paddle/fluid/framework/fleet/heter_ps/hashtable_kernel.kps b/paddle/fluid/framework/fleet/heter_ps/hashtable_kernel.kps index e879d817b14dd..cd43a73b44ec3 100644 --- a/paddle/fluid/framework/fleet/heter_ps/hashtable_kernel.kps +++ b/paddle/fluid/framework/fleet/heter_ps/hashtable_kernel.kps @@ -14,41 +14,21 @@ limitations under the License. */ #ifdef PADDLE_WITH_HETERPS #include "paddle/fluid/framework/fleet/heter_ps/hashtable.h" - -namespace optimizer_config { -extern _global_ptr_ float* nonclk_coeff; -extern _global_ptr_ float* clk_coeff; - -extern _global_ptr_ float* min_bound; -extern _global_ptr_ float* max_bound; -extern _global_ptr_ float* learning_rate; -extern _global_ptr_ float* initial_g2sum; -extern _global_ptr_ float* initial_range; - -extern _global_ptr_ float* mf_create_thresholds; -extern _global_ptr_ float* mf_learning_rate; -extern _global_ptr_ float* mf_initial_g2sum; -extern _global_ptr_ float* mf_initial_range; -extern _global_ptr_ float* mf_min_bound; -extern _global_ptr_ float* mf_max_bound; -} +#include "paddle/fluid/framework/fleet/heter_ps/optimizer_conf.h" namespace paddle { namespace framework { #if defined(PADDLE_WITH_XPU_KP) -__device__ void update_lr(float& w, float& g2sum, float g, // NOLINT +__device__ void update_lr(OptimizerConfig& optimizer_config, float& w, + float& g2sum, + float g, // NOLINT float scale) { - __local__ float local_learning_rate; - __local__ float local_initial_g2sum; - __local__ float local_min_bound; - __local__ float local_max_bound; - - GM2LM(optimizer_config::learning_rate, &local_learning_rate, sizeof(float)); - GM2LM(optimizer_config::initial_g2sum, &local_initial_g2sum, sizeof(float)); - GM2LM(optimizer_config::min_bound, &local_min_bound, sizeof(float)); - GM2LM(optimizer_config::max_bound, &local_max_bound, sizeof(float)); + float local_learning_rate = optimizer_config.learning_rate; + float local_initial_g2sum = optimizer_config.initial_g2sum; + float local_min_bound = optimizer_config.min_bound; + float local_max_bound = optimizer_config.max_bound; double add_g2sum = 0; double ratio = local_learning_rate * @@ -65,19 +45,12 @@ __device__ void update_lr(float& w, float& g2sum, float g, // NOLINT g2sum += add_g2sum; } -__device__ void update_mf(int n, float* w, float& g2sum, const float* g, - float scale) { - __local__ float local_mf_learning_rate; - __local__ float local_mf_initial_g2sum; - __local__ float local_mf_min_bound; - __local__ float local_mf_max_bound; - - GM2LM(optimizer_config::mf_learning_rate, &local_mf_learning_rate, - sizeof(float)); - GM2LM(optimizer_config::mf_initial_g2sum, &local_mf_initial_g2sum, - sizeof(float)); - GM2LM(optimizer_config::mf_min_bound, &local_mf_min_bound, sizeof(float)); - GM2LM(optimizer_config::mf_max_bound, &local_mf_max_bound, sizeof(float)); +__device__ void update_mf(OptimizerConfig& optimizer_config, int n, float* w, + float& g2sum, const float* g, float scale) { + float local_mf_learning_rate = optimizer_config.mf_learning_rate; + float local_mf_initial_g2sum = optimizer_config.mf_initial_g2sum; + float local_mf_min_bound = optimizer_config.mf_min_bound; + float local_mf_max_bound = optimizer_config.mf_max_bound; double add_g2sum = 0; double ratio = @@ -98,26 +71,22 @@ __device__ void update_mf(int n, float* w, float& g2sum, const float* g, __device__ float xpu_rand_uniform() { return 0.1; } template -__device__ void update_value(ValType& val, const GradType& grad) { // NOLINT +__device__ void update_value(OptimizerConfig& optimizer_config, ValType& val, + const GradType& grad) { // NOLINT val.slot = grad.slot; val.show += grad.show; val.clk += grad.clk; - __local__ float local_nonclk_coeff; - __local__ float local_clk_coeff; + float local_nonclk_coeff = optimizer_config.nonclk_coeff; + float local_clk_coeff = optimizer_config.clk_coeff; - __local__ float local_mf_create_thresholds; - __local__ float local_mf_initial_range; - - GM2LM(optimizer_config::nonclk_coeff, &local_nonclk_coeff, sizeof(float)); - GM2LM(optimizer_config::clk_coeff, &local_clk_coeff, sizeof(float)); - GM2LM(optimizer_config::mf_create_thresholds, &local_mf_create_thresholds, - sizeof(float)); + float local_mf_create_thresholds = optimizer_config.mf_create_thresholds; + float local_mf_initial_range = optimizer_config.mf_initial_range; val.delta_score += local_nonclk_coeff * (grad.show - grad.clk) + local_clk_coeff * grad.clk; - update_lr(val.lr, val.lr_g2sum, grad.lr_g, grad.show); + update_lr(optimizer_config, val.lr, val.lr_g2sum, grad.lr_g, grad.show); if (val.mf_size == 0) { if (local_mf_create_thresholds <= @@ -130,12 +99,13 @@ __device__ void update_value(ValType& val, const GradType& grad) { // NOLINT } } } else { - update_mf(MF_DIM, &val.mf[1], val.mf[0], grad.mf_g, grad.show); + update_mf(optimizer_config, MF_DIM, &val.mf[1], val.mf[0], grad.mf_g, + grad.show); } } template -__global__ void insert_kernel(Table* table, const KeyType* const keys, +__global__ void insert_kernel(Table& table, const KeyType* const keys, const ValType* const vals, long long len) { int cid = core_id(); int ncores = core_num(); @@ -156,14 +126,14 @@ __global__ void insert_kernel(Table* table, const KeyType* const keys, GM2LM(keys, local_keys, read_len * sizeof(KeyType)); GM2LM(vals, local_vals, read_len * sizeof(ValType)); for (int k = 0; k < read_len; k++) { - // auto status = table->insert(local_keys[k], local_vals[k]); - // assert(status != false && "error: insert fails: table is full"); + auto status = table.insert(local_keys[k], local_vals[k]); + assert(status != false && "error: insert fails: table is full"); } } } template -__global__ void search_kernel(Table* table, const KeyType* const keys, +__global__ void search_kernel(Table& table, const KeyType* const keys, ValType* const vals, long long len) { int cid = core_id(); int ncores = core_num(); @@ -183,17 +153,18 @@ __global__ void search_kernel(Table* table, const KeyType* const keys, int read_len = min(len_per_loop, len - i); GM2LM(keys, local_keys, read_len * sizeof(KeyType)); for (int k = 0; k < read_len; k++) { - // ValType* val = table->find(local_keys[k]); - // if (val != NULL) { - // local_vals[k] = *val; - // } + ValType* val = table.find(local_keys[k]); + if (val != NULL) { + local_vals[k] = *val; + } } LM2GM(local_vals, vals + i, read_len * sizeof(ValType)); } } template -__global__ void update_kernel(Table* table, const KeyType* const keys, +__global__ void update_kernel(OptimizerConfig& optimizer_config, Table& table, + const KeyType* const keys, const GradType* const grads, long long len) { int cid = core_id(); int ncores = core_num(); @@ -216,10 +187,10 @@ __global__ void update_kernel(Table* table, const KeyType* const keys, GM2LM(grads, local_grads, read_len * sizeof(GradType)); for (int k = 0; k < read_len; k++) { - // ValType* val = table->find(local_keys[k]); - // if (val != NULL) { - // update_value(*val, grads[i]); - //} + ValType* val = table.find(local_keys[k]); + if (val != NULL) { + update_value(optimizer_config, *val, local_grads[i]); + } } } } @@ -229,14 +200,23 @@ HashTable::HashTable(size_t capacity) { auto tmp_container = XPUCacheArray(capacity); xpu_malloc(reinterpret_cast(&container_), sizeof(XPUCacheArray)); - xpu_memcpy(container_, &tmp_container, + xpu_memcpy((void*)container_, &tmp_container, sizeof(XPUCacheArray), XPU_HOST_TO_DEVICE); + + OptimizerConfig tmp_opt_config; + xpu_malloc(reinterpret_cast(&xpu_optimizer_config_), + sizeof(OptimizerConfig)); + + xpu_memcpy((void*)xpu_optimizer_config_, &tmp_opt_config, + sizeof(OptimizerConfig), XPU_HOST_TO_DEVICE); + rwlock_.reset(new phi::RWLock); } template HashTable::~HashTable() { xpu_free((void*)container_); + xpu_free((void*)xpu_optimizer_config_); } template @@ -244,6 +224,34 @@ void HashTable::show() { container_->print(); } +template +void HashTable::set_sparse_sgd( + const OptimizerConfig& optimizer_config) { + cpu_optimizer_config_.nonclk_coeff = optimizer_config.nonclk_coeff; + cpu_optimizer_config_.clk_coeff = optimizer_config.clk_coeff; + cpu_optimizer_config_.min_bound = optimizer_config.min_bound; + cpu_optimizer_config_.max_bound = optimizer_config.max_bound; + cpu_optimizer_config_.learning_rate = optimizer_config.learning_rate; + cpu_optimizer_config_.initial_g2sum = optimizer_config.initial_g2sum; + cpu_optimizer_config_.initial_range = optimizer_config.initial_range; + xpu_memcpy((void*)xpu_optimizer_config_, &cpu_optimizer_config_, + sizeof(OptimizerConfig), XPU_HOST_TO_DEVICE); +} + +template +void HashTable::set_embedx_sgd( + const OptimizerConfig& optimizer_config) { + cpu_optimizer_config_.mf_create_thresholds = + optimizer_config.mf_create_thresholds; + cpu_optimizer_config_.mf_learning_rate = optimizer_config.mf_learning_rate; + cpu_optimizer_config_.mf_initial_g2sum = optimizer_config.mf_initial_g2sum; + cpu_optimizer_config_.mf_initial_range = optimizer_config.mf_initial_range; + cpu_optimizer_config_.mf_min_bound = optimizer_config.mf_min_bound; + cpu_optimizer_config_.mf_max_bound = optimizer_config.mf_max_bound; + xpu_memcpy((void*)xpu_optimizer_config_, &cpu_optimizer_config_, + sizeof(OptimizerConfig), XPU_HOST_TO_DEVICE); +} + template template void HashTable::get(const KeyType* d_keys, ValType* d_vals, @@ -254,7 +262,7 @@ void HashTable::get(const KeyType* d_keys, ValType* d_vals, long long c_len = (long long)len; search_kernel><<<4, 64, stream>>>( - container_, d_keys, d_vals, c_len); + *container_, d_keys, d_vals, c_len); } template @@ -278,7 +286,7 @@ void HashTable::insert(const KeyType* d_keys, long long c_len = (long long)len; insert_kernel><<<4, 64, stream>>>( - container_, d_keys, d_vals, c_len); + *container_, d_keys, d_vals, c_len); } template @@ -297,8 +305,8 @@ void HashTable::update(const KeyType* d_keys, } long long c_len = (long long)len; update_kernel, - GradType><<<4, 64, stream>>>(container_, d_keys, d_grads, - c_len); + GradType><<<4, 64, stream>>>( + *xpu_optimizer_config_, *container_, d_keys, d_grads, c_len); } template diff --git a/paddle/fluid/framework/fleet/heter_ps/heter_comm.h b/paddle/fluid/framework/fleet/heter_ps/heter_comm.h index 338009250bc4f..6379f7ee91264 100644 --- a/paddle/fluid/framework/fleet/heter_ps/heter_comm.h +++ b/paddle/fluid/framework/fleet/heter_ps/heter_comm.h @@ -21,6 +21,7 @@ limitations under the License. */ #include "paddle/fluid/platform/dynload/nccl.h" #include "thrust/pair.h" #elif defined(PADDLE_WITH_XPU_KP) +// #include "paddle/fluid/framework/fleet/heter_ps/optimizer_conf.h" #include #include "paddle/fluid/platform/device/xpu/enforce_xpu.h" #endif @@ -64,6 +65,11 @@ class HeterComm { void push_sparse(int num, KeyType* d_keys, GradType* d_grads, size_t len); #endif +#if defined(PADDLE_WITH_XPU_KP) + void set_sparse_sgd(const OptimizerConfig& optimizer_config); + void set_embedx_sgd(const OptimizerConfig& optimizer_config); +#endif + int log2i(int x); template diff --git a/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h b/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h index 551b5c38895a9..870bad8d19a6f 100644 --- a/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h +++ b/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h @@ -338,6 +338,24 @@ int HeterComm::get_index_by_devid(int devid) { return resource_->get_index_by_devid(devid); } +#if defined(PADDLE_WITH_XPU_KP) +template +void HeterComm::set_sparse_sgd( + const OptimizerConfig& optimizer_config) { + for (auto& table : tables_) { + table->set_sparse_sgd(optimizer_config); + } +} + +template +void HeterComm::set_embedx_sgd( + const OptimizerConfig& optimizer_config) { + for (auto& table : tables_) { + table->set_embedx_sgd(optimizer_config); + } +} +#endif + template void HeterComm::build_ps( int dev_num, KeyType* h_keys, ValType* h_vals, size_t len, diff --git a/paddle/fluid/framework/fleet/heter_ps/heter_ps.cu b/paddle/fluid/framework/fleet/heter_ps/heter_ps.cu index 583eb926a26a5..8a877f85076ef 100644 --- a/paddle/fluid/framework/fleet/heter_ps/heter_ps.cu +++ b/paddle/fluid/framework/fleet/heter_ps/heter_ps.cu @@ -50,6 +50,16 @@ int HeterPs::get_index_by_devid(int devid) { return comm_->get_index_by_devid(devid); } +#if defined(PADDLE_WITH_XPU_KP) +void HeterPs::set_sparse_sgd(const OptimizerConfig& optimizer_config) { + comm_->set_sparse_sgd(optimizer_config); +} + +void HeterPs::set_embedx_sgd(const OptimizerConfig& optimizer_config) { + comm_->set_embedx_sgd(optimizer_config); +} +#endif + void HeterPs::end_pass() { comm_->end_pass(); } void HeterPs::show_one_table(int gpu_num) { comm_->show_one_table(gpu_num); } diff --git a/paddle/fluid/framework/fleet/heter_ps/heter_ps.h b/paddle/fluid/framework/fleet/heter_ps/heter_ps.h index 7fb50f4da1fce..7060817be91eb 100644 --- a/paddle/fluid/framework/fleet/heter_ps/heter_ps.h +++ b/paddle/fluid/framework/fleet/heter_ps/heter_ps.h @@ -33,22 +33,27 @@ class HeterPs : public HeterPsBase { HeterPs(const HeterPs&) = delete; HeterPs& operator=(const HeterPs&) = delete; - virtual void pull_sparse(int num, FeatureKey* d_keys, FeatureValue* d_vals, - size_t len) override; - virtual void build_ps(int num, FeatureKey* h_keys, FeatureValue* h_vals, - size_t len, size_t chunk_size, int stream_num) override; + void pull_sparse(int num, FeatureKey* d_keys, FeatureValue* d_vals, + size_t len) override; + void build_ps(int num, FeatureKey* h_keys, FeatureValue* h_vals, size_t len, + size_t chunk_size, int stream_num) override; #if defined(PADDLE_WITH_CUDA) - virtual void set_nccl_comm_and_size( - const std::vector& inner_comms, - const std::vector& inter_comms, int comm_size) override; + void set_nccl_comm_and_size(const std::vector& inner_comms, + const std::vector& inter_comms, + int comm_size) override; #endif - virtual void end_pass() override; - virtual int get_index_by_devid(int devid) override; - virtual void show_one_table(int gpu_num) override; - virtual void push_sparse(int num, FeatureKey* d_keys, - FeaturePushValue* d_grads, size_t len) override; +#if defined(PADDLE_WITH_XPU_KP) + void set_sparse_sgd(const OptimizerConfig& optimizer_config) override; + void set_embedx_sgd(const OptimizerConfig& optimizer_config) override; +#endif + + void end_pass() override; + int get_index_by_devid(int devid) override; + void show_one_table(int gpu_num) override; + void push_sparse(int num, FeatureKey* d_keys, FeaturePushValue* d_grads, + size_t len) override; private: std::shared_ptr> comm_; diff --git a/paddle/fluid/framework/fleet/heter_ps/heter_ps_base.h b/paddle/fluid/framework/fleet/heter_ps/heter_ps_base.h index ddbf02df6c578..79061ab66af1c 100644 --- a/paddle/fluid/framework/fleet/heter_ps/heter_ps_base.h +++ b/paddle/fluid/framework/fleet/heter_ps/heter_ps_base.h @@ -16,6 +16,9 @@ limitations under the License. */ #include #include "paddle/fluid/framework/fleet/heter_ps/feature_value.h" #include "paddle/fluid/framework/fleet/heter_ps/heter_resource.h" +#if defined(PADDLE_WITH_XPU_KP) +#include "paddle/fluid/framework/fleet/heter_ps/optimizer_conf.h" +#endif #ifdef PADDLE_WITH_HETERPS @@ -24,9 +27,9 @@ namespace framework { class HeterPsBase { public: - HeterPsBase(){}; - HeterPsBase(size_t capacity, std::shared_ptr resource){}; - virtual ~HeterPsBase(){}; + HeterPsBase() {} + HeterPsBase(size_t capacity, std::shared_ptr resource) {} + virtual ~HeterPsBase() {} HeterPsBase(const HeterPsBase&) = delete; HeterPsBase& operator=(const HeterPsBase&) = delete; @@ -44,6 +47,12 @@ class HeterPsBase { virtual void show_one_table(int gpu_num) = 0; virtual void push_sparse(int num, FeatureKey* d_keys, FeaturePushValue* d_grads, size_t len) = 0; + +#if defined(PADDLE_WITH_XPU_KP) + virtual void set_sparse_sgd(const OptimizerConfig& optimizer_config) {} + virtual void set_embedx_sgd(const OptimizerConfig& optimizer_config) {} +#endif + static HeterPsBase* get_instance(size_t capacity, std::shared_ptr resource); }; diff --git a/paddle/fluid/framework/fleet/heter_ps/optimizer_conf.h b/paddle/fluid/framework/fleet/heter_ps/optimizer_conf.h index 6d924a395e19a..2a80aa4b52d91 100644 --- a/paddle/fluid/framework/fleet/heter_ps/optimizer_conf.h +++ b/paddle/fluid/framework/fleet/heter_ps/optimizer_conf.h @@ -14,16 +14,10 @@ limitations under the License. */ #pragma once -#if defined(PADDLE_WITH_XPU_KP) -#include "xpu/kernel/cluster_header.h" -#include "xpu/kernel/debug.h" -#include "xpu/kernel/math.h" -#endif +#if defined(PADDLE_WITH_CUDA) namespace optimizer_config { -#if defined(PADDLE_WITH_CUDA) - __constant__ float nonclk_coeff = 0.1; __constant__ float clk_coeff = 1; @@ -39,24 +33,31 @@ __constant__ float mf_initial_g2sum = 3.0; __constant__ float mf_initial_range = 1e-4; __constant__ float mf_min_bound = -10; __constant__ float mf_max_bound = 10; +} // namespace optimizer_config #elif defined(PADDLE_WITH_XPU_KP) - -_global_ptr_ float* nonclk_coeff; -_global_ptr_ float* clk_coeff; - -_global_ptr_ float* min_bound; -_global_ptr_ float* max_bound; -_global_ptr_ float* learning_rate; -_global_ptr_ float* initial_g2sum; -_global_ptr_ float* initial_range; - -_global_ptr_ float* mf_create_thresholds; -_global_ptr_ float* mf_learning_rate; -_global_ptr_ float* mf_initial_g2sum; -_global_ptr_ float* mf_initial_range; -_global_ptr_ float* mf_min_bound; -_global_ptr_ float* mf_max_bound; +namespace paddle { +namespace framework { + +class OptimizerConfig { + public: + float nonclk_coeff; + float clk_coeff; + + float min_bound; + float max_bound; + float learning_rate; + float initial_g2sum; + float initial_range; + + float mf_create_thresholds; + float mf_learning_rate; + float mf_initial_g2sum; + float mf_initial_range; + float mf_min_bound; + float mf_max_bound; +}; +} // namespace framework +} // namespace paddle #endif -} // namespace optimizer_config diff --git a/paddle/fluid/framework/fleet/ps_gpu_wrapper.kps b/paddle/fluid/framework/fleet/ps_gpu_wrapper.kps index 6d69ae0136d68..571a090b9b4a6 100644 --- a/paddle/fluid/framework/fleet/ps_gpu_wrapper.kps +++ b/paddle/fluid/framework/fleet/ps_gpu_wrapper.kps @@ -18,7 +18,6 @@ limitations under the License. */ #include #include #include -#include "paddle/fluid/framework/fleet/heter_ps/optimizer_conf.h" #include "paddle/fluid/framework/fleet/ps_gpu_wrapper.h" #include "paddle/fluid/framework/lod_tensor.h" #include "xpu/kernel/cluster_header.h" // NOLINT @@ -162,23 +161,7 @@ __global__ void PushCopy(FeaturePushValue* dest, float** src, long long* len, } } -PSGPUWrapper::~PSGPUWrapper() { - delete HeterPs_; - xpu_free((void*)optimizer_config::nonclk_coeff); - xpu_free((void*)optimizer_config::clk_coeff); - xpu_free((void*)optimizer_config::min_bound); - xpu_free((void*)optimizer_config::max_bound); - xpu_free((void*)optimizer_config::learning_rate); - xpu_free((void*)optimizer_config::initial_g2sum); - xpu_free((void*)optimizer_config::initial_range); - - xpu_free((void*)optimizer_config::mf_create_thresholds); - xpu_free((void*)optimizer_config::mf_learning_rate); - xpu_free((void*)optimizer_config::mf_initial_g2sum); - xpu_free((void*)optimizer_config::mf_initial_range); - xpu_free((void*)optimizer_config::mf_min_bound); - xpu_free((void*)optimizer_config::mf_max_bound); -} +PSGPUWrapper::~PSGPUWrapper() { delete HeterPs_; } void PSGPUWrapper::CopyForPull(const paddle::platform::Place& place, uint64_t** gpu_keys, @@ -272,66 +255,29 @@ void PSGPUWrapper::SetSparseSGD(float nonclk_coeff, float clk_coeff, float min_bound, float max_bound, float learning_rate, float initial_g2sum, float initial_range) { - xpu_malloc(reinterpret_cast(&optimizer_config::nonclk_coeff), - sizeof(float)); - xpu_malloc(reinterpret_cast(&optimizer_config::clk_coeff), - sizeof(float)); - xpu_malloc(reinterpret_cast(&optimizer_config::min_bound), - sizeof(float)); - xpu_malloc(reinterpret_cast(&optimizer_config::max_bound), - sizeof(float)); - xpu_malloc(reinterpret_cast(&optimizer_config::learning_rate), - sizeof(float)); - xpu_malloc(reinterpret_cast(&optimizer_config::initial_g2sum), - sizeof(float)); - xpu_malloc(reinterpret_cast(&optimizer_config::initial_range), - sizeof(float)); - - xpu_memcpy((void*)optimizer_config::nonclk_coeff, &nonclk_coeff, - sizeof(float), XPU_HOST_TO_DEVICE); - xpu_memcpy((void*)optimizer_config::clk_coeff, &clk_coeff, sizeof(float), - XPU_HOST_TO_DEVICE); - xpu_memcpy((void*)optimizer_config::min_bound, &min_bound, sizeof(float), - XPU_HOST_TO_DEVICE); - xpu_memcpy((void*)optimizer_config::max_bound, &max_bound, sizeof(float), - XPU_HOST_TO_DEVICE); - xpu_memcpy((void*)optimizer_config::learning_rate, &learning_rate, - sizeof(float), XPU_HOST_TO_DEVICE); - xpu_memcpy((void*)optimizer_config::initial_g2sum, &initial_g2sum, - sizeof(float), XPU_HOST_TO_DEVICE); - xpu_memcpy((void*)optimizer_config::initial_range, &initial_range, - sizeof(float), XPU_HOST_TO_DEVICE); + OptimizerConfig optimizer_config; + optimizer_config.nonclk_coeff = nonclk_coeff; + optimizer_config.clk_coeff = clk_coeff; + optimizer_config.min_bound = min_bound; + optimizer_config.max_bound = max_bound; + optimizer_config.learning_rate = learning_rate; + optimizer_config.initial_g2sum = initial_g2sum; + optimizer_config.initial_range = initial_range; + HeterPs_->set_sparse_sgd(optimizer_config); } void PSGPUWrapper::SetEmbedxSGD(float mf_create_thresholds, float mf_learning_rate, float mf_initial_g2sum, float mf_initial_range, float mf_min_bound, float mf_max_bound) { - xpu_malloc(reinterpret_cast(&optimizer_config::mf_create_thresholds), - sizeof(float)); - xpu_malloc(reinterpret_cast(&optimizer_config::mf_learning_rate), - sizeof(float)); - xpu_malloc(reinterpret_cast(&optimizer_config::mf_initial_g2sum), - sizeof(float)); - xpu_malloc(reinterpret_cast(&optimizer_config::mf_initial_range), - sizeof(float)); - xpu_malloc(reinterpret_cast(&optimizer_config::mf_min_bound), - sizeof(float)); - xpu_malloc(reinterpret_cast(&optimizer_config::mf_max_bound), - sizeof(float)); - - xpu_memcpy((void*)optimizer_config::mf_create_thresholds, - &mf_create_thresholds, sizeof(float), XPU_HOST_TO_DEVICE); - xpu_memcpy((void*)optimizer_config::mf_initial_g2sum, &mf_initial_g2sum, - sizeof(float), XPU_HOST_TO_DEVICE); - xpu_memcpy((void*)optimizer_config::mf_initial_range, &mf_initial_range, - sizeof(float), XPU_HOST_TO_DEVICE); - xpu_memcpy((void*)optimizer_config::mf_min_bound, &mf_min_bound, - sizeof(float), XPU_HOST_TO_DEVICE); - xpu_memcpy((void*)optimizer_config::mf_max_bound, &mf_max_bound, - sizeof(float), XPU_HOST_TO_DEVICE); - xpu_memcpy((void*)optimizer_config::mf_learning_rate, &mf_learning_rate, - sizeof(float), XPU_HOST_TO_DEVICE); + OptimizerConfig optimizer_config; + optimizer_config.mf_create_thresholds = mf_create_thresholds; + optimizer_config.mf_learning_rate = mf_learning_rate; + optimizer_config.mf_initial_g2sum = mf_initial_g2sum; + optimizer_config.mf_initial_range = mf_initial_range; + optimizer_config.mf_min_bound = mf_min_bound; + optimizer_config.mf_max_bound = mf_max_bound; + HeterPs_->set_embedx_sgd(optimizer_config); } } // end namespace framework From 5439f07dd787ec79048aa37cd734cbf3b42624bb Mon Sep 17 00:00:00 2001 From: qipengh Date: Thu, 21 Apr 2022 21:31:38 +0800 Subject: [PATCH 20/66] [MLU]:add elementwise_div op (#41810) --- .../elementwise/elementwise_div_op_mlu.cc | 141 ++++++++++ .../mlu/test_elementwise_div_op_mlu.py | 253 ++++++++++++++++++ 2 files changed, 394 insertions(+) create mode 100644 paddle/fluid/operators/elementwise/elementwise_div_op_mlu.cc create mode 100644 python/paddle/fluid/tests/unittests/mlu/test_elementwise_div_op_mlu.py diff --git a/paddle/fluid/operators/elementwise/elementwise_div_op_mlu.cc b/paddle/fluid/operators/elementwise/elementwise_div_op_mlu.cc new file mode 100644 index 0000000000000..1a7d757a27d13 --- /dev/null +++ b/paddle/fluid/operators/elementwise/elementwise_div_op_mlu.cc @@ -0,0 +1,141 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include + +#include "paddle/fluid/operators/elementwise/elementwise_div_op.h" +#include "paddle/fluid/operators/elementwise/elementwise_mlu.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +template +class ElementwiseDivMLUKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + MLUBinaryOp(ctx); + } +}; + +template +class ElementwiseDivGradMLUKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto* out = ctx.Input("Out"); + auto* x = ctx.Input("X"); + auto* y = ctx.Input("Y"); + auto* dout = ctx.Input(framework::GradVarName("Out")); + auto* dx = ctx.Output(framework::GradVarName("X")); + auto* dy = ctx.Output(framework::GradVarName("Y")); + int axis = ctx.Attr("axis"); + + const auto& x_dims = x->dims(); + const auto& y_dims = y->dims(); + axis = (axis < 0 ? (std::abs(x_dims.size() - y_dims.size()) + axis + 1) + : axis); + int max_dim = std::max(x_dims.size(), y_dims.size()); + std::vector x_dims_array(max_dim); + std::vector y_dims_array(max_dim); + std::vector out_dims_array(max_dim); + GetBroadcastDimsArrays(x_dims, y_dims, x_dims_array.data(), + y_dims_array.data(), out_dims_array.data(), max_dim, + axis); + + MLUCnnlTensorDesc x_desc(max_dim, x_dims_array.data(), ToCnnlDataType()); + MLUCnnlTensorDesc y_desc(max_dim, y_dims_array.data(), ToCnnlDataType()); + MLUCnnlTensorDesc dout_desc(*dout); + MLUCnnlOpTensorDesc mul_op_desc(CNNL_OP_TENSOR_MUL, ToCnnlDataType(), + CNNL_NOT_PROPAGATE_NAN); + + // compute dout/y == 1/y * dout + Tensor dout_div_y(dout->dtype()); + dout_div_y.Resize(dout->dims()); + dout_div_y.mutable_data(ctx.GetPlace()); + MLUBinary
(ctx, CNNL_COMPUTATION_HIGH_PRECISION, dout_desc.get(), + GetBasePtr(dout), y_desc.get(), GetBasePtr(y), + dout_desc.get(), GetBasePtr(&dout_div_y)); + + if (dx) { + // compute dx = dout/y = 1/y * dout + if (dx->dims() != dout->dims()) { + dx->mutable_data(ctx.GetPlace()); + + std::vector reduce_axes; + GetReduceAxes(axis, dout_div_y.dims(), dx->dims(), &reduce_axes); + MLUCnnlReduceDesc reduction_desc( + reduce_axes, CNNL_REDUCE_ADD, ToCnnlDataType(), + CNNL_NOT_PROPAGATE_NAN, CNNL_REDUCE_NO_INDICES, CNNL_32BIT_INDICES); + MLUCnnlTensorDesc dx_desc(*dx); + MLUCnnl::Reduce(ctx, true /*need_workspace*/, reduction_desc.get(), + nullptr, dout_desc.get(), GetBasePtr(&dout_div_y), 0, + nullptr, nullptr, dx_desc.get(), GetBasePtr(dx)); + } else { + dx->ShareDataWith(dout_div_y); + } + } + + if (dy) { + // compute dy = -out * (dout/y) = -out/y * dout + Tensor neg_out(out->type()); + neg_out.mutable_data(out->dims(), ctx.GetPlace()); + + MLUCnnlTensorDesc out_desc(*out); + MLUUnary(ctx, CNNL_COMPUTATION_HIGH_PRECISION, out_desc.get(), + GetBasePtr(out), out_desc.get(), GetBasePtr(&neg_out)); + + Tensor dy_temp(y->dtype()); + dy_temp.Resize(dout->dims()); + dy_temp.mutable_data(ctx.GetPlace()); + + MLUCnnl::OpTensor(ctx, mul_op_desc.get(), dout_desc.get(), + GetBasePtr(&neg_out), dout_desc.get(), + GetBasePtr(&dout_div_y), dout_desc.get(), + GetBasePtr(&dy_temp), ToCnnlDataType()); + + if (dy->dims() != dout->dims()) { + dy->mutable_data(ctx.GetPlace()); + + std::vector reduce_axes; + GetReduceAxes(axis, dy_temp.dims(), dy->dims(), &reduce_axes); + MLUCnnlReduceDesc reduction_desc( + reduce_axes, CNNL_REDUCE_ADD, ToCnnlDataType(), + CNNL_NOT_PROPAGATE_NAN, CNNL_REDUCE_NO_INDICES, CNNL_32BIT_INDICES); + MLUCnnlTensorDesc dy_desc(*dy); + MLUCnnl::Reduce(ctx, true /*need_workspace*/, reduction_desc.get(), + nullptr, dout_desc.get(), GetBasePtr(&dy_temp), 0, + nullptr, nullptr, dy_desc.get(), GetBasePtr(dy)); + } else { + dy->ShareDataWith(dy_temp); + } + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +namespace plat = paddle::platform; + +REGISTER_OP_MLU_KERNEL(elementwise_div, ops::ElementwiseDivMLUKernel, + ops::ElementwiseDivMLUKernel, + ops::ElementwiseDivMLUKernel); + +REGISTER_OP_MLU_KERNEL(elementwise_div_grad, + ops::ElementwiseDivGradMLUKernel, + ops::ElementwiseDivGradMLUKernel, + ops::ElementwiseDivGradMLUKernel); diff --git a/python/paddle/fluid/tests/unittests/mlu/test_elementwise_div_op_mlu.py b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_div_op_mlu.py new file mode 100644 index 0000000000000..8fdac75c4c1a8 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/mlu/test_elementwise_div_op_mlu.py @@ -0,0 +1,253 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import numpy as np +import unittest +import sys +sys.path.append("..") +from op_test import OpTest, skip_check_grad_ci +import paddle +import paddle.fluid as fluid +from paddle.fluid.core import ops + +paddle.enable_static() +SEED = 2022 + + +class TestElementwiseDiv(OpTest): + def setUp(self): + self.set_mlu() + self.op_type = "elementwise_div" + + self.init_dtype() + np.random.seed(SEED) + x = np.random.uniform(1, 2, [11, 17]).astype(self.dtype) + y = np.random.uniform(1, 2, [11, 17]).astype(self.dtype) + out = np.divide(x, y) + + self.inputs = { + 'X': OpTest.np_dtype_to_fluid_dtype(x), + 'Y': OpTest.np_dtype_to_fluid_dtype(y) + } + self.attrs = {} + self.outputs = {'Out': out} + + def set_mlu(self): + self.__class__.use_mlu = True + self.place = paddle.device.MLUPlace(0) + + def init_dtype(self): + self.dtype = np.float32 + + def test_check_output(self): + self.check_output_with_place(self.place) + + def test_check_grad_normal(self): + self.check_grad_with_place( + self.place, ['X', 'Y'], 'Out', max_relative_error=0.05) + + def test_check_grad_ingore_x(self): + self.check_grad_with_place( + self.place, ['Y'], + 'Out', + max_relative_error=0.05, + no_grad_set=set("X")) + + def test_check_grad_ingore_y(self): + self.check_grad_with_place( + self.place, ['X'], + 'Out', + max_relative_error=0.05, + no_grad_set=set("Y")) + + +class TestElementwiseDivFp16(OpTest): + def setUp(self): + self.set_mlu() + self.op_type = "elementwise_div" + + self.init_dtype() + np.random.seed(SEED) + x = np.random.uniform(1, 2, [3, 4]).astype(self.dtype) + y = np.random.uniform(1, 2, [3, 4]).astype(self.dtype) + out = np.divide(x, y) + + self.inputs = { + 'X': OpTest.np_dtype_to_fluid_dtype(x), + 'Y': OpTest.np_dtype_to_fluid_dtype(y) + } + self.attrs = {} + self.outputs = {'Out': out} + + def set_mlu(self): + self.__class__.use_mlu = True + self.__class__.no_need_check_grad = True + self.place = paddle.device.MLUPlace(0) + + def init_dtype(self): + self.dtype = np.float16 + + def test_check_output(self): + self.check_output_with_place(self.place, atol=1e-5) + + +@skip_check_grad_ci( + reason="[skip shape check] Use y_shape(1) to test broadcast.") +class TestTestElementwiseDiv_scalar(TestElementwiseDiv): + def setUp(self): + self.set_mlu() + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [20, 3, 4]).astype(np.float32), + 'Y': np.random.uniform(0.1, 1, [1]).astype(np.float32) + } + self.outputs = {'Out': self.inputs['X'] / self.inputs['Y']} + + +class TestTestElementwiseDiv_Vector(TestElementwiseDiv): + def setUp(self): + self.set_mlu() + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [100]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [100]).astype("float32") + } + self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} + + +class TestTestElementwiseDiv_broadcast_0(TestElementwiseDiv): + def setUp(self): + self.set_mlu() + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [100, 3, 4]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [100]).astype("float32") + } + + self.attrs = {'axis': 0} + self.outputs = { + 'Out': + np.divide(self.inputs['X'], self.inputs['Y'].reshape(100, 1, 1)) + } + + +class TestTestElementwiseDiv_broadcast_1(TestElementwiseDiv): + def setUp(self): + self.set_mlu() + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [2, 100, 4]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [100]).astype("float32") + } + + self.attrs = {'axis': 1} + self.outputs = { + 'Out': + np.divide(self.inputs['X'], self.inputs['Y'].reshape(1, 100, 1)) + } + + +class TestTestElementwiseDiv_broadcast_2(TestElementwiseDiv): + def setUp(self): + self.set_mlu() + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [2, 3, 100]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [100]).astype("float32") + } + + self.outputs = { + 'Out': + np.divide(self.inputs['X'], self.inputs['Y'].reshape(1, 1, 100)) + } + + +class TestTestElementwiseDiv_broadcast_3(TestElementwiseDiv): + def setUp(self): + self.set_mlu() + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [2, 10, 12, 5]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [10, 12]).astype("float32") + } + + self.attrs = {'axis': 1} + self.outputs = { + 'Out': + np.divide(self.inputs['X'], self.inputs['Y'].reshape(1, 10, 12, 1)) + } + + +class TestTestElementwiseDiv_broadcast_4(TestElementwiseDiv): + def setUp(self): + self.set_mlu() + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [2, 3, 50]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [2, 1, 50]).astype("float32") + } + self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} + + +class TestTestElementwiseDiv_broadcast_5(TestElementwiseDiv): + def setUp(self): + self.set_mlu() + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [2, 3, 4, 20]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [2, 3, 1, 20]).astype("float32") + } + self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} + + +class TestTestElementwiseDiv_commonuse_1(TestElementwiseDiv): + def setUp(self): + self.set_mlu() + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [2, 3, 100]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [1, 1, 100]).astype("float32"), + } + self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} + + +class TestTestElementwiseDiv_commonuse_2(TestElementwiseDiv): + def setUp(self): + self.set_mlu() + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [30, 3, 1, 5]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [30, 1, 4, 1]).astype("float32"), + } + self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} + + +class TestTestElementwiseDiv_xsize_lessthan_ysize(TestElementwiseDiv): + def setUp(self): + self.set_mlu() + self.op_type = "elementwise_div" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [10, 12]).astype("float32"), + 'Y': np.random.uniform(0.1, 1, [2, 3, 10, 12]).astype("float32"), + } + + self.attrs = {'axis': 2} + + self.outputs = {'Out': np.divide(self.inputs['X'], self.inputs['Y'])} + + +if __name__ == '__main__': + unittest.main() From f1704b204363052a771f6584412847627a44545d Mon Sep 17 00:00:00 2001 From: zyfncg Date: Thu, 21 Apr 2022 22:13:38 +0800 Subject: [PATCH 21/66] optimiaze performance of PreparePhiData (#42093) --- paddle/fluid/imperative/prepared_operator.h | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/imperative/prepared_operator.h b/paddle/fluid/imperative/prepared_operator.h index b3c5a6b5fa220..cb3275674ed49 100644 --- a/paddle/fluid/imperative/prepared_operator.h +++ b/paddle/fluid/imperative/prepared_operator.h @@ -581,10 +581,11 @@ void PreparePhiData(const phi::Kernel& pt_kernel, for (size_t i = 0; i < input_names.size(); ++i) { auto& in_def = input_defs.at(i); - if (ins.find(input_names[i]) == ins.end()) { + auto iter = ins.find(input_names[i]); + if (iter == ins.end()) { continue; } - auto& ins_vector = ins.at(input_names[i]); + auto& ins_vector = iter->second; for (size_t offset = 0; offset < ins_vector.size(); ++offset) { auto& var = ins_vector[offset]; @@ -593,11 +594,15 @@ void PreparePhiData(const phi::Kernel& pt_kernel, if (in_def.backend == phi::Backend::ALL_BACKEND) { continue; } - auto expected_place = phi::TransToPhiPlace(in_def.backend); - if (platform::is_same_place(tensor_in->place(), expected_place)) { + auto tensor_backend = phi::TransToPhiBackend(tensor_in->place()); + if (in_def.backend == tensor_backend || + (in_def.backend == phi::Backend::GPUDNN && + tensor_backend == phi::Backend::GPU)) { continue; } + auto expected_place = phi::TransToPhiPlace(in_def.backend); + VLOG(3) << "Phi Transform Variable " << input_names[i] << " from " << tensor_in->place() << " to " << expected_place; From c51f55f9bcb8aad17047f7430fe94268568e4471 Mon Sep 17 00:00:00 2001 From: heliqi <1101791222@qq.com> Date: Thu, 21 Apr 2022 22:55:08 +0800 Subject: [PATCH 22/66] fix onnxruntime bug (#42095) --- paddle/fluid/inference/api/details/zero_copy_tensor.cc | 7 +++---- paddle/fluid/inference/api/onnxruntime_predictor.cc | 6 ++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/inference/api/details/zero_copy_tensor.cc b/paddle/fluid/inference/api/details/zero_copy_tensor.cc index 7461724afb4dd..5e1a9b85ff586 100644 --- a/paddle/fluid/inference/api/details/zero_copy_tensor.cc +++ b/paddle/fluid/inference/api/details/zero_copy_tensor.cc @@ -693,10 +693,9 @@ void Tensor::ORTCopyToCpu(T *data) const { if (place_ == PlaceType::kCPU) { std::memcpy(static_cast(data), value.GetTensorData(), size); } else { - paddle::memory::Copy(paddle::platform::CPUPlace(), - static_cast(data), - paddle::platform::CUDAPlace(device_), - value.GetTensorData(), size, nullptr); + PADDLE_THROW(paddle::platform::errors::Unavailable( + "CopyToCpu error.The current ONNXRuntime backend doesn't support " + "GPU.")); } } diff --git a/paddle/fluid/inference/api/onnxruntime_predictor.cc b/paddle/fluid/inference/api/onnxruntime_predictor.cc index eb561667fe1f3..e42e395ce90f8 100644 --- a/paddle/fluid/inference/api/onnxruntime_predictor.cc +++ b/paddle/fluid/inference/api/onnxruntime_predictor.cc @@ -279,6 +279,12 @@ bool ONNXRuntimePredictor::Run(const std::vector &inputs, bool ONNXRuntimePredictor::ZeroCopyRun() { try { + const char *device_name = place_ == PlaceType::kCPU ? "Cpu" : "Cuda"; + for (auto output : output_desc_) { + Ort::MemoryInfo out_memory_info(device_name, OrtDeviceAllocator, + place_.GetDeviceId(), OrtMemTypeDefault); + binding_->BindOutput(output.name.c_str(), out_memory_info); + } session_.Run({}, *(binding_.get())); } catch (const std::exception &e) { LOG(ERROR) << e.what(); From 79303c2ac1305b6f506ada5b767639392c2cd695 Mon Sep 17 00:00:00 2001 From: Aganlengzi Date: Thu, 21 Apr 2022 23:22:24 +0800 Subject: [PATCH 23/66] [CustomDevice] fix exit order (#42088) --- python/paddle/fluid/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py index 13b964274fde2..8dbeb3eeb27c3 100644 --- a/python/paddle/fluid/__init__.py +++ b/python/paddle/fluid/__init__.py @@ -227,7 +227,9 @@ def remove_flag_if_exists(name): atexit.register(core.npu_finalize) # NOTE(Aurelius84): clean up ExecutorCacheInfo in advance manually. atexit.register(core.clear_executor_cache) + # NOTE(Aganlengzi): clean up KernelFactory in advance manually. -atexit.register(core.clear_kernel_factory) # NOTE(wangran16): clean up DeviceManger in advance manually. +# Keep clear_kernel_factory running before clear_device_manager atexit.register(core.clear_device_manager) +atexit.register(core.clear_kernel_factory) From 86a8863191f52b40f924792bc687038f25fcedc4 Mon Sep 17 00:00:00 2001 From: Huihuang Zheng Date: Fri, 22 Apr 2022 10:55:28 +0800 Subject: [PATCH 24/66] Change CINN tag, prepare for CINN release/v0.2 (#42063) As the title --- cmake/external/cinn.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/cinn.cmake b/cmake/external/cinn.cmake index 004bf353d34e8..2ec9a3faa07b7 100644 --- a/cmake/external/cinn.cmake +++ b/cmake/external/cinn.cmake @@ -26,7 +26,7 @@ add_definitions(-w) ###################################### include(ExternalProject) set(CINN_PREFIX_DIR ${THIRD_PARTY_PATH}/CINN) -set(CINN_GIT_TAG eedb801ca39bfc6b9621bc76c24a0bf98cb8404b) +set(CINN_GIT_TAG release/v0.2) set(CINN_OPTIONAL_ARGS -DPY_VERSION=${PY_VERSION} -DWITH_CUDA=${WITH_GPU} -DWITH_CUDNN=${WITH_GPU} From 1b8fd85d4460b5cf9dab3ce68897b130f83ebfb2 Mon Sep 17 00:00:00 2001 From: YuanRisheng Date: Fri, 22 Apr 2022 11:12:23 +0800 Subject: [PATCH 25/66] Support double grad check of op in Eager mode and Add log double grad yaml (#42090) * Support double grad check of op in Eager mode * fix bugs of backward yaml * adjust code format --- .../fluid/tests/unittests/gradient_checker.py | 224 ++++++++++++++++++ .../unittests/test_activation_nn_grad.py | 20 +- python/paddle/utils/code_gen/backward.yaml | 13 +- 3 files changed, 255 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/gradient_checker.py b/python/paddle/fluid/tests/unittests/gradient_checker.py index dff2b7aa8d8d6..562d52668ce5b 100644 --- a/python/paddle/fluid/tests/unittests/gradient_checker.py +++ b/python/paddle/fluid/tests/unittests/gradient_checker.py @@ -20,11 +20,13 @@ import collections import numpy as np from itertools import product +import paddle import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.executor import Executor from paddle.fluid.backward import _append_grad_suffix_, _as_list +from paddle.fluid.framework import _test_eager_guard def _product(t): @@ -58,6 +60,19 @@ def _get_item(t, i, np_dtype): raise ValueError("Not supported data type " + str(np_dtype)) +def _get_item_for_dygraph(t, i, np_dtype): + if np_dtype == np.float16: + np_t = t.numpy().astype(np.float16) + elif np_dtype == np.float32: + np_t = t.numpy().astype(np.float32) + elif np_dtype == np.float64: + np_t = t.numpy().astype(np.float64) + else: + raise ValueError("Not supported data type " + str(np_dtype)) + np_t = np_t.flatten() + return np_t[i] + + def _set_item(t, i, e, np_dtype): if np_dtype == np.float16: np_t = np.array(t).astype(np.float16) @@ -74,6 +89,22 @@ def _set_item(t, i, e, np_dtype): raise ValueError("Not supported data type " + str(np_dtype)) +def _set_item_for_dygraph(t, i, e, np_dtype): + if np_dtype == np.float16: + np_t = t.numpy().astype(np.float16) + elif np_dtype == np.float32: + np_t = t.numpy().astype(np.float32) + elif np_dtype == np.float64: + np_t = t.numpy().astype(np.float64) + else: + raise ValueError("Not supported data type " + str(np_dtype)) + shape = np_t.shape + np_t = np_t.flatten() + np_t[i] = e + np_t = np_t.reshape(shape) + paddle.assign(np_t, t) + + def set_var_in_scope(scope, place, name, value, recursive_seq_len=None): t = scope.var(name).get_tensor() t.set(value, place) @@ -138,6 +169,8 @@ def run(): np_type = dtype_to_np_dtype(x.dtype) jacobian = [make_jacobian(x, _product(yi.shape), np_type) for yi in y] + if np_type == np.float64: + delta = 1e-5 for i in six.moves.xrange(x_size): orig = _get_item(x_t, i, np_type) x_pos = orig + delta @@ -510,3 +543,194 @@ def triple_grad_check(x, eps=eps, atol=atol, rtol=rtol) + + +def get_static_double_grad(x, y, x_init=None, dy_init=None, place=None): + """ + Get Double Grad result of static graph. + + Args: + x (Variable|list[Variable]): input variables to the program. + y (Variable|list[Variable]): output variables to the program. + x_init (numpy.array|list[numpy.array]|None): the init value for input x. + dy_init (numpy.array|list[numpy.array]|None): the init value for output y. + place (fluid.CPUPlace or fluid.CUDAPlace): the device. + Returns: + A list of numpy array that stores second derivative result calulated by static graph. + """ + + program = fluid.default_main_program() + scope = fluid.executor.global_scope() + y_grads = [] + for i in six.moves.xrange(len(y)): + yi = y[i] + dyi_name = _append_grad_suffix_(yi.name) + np_type = dtype_to_np_dtype(yi.dtype) + dy = program.global_block().create_var( + name=dyi_name, shape=yi.shape, dtype=np_type, persistable=True) + dy.stop_gradient = False + set_var_in_scope(scope, place, dyi_name, dy_init[i]) + y_grads.append(dy) + + # append first order grads + dx = fluid.gradients(y, x, y_grads) + + # y_grads are the input of first-order backward, + # so, they are also the input of second-order backward. + x += y_grads + x_init += dy_init + y = dx + + # check input arguments + x = _as_list(x) + y = _as_list(y) + + for v in x: + v.stop_gradient = False + v.persistable = True + if place is None: + place = fluid.CPUPlace() + if program is None: + program = fluid.default_main_program() + + # init variable in strtup program + scope = fluid.executor.global_scope() + exe = fluid.Executor(place) + exe.run(fluid.default_startup_program()) + + x_init = _as_list(x_init) + # init inputs if x_init is not None + if x_init: + if len(x_init) != len(x): + raise ValueError('len(x_init) (=%d) is not the same' + ' as len(x) (= %d)' % (len(x_init), len(x))) + # init variable in main program + for var, arr in zip(x, x_init): + assert var.shape == arr.shape + feeds = {k.name: v for k, v in zip(x, x_init)} + exe.run(program, feed=feeds, scope=scope) + + dys = [] + for yi in y: + np_type = dtype_to_np_dtype(yi.dtype) + dy_name = _append_grad_suffix_(yi.name) + # create dy Variable in Program + dy = program.global_block().create_var( + name=dy_name, shape=yi.shape, dtype=np_type, persistable=True) + # init dy tensor in scope + value = np.ones(yi.shape, dtype=np_type) + dy_t = set_var_in_scope(scope, place, dy_name, value) + dys.append(dy) + + # append second order backward + ddx = fluid.gradients(y, x, dys) + exe = fluid.Executor(place) + + # filter None in dx for DX/DY may be None in kernel + # only fetch not None dx in exe.run + filted = [(i, dxi) for i, dxi in enumerate(ddx) if dxi is not None] + filted_idx, filted_ddx = zip(*filted) + ddx_res = exe.run(program, scope=scope, fetch_list=filted_ddx) + + return ddx_res + + +def get_eager_double_grad(func, x_init=None, dy_init=None): + """ + Get Double Grad result of dygraph. + + Args: + func: A wrapped dygraph function that its logic is equal to static program + x_init (numpy.array|list[numpy.array]|None): the init value for input x. + dy_init (numpy.array|list[numpy.array]|None): the init value for gradient of output. + Returns: + A list of numpy array that stores second derivative result calulated by dygraph + """ + inputs = [] + dys = [] + for x in x_init: + input_tensor = paddle.to_tensor(x) + input_tensor.stop_gradient = False + inputs.append(input_tensor) + for dy in dy_init: + dy_tensor = paddle.to_tensor(dy) + dy_tensor.stop_gradient = False + dys.append(dy_tensor) + # calculate first derivative + outputs = func(inputs) + d_inputs = paddle.grad( + outputs=outputs, inputs=inputs, grad_outputs=dys, create_graph=True) + + # calcluate second derivative + inputs = inputs + dys + ddys = [] + for d_input in d_inputs: + d_input.stop_gradient = False + ddy = paddle.ones(shape=d_input.shape, dtype=d_input.dtype) + ddy.stop_gradient = False + ddys.append(ddy) + dd_inputs = paddle.grad(outputs=d_inputs, inputs=inputs, grad_outputs=ddys) + return [dd_input.numpy() for dd_input in dd_inputs] + + +def double_grad_check_for_dygraph(func, + x, + y, + x_init=None, + place=None, + atol=1e-5, + rtol=1e-3, + raise_exception=True): + """ + Check gradients of gradients. This function will append backward to the + program before second order gradient check. + + Args: + func: A wrapped dygraph function that its logic is equal to static program + x (Variable|list[Variable]): input variables to the program. + y (Variable|list[Variable]): output variables to the program. + x_init (numpy.array|list[numpy.array]|None): the init value for input x. + place (fluid.CPUPlace or fluid.CUDAPlace): the device. + eps (float): perturbation for finite differences. + atol (float): absolute tolerance. + rtol (float): relative tolerance. + raise_exception (bool): whether to raise an exception if + the check fails. Default is True. + """ + + def fail_test(msg): + if raise_exception: + raise RuntimeError(msg) + return False + + # check input arguments + x = _as_list(x) + for v in x: + v.stop_gradient = False + v.persistable = True + y = _as_list(y) + + y_grads_init = [] + for yi in y: + np_type = dtype_to_np_dtype(yi.dtype) + v = np.random.random(size=yi.shape).astype(np_type) + y_grads_init.append(v) + + x_init = _as_list(x_init) + + paddle.disable_static() + with _test_eager_guard(): + eager_double_grad = get_eager_double_grad(func, x_init, y_grads_init) + paddle.enable_static() + + static_double_grad = get_static_double_grad(x, y, x_init, y_grads_init, + place) + + for i in six.moves.xrange(len(static_double_grad)): + if not np.allclose(static_double_grad[i], eager_double_grad[i], rtol, + atol): + msg = 'Check eager double result fail. Mismatch between static_graph double grad %s ' \ + 'and eager double grad %s on %s,\n' \ + 'static:%s\n eager:%s\n' \ + % (static_double_grad[i].name, eager_double_grad[i].name, str(place), static_double_grad[i], eager_double_grad[i]) + return fail_test(msg) diff --git a/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py b/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py index eb4243ef1cbf1..72240be41dd49 100644 --- a/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_activation_nn_grad.py @@ -23,6 +23,7 @@ import paddle.fluid.core as core import gradient_checker import paddle.nn.functional as F +from paddle.fluid.framework import _test_eager_guard from decorator_helper import prog_scope @@ -42,6 +43,7 @@ def func(self, place): [x], y, x_init=x_arr, place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -64,6 +66,7 @@ def func(self, place): [x], y, x_init=x_arr, place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -86,6 +89,7 @@ def func(self, place): [x], y, x_init=x_arr, place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -108,6 +112,7 @@ def func(self, place): [x], y, x_init=x_arr, place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -132,6 +137,7 @@ def func(self, place): [x], y, x_init=x_arr, place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -158,6 +164,7 @@ def func(self, place): [x], y, x_init=x_arr, place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places = [fluid.CUDAPlace(0)] @@ -184,6 +191,7 @@ def func(self, place): [x], y, x_init=x_arr, place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -210,6 +218,7 @@ def func(self, place): [x], y, x_init=x_arr, place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -234,6 +243,7 @@ def func(self, place): [x], y, x_init=x_arr, place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places = [fluid.CUDAPlace(0)] @@ -258,6 +268,7 @@ def func(self, place): [x], y, x_init=x_arr, place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places = [fluid.CUDAPlace(0)] @@ -282,6 +293,7 @@ def func(self, place): [x], y, x_init=x_arr, place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -310,6 +322,7 @@ def func(self, place): [x], y, x_init=x_arr, place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -318,6 +331,9 @@ def test_grad(self): class TestLogDoubleGradCheck(unittest.TestCase): + def log_wrapper(self, x): + return paddle.log(x[0]) + @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -332,8 +348,11 @@ def func(self, place): gradient_checker.double_grad_check( [x], y, x_init=x_arr, place=place, eps=eps) + gradient_checker.double_grad_check_for_dygraph( + self.log_wrapper, [x], y, x_init=x_arr, place=place) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -342,5 +361,4 @@ def test_grad(self): if __name__ == "__main__": - paddle.enable_static() unittest.main() diff --git a/python/paddle/utils/code_gen/backward.yaml b/python/paddle/utils/code_gen/backward.yaml index 64acc140c2117..dfdc2335ae180 100644 --- a/python/paddle/utils/code_gen/backward.yaml +++ b/python/paddle/utils/code_gen/backward.yaml @@ -839,6 +839,16 @@ kernel : func : log2_grad +- backward_api : log_double_grad + forward : log_grad (Tensor x, Tensor grad_out) -> Tensor(grad_x) + args : (Tensor x, Tensor grad_out, Tensor grad_x_grad) + output : Tensor(x_grad), Tensor(grad_out_grad) + infer_meta : + func : GeneralBinaryGradInferMeta + param : [x, x] + kernel : + func : log_double_grad + - backward_api : log_grad forward : log (Tensor x) -> Tensor(out) args : (Tensor x, Tensor out_grad) @@ -848,6 +858,7 @@ param : [x] kernel : func : log_grad + backward : log_double_grad - backward_api : log_loss_grad forward : log_loss (Tensor input, Tensor label, float epsilon) -> Tensor(out) @@ -1473,7 +1484,7 @@ func : UnchangedInferMeta param : [x] kernel : - func : sigmoid_cross_entropy_with_logits_grad + func : sigmoid_cross_entropy_with_logits_grad - backward_api : sigmoid_double_grad forward : sigmoid_grad (Tensor out, Tensor fwd_grad_out) -> Tensor(grad_x) From 23d1b3e8ed8187bfb3bd926934dd6cc71e691e53 Mon Sep 17 00:00:00 2001 From: Jiabin Yang <360788950@qq.com> Date: Fri, 22 Apr 2022 11:19:45 +0800 Subject: [PATCH 26/66] [Eager] fix memory issue for eager (#42086) * fix memory issue for eager * fix bug --- paddle/fluid/eager/tensor_wrapper.h | 14 ++++++++++++++ paddle/phi/api/lib/tensor.cc | 6 +++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/eager/tensor_wrapper.h b/paddle/fluid/eager/tensor_wrapper.h index 405105771b9b1..3ee1603a53ab4 100644 --- a/paddle/fluid/eager/tensor_wrapper.h +++ b/paddle/fluid/eager/tensor_wrapper.h @@ -55,6 +55,20 @@ class TensorWrapper { if (full_reserved_) { VLOG(6) << "Fully reserved tensor: " << tensor.name(); intermidiate_tensor_ = tensor; + if (no_need_buffer_) { + if (phi::DenseTensor::classof(tensor.impl().get())) { + // Only Copy Meta + phi::DenseTensor* dense_tensor = + static_cast(tensor.impl().get()); + auto tw_dense_tensor = + std::make_shared(*dense_tensor); + tw_dense_tensor->clear(); + intermidiate_tensor_.set_impl(tw_dense_tensor); + } else { + PADDLE_THROW(paddle::platform::errors::Fatal( + "Unrecognized tensor type for no_need_buffer feature")); + } + } return; } diff --git a/paddle/phi/api/lib/tensor.cc b/paddle/phi/api/lib/tensor.cc index be0a937c91e4f..a7b89d7a4dca9 100644 --- a/paddle/phi/api/lib/tensor.cc +++ b/paddle/phi/api/lib/tensor.cc @@ -341,7 +341,11 @@ bool Tensor::is_initialized() const { return defined() && impl_->initialized(); } -void Tensor::reset() { impl_.reset(); } +void Tensor::reset() { + impl_.reset(); + autograd_meta_.reset(); + name_ = ""; +} /* Part 6: Operator overloading */ From f0ec580e64c25cb339796b4e22dc70185b0bb98f Mon Sep 17 00:00:00 2001 From: niuliling123 <51102941+niuliling123@users.noreply.github.com> Date: Fri, 22 Apr 2022 12:31:39 +0800 Subject: [PATCH 27/66] Add AutoTune to reader.py for DataLoader (#41202) --- python/paddle/fluid/reader.py | 133 +++++++++++++++++- .../unittests/test_dataloader_autotune.py | 76 ++++++++++ 2 files changed, 206 insertions(+), 3 deletions(-) create mode 100755 python/paddle/fluid/tests/unittests/test_dataloader_autotune.py diff --git a/python/paddle/fluid/reader.py b/python/paddle/fluid/reader.py index 0f5f217442135..841c58821d7a1 100644 --- a/python/paddle/fluid/reader.py +++ b/python/paddle/fluid/reader.py @@ -18,11 +18,13 @@ import numpy as np import threading import paddle +import time + from .framework import Program, Variable, program_guard, default_main_program, default_startup_program, _non_static_mode, cpu_places, _current_expected_place, _in_eager_without_dygraph_check from .executor import global_scope from .data_feeder import DataFeeder, BatchedTensorProvider from .multiprocess_utils import multiprocess_queue_set, CleanupFuncRegistrar, _cleanup_mmap, _cleanup, _set_SIGCHLD_handler -from .dataloader import BatchSampler, Dataset, IterableDataset +from .dataloader import BatchSampler, Dataset, IterableDataset, Subset from .dataloader.dataloader_iter import _DataLoaderIterSingleProcess, _DataLoaderIterMultiProcess, _DatasetKind, default_collate_fn from .dataloader.batch_sampler import _InfiniteIterableSampler from .layers.io import monkey_patch_reader_methods, _copy_reader_var_, double_buffer @@ -36,10 +38,8 @@ import os import multiprocessing import signal - # NOTE: queue has a different name in python2 and python3 import queue - # NOTE: [ avoid hanging & failed quickly ] These value is used in getting data from another process QUEUE_GET_TIMEOUT = 60 @@ -49,6 +49,16 @@ KEEP_DATA_LOADER_ORDER = True USE_PINNED_MEMORY = None +# AutoTune Flags +USE_AUTOTUNE = False +TUNING_STEPS = 500 + + +def set_autotune_config(use_autotune, tuning_steps=500): + global USE_AUTOTUNE + USE_AUTOTUNE = use_autotune + global TUNING_STEPS + TUNING_STEPS = tuning_steps def keep_data_loader_order(*args): @@ -143,6 +153,122 @@ def _check_input_array(cls, item): return arr +class AuToTune(object): + def __init__(self, loader): + self.loader = loader + self.max_num_worker = multiprocessing.cpu_count() / 2 + + def __call__(self): + # use default loader + if (not USE_AUTOTUNE) or (not self.need_autotune()): + return self.loader.num_workers + + # get autotune loader + auto_tune_loader = self.get_autotune_loader() + if auto_tune_loader is None: + return self.loader.num_workers + + # pick the best num_workers + auto_tune_start = time.time() + logging.debug("========= DataLoader Auto Tune =========") + logging.debug("User config for DataLoader: " + str( + self.loader.num_workers)) + best_num_workers = 0 + min_cost = float("inf") + logging.debug("Tuning Range for num_workers: 0 ~ " + str( + self.max_num_worker)) + num_workers = 0 + while num_workers < self.max_num_worker: + auto_tune_loader.num_workers = num_workers + avg_cost = self.evaluate_reader_cost(auto_tune_loader) + if min_cost * 0.75 > avg_cost: + min_cost = avg_cost + best_num_workers = num_workers + else: + update_num = self.is_best(auto_tune_loader, best_num_workers, + min_cost, self.max_num_worker) + if update_num == best_num_workers: + break + else: + best_num_workers = update_num + logging.debug("num_workers: " + str(num_workers) + " avg_cost: " + + str(avg_cost)) + num_workers += 2 + logging.info("auto_tune dataLoader best_num_workers: " + str( + best_num_workers)) + logging.debug("AutoTuning Cost for DataLoader: " + str(time.time( + ) - auto_tune_start) + ' seconds') + + # tune the default loader's num_workers + return best_num_workers + + def need_autotune(self): + if (sys.platform == 'darwin' or sys.platform == 'win32'): + return False + else: + return True + + def get_sub_dataset(self, dataset, batch_size): + num_samples = min(batch_size * TUNING_STEPS, len(dataset)) + sub_dataset = Subset(dataset, indices=list(range(num_samples))) + return sub_dataset + + def get_autotune_loader(self): + loader = self.loader + batch_size = self.loader.batch_sampler.batch_size + if isinstance(self.loader.batch_sampler, + paddle.io.DistributedBatchSampler): + dataset = self.loader.batch_sampler.dataset + sub_dataset = self.get_sub_dataset(dataset, batch_size) + loader.batch_sampler = paddle.io.DistributedBatchSampler( + dataset=sub_dataset, + batch_size=batch_size, + num_replicas=self.loader.batch_sampler.nranks, + rank=self.loader.batch_sampler.local_rank, + shuffle=self.loader.batch_sampler.shuffle, + drop_last=self.loader.batch_sampler.drop_last) + elif isinstance(self.loader.batch_sampler, paddle.io.BatchSampler): + dataset = self.loader.batch_sampler.sampler.data_source + sub_dataset = self.get_sub_dataset(dataset, batch_size) + loader.batch_sampler = paddle.io.BatchSampler( + dataset=sub_dataset, + batch_size=batch_size, + drop_last=self.loader.batch_sampler.drop_last) + else: + loader = None + return loader + + def evaluate_reader_cost(self, reader): + costs = [] + avg_cost = 0 + start = time.time() + for i, data in enumerate(reader): + costs.append(time.time() - start) + start = time.time() + if len(costs) > 2: + avg_cost = sum(costs[2:]) / len(costs[2:]) + else: + avg_cost = sum(costs[0:]) / len(costs[0:]) + return avg_cost + + def is_best(self, reader, best_workers, best_time, num_work_boundary): + step = 0 + num_workers = best_workers + 1 + boundary = 1 + while num_workers < num_work_boundary and step < 5: + self.loader.num_workers = num_workers + time = self.evaluate_reader_cost(reader) + logging.debug("for back num_workers: " + str(num_workers) + + " avg_cost: " + str(time)) + step += 1 + if (time < best_time * 0.70 * boundary): + return num_workers + else: + num_workers += 1 + boundary *= 0.80 + return best_workers + + class DataLoader(object): """ DataLoader prodives an iterator which iterates given dataset @@ -409,6 +535,7 @@ def __init__(self, self._persistent_workers = persistent_workers self._iterator = None + self.num_workers = AuToTune(self).__call__() def __len__(self): if self.dataset_kind == _DatasetKind.ITER: diff --git a/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py b/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py new file mode 100755 index 0000000000000..a140bb5c79c93 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_dataloader_autotune.py @@ -0,0 +1,76 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import unittest +import numpy as np + +import paddle +import paddle.nn as nn +from paddle.io import Dataset, DataLoader, BatchSampler, SequenceSampler +from paddle.fluid.reader import set_autotune_config +import sys + + +class RandomDataset(Dataset): + def __init__(self, num_samples): + self.num_samples = num_samples + + def __getitem__(self, idx): + image = np.random.random([10]).astype('float32') + label = np.random.randint(0, 10 - 1, (1, )).astype('int64') + return image, label + + def __len__(self): + return self.num_samples + + +class SimpleNet(nn.Layer): + def __init__(self): + super(SimpleNet, self).__init__() + self.fc = nn.Linear(10, 10) + + def forward(self, image): + return self.fc(image) + + +class TestAutoTune(unittest.TestCase): + def setUp(self): + self.batch_size = 1 + self.dataset = RandomDataset(10) + + def test_dataloader_use_autotune(self): + set_autotune_config(True, 1) + loader = DataLoader( + self.dataset, batch_size=self.batch_size, num_workers=0) + + def test_dataloader_disable_autotune(self): + set_autotune_config(False) + loader = DataLoader( + self.dataset, batch_size=self.batch_size, num_workers=2) + if (sys.platform == 'darwin' or sys.platform == 'win32'): + self.assertEqual(loader.num_workers, 0) + else: + self.assertEqual(loader.num_workers, 2) + + def test_distributer_batch_sampler_autotune(self): + set_autotune_config(True, 1) + batch_sampler = paddle.io.DistributedBatchSampler( + self.dataset, batch_size=self.batch_size) + loader = DataLoader( + self.dataset, batch_sampler=batch_sampler, num_workers=2) + + +if __name__ == '__main__': + unittest.main() From c79d1186612737560e21b867d89ce0b8f3510b34 Mon Sep 17 00:00:00 2001 From: zyfncg Date: Fri, 22 Apr 2022 13:21:59 +0800 Subject: [PATCH 28/66] Dygraph performance optimization (v2) (#42103) * optimiaze performance of PreparePhiData * dygraph performance optimization --- paddle/fluid/framework/infershape_utils.cc | 6 ++--- paddle/fluid/framework/operator.cc | 22 ++++++++++++------- paddle/fluid/imperative/prepared_operator.h | 8 +++---- paddle/fluid/pybind/imperative.cc | 6 ++--- .../pybind/kernel_signature_generator.cc | 8 +++---- .../dialect/phi/pass/phi_op_convert_pass.cc | 4 ++-- paddle/phi/core/compat/arg_map_context.cc | 6 ++--- paddle/phi/core/compat/arg_map_context.h | 18 +++++++++++---- paddle/phi/tests/ops/test_op_signature.cc | 6 ++--- 9 files changed, 49 insertions(+), 35 deletions(-) diff --git a/paddle/fluid/framework/infershape_utils.cc b/paddle/fluid/framework/infershape_utils.cc index bd71ade7e9311..68ee68fdd076a 100644 --- a/paddle/fluid/framework/infershape_utils.cc +++ b/paddle/fluid/framework/infershape_utils.cc @@ -414,9 +414,9 @@ CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, CompatInferMetaContext infer_meta_context( {ctx->IsRuntime(), ctx->IsRunMKLDNNKernel()}); - auto& input_names = std::get<0>(signature.args); - auto& attr_names = std::get<1>(signature.args); - auto& output_names = std::get<2>(signature.args); + const auto& input_names = signature.input_names; + const auto& attr_names = signature.attr_names; + const auto& output_names = signature.output_names; const auto& args_def = phi::KernelFactory::Instance().GetFirstKernelArgsDef(signature.name); diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index 871c459c71764..0c35786394a43 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -1198,8 +1198,10 @@ bool OperatorWithKernel::SupportsMKLDNN( bool OperatorWithKernel::CanMKLDNNBeUsed(const framework::ExecutionContext& ctx, proto::VarType::Type data_type) const { - bool use_mkldnn_ctx = ctx.HasAttr("use_mkldnn") && - ctx.Attr("use_mkldnn") && + const auto& attrs_map = ctx.Attrs(); + auto iter = attrs_map.find("use_mkldnn"); + bool use_mkldnn_ctx = iter != attrs_map.end() && + BOOST_GET_CONST(bool, iter->second) && platform::is_cpu_place(ctx.GetPlace()); return use_mkldnn_ctx && this->SupportsMKLDNN(data_type); } @@ -2124,7 +2126,7 @@ KernelSignature OperatorWithKernel::GetExpectedPhiKernelArgs( Scope* OperatorWithKernel::PreparePhiData( const Scope& scope, const phi::Kernel& pt_kernel, const KernelSignature& pt_kernel_signature, RuntimeContext* ctx) const { - auto& input_names = std::get<0>(pt_kernel_signature.args); + const auto& input_names = pt_kernel_signature.input_names; auto input_defs = pt_kernel.args_def().input_defs(); PADDLE_ENFORCE_EQ(input_names.size(), input_defs.size(), platform::errors::InvalidArgument( @@ -2176,11 +2178,15 @@ Scope* OperatorWithKernel::PreparePhiData( if (in_def.backend == phi::Backend::ALL_BACKEND) { continue; } - auto expected_place = phi::TransToPhiPlace(in_def.backend); - if (platform::is_same_place(tensor_in->place(), expected_place)) { + + auto tensor_backend = phi::TransToPhiBackend(tensor_in->place()); + if (in_def.backend == tensor_backend || + (in_def.backend == phi::Backend::GPUDNN && + tensor_backend == phi::Backend::GPU)) { continue; } + auto expected_place = phi::TransToPhiPlace(in_def.backend); VLOG(3) << "phi Transform Variable " << input_names[i] << " from " << tensor_in->place() << " to " << expected_place; @@ -2217,9 +2223,9 @@ void OperatorWithKernel::BuildPhiKernelContext( phi::KernelContext* pt_kernel_context) const { pt_kernel_context->SetDeviceContext(dev_ctx); - auto& input_names = std::get<0>(pt_kernel_signature_->args); - auto& attr_names = std::get<1>(pt_kernel_signature_->args); - auto& output_names = std::get<2>(pt_kernel_signature_->args); + auto& input_names = pt_kernel_signature_->input_names; + auto& attr_names = pt_kernel_signature_->attr_names; + auto& output_names = pt_kernel_signature_->output_names; auto input_defs = pt_kernel_->args_def().input_defs(); auto attr_defs = pt_kernel_->args_def().attribute_defs(); diff --git a/paddle/fluid/imperative/prepared_operator.h b/paddle/fluid/imperative/prepared_operator.h index cb3275674ed49..754b553bd192f 100644 --- a/paddle/fluid/imperative/prepared_operator.h +++ b/paddle/fluid/imperative/prepared_operator.h @@ -233,9 +233,9 @@ void BuildDygraphPhiKernelContext( platform::DeviceContext* dev_ctx, phi::KernelContext* kernel_ctx) { kernel_ctx->SetDeviceContext(dev_ctx); - auto& input_names = std::get<0>(pt_kernel_signature.args); - auto& attr_names = std::get<1>(pt_kernel_signature.args); - auto& output_names = std::get<2>(pt_kernel_signature.args); + const auto& input_names = pt_kernel_signature.input_names; + const auto& attr_names = pt_kernel_signature.attr_names; + const auto& output_names = pt_kernel_signature.output_names; auto& input_defs = pt_kernel.args_def().input_defs(); auto& output_defs = pt_kernel.args_def().output_defs(); @@ -570,7 +570,7 @@ template void PreparePhiData(const phi::Kernel& pt_kernel, const framework::KernelSignature& pt_kernel_signature, const NameVarMap& ins) { - auto& input_names = std::get<0>(pt_kernel_signature.args); + const auto& input_names = pt_kernel_signature.input_names; auto& input_defs = pt_kernel.args_def().input_defs(); PADDLE_ENFORCE_EQ(input_names.size(), input_defs.size(), diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc index 4caf51ecc4bf8..145c116fa14c3 100644 --- a/paddle/fluid/pybind/imperative.cc +++ b/paddle/fluid/pybind/imperative.cc @@ -2050,9 +2050,9 @@ void BindImperative(py::module *m_ptr) { }; auto ret = self.GetExpectedKernelSignature(type, ins_map, outs_map, attrs); - auto kernelsig_ins = input_to_vector(std::get<0>(ret.args)); - auto kernelsig_attrs = attr_to_vector(std::get<1>(ret.args)); - auto kernelsig_outs = output_to_vector(std::get<2>(ret.args)); + auto kernelsig_ins = input_to_vector(ret.input_names); + auto kernelsig_attrs = attr_to_vector(ret.attr_names); + auto kernelsig_outs = output_to_vector(ret.output_names); return std::make_tuple(kernelsig_ins, kernelsig_attrs, kernelsig_outs); } diff --git a/paddle/fluid/pybind/kernel_signature_generator.cc b/paddle/fluid/pybind/kernel_signature_generator.cc index 1520174fba288..0b0a8628b14f1 100644 --- a/paddle/fluid/pybind/kernel_signature_generator.cc +++ b/paddle/fluid/pybind/kernel_signature_generator.cc @@ -58,10 +58,10 @@ int main(int argc, char **argv) { if (kernel_signature_map.Has(op_name)) { kernel_signature_map_str = kernel_signature_map_str + "\"" + op_kernel_pair.first + "\":{"; - auto &args = kernel_signature_map.Get(op_name).args; + const auto &args = kernel_signature_map.Get(op_name); kernel_signature_map_str += "\"inputs\":["; - auto inputs_ = std::get<0>(args); + auto inputs_ = args.input_names; for (size_t i = 0; i < inputs_.size(); i++) { kernel_signature_map_str = kernel_signature_map_str + "\"" + inputs_[i] + "\","; @@ -69,14 +69,14 @@ int main(int argc, char **argv) { if (inputs_.size()) kernel_signature_map_str.pop_back(); kernel_signature_map_str += "],\"attrs\":["; - auto attrs_ = std::get<1>(args); + auto attrs_ = args.attr_names; for (size_t i = 0; i < attrs_.size(); i++) { kernel_signature_map_str = kernel_signature_map_str + "\"" + attrs_[i] + "\","; } if (attrs_.size()) kernel_signature_map_str.pop_back(); kernel_signature_map_str += "],\"outputs\":["; - auto outputs_ = std::get<2>(args); + auto outputs_ = args.output_names; for (size_t i = 0; i < outputs_.size(); i++) { kernel_signature_map_str = kernel_signature_map_str + "\"" + outputs_[i] + "\","; diff --git a/paddle/infrt/dialect/phi/pass/phi_op_convert_pass.cc b/paddle/infrt/dialect/phi/pass/phi_op_convert_pass.cc index 4bf39d4f66094..76a4b84d06f21 100644 --- a/paddle/infrt/dialect/phi/pass/phi_op_convert_pass.cc +++ b/paddle/infrt/dialect/phi/pass/phi_op_convert_pass.cc @@ -200,7 +200,7 @@ void PhiOpConvertPass::convertStage() { // resort input&output according to kernel_sign ::llvm::SmallVector inputs, ori_output; ::llvm::SmallVector output_types; - for (const std::string &str : std::get<0>(kernel_sign.args)) { + for (const std::string &str : kernel_sign.input_names) { if (pd_dialect_inputs_info_map_.at(op_name).count(str) == 0) { LOG(ERROR) << "No input info for Op " << op_name << " and argument " << str; @@ -210,7 +210,7 @@ void PhiOpConvertPass::convertStage() { inputs.push_back(op->getOperands()[index]); } - for (const std::string &str : std::get<2>(kernel_sign.args)) { + for (const std::string &str : kernel_sign.output_names) { if (pd_dialect_outputs_info_map_.at(op_name).count(str) == 0) { LOG(ERROR) << "No output info for Op " << op_name << " and argument " << str; diff --git a/paddle/phi/core/compat/arg_map_context.cc b/paddle/phi/core/compat/arg_map_context.cc index 6f678966badd9..800245406afd3 100644 --- a/paddle/phi/core/compat/arg_map_context.cc +++ b/paddle/phi/core/compat/arg_map_context.cc @@ -20,11 +20,11 @@ limitations under the License. */ namespace phi { std::ostream& operator<<(std::ostream& os, KernelSignature signature) { os << "Kernel Signature - name: " << signature.name << "; inputs: " - << paddle::string::join_strings(std::get<0>(signature.args), ", ") + << paddle::string::join_strings(signature.input_names, ", ") << "; attributes: " - << paddle::string::join_strings(std::get<1>(signature.args), ", ") + << paddle::string::join_strings(signature.attr_names, ", ") << "; outputs: " - << paddle::string::join_strings(std::get<2>(signature.args), ", "); + << paddle::string::join_strings(signature.output_names, ", "); return os; } diff --git a/paddle/phi/core/compat/arg_map_context.h b/paddle/phi/core/compat/arg_map_context.h index 122ebed21942a..102dca48b998b 100644 --- a/paddle/phi/core/compat/arg_map_context.h +++ b/paddle/phi/core/compat/arg_map_context.h @@ -33,7 +33,9 @@ using KernelArgsTuple = std::tuple, struct KernelSignature { const char* name; - KernelArgsTuple args; + paddle::SmallVector input_names; + paddle::SmallVector attr_names; + paddle::SmallVector output_names; KernelSignature() = default; @@ -41,18 +43,26 @@ struct KernelSignature { paddle::SmallVector&& inputs, paddle::SmallVector&& attrs, paddle::SmallVector&& outputs) - : name(kernel_name), args(std::make_tuple(inputs, attrs, outputs)) {} + : name(kernel_name), + input_names(std::move(inputs)), + attr_names(std::move(attrs)), + output_names(std::move(outputs)) {} KernelSignature(const char* kernel_name, const paddle::SmallVector& inputs, const paddle::SmallVector& attrs, const paddle::SmallVector& outputs) - : name(kernel_name), args(std::make_tuple(inputs, attrs, outputs)) {} + : name(kernel_name), + input_names(inputs), + attr_names(attrs), + output_names(outputs) {} // TODO(chenweihang): add assign constructor to solve windows compile // problem, remove it later KernelSignature& operator=(const KernelSignature& other) { name = other.name; - args = other.args; + input_names = other.input_names; + attr_names = other.attr_names; + output_names = other.output_names; return *this; } }; diff --git a/paddle/phi/tests/ops/test_op_signature.cc b/paddle/phi/tests/ops/test_op_signature.cc index 6acf3916a1866..6c9f36a5e573f 100644 --- a/paddle/phi/tests/ops/test_op_signature.cc +++ b/paddle/phi/tests/ops/test_op_signature.cc @@ -560,8 +560,7 @@ TEST(ARG_MAP, allclose) { auto signature1 = OpUtilsMap::Instance().GetArgumentMappingFn("allclose")(arg_case1); ASSERT_EQ(signature1.name, "allclose"); - auto attr_names1 = std::get<1>(signature1.args); - ASSERT_EQ(attr_names1[0], "Rtol"); + ASSERT_EQ(signature1.attr_names[0], "Rtol"); TestArgumentMappingContext arg_case2( {"Input", "Other", "Atol"}, @@ -573,8 +572,7 @@ TEST(ARG_MAP, allclose) { auto signature2 = OpUtilsMap::Instance().GetArgumentMappingFn("allclose")(arg_case2); ASSERT_EQ(signature2.name, "allclose"); - auto attr_names2 = std::get<1>(signature2.args); - ASSERT_EQ(attr_names2[1], "Atol"); + ASSERT_EQ(signature2.attr_names[1], "Atol"); } TEST(ARG_MAP, reshape) { From e49b7b64a92869e54e96d0f816b130dd7e488ba2 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Fri, 22 Apr 2022 14:38:08 +0800 Subject: [PATCH 29/66] add build pylayer depend pybind (#42099) --- paddle/fluid/eager/pylayer/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/fluid/eager/pylayer/CMakeLists.txt b/paddle/fluid/eager/pylayer/CMakeLists.txt index 1e5f2dc6ccc31..8c660fa9694ed 100644 --- a/paddle/fluid/eager/pylayer/CMakeLists.txt +++ b/paddle/fluid/eager/pylayer/CMakeLists.txt @@ -1 +1 @@ -cc_library(py_layer_node SRCS py_layer_node.cc DEPS phi phi_api grad_node_info) +cc_library(py_layer_node SRCS py_layer_node.cc DEPS pybind phi phi_api grad_node_info) From 281a5be7ac7e7a17a9b613a513acf5148d2dcb95 Mon Sep 17 00:00:00 2001 From: Weilong Wu Date: Fri, 22 Apr 2022 15:56:48 +0800 Subject: [PATCH 30/66] [Eager] Fix CastPyArg2scalar for max value of int64 (#42098) * [Eager] Fix CastPyArg2Scalar in Long case * Add more test cases for paddle.clip * Use PyLong_AsLongLong --- paddle/fluid/pybind/eager_utils.cc | 2 +- paddle/fluid/pybind/op_function_common.cc | 2 +- .../fluid/tests/unittests/test_clip_op.py | 26 ++++++++++++++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc index 9719963d51da0..78db1a6f1b991 100644 --- a/paddle/fluid/pybind/eager_utils.cc +++ b/paddle/fluid/pybind/eager_utils.cc @@ -1058,7 +1058,7 @@ paddle::experimental::Scalar CastPyArg2Scalar(PyObject* obj, bool value = CastPyArg2Boolean(obj, op_type, arg_pos); return paddle::experimental::Scalar(value); } else if (PyLong_Check(obj)) { - int value = CastPyArg2Int(obj, op_type, arg_pos); + int64_t value = CastPyArg2Long(obj, op_type, arg_pos); return paddle::experimental::Scalar(value); } else if (PyFloat_Check(obj)) { float value = CastPyArg2Float(obj, op_type, arg_pos); diff --git a/paddle/fluid/pybind/op_function_common.cc b/paddle/fluid/pybind/op_function_common.cc index 50e0daf8508e3..5eed63d0800b3 100644 --- a/paddle/fluid/pybind/op_function_common.cc +++ b/paddle/fluid/pybind/op_function_common.cc @@ -153,7 +153,7 @@ void CastPyArg2AttrInt(PyObject* obj, int64_t CastPyArg2Long(PyObject* obj, const std::string& op_type, ssize_t arg_pos) { if (PyObject_CheckLongOrToLong(&obj)) { - return (int64_t)PyLong_AsLong(obj); // NOLINT + return (int64_t)PyLong_AsLongLong(obj); // NOLINT } else { PADDLE_THROW(platform::errors::InvalidArgument( "%s(): argument (position %d) must be " diff --git a/python/paddle/fluid/tests/unittests/test_clip_op.py b/python/paddle/fluid/tests/unittests/test_clip_op.py index 37b1cfd02faf7..121b91d741546 100644 --- a/python/paddle/fluid/tests/unittests/test_clip_op.py +++ b/python/paddle/fluid/tests/unittests/test_clip_op.py @@ -200,7 +200,7 @@ def test_clip(self): np.allclose(res11, (data * 10).astype(np.int64).clip(2, 8))) paddle.disable_static() - def test_clip_dygraph(self): + def func_clip_dygraph(self): paddle.disable_static() place = fluid.CUDAPlace(0) if fluid.core.is_compiled_with_cuda( ) else fluid.CPUPlace() @@ -233,9 +233,29 @@ def test_clip_dygraph(self): np.allclose(out_5.numpy(), (data * 10).astype(np.int64).clip(2, 8))) self.assertTrue(np.allclose(out_6.numpy(), data.clip(0.2, 0.8))) - def test_eager(self): + def test_clip_dygraph(self): + with _test_eager_guard(): + self.func_clip_dygraph() + self.func_clip_dygraph() + + def test_clip_dygraph_default_max(self): + paddle.disable_static() with _test_eager_guard(): - self.test_clip_dygraph() + x_int32 = paddle.to_tensor([1, 2, 3], dtype="int32") + x_int64 = paddle.to_tensor([1, 2, 3], dtype="int64") + x_f32 = paddle.to_tensor([1, 2, 3], dtype="float32") + egr_out1 = paddle.clip(x_int32, min=1) + egr_out2 = paddle.clip(x_int64, min=1) + egr_out3 = paddle.clip(x_f32, min=1) + x_int32 = paddle.to_tensor([1, 2, 3], dtype="int32") + x_int64 = paddle.to_tensor([1, 2, 3], dtype="int64") + x_f32 = paddle.to_tensor([1, 2, 3], dtype="float32") + out1 = paddle.clip(x_int32, min=1) + out2 = paddle.clip(x_int64, min=1) + out3 = paddle.clip(x_f32, min=1) + self.assertTrue(np.allclose(out1.numpy(), egr_out1.numpy())) + self.assertTrue(np.allclose(out2.numpy(), egr_out2.numpy())) + self.assertTrue(np.allclose(out3.numpy(), egr_out3.numpy())) def test_errors(self): paddle.enable_static() From 8a6456db022e562253920da0303573065c74fc01 Mon Sep 17 00:00:00 2001 From: zhangkaihuo Date: Fri, 22 Apr 2022 16:46:47 +0800 Subject: [PATCH 31/66] Add Sparse BatchNorm and fix two bugs (#42013) --- .../kernels/sparse/cpu/coalesced_kernel.cc | 2 +- .../kernels/sparse/cpu/sparse_mask_kernel.cc | 2 +- .../kernels/sparse/gpu/coalesced_kernel.cu | 2 +- .../kernels/sparse/gpu/sparse_mask_kernel.cu | 2 +- .../tests/unittests/test_sparse_conv_op.py | 6 +- .../tests/unittests/test_sparse_norm_op.py | 87 ++++++++++ .../tests/unittests/test_sparse_utils_op.py | 39 +++++ python/paddle/sparse/__init__.py | 5 +- python/paddle/sparse/creation.py | 14 +- python/paddle/sparse/functional/conv.py | 17 +- python/paddle/sparse/layer/__init__.py | 1 + python/paddle/sparse/layer/norm.py | 160 ++++++++++++++++++ 12 files changed, 323 insertions(+), 14 deletions(-) create mode 100644 python/paddle/fluid/tests/unittests/test_sparse_norm_op.py create mode 100644 python/paddle/sparse/layer/norm.py diff --git a/paddle/phi/kernels/sparse/cpu/coalesced_kernel.cc b/paddle/phi/kernels/sparse/cpu/coalesced_kernel.cc index 0ebddf9b683f0..22c5e14b35f56 100644 --- a/paddle/phi/kernels/sparse/cpu/coalesced_kernel.cc +++ b/paddle/phi/kernels/sparse/cpu/coalesced_kernel.cc @@ -44,7 +44,7 @@ void CoalescedCPUKernel(const CPUContext& dev_ctx, const T* x_values_ptr = x_values.data(); const int64_t stride = - x.dims().size() == sparse_dim ? 1 : x.dims().size() - sparse_dim; + x.dims().size() == sparse_dim ? 1 : x.non_zero_elements().dims()[1]; std::map> indices_to_index; for (uint64_t i = 0; i < x_indexs.size(); i++) { diff --git a/paddle/phi/kernels/sparse/cpu/sparse_mask_kernel.cc b/paddle/phi/kernels/sparse/cpu/sparse_mask_kernel.cc index 1508de407caa7..0ec8b808ba838 100644 --- a/paddle/phi/kernels/sparse/cpu/sparse_mask_kernel.cc +++ b/paddle/phi/kernels/sparse/cpu/sparse_mask_kernel.cc @@ -125,7 +125,7 @@ void SparseMaskHelperCPUKernel(const CPUContext& dev_ctx, T* out_ptr = out->data(); memset(out_ptr, static_cast(0), out->numel() * sizeof(T)); const int64_t stride = - x.dims().size() == sparse_dim ? 1 : x.dims().size() - sparse_dim; + x.dims().size() == sparse_dim ? 1 : x.non_zero_elements().dims()[1]; const T* in_ptr = x.non_zero_elements().data(); // TODO(zhangkaihuo): multithreading can be used for acceleration for (uint64_t i = 0; i < mask_indexs.size(); i++) { diff --git a/paddle/phi/kernels/sparse/gpu/coalesced_kernel.cu b/paddle/phi/kernels/sparse/gpu/coalesced_kernel.cu index 3ffcd28955a53..b2e7884580c74 100644 --- a/paddle/phi/kernels/sparse/gpu/coalesced_kernel.cu +++ b/paddle/phi/kernels/sparse/gpu/coalesced_kernel.cu @@ -76,7 +76,7 @@ void CoalescedGPUKernel(const GPUContext& dev_ctx, // 2. get the address of each non-zero values const T* x_values_ptr = x_values.data(); const int64_t stride = - x.dims().size() == sparse_dim ? 1 : x.dims().size() - sparse_dim; + x.dims().size() == sparse_dim ? 1 : x.non_zero_elements().dims()[1]; DenseTensor values_indexs = phi::Empty( dev_ctx, DenseTensorMeta(DataType::INT32, {nnz}, DataLayout::NCHW)); int* values_indexs_ptr = values_indexs.data(); diff --git a/paddle/phi/kernels/sparse/gpu/sparse_mask_kernel.cu b/paddle/phi/kernels/sparse/gpu/sparse_mask_kernel.cu index 4e2d12f33955e..4253845956ea7 100644 --- a/paddle/phi/kernels/sparse/gpu/sparse_mask_kernel.cu +++ b/paddle/phi/kernels/sparse/gpu/sparse_mask_kernel.cu @@ -231,7 +231,7 @@ void SparseMaskHelperGPUKernel(const GPUContext& dev_ctx, T* out_ptr = out->data(); const int64_t stride = - x.dims().size() == sparse_dim ? 1 : x.dims().size() - sparse_dim; + x.dims().size() == sparse_dim ? 1 : x.non_zero_elements().dims()[1]; SparseMaskCopyKernel<< 1: + lens = np.append(lens, values.shape[1:]) + return list(lens) def _get_place(place): @@ -106,7 +111,7 @@ def sparse_coo_tensor(indices, with _test_eager_guard(): indices = [[0, 1, 2], [1, 2, 0]] values = [1.0, 2.0, 3.0] - dense_shape = [2, 3] + dense_shape = [3, 3] coo = paddle.sparse.sparse_coo_tensor(indices, values, dense_shape) # print(coo) # Tensor(shape=[2, 3], dtype=paddle.float32, place=Place(gpu:0), stop_gradient=True, @@ -145,7 +150,8 @@ def sparse_coo_tensor(indices, values = _handle_dtype(values, dtype) values.stop_gradient = stop_gradient - min_shape = _infer_dense_shape(indices) + min_shape = _infer_dense_shape(indices, values) + if shape is None: shape = min_shape else: diff --git a/python/paddle/sparse/functional/conv.py b/python/paddle/sparse/functional/conv.py index d8c0e5c914ccb..42b7b49835cf0 100644 --- a/python/paddle/sparse/functional/conv.py +++ b/python/paddle/sparse/functional/conv.py @@ -16,6 +16,8 @@ from paddle import _C_ops, in_dynamic_mode from ...fluid.layers.utils import convert_to_list +from ...fluid.layers.nn import elementwise_add +from .. import sparse_coo_tensor from paddle.nn.functional.conv import _update_padding_nd @@ -30,7 +32,6 @@ def _conv3d(x, data_format="NDHWC", name=None): assert in_dynamic_mode(), "Currently, only support dynamic mode" - assert bias == None, "Currently, sparse_conv3d does not support bias" assert groups == 1, "Currently, only support groups=1" dims = 3 @@ -61,8 +62,18 @@ def _conv3d(x, dilation = convert_to_list(dilation, dims, 'dilation') op_type = "conv3d" - return _C_ops.final_state_sparse_conv3d(x, weight, padding, dilation, - stride, groups, subm) + pre_bias = _C_ops.final_state_sparse_conv3d(x, weight, padding, dilation, + stride, groups, subm) + if bias is not None: + values = pre_bias.values() + add_bias = elementwise_add(values, bias, axis=1) + return sparse_coo_tensor( + pre_bias.indices(), + add_bias, + shape=pre_bias.shape, + stop_gradient=pre_bias.stop_gradient) + else: + return pre_bias def conv3d(x, diff --git a/python/paddle/sparse/layer/__init__.py b/python/paddle/sparse/layer/__init__.py index a0f9d068e677c..ee32e5027b50f 100644 --- a/python/paddle/sparse/layer/__init__.py +++ b/python/paddle/sparse/layer/__init__.py @@ -13,6 +13,7 @@ # limitations under the License. from .activation import ReLU +from .norm import BatchNorm from .conv import Conv3D from .conv import SubmConv3D diff --git a/python/paddle/sparse/layer/norm.py b/python/paddle/sparse/layer/norm.py new file mode 100644 index 0000000000000..83b738a5dc354 --- /dev/null +++ b/python/paddle/sparse/layer/norm.py @@ -0,0 +1,160 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle +import warnings + + +class BatchNorm(paddle.nn.BatchNorm1D): + r""" + Applies Batch Normalization over a SparseCooTensor as described in the paper Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift . + + When use_global_stats = False, the :math:`\mu_{\beta}` + and :math:`\sigma_{\beta}^{2}` are the statistics of one mini-batch. + Calculated as follows: + + .. math:: + + \mu_{\beta} &\gets \frac{1}{m} \sum_{i=1}^{m} x_i \qquad &//\ + \ mini-batch\ mean \\ + \sigma_{\beta}^{2} &\gets \frac{1}{m} \sum_{i=1}^{m}(x_i - \ + \mu_{\beta})^2 \qquad &//\ mini-batch\ variance \\ + + When use_global_stats = True, the :math:`\mu_{\beta}` + and :math:`\sigma_{\beta}^{2}` are not the statistics of one mini-batch. + They are global or running statistics (moving_mean and moving_variance). It usually got from the + pre-trained model. Calculated as follows: + + .. math:: + moving\_mean = moving\_mean * momentum + \mu_{\beta} * (1. - momentum) \quad &// global \ mean \\ + moving\_variance = moving\_variance * momentum + \sigma_{\beta}^{2} * (1. - momentum) \quad &// global \ variance \\ + + The normalization function formula is as follows: + + .. math:: + + \hat{x_i} &\gets \frac{x_i - \mu_\beta} {\sqrt{\sigma_{\beta}^{2} + \epsilon}} \qquad &//\ normalize \\ + y_i &\gets \gamma \hat{x_i} + \beta \qquad &//\ scale\ and\ shift + + - :math:`\epsilon` : add a smaller value to the variance to prevent division by zero + - :math:`\gamma` : trainable proportional parameter + - :math:`\beta` : trainable deviation parameter + + Parameters: + num_features(int): Indicate the number of channels of the input ``Tensor``. + momentum(float, optional): The value used for the moving_mean and moving_var computation. Default: 0.9. + epsilon(float, optional): The small value added to the variance to prevent division by zero. Default: 1e-5. + weight_attr(ParamAttr|bool, optional): The parameter attribute for Parameter `scale` + of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm + will create ParamAttr as weight_attr. If it is set to Fasle, the weight is not learnable. + If the Initializer of the weight_attr is not set, the parameter is initialized with Xavier. Default: None. + bias_attr(ParamAttr|bool, optional): The parameter attribute for the bias of batch_norm. + If it is set to None or one attribute of ParamAttr, batch_norm + will create ParamAttr as bias_attr. If it is set to Fasle, the weight is not learnable. + If the Initializer of the bias_attr is not set, the bias is initialized zero. Default: None. + data_format(str, optional): Specify the input data format, may be "NC", "NCL" or "NLC". Defalut "NCL". + use_global_stats(bool|None, optional): Whether to use global mean and variance. If set to False, use the statistics of one mini-batch, if set to True, use the global statistics, if set to None, use global statistics in the test phase and use the statistics of one mini-batch in the training phase. Default: None. + name(str, optional): Name for the BatchNorm, default is None. For more information, please refer to :ref:`api_guide_Name`.. + + Shape: + - x: A SparseCooTensor with layout = 'NDHWC'. + - output: SparseCooTensor with same shape as input x. + + Returns: + None. + + + Examples: + .. code-block:: python + + import paddle + from paddle.fluid.framework import _test_eager_guard + + with _test_eager_guard(): + paddle.seed(123) + channels = 3 + x_data = paddle.randn((1, 6, 6, 6, channels)).astype('float32') + dense_x = paddle.to_tensor(x_data) + sparse_x = dense_x.to_sparse_coo(4) + batch_norm = paddle.sparse.BatchNorm(channels) + batch_norm_out = batch_norm(sparse_x) + print(batch_norm_out.shape) + # [1, 6, 6, 6, 3] + """ + + def __init__(self, + num_features, + momentum=0.9, + epsilon=1e-05, + weight_attr=None, + bias_attr=None, + data_format='NDHWC', + use_global_stats=None, + name=None): + super(BatchNorm, self).__init__( + num_features, + momentum=momentum, + epsilon=epsilon, + weight_attr=weight_attr, + bias_attr=bias_attr, + data_format=data_format, + use_global_stats=use_global_stats, + name=name) + + def _check_data_format(self, input): + if input != "NDHWC": + raise ValueError('sparse BatchNorm only support layout of "NDHWC"') + + def forward(self, input): + values = input.values() + self._check_data_format(self._data_format) + + if len(values.shape) != 2: + raise ValueError('expected 2D input.values() (got {}D)'.format( + len(values.shape))) + + if self.training: + warnings.warn( + "When training, we now always track global mean and variance.") + + batch_norm_out = paddle.nn.functional.batch_norm( + values, + self._mean, + self._variance, + weight=self.weight, + bias=self.bias, + training=self.training, + momentum=self._momentum, + epsilon=self._epsilon, + data_format='NC', + use_global_stats=self._use_global_stats) + + return paddle.sparse.sparse_coo_tensor( + input.indices(), + batch_norm_out, + shape=input.shape, + stop_gradient=input.stop_gradient) From 9e3cfdfacf51d1c1f97c3a758c2c311a0f211291 Mon Sep 17 00:00:00 2001 From: chenjian Date: Fri, 22 Apr 2022 17:44:33 +0800 Subject: [PATCH 32/66] fix kenrel name apperance (#42071) --- python/paddle/profiler/profiler_statistic.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/paddle/profiler/profiler_statistic.py b/python/paddle/profiler/profiler_statistic.py index 422dbe4ce359f..50aa3a1f11f85 100755 --- a/python/paddle/profiler/profiler_statistic.py +++ b/python/paddle/profiler/profiler_statistic.py @@ -13,6 +13,7 @@ # limitations under the License. import collections from enum import Enum +import re from paddle.fluid.core import TracerEventType @@ -1317,10 +1318,11 @@ def format_ratio(ratio, indent=0): append(header_sep) append(row_format.format(*headers)) append(header_sep) + kernel_name_pattern = re.compile('(.+?)(<.*>)(\(.*\))') for row_values in all_row_values: - indx = row_values[0].find('(') - if indx != -1: - name = row_values[0][:indx] + match = kernel_name_pattern.match(row_values[0]) + if match: + name = match.group(1) + match.group(2) else: name = row_values[0] if len(name) > name_column_width: From 19650d722a7f6ca79ce43305328fd83ef0aca597 Mon Sep 17 00:00:00 2001 From: Ming-Xu Huang Date: Fri, 22 Apr 2022 18:16:04 +0800 Subject: [PATCH 33/66] [WIP] Algorithm Cache of cuBlasLt Epilogue (#41010) * Fix leading dimension setting error in fused_gemm_epilogue_grad_op. * Add dyload to cuBlasLt functions. * Added cublasLtMatmulAlgoGetHeuristic to improve performance. * Added FLAGS_cublaslt_exhaustive_search_times to cublasLt epilogue * Added UTs to FLAGS_cublaslt_exhaustive_search_times * Added warmup runs in algo searching of Gemm epilogue. * Update copyright and documents. * Fixed error handling. --- .../operators/fused/fused_gemm_epilogue_op.cu | 68 ++++- .../operators/fused/fused_gemm_epilogue_op.h | 271 ++++++++++++++++++ paddle/fluid/platform/dynload/cublasLt.h | 33 ++- paddle/fluid/platform/flags.cc | 24 ++ paddle/phi/backends/dynload/cublasLt.h | 33 ++- .../fluid/tests/unittests/CMakeLists.txt | 22 +- .../unittests/test_fuse_gemm_epilogue_pass.py | 4 +- 7 files changed, 405 insertions(+), 50 deletions(-) create mode 100644 paddle/fluid/operators/fused/fused_gemm_epilogue_op.h diff --git a/paddle/fluid/operators/fused/fused_gemm_epilogue_op.cu b/paddle/fluid/operators/fused/fused_gemm_epilogue_op.cu index e16c9e8f483cc..9bf3d1a485efc 100644 --- a/paddle/fluid/operators/fused/fused_gemm_epilogue_op.cu +++ b/paddle/fluid/operators/fused/fused_gemm_epilogue_op.cu @@ -15,6 +15,7 @@ limitations under the License. */ #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/framework/op_version_registry.h" +#include "paddle/fluid/operators/fused/fused_gemm_epilogue_op.h" #include "paddle/fluid/platform/dynload/cublasLt.h" #include "paddle/fluid/platform/float16.h" @@ -56,7 +57,6 @@ class FusedGemmEpilogueKernel : public framework::OpKernel { cublasComputeType_t compute_type = CUBLAS_COMPUTE_32F; if (std::is_same::value) { mat_type = CUDA_R_16F; - scale_type = CUDA_R_16F; } if (std::is_same::value) { mat_type = CUDA_R_64F; @@ -130,7 +130,7 @@ class FusedGemmEpilogueKernel : public framework::OpKernel { cublasLtHandle_t lt_handle = dev_ctx.cublaslt_handle(); size_t workspace_size = 4 * 1024 * 1024; - const cublasLtMatmulAlgo_t* algo = nullptr; + cudaStream_t stream = dev_ctx.stream(); memory::allocation::AllocationPtr workspace = memory::Alloc(dev_ctx, workspace_size); @@ -146,10 +146,26 @@ class FusedGemmEpilogueKernel : public framework::OpKernel { beta = &beta32; } + const auto* y_data = y->data(); + const auto* x_data = x->data(); + + cublasLtMatmulAlgo_t algo = GemmEpilogueAlgoCache::Instance().GetGemmAlgo( + lt_handle, operation_desc, y_desc, x_desc, out_desc, alpha, beta, + y_data, x_data, out_data, stream, workspace->ptr(), workspace_size); + PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cublasLtMatmul( - lt_handle, operation_desc, alpha, y->data(), y_desc, x->data(), - x_desc, beta, out_data, out_desc, out_data, out_desc, algo, - workspace->ptr(), workspace_size, stream)); + lt_handle, operation_desc, alpha, y_data, y_desc, x_data, x_desc, beta, + out_data, out_desc, out_data, out_desc, &algo, workspace->ptr(), + workspace_size, stream)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulDescDestroy(operation_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(y_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(x_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(out_desc)); } private: @@ -205,7 +221,6 @@ class FusedGemmEpilogueGradKernel : public framework::OpKernel { cublasComputeType_t compute_type = CUBLAS_COMPUTE_32F; if (std::is_same::value) { mat_type = CUDA_R_16F; - scale_type = CUDA_R_16F; } if (std::is_same::value) { mat_type = CUDA_R_64F; @@ -215,7 +230,6 @@ class FusedGemmEpilogueGradKernel : public framework::OpKernel { cublasLtHandle_t lt_handle = dev_ctx.cublaslt_handle(); size_t workspace_size = 4 * 1024 * 1024; - const cublasLtMatmulAlgo_t* algo = nullptr; cudaStream_t stream = dev_ctx.stream(); double alpha64 = 1.0, beta64 = 0.0; @@ -262,8 +276,8 @@ class FusedGemmEpilogueGradKernel : public framework::OpKernel { &aux_data, sizeof(aux_data))); PADDLE_ENFORCE_GPU_SUCCESS( platform::dynload::cublasLtMatmulDescSetAttribute( - dx_operation_desc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_LD, &N, - sizeof(N))); + dx_operation_desc, CUBLASLT_MATMUL_DESC_EPILOGUE_AUX_LD, &K, + sizeof(K))); } cublasLtMatrixLayout_t y_desc = NULL, dx_desc = NULL; @@ -277,10 +291,24 @@ class FusedGemmEpilogueGradKernel : public framework::OpKernel { dx->mutable_data(ctx.GetPlace()); auto* dx_data = dx->data(); + const auto* y_data = y->data(); + const auto* dout_data = dout->data(); + + cublasLtMatmulAlgo_t algo = GemmEpilogueAlgoCache::Instance().GetGemmAlgo( + lt_handle, dx_operation_desc, y_desc, dout_desc, dx_desc, alpha, beta, + y_data, dout_data, dx_data, stream, dx_workspace->ptr(), + workspace_size); + PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cublasLtMatmul( lt_handle, dx_operation_desc, alpha, y->data(), y_desc, dout->data(), dout_desc, beta, dx_data, dx_desc, dx_data, dx_desc, - algo, dx_workspace->ptr(), workspace_size, stream)); + &algo, dx_workspace->ptr(), workspace_size, stream)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulDescDestroy(dx_operation_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(y_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(dx_desc)); } if (dy) { @@ -324,11 +352,27 @@ class FusedGemmEpilogueGradKernel : public framework::OpKernel { dy->mutable_data(ctx.GetPlace()); auto* dy_data = dy->data(); + const auto* dout_data = dout->data(); + const auto* x_data = x->data(); + + cublasLtMatmulAlgo_t algo = GemmEpilogueAlgoCache::Instance().GetGemmAlgo( + lt_handle, dy_operation_desc, dout_desc, x_desc, dy_desc, alpha, beta, + dout_data, x_data, dy_data, stream, dy_workspace->ptr(), + workspace_size); + PADDLE_ENFORCE_GPU_SUCCESS(platform::dynload::cublasLtMatmul( - lt_handle, dy_operation_desc, alpha, dout->data(), dout_desc, - x->data(), x_desc, beta, dy_data, dy_desc, dy_data, dy_desc, algo, + lt_handle, dy_operation_desc, alpha, dout_data, dout_desc, x_data, + x_desc, beta, dy_data, dy_desc, dy_data, dy_desc, &algo, dy_workspace->ptr(), workspace_size, stream)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulDescDestroy(dy_operation_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(x_desc)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(dy_desc)); } + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutDestroy(dout_desc)); } private: diff --git a/paddle/fluid/operators/fused/fused_gemm_epilogue_op.h b/paddle/fluid/operators/fused/fused_gemm_epilogue_op.h new file mode 100644 index 0000000000000..c90a6966fe0a8 --- /dev/null +++ b/paddle/fluid/operators/fused/fused_gemm_epilogue_op.h @@ -0,0 +1,271 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +Copyright (c) 2022 NVIDIA Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include +#include +#include +#include "gflags/gflags.h" +#include "paddle/fluid/platform/dynload/cublasLt.h" +#include "paddle/fluid/platform/float16.h" + +DECLARE_int64(cublaslt_exhaustive_search_times); + +namespace paddle { +namespace operators { + +class GemmEpilogueAlgoCache { + public: + static GemmEpilogueAlgoCache &Instance() { + static GemmEpilogueAlgoCache instance( + FLAGS_cublaslt_exhaustive_search_times); + return instance; + } + + GemmEpilogueAlgoCache(GemmEpilogueAlgoCache const &) = delete; + void operator=(GemmEpilogueAlgoCache const &) = delete; + + cublasLtMatmulAlgo_t GetGemmAlgo( + cublasLtHandle_t lt_handle, cublasLtMatmulDesc_t op_desc, + cublasLtMatrixLayout_t a_desc, cublasLtMatrixLayout_t b_desc, + cublasLtMatrixLayout_t c_desc, const void *alpha, const void *beta, + const void *a, const void *b, void *c, cudaStream_t stream, + void *workspace, size_t workspace_size) { + int64_t seed = 0; + std::hash hash_fn; + + HashMatmulDesc_(op_desc, &seed, hash_fn); + HashMatrixLayoutDesc_(a_desc, &seed, hash_fn); + HashMatrixLayoutDesc_(b_desc, &seed, hash_fn); + HashMatrixLayoutDesc_(c_desc, &seed, hash_fn); + + cublasLtMatmulAlgo_t ret; + auto it = map_.end(); + bool have_found = false; + { + std::lock_guard lock(cache_mutex_); + it = map_.find(seed); + + if (it != map_.end()) { + ret = it->second; + have_found = true; + } + } + + if (!have_found) { + cublasLtMatmulPreference_t preference; + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulPreferenceCreate(&preference)); + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulPreferenceSetAttribute( + preference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, + &workspace_size, sizeof(workspace_size))); + + int returned_results = 0; + cublasLtMatmulHeuristicResult_t heuristic_results[requested_algo_count_] = + {0}; + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulAlgoGetHeuristic( + lt_handle, op_desc, a_desc, b_desc, c_desc, c_desc, preference, + requested_algo_count_, heuristic_results, &returned_results)); + + PADDLE_ENFORCE_GT( + returned_results, 0, + platform::errors::Unavailable("No GEMM epilogue algorithm support!")); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulPreferenceDestroy(preference)); + + if (search_times_ > 0) { + int best_algo_idx = -1; + float best_algo_time = 0; + + // Run 100 times for warmup + int warmup_algo_idx = 0; + for (int t = 0; t < 100; t++) { + cublasStatus_t status = platform::dynload::cublasLtMatmul( + lt_handle, op_desc, alpha, a, a_desc, b, b_desc, beta, c, c_desc, + c, c_desc, &heuristic_results[warmup_algo_idx].algo, workspace, + workspace_size, stream); + if (status != CUBLAS_STATUS_SUCCESS) { + t = -1; + warmup_algo_idx += 1; + if (warmup_algo_idx == requested_algo_count_) { + PADDLE_THROW(platform::errors::Unavailable( + "No GEMM epilogue algorithm support!")); + } + } + } + + cudaEvent_t start_event, stop_event; + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventCreate(&start_event)); + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventCreate(&stop_event)); + + for (int algo_idx = 0; algo_idx < returned_results; ++algo_idx) { + float curr_time = 0; + for (int check_idx = 0; check_idx < search_times_; check_idx++) { + float time = 0; + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventRecord(start_event, stream)); + + cublasStatus_t status = platform::dynload::cublasLtMatmul( + lt_handle, op_desc, alpha, a, a_desc, b, b_desc, beta, c, + c_desc, c, c_desc, &heuristic_results[algo_idx].algo, workspace, + workspace_size, stream); + + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventRecord(stop_event, stream)); + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventSynchronize(stop_event)); + PADDLE_ENFORCE_GPU_SUCCESS( + cudaEventElapsedTime(&time, start_event, stop_event)); + curr_time += time; + if (status != CUBLAS_STATUS_SUCCESS) { + curr_time = 3.40282e+038; // Max Value of float + break; + } + } + + curr_time = curr_time / search_times_; + if (curr_time < best_algo_time || algo_idx == 0) { + best_algo_idx = algo_idx; + best_algo_time = curr_time; + } + } + + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventDestroy(start_event)); + PADDLE_ENFORCE_GPU_SUCCESS(cudaEventDestroy(stop_event)); + + if (best_algo_idx == -1) { + PADDLE_THROW(platform::errors::Unavailable( + "No GEMM epilogue algorithm support!")); + } + + ret = heuristic_results[best_algo_idx].algo; + } else { + int decided_algo_idx = -1; + for (int algo_idx = 0; algo_idx < returned_results; ++algo_idx) { + cublasStatus_t status = platform::dynload::cublasLtMatmul( + lt_handle, op_desc, alpha, a, a_desc, b, b_desc, beta, c, c_desc, + c, c_desc, &heuristic_results[algo_idx].algo, workspace, + workspace_size, stream); + if (status == CUBLAS_STATUS_SUCCESS) { + decided_algo_idx = algo_idx; + break; + } + } + if (decided_algo_idx == -1) { + PADDLE_THROW(platform::errors::Unavailable( + "No GEMM epilogue algorithm support!")); + } + ret = heuristic_results[decided_algo_idx].algo; + } + + std::lock_guard lock(cache_mutex_); + map_[seed] = ret; + } + + VLOG(4) << "Search time:" << search_times_ << ", Is hash-key (" << seed + << ") found in GemmEpilogueAlgoCache? " << have_found; + + return ret; + } + + private: + explicit GemmEpilogueAlgoCache(int search_times) + : search_times_(search_times) { + map_.clear(); + } + std::unordered_map map_; + int search_times_; + const int requested_algo_count_ = 10; + std::mutex cache_mutex_; + + void HashMatmulDesc_(cublasLtMatmulDesc_t desc, int64_t *seed, + const std::hash &hash_fn) { + size_t size_to_write; + int trans_a, trans_b; + uint32_t epilogue; + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulDescGetAttribute( + desc, CUBLASLT_MATMUL_DESC_TRANSA, &trans_a, sizeof(trans_a), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(trans_a)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulDescGetAttribute( + desc, CUBLASLT_MATMUL_DESC_TRANSB, &trans_b, sizeof(trans_b), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(trans_b)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatmulDescGetAttribute( + desc, CUBLASLT_MATMUL_DESC_EPILOGUE, &epilogue, sizeof(epilogue), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(epilogue)); + } + + void HashMatrixLayoutDesc_(cublasLtMatrixLayout_t desc, int64_t *seed, + const std::hash &hash_fn) { + size_t size_to_write; + uint32_t dtype; + int32_t batch; + uint64_t row, col; + int64_t ld, batch_offset; + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutGetAttribute( + desc, CUBLASLT_MATRIX_LAYOUT_TYPE, &dtype, sizeof(dtype), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(dtype)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutGetAttribute( + desc, CUBLASLT_MATRIX_LAYOUT_BATCH_COUNT, &batch, sizeof(batch), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(batch)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutGetAttribute( + desc, CUBLASLT_MATRIX_LAYOUT_ROWS, &row, sizeof(row), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(row)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutGetAttribute( + desc, CUBLASLT_MATRIX_LAYOUT_COLS, &col, sizeof(col), + &size_to_write)); + HashValue_(seed, hash_fn, static_cast(col)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutGetAttribute( + desc, CUBLASLT_MATRIX_LAYOUT_LD, &ld, sizeof(ld), &size_to_write)); + HashValue_(seed, hash_fn, static_cast(ld)); + + PADDLE_ENFORCE_GPU_SUCCESS( + platform::dynload::cublasLtMatrixLayoutGetAttribute( + desc, CUBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, &batch_offset, + sizeof(batch_offset), &size_to_write)); + HashValue_(seed, hash_fn, static_cast(batch_offset)); + } + + void HashValue_(int64_t *seed, const std::hash &hash_fn, + int64_t value) { + *seed ^= hash_fn(value) + 0x9e3779b9 + (*seed << 6) + (*seed >> 2); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/platform/dynload/cublasLt.h b/paddle/fluid/platform/dynload/cublasLt.h index c9a59751a320a..5157cfdad2e59 100644 --- a/paddle/fluid/platform/dynload/cublasLt.h +++ b/paddle/fluid/platform/dynload/cublasLt.h @@ -1,4 +1,5 @@ /* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +Copyright (c) 2022 NVIDIA Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -38,19 +39,25 @@ namespace dynload { // APIs available after CUDA 10.1 // #if CUDA_VERSION >= 10100 -#define CUBLASLT_BLAS_ROUTINE_EACH(__macro) \ - __macro(cublasLtCreate); \ - __macro(cublasLtDestroy); \ - __macro(cublasLtMatmul); \ - __macro(cublasLtMatmulDescCreate); \ - __macro(cublasLtMatmulDescDestroy); \ - __macro(cublasLtMatmulDescSetAttribute); \ - __macro(cublasLtMatrixLayoutCreate); \ - __macro(cublasLtMatrixLayoutDestroy); \ - __macro(cublasLtMatrixLayoutSetAttribute); \ - __macro(cublasLtMatrixTransform); \ - __macro(cublasLtMatrixTransformDescCreate); \ - __macro(cublasLtMatrixTransformDescDestroy); \ +#define CUBLASLT_BLAS_ROUTINE_EACH(__macro) \ + __macro(cublasLtCreate); \ + __macro(cublasLtDestroy); \ + __macro(cublasLtMatmul); \ + __macro(cublasLtMatmulDescCreate); \ + __macro(cublasLtMatmulDescDestroy); \ + __macro(cublasLtMatmulDescSetAttribute); \ + __macro(cublasLtMatmulDescGetAttribute); \ + __macro(cublasLtMatrixLayoutCreate); \ + __macro(cublasLtMatrixLayoutDestroy); \ + __macro(cublasLtMatrixLayoutSetAttribute); \ + __macro(cublasLtMatrixLayoutGetAttribute); \ + __macro(cublasLtMatmulPreferenceCreate); \ + __macro(cublasLtMatmulPreferenceDestroy); \ + __macro(cublasLtMatmulPreferenceSetAttribute); \ + __macro(cublasLtMatmulAlgoGetHeuristic); \ + __macro(cublasLtMatrixTransform); \ + __macro(cublasLtMatrixTransformDescCreate); \ + __macro(cublasLtMatrixTransformDescDestroy); \ __macro(cublasLtMatrixTransformDescSetAttribute); CUBLASLT_BLAS_ROUTINE_EACH(PLATFORM_DECLARE_DYNAMIC_LOAD_CUBLASLT_WRAP) diff --git a/paddle/fluid/platform/flags.cc b/paddle/fluid/platform/flags.cc index f89452853b49b..054a804e6b38e 100644 --- a/paddle/fluid/platform/flags.cc +++ b/paddle/fluid/platform/flags.cc @@ -1,4 +1,5 @@ // Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// Copyright (c) 2022 NVIDIA Authors. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -107,6 +108,29 @@ PADDLE_DEFINE_EXPORTED_string( "share-memory only."); #endif +#if defined(PADDLE_WITH_CUDA) +/** + * CUDA related FLAG + * Name: FLAGS_cublaslt_exhaustive_search_times + * Since Version: 2.3.0 + * Value Range: int64_t, default=0 + * Example: + * Note: Represents times of exhaustive search to evaluate performance of + * cuBlasLt matmul algorithm (with/without epilogue). Set this flag + * with value > 0 to enable exhaustive search. Default is 0, means + * getting algorithms via heuristic search. There are two search methods + * in cuBlasLt, heuristic search and exhaustive search. Exhaustive search + * attempts all cuBlasLt algorithms to select the fastest, which is very + * time-consuming, and the selected algorithm will be cached for a given + * layer specification Once you change the layer specifications + * (such as M, N and K), it will re-search again. + */ +PADDLE_DEFINE_EXPORTED_int64( + cublaslt_exhaustive_search_times, 0, + "The times of exhaustive search for cuBlasLt matmul with/without " + " epilogue algorithms, default is 0, means disabling exhaustive search."); +#endif + #if defined(PADDLE_WITH_ASCEND_CL) PADDLE_DEFINE_EXPORTED_string( selected_npus, "", diff --git a/paddle/phi/backends/dynload/cublasLt.h b/paddle/phi/backends/dynload/cublasLt.h index a1562370c377b..4c7ac9c3f21c4 100644 --- a/paddle/phi/backends/dynload/cublasLt.h +++ b/paddle/phi/backends/dynload/cublasLt.h @@ -1,4 +1,5 @@ /* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +Copyright (c) 2022 NVIDIA Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -52,19 +53,25 @@ extern void *cublasLt_dso_handle; // APIs available after CUDA 10.1 // #if CUDA_VERSION >= 10100 -#define CUBLASLT_BLAS_ROUTINE_EACH(__macro) \ - __macro(cublasLtCreate); \ - __macro(cublasLtDestroy); \ - __macro(cublasLtMatmul); \ - __macro(cublasLtMatmulDescCreate); \ - __macro(cublasLtMatmulDescDestroy); \ - __macro(cublasLtMatmulDescSetAttribute); \ - __macro(cublasLtMatrixLayoutCreate); \ - __macro(cublasLtMatrixLayoutDestroy); \ - __macro(cublasLtMatrixLayoutSetAttribute); \ - __macro(cublasLtMatrixTransform); \ - __macro(cublasLtMatrixTransformDescCreate); \ - __macro(cublasLtMatrixTransformDescDestroy); \ +#define CUBLASLT_BLAS_ROUTINE_EACH(__macro) \ + __macro(cublasLtCreate); \ + __macro(cublasLtDestroy); \ + __macro(cublasLtMatmul); \ + __macro(cublasLtMatmulDescCreate); \ + __macro(cublasLtMatmulDescDestroy); \ + __macro(cublasLtMatmulDescSetAttribute); \ + __macro(cublasLtMatmulDescGetAttribute); \ + __macro(cublasLtMatrixLayoutCreate); \ + __macro(cublasLtMatrixLayoutDestroy); \ + __macro(cublasLtMatrixLayoutSetAttribute); \ + __macro(cublasLtMatrixLayoutGetAttribute); \ + __macro(cublasLtMatmulPreferenceCreate); \ + __macro(cublasLtMatmulPreferenceDestroy); \ + __macro(cublasLtMatmulPreferenceSetAttribute); \ + __macro(cublasLtMatmulAlgoGetHeuristic); \ + __macro(cublasLtMatrixTransform); \ + __macro(cublasLtMatrixTransformDescCreate); \ + __macro(cublasLtMatrixTransformDescDestroy); \ __macro(cublasLtMatrixTransformDescSetAttribute); CUBLASLT_BLAS_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_CUBLASLT_WRAP) diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index 5235b7f1e88ab..32d8f5e3847c8 100755 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -129,18 +129,11 @@ if(NOT WITH_GPU) LIST(REMOVE_ITEM TEST_OPS test_fused_attention_op) LIST(REMOVE_ITEM TEST_OPS test_fused_attention_op_api) LIST(REMOVE_ITEM TEST_OPS test_fused_transformer_encoder_layer) - LIST(REMOVE_ITEM TEST_OPS test_fused_gemm_epilogue_op) - LIST(REMOVE_ITEM TEST_OPS test_fused_gemm_epilogue_grad_op) - LIST(REMOVE_ITEM TEST_OPS test_fuse_gemm_epilogue_pass) endif() -if (WITH_GPU) - if (CUDA_VERSION LESS 11.6) - LIST(REMOVE_ITEM TEST_OPS test_fused_gemm_epilogue_op) - LIST(REMOVE_ITEM TEST_OPS test_fused_gemm_epilogue_grad_op) - LIST(REMOVE_ITEM TEST_OPS test_fuse_gemm_epilogue_pass) - endif() -endif() +LIST(REMOVE_ITEM TEST_OPS test_fused_gemm_epilogue_op) +LIST(REMOVE_ITEM TEST_OPS test_fused_gemm_epilogue_grad_op) +LIST(REMOVE_ITEM TEST_OPS test_fuse_gemm_epilogue_pass) if(((NOT WITH_ROCM) AND (NOT WITH_GPU)) OR WIN32) LIST(REMOVE_ITEM TEST_OPS test_c_comm_init_all_op) @@ -644,6 +637,15 @@ py_test_modules(test_imperative_static_runner_mnist MODULES test_imperative_stat FLAGS_cudnn_deterministic=1) py_test_modules(test_imperative_static_runner_while MODULES test_imperative_static_runner_while ENVS FLAGS_cudnn_deterministic=1) + +if ((WITH_GPU) AND (CUDA_VERSION GREATER_EQUAL 11.6)) + py_test_modules(test_fused_gemm_epilogue_op MODULES test_fused_gemm_epilogue_op) + py_test_modules(test_fused_gemm_epilogue_grad_op MODULES test_fused_gemm_epilogue_grad_op) + py_test_modules(test_fused_gemm_epilogue_op_with_es MODULES test_fused_gemm_epilogue_op ENVS FLAGS_cublaslt_exhaustive_search_times=30) + py_test_modules(test_fused_gemm_epilogue_grad_op_with_es MODULES test_fused_gemm_epilogue_grad_op ENVS FLAGS_cublaslt_exhaustive_search_times=30) + py_test_modules(test_fuse_gemm_epilogue_pass MODULES test_fuse_gemm_epilogue_pass) +endif() + set_tests_properties(test_conv2d_op PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE") set_tests_properties(test_faster_tokenizer_op PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE") set_tests_properties(test_conv2d_op_depthwise_conv PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE") diff --git a/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py index 7f3180e21d8c6..00d91b1fab0f1 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_gemm_epilogue_pass.py @@ -49,8 +49,8 @@ def verify_node_count(graph, node_name, target_count): class MultiFCLayer(paddle.nn.Layer): def __init__(self, hidden, Activation): super(MultiFCLayer, self).__init__() - self.linear1 = paddle.nn.Linear(hidden, hidden) - self.linear2 = paddle.nn.Linear(hidden, hidden) + self.linear1 = paddle.nn.Linear(hidden, 4 * hidden) + self.linear2 = paddle.nn.Linear(4 * hidden, hidden) self.linear3 = paddle.nn.Linear(hidden, hidden) self.relu1 = Activation() From 4fd190d5141d56445d5e6e46e6cb603eeddee507 Mon Sep 17 00:00:00 2001 From: chenjian Date: Fri, 22 Apr 2022 19:07:59 +0800 Subject: [PATCH 34/66] Reduce performance influence by record event in python (#42040) * optimize performance * fix * improve coverage * fix * fix --- .../fluid/dataloader/dataloader_iter.py | 25 +++++++------ python/paddle/fluid/dygraph/layers.py | 10 ++++-- .../fluid/dygraph/varbase_patch_methods.py | 11 +++--- .../fluid/tests/unittests/test_newprofiler.py | 36 +++++++++++++++++++ python/paddle/profiler/utils.py | 18 ++++++++-- 5 files changed, 80 insertions(+), 20 deletions(-) diff --git a/python/paddle/fluid/dataloader/dataloader_iter.py b/python/paddle/fluid/dataloader/dataloader_iter.py index bbf2a4377c767..430578db51022 100644 --- a/python/paddle/fluid/dataloader/dataloader_iter.py +++ b/python/paddle/fluid/dataloader/dataloader_iter.py @@ -31,6 +31,7 @@ import paddle import paddle.profiler as profiler +from paddle.profiler.utils import in_profiler_mode from .. import core, layers from ..framework import _non_static_mode, in_dygraph_mode, _in_legacy_dygraph from ..multiprocess_utils import _set_SIGCHLD_handler, MP_STATUS_CHECK_INTERVAL, CleanupFuncRegistrar @@ -252,10 +253,11 @@ def _thread_loop(self, legacy_expected_place): self._exit_thread_expectedly() def __next__(self): - trace_event = profiler.RecordEvent( - name="_DataLoaderIterSingleProcess", - event_type=profiler.TracerEventType.Dataloader) - trace_event.begin() + if in_profiler_mode(): + trace_event = profiler.RecordEvent( + name="_DataLoaderIterSingleProcess", + event_type=profiler.TracerEventType.Dataloader) + trace_event.begin() try: benchmark().check_if_need_record(self) benchmark().before_reader() @@ -294,7 +296,8 @@ def __next__(self): self._try_shutdown_all() six.reraise(*sys.exc_info()) finally: - trace_event.end() + if in_profiler_mode(): + trace_event.end() def _shutdown_thread(self): if self._thread: @@ -708,10 +711,11 @@ def _shutdown_on_exit(self): self._try_shutdown_all(1) def __next__(self): - trace_event = profiler.RecordEvent( - name="_DataLoaderIterMultiProcess", - event_type=profiler.TracerEventType.Dataloader) - trace_event.begin() + if in_profiler_mode(): + trace_event = profiler.RecordEvent( + name="_DataLoaderIterMultiProcess", + event_type=profiler.TracerEventType.Dataloader) + trace_event.begin() try: benchmark().check_if_need_record(self) benchmark().before_reader() @@ -765,7 +769,8 @@ def __next__(self): self._try_shutdown_all() six.reraise(*sys.exc_info()) finally: - trace_event.end() + if in_profiler_mode(): + trace_event.end() # python2 compatibility def next(self): diff --git a/python/paddle/fluid/dygraph/layers.py b/python/paddle/fluid/dygraph/layers.py index 41c1a0aa5808e..088fed03c3595 100644 --- a/python/paddle/fluid/dygraph/layers.py +++ b/python/paddle/fluid/dygraph/layers.py @@ -26,6 +26,7 @@ import paddle import paddle.profiler as profiler +from paddle.profiler.utils import in_profiler_mode from . import parallel_helper from .. import unique_name @@ -906,8 +907,11 @@ def _dygraph_call_func(self, *inputs, **kwargs): self._built = True - with profiler.RecordEvent(self.full_name(), - profiler.TracerEventType.Forward): + if in_profiler_mode(): + with profiler.RecordEvent(self.full_name(), + profiler.TracerEventType.Forward): + outputs = self.forward(*inputs, **kwargs) + else: outputs = self.forward(*inputs, **kwargs) for forward_post_hook in self._forward_post_hooks.values(): @@ -919,7 +923,7 @@ def _dygraph_call_func(self, *inputs, **kwargs): def __call__(self, *inputs, **kwargs): if (not in_declarative_mode()) and (not self._forward_pre_hooks) \ - and (not self._forward_post_hooks) and (not self._built) and in_dygraph_mode(): + and (not self._forward_post_hooks) and (not self._built) and in_dygraph_mode() and (not in_profiler_mode()): self._build_once(*inputs, **kwargs) return self.forward(*inputs, **kwargs) else: diff --git a/python/paddle/fluid/dygraph/varbase_patch_methods.py b/python/paddle/fluid/dygraph/varbase_patch_methods.py index db6af87635ccb..a93facbc34a5b 100644 --- a/python/paddle/fluid/dygraph/varbase_patch_methods.py +++ b/python/paddle/fluid/dygraph/varbase_patch_methods.py @@ -30,6 +30,7 @@ from paddle.fluid.data_feeder import convert_dtype, _PADDLE_DTYPE_2_NUMPY_DTYPE import paddle.utils.deprecated as deprecated import paddle.profiler as profiler +from paddle.profiler.utils import in_profiler_mode from paddle import _C_ops _grad_scalar = None @@ -247,9 +248,10 @@ def backward(self, grad_tensor=None, retain_graph=False): """ if framework._non_static_mode(): - record_event = profiler.RecordEvent( - "Gradient Backward", profiler.TracerEventType.Backward) - record_event.begin() + if in_profiler_mode(): + record_event = profiler.RecordEvent( + "Gradient Backward", profiler.TracerEventType.Backward) + record_event.begin() if grad_tensor is not None: if framework._in_eager_mode_: assert isinstance( @@ -289,7 +291,8 @@ def backward(self, grad_tensor=None, retain_graph=False): core.dygraph_run_backward([self], [grad_tensor], retain_graph, framework._dygraph_tracer()) - record_event.end() + if in_profiler_mode(): + record_event.end() else: raise ValueError( "Variable.backward() is only available in DyGraph mode") diff --git a/python/paddle/fluid/tests/unittests/test_newprofiler.py b/python/paddle/fluid/tests/unittests/test_newprofiler.py index ae804f82b90f7..53ade0dfb79c1 100755 --- a/python/paddle/fluid/tests/unittests/test_newprofiler.py +++ b/python/paddle/fluid/tests/unittests/test_newprofiler.py @@ -134,6 +134,42 @@ def my_sheduler1(num_step): prof.export(path='./test_profiler_pb.pb', format='pb') prof.summary() result = profiler.utils.load_profiler_result('./test_profiler_pb.pb') + prof = None + dataset = RandomDataset(10 * 4) + simple_net = SimpleNet() + opt = paddle.optimizer.SGD(learning_rate=1e-3, + parameters=simple_net.parameters()) + loader = DataLoader( + dataset, batch_size=4, shuffle=True, drop_last=True, num_workers=2) + prof = profiler.Profiler(on_trace_ready=lambda prof: None) + prof.start() + for i, (image, label) in enumerate(loader()): + out = simple_net(image) + loss = F.cross_entropy(out, label) + avg_loss = paddle.mean(loss) + avg_loss.backward() + opt.minimize(avg_loss) + simple_net.clear_gradients() + prof.step() + prof.stop() + prof.summary() + prof = None + dataset = RandomDataset(10 * 4) + simple_net = SimpleNet() + loader = DataLoader(dataset, batch_size=4, shuffle=True, drop_last=True) + opt = paddle.optimizer.Adam( + learning_rate=1e-3, parameters=simple_net.parameters()) + prof = profiler.Profiler(on_trace_ready=lambda prof: None) + prof.start() + for i, (image, label) in enumerate(loader()): + out = simple_net(image) + loss = F.cross_entropy(out, label) + avg_loss = paddle.mean(loss) + avg_loss.backward() + opt.step() + simple_net.clear_gradients() + prof.step() + prof.stop() class TestNvprof(unittest.TestCase): diff --git a/python/paddle/profiler/utils.py b/python/paddle/profiler/utils.py index 6ae3fe4e60b92..fba1aeabf28bd 100644 --- a/python/paddle/profiler/utils.py +++ b/python/paddle/profiler/utils.py @@ -21,6 +21,7 @@ from paddle.fluid.core import (_RecordEvent, TracerEventType) _is_profiler_used = False +_has_optimizer_wrapped = False _AllowedEventTypeList = [ TracerEventType.Dataloader, TracerEventType.ProfileStep, @@ -154,20 +155,31 @@ def load_profiler_result(filename: str): return core.load_profiler_result(filename) +def in_profiler_mode(): + return _is_profiler_used == True + + def wrap_optimizers(): def optimizer_warpper(func): @functools.wraps(func) def warpper(*args, **kwargs): - with RecordEvent( - 'Optimization Step', - event_type=TracerEventType.Optimization): + if in_profiler_mode(): + with RecordEvent( + 'Optimization Step', + event_type=TracerEventType.Optimization): + return func(*args, **kwargs) + else: return func(*args, **kwargs) return warpper + global _has_optimizer_wrapped + if _has_optimizer_wrapped == True: + return import paddle.optimizer as optimizer for classname in optimizer.__all__: if classname != 'Optimizer': classobject = getattr(optimizer, classname) if getattr(classobject, 'step', None) != None: classobject.step = optimizer_warpper(classobject.step) + _has_optimizer_wrapped = True From cca57c4ac4856cde401071edd7e6a5219524270d Mon Sep 17 00:00:00 2001 From: zhaocaibei123 <48509226+zhaocaibei123@users.noreply.github.com> Date: Fri, 22 Apr 2022 19:15:31 +0800 Subject: [PATCH 35/66] Ssd sparse table (#41812) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [cherry-pick2.3]fix compile bug of windows cuda11.5 (#41464) cherry-pick fix compile bug of windows cuda11.5 #41433 * fix bug of missing boost when compile cache.cc (#41449) 【chery-pick #41430】fix bug of random compile failure, due to incorrect compile order of dependencies * Fix eager try catch (#41438) (#41477) [Cherry-Pick]Fix eager try catch (#41438) * Cherry-pick-PR41407, fix device_id bug for final_state op in multiprocess testcase (#41407) (#41475) Cherry-pick PR #41407 * [BugFix] Add error hint for one_hot gpu version (#41335) (#41495) * add one_hot gpu hint * move allow_out_of_range judgement * delete useless unittest * fix bugs of reshape double grad infermeta (#41459) (#41493) * [cherrypick-2.3] modify infer gpu memory strategy (#41427), remove cudnn_deterministic=True (#41341) (#41491) Co-authored-by: JingZhuangzhuang <75348594+JZZ-NOTE@users.noreply.github.com> * [Cherry-pick][ROCm] fix dcu error in device event base, test=develop (#41523) Cherry-pick of #41521 * [Cherry-Pick]Cherry pick PR41200, PR41474, PR41382 (#41509) * Use `self`as a parameter of _hash_with_id function to avoid error caused by hash_id reuse (#41200) * Add fill_constant_batch_size YAML and UT (#41474) * Switch some dy2st UT to eager mode (#41382) * Sitch some dy2st UT to eager mode * Fix test_lstm and remove test_transformer * Run test_resnet_v2 in old dy mode * Unittest recover (#41431) * update name * update name * fix test * fix fleet bind * update name * update name * fix test * fix gpups wrapper * remove Push/Pull/Load/Save with context in client and wrapper base class * fix * fix * remove some interface * fix * remove * code style * recover * fix * remove code unused * remove some unused table & accessor & CommonDenseTable => MemoryDenseTable * fix * fix * fix * recover * remove unused code * recover unittest * fix * remove * fix * remove code unuseful * remove * fix * recover * remove Co-authored-by: esythan * add ssd sparse table * fix * add cache shuffle * fix * fix * fix * fix * fix * fix * add unit test * fix Co-authored-by: Zhou Wei <1183042833@qq.com> Co-authored-by: Sing_chan <51314274+betterpig@users.noreply.github.com> Co-authored-by: 0x45f <23097963+0x45f@users.noreply.github.com> Co-authored-by: pangyoki Co-authored-by: Siming Dai <908660116@qq.com> Co-authored-by: YuanRisheng Co-authored-by: Zhang Jun Co-authored-by: JingZhuangzhuang <75348594+JZZ-NOTE@users.noreply.github.com> Co-authored-by: Qi Li Co-authored-by: esythan --- cmake/third_party.cmake | 6 +- .../distributed/common/topk_calculator.h | 70 ++ .../distributed/ps/service/CMakeLists.txt | 6 +- .../distributed/ps/service/brpc_ps_client.cc | 76 ++ .../distributed/ps/service/brpc_ps_client.h | 14 + .../distributed/ps/service/brpc_ps_server.cc | 213 ++++- .../distributed/ps/service/brpc_ps_server.h | 16 + .../fluid/distributed/ps/service/ps_client.h | 41 + .../distributed/ps/service/sendrecv.proto | 2 + paddle/fluid/distributed/ps/service/server.cc | 2 + paddle/fluid/distributed/ps/service/server.h | 40 + .../fluid/distributed/ps/table/CMakeLists.txt | 21 +- paddle/fluid/distributed/ps/table/accessor.h | 5 + .../distributed/ps/table/common_graph_table.h | 2 +- .../distributed/ps/table/ctr_accessor.cc | 21 + .../fluid/distributed/ps/table/ctr_accessor.h | 3 + .../ps/table/ctr_double_accessor.cc | 27 +- .../ps/table/ctr_double_accessor.h | 2 + .../ps/table/depends/rocksdb_warpper.h | 8 +- .../ps/table/memory_sparse_table.cc | 19 +- .../ps/table/memory_sparse_table.h | 8 +- .../distributed/ps/table/sparse_accessor.h | 5 + .../distributed/ps/table/ssd_sparse_table.cc | 759 ++++++++++++++++++ .../distributed/ps/table/ssd_sparse_table.h | 94 +++ paddle/fluid/distributed/ps/table/table.cc | 2 + paddle/fluid/distributed/ps/table/table.h | 21 + .../distributed/ps/table/tensor_accessor.h | 6 + paddle/fluid/distributed/ps/wrapper/fleet.cc | 40 + paddle/fluid/distributed/ps/wrapper/fleet.h | 5 + paddle/fluid/distributed/the_one_ps.proto | 4 + paddle/fluid/pybind/fleet_py.cc | 6 +- paddle/utils/string/string_helper.h | 8 + python/paddle/distributed/fleet/__init__.py | 1 + .../distributed/fleet/base/fleet_base.py | 5 + python/paddle/distributed/ps/the_one_ps.py | 24 + .../fluid/tests/unittests/dist_fleet_ctr.py | 4 + .../tests/unittests/test_dist_fleet_ctr.py | 2 + 37 files changed, 1526 insertions(+), 62 deletions(-) create mode 100644 paddle/fluid/distributed/common/topk_calculator.h create mode 100644 paddle/fluid/distributed/ps/table/ssd_sparse_table.cc create mode 100644 paddle/fluid/distributed/ps/table/ssd_sparse_table.h diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake index f8a841fecbc0a..c8ef4ad16ea9d 100755 --- a/cmake/third_party.cmake +++ b/cmake/third_party.cmake @@ -357,10 +357,8 @@ if (WITH_PSCORE) include(external/libmct) # download, build, install libmct list(APPEND third_party_deps extern_libmct) - if (WITH_HETERPS) - include(external/rocksdb) # download, build, install libmct - list(APPEND third_party_deps extern_rocksdb) - endif() + include(external/rocksdb) # download, build, install libmct + list(APPEND third_party_deps extern_rocksdb) endif() if(WITH_XBYAK) diff --git a/paddle/fluid/distributed/common/topk_calculator.h b/paddle/fluid/distributed/common/topk_calculator.h new file mode 100644 index 0000000000000..326f0f718e9bd --- /dev/null +++ b/paddle/fluid/distributed/common/topk_calculator.h @@ -0,0 +1,70 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include +#include + +namespace paddle { +namespace distributed { +class TopkCalculator { + public: + TopkCalculator(int shard_num, size_t k) + : _shard_num(shard_num), _total_max_size(k) { + _shard_max_size = _total_max_size / shard_num; + _shard_max_size = _shard_max_size > 1 ? _shard_max_size : 1; + for (int i = 0; i < shard_num; ++i) { + _mpq.emplace(i, std::priority_queue, + std::greater>()); + } + } + ~TopkCalculator() {} + bool push(int shard_id, double value) { + if (_mpq.find(shard_id) == _mpq.end()) { + return false; + } + auto &pq = _mpq[shard_id]; + if (pq.size() < _shard_max_size) { + pq.push(value); + } else { + if (pq.top() < value) { + pq.pop(); + pq.push(value); + } + } + return true; + } + // TODO 再进行一次堆排序merge各个shard的结果 + int top() { + double total = 0; + for (const auto &item : _mpq) { + auto &pq = item.second; + if (!pq.empty()) { + total += pq.top(); + } + } + return total / _shard_num; + } + + private: + std::unordered_map, + std::greater>> + _mpq; + int _shard_num; + size_t _total_max_size; + size_t _shard_max_size; +}; + +} // namespace distributed +} // namespace paddle diff --git a/paddle/fluid/distributed/ps/service/CMakeLists.txt b/paddle/fluid/distributed/ps/service/CMakeLists.txt index b8de291072a1f..f0ac7bc6a0635 100755 --- a/paddle/fluid/distributed/ps/service/CMakeLists.txt +++ b/paddle/fluid/distributed/ps/service/CMakeLists.txt @@ -1,7 +1,11 @@ set(BRPC_SRCS ps_client.cc server.cc) set_source_files_properties(${BRPC_SRCS}) -set(BRPC_DEPS brpc ssl crypto protobuf gflags glog zlib leveldb snappy gflags glog device_context) +if(WITH_HETERPS) + set(BRPC_DEPS brpc ssl crypto protobuf gflags glog zlib leveldb snappy gflags glog device_context rocksdb) +else() + set(BRPC_DEPS brpc ssl crypto protobuf gflags glog zlib leveldb snappy gflags glog device_context) +endif() brpc_library(sendrecv_rpc SRCS ${BRPC_SRCS} diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_client.cc b/paddle/fluid/distributed/ps/service/brpc_ps_client.cc index 971c448bf2714..921a110984a4a 100644 --- a/paddle/fluid/distributed/ps/service/brpc_ps_client.cc +++ b/paddle/fluid/distributed/ps/service/brpc_ps_client.cc @@ -429,6 +429,82 @@ std::future BrpcPsClient::Save(uint32_t table_id, return SendSaveCmd(table_id, PS_SAVE_ONE_TABLE, {epoch, mode}); } +std::future BrpcPsClient::CacheShuffle( + uint32_t table_id, const std::string &path, const std::string &mode, + const std::string &cache_threshold) { + VLOG(1) << "BrpcPsClient send cmd for cache shuffle"; + return SendSaveCmd(table_id, PS_CACHE_SHUFFLE, {path, mode, cache_threshold}); +} + +std::future BrpcPsClient::CacheShuffleMultiTable( + std::vector tables, const std::string &path, const std::string &mode, + const std::string &cache_threshold) { + VLOG(1) << "BrpcPsClient send cmd for cache shuffle multi table one path"; + std::vector param; + param.push_back(path); + param.push_back(mode); + param.push_back(cache_threshold); + for (size_t i = 0; i < tables.size(); i++) { + param.push_back(std::to_string(tables[i])); + } + return SendSaveCmd(0, PS_CACHE_SHUFFLE, param); +} + +std::future BrpcPsClient::SaveCache(uint32_t table_id, + const std::string &path, + const std::string &mode) { + return SendSaveCmd(table_id, PS_SAVE_ONE_CACHE_TABLE, {path, mode}); +} + +std::future BrpcPsClient::GetCacheThreshold(uint32_t table_id, + double &cache_threshold) { + int cmd_id = PS_GET_CACHE_THRESHOLD; + size_t request_call_num = _server_channels.size(); + DownpourBrpcClosure *closure = new DownpourBrpcClosure( + request_call_num, + [request_call_num, cmd_id, &cache_threshold](void *done) { + int ret = 0; + auto *closure = (DownpourBrpcClosure *)done; + std::vector cache_thresholds(request_call_num, 0); + for (size_t i = 0; i < request_call_num; ++i) { + if (closure->check_response(i, cmd_id) != 0) { + ret = -1; + break; + } + std::string cur_res = closure->get_response(i, cmd_id); + cache_thresholds[i] = std::stod(cur_res); + } + double sum_threshold = 0.0; + int count = 0; + for (auto t : cache_thresholds) { + if (t >= 0) { + sum_threshold += t; + ++count; + } + } + if (count == 0) { + cache_threshold = 0; + } else { + cache_threshold = sum_threshold / count; + } + VLOG(1) << "client get cache threshold: " << cache_threshold; + closure->set_promise_value(ret); + }); + auto promise = std::make_shared>(); + closure->add_promise(promise); + std::future fut = promise->get_future(); + for (size_t i = 0; i < request_call_num; ++i) { + closure->request(i)->set_cmd_id(cmd_id); + closure->request(i)->set_table_id(table_id); + closure->request(i)->set_client_id(_client_id); + PsService_Stub rpc_stub(GetCmdChannel(i)); + closure->cntl(i)->set_timeout_ms(10800000); + rpc_stub.service(closure->cntl(i), closure->request(i), + closure->response(i), closure); + } + return fut; +} + std::future BrpcPsClient::Clear() { return SendCmd(-1, PS_CLEAR_ALL_TABLE, {}); } diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_client.h b/paddle/fluid/distributed/ps/service/brpc_ps_client.h index f109b473ca1f4..e2c16d496c42c 100644 --- a/paddle/fluid/distributed/ps/service/brpc_ps_client.h +++ b/paddle/fluid/distributed/ps/service/brpc_ps_client.h @@ -219,6 +219,20 @@ class BrpcPsClient : public PSClient { virtual int32_t RecvAndSaveTable(const uint64_t table_id, const std::string &path); + std::future CacheShuffle( + uint32_t table_id, const std::string &path, const std::string &mode, + const std::string &cache_threshold) override; + + std::future CacheShuffleMultiTable( + std::vector tables, const std::string &path, const std::string &mode, + const std::string &cache_threshold); + + std::future SaveCache(uint32_t table_id, const std::string &path, + const std::string &mode) override; + + std::future GetCacheThreshold(uint32_t table_id, + double &cache_threshold) override; + void PrintQueueSize(); void PrintQueueSizeThread(); diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_server.cc b/paddle/fluid/distributed/ps/service/brpc_ps_server.cc index d22cca91f7816..d0bf06d49504a 100644 --- a/paddle/fluid/distributed/ps/service/brpc_ps_server.cc +++ b/paddle/fluid/distributed/ps/service/brpc_ps_server.cc @@ -28,6 +28,13 @@ class RpcController; } // namespace protobuf } // namespace google +DEFINE_int32(pserver_timeout_ms_s2s, 10000, + "pserver request server timeout_ms"); +DEFINE_int32(pserver_connect_timeout_ms_s2s, 10000, + "pserver connect server timeout_ms"); +DEFINE_string(pserver_connection_type_s2s, "pooled", + "pserver connection_type[pooled:single]"); + namespace paddle { namespace distributed { @@ -93,6 +100,84 @@ uint64_t BrpcPsServer::Start(const std::string &ip, uint32_t port) { return host.rank; } +int32_t BrpcPsServer::StartS2S() { + brpc::ChannelOptions options; + options.protocol = "baidu_std"; + options.timeout_ms = FLAGS_pserver_timeout_ms_s2s; + options.connection_type = FLAGS_pserver_connection_type_s2s; + options.connect_timeout_ms = FLAGS_pserver_connect_timeout_ms_s2s; + options.max_retry = 3; + + std::vector pserver_list = _environment->GetPsServers(); + _pserver_channels.resize(pserver_list.size()); + VLOG(2) << "pserver start s2s server_list size: " << _pserver_channels.size(); + + std::ostringstream os; + std::string server_ip_port; + + for (size_t i = 0; i < pserver_list.size(); ++i) { + server_ip_port.assign(pserver_list[i].ip.c_str()); + server_ip_port.append(":"); + server_ip_port.append(std::to_string(pserver_list[i].port)); + _pserver_channels[i].reset(new brpc::Channel()); + if (_pserver_channels[i]->Init(server_ip_port.c_str(), "", &options) != 0) { + LOG(ERROR) << "pserver connect to pserver:" << server_ip_port + << " Failed!"; + } + os << server_ip_port << ","; + } + LOG(INFO) << "pserver connect success: " << os.str(); + return 0; +} + +std::future BrpcPsServer::SendPServer2PServerMsg( + int msg_type, int to_pserver_id, const std::string &msg) { + auto promise = std::make_shared>(); + std::future fut = promise->get_future(); + if (to_pserver_id >= _pserver_channels.size()) { + LOG(FATAL) << "to_pserver_id is out of range pservers, which size is " + << _pserver_channels.size(); + promise->set_value(-1); + return fut; + } + auto *closure = new DownpourPServerBrpcClosure(1, [msg_type](void *done) { + auto *closure = (DownpourPServerBrpcClosure *)done; + int32_t ret = closure->check_response(0, msg_type + 1000); + closure->set_promise_value(ret); + }); + + closure->add_promise(promise); + closure->request(0)->set_cmd_id(101); + closure->request(0)->set_client_id(_rank); + closure->request(0)->set_table_id(0); + closure->request(0)->set_data(msg); + PsService_Stub rpc_stub(_pserver_channels[to_pserver_id].get()); + rpc_stub.service(closure->cntl(0), closure->request(0), closure->response(0), + closure); + return fut; +} + +int32_t BrpcPsServer::ReceiveFromPServer(int msg_type, int pserver_id, + const std::string &msg) { + if (msg.length() == 0) { + LOG(WARNING) << "SERVER>>RESPONSE>>msg = 0 Finish S2S Response"; + return 0; + } + paddle::framework::BinaryArchive ar; + ar.SetReadBuffer(const_cast(msg.c_str()), msg.length(), nullptr); + if (ar.Cursor() == ar.Finish()) { + LOG(WARNING) << "SERVER>>RESPONSE ar = 0>> Finish S2S Response"; + return 0; + } + std::vector> data; + while (ar.Cursor() < ar.Finish()) { + data.push_back(ar.Get>()); + } + CHECK(ar.Cursor() == ar.Finish()); + this->_shuffled_ins->Write(std::move(data)); + return 0; +} + int32_t BrpcPsServer::Port() { return _server.listen_address().port; } int32_t BrpcPsService::Initialize() { @@ -117,6 +202,14 @@ int32_t BrpcPsService::Initialize() { _service_handler_map[PS_START_PROFILER] = &BrpcPsService::StartProfiler; _service_handler_map[PS_STOP_PROFILER] = &BrpcPsService::StopProfiler; _service_handler_map[PS_PUSH_GLOBAL_STEP] = &BrpcPsService::PushGlobalStep; + // for save cache + + _service_handler_map[PS_SAVE_ONE_CACHE_TABLE] = + &BrpcPsService::SaveCacheTable; + _service_handler_map[PS_GET_CACHE_THRESHOLD] = + &BrpcPsService::GetCacheThreshold; + _service_handler_map[PS_CACHE_SHUFFLE] = &BrpcPsService::CacheShuffle; + auto &profiler = CostProfiler::instance(); profiler.register_profiler("pserver_server_pull_dense"); profiler.register_profiler("pserver_server_push_dense"); @@ -168,19 +261,29 @@ void BrpcPsService::service(google::protobuf::RpcController *cntl_base, response->set_err_msg(""); auto *table = _server->GetTable(request->table_id()); brpc::Controller *cntl = static_cast(cntl_base); - auto itr = _service_handler_map.find(request->cmd_id()); - if (itr == _service_handler_map.end()) { - std::string err_msg( - "undefined cmd_id, should match PsCmdID in ps.proto, cmd_id:"); - err_msg.append(std::to_string(request->cmd_id())); - set_response_code(*response, -1, err_msg.c_str()); - return; - } - serviceHandlerFunc handler_func = itr->second; - int service_ret = (this->*handler_func)(table, *request, *response, cntl); - if (service_ret != 0) { - response->set_err_code(service_ret); - response->set_err_msg("server internal error"); + + if (request->cmd_id() < 100) { + auto itr = _service_handler_map.find(request->cmd_id()); + if (itr == _service_handler_map.end()) { + std::string err_msg( + "undefined cmd_id, should match PsCmdID in ps.proto, cmd_id:"); + err_msg.append(std::to_string(request->cmd_id())); + set_response_code(*response, -1, err_msg.c_str()); + return; + } + serviceHandlerFunc handler_func = itr->second; + int service_ret = (this->*handler_func)(table, *request, *response, cntl); + if (service_ret != 0) { + response->set_err_code(service_ret); + response->set_err_msg("server internal error"); + } + } else { + int service_ret = _server->HandlePServer2PServerMsg( + request->cmd_id(), request->client_id(), request->data()); + if (service_ret != 0) { + response->set_err_code(-1); + response->set_err_msg("handle_pserver2pserver_msg failed"); + } } } @@ -561,6 +664,90 @@ int32_t BrpcPsService::SaveAllTable(Table *table, return 0; } +int32_t BrpcPsService::SaveCacheTable(Table *table, + const PsRequestMessage &request, + PsResponseMessage &response, + brpc::Controller *cntl) { + CHECK_TABLE_EXIST(table, request, response) + if (request.params_size() < 2) { + set_response_code( + response, -1, + "PsRequestMessage.datas is requeired at least 3, path&mode"); + return -1; + } + table->Flush(); + int32_t feasign_size = 0; + // if (_server->_shuffled_ins->size() <= 0) { + // LOG(WARNING) << "shuffled ins size <= 0"; + //} + feasign_size = table->SaveCache(request.params(0), request.params(1), + _server->_shuffled_ins); + if (feasign_size < 0) { + set_response_code(response, -1, "table save failed"); + return -1; + } + return feasign_size; +} + +int32_t BrpcPsService::CacheShuffle(Table *table, + const PsRequestMessage &request, + PsResponseMessage &response, + brpc::Controller *cntl) { + // start cache shuffle + CHECK_TABLE_EXIST(table, request, response) + if (request.params_size() < 3) { + set_response_code(response, -1, + "PsRequestMessage.datas is requeired at least 3, " + "path&mode&cache_threshold"); + return -1; + } + table->Flush(); + double cache_threshold = std::stod(request.params(2)); + LOG(INFO) << "cache threshold for cache shuffle: " << cache_threshold; + // auto shuffled_ins = paddle::ps::make_channel>(); + // shuffled_ins->set_block_size(80000); + _server->StartS2S(); + std::function(int msg_type, int to_pserver_id, + const std::string &msg)> + send_msg_func = [this](int msg_type, int to_pserver_id, + const std::string &msg) -> std::future { + return this->_server->SendPServer2PServerMsg(msg_type, to_pserver_id, msg); + }; + + std::vector table_ptrs; + for (size_t i = 3; i < request.params_size(); ++i) { + int table_id = std::stoi(request.params(i)); + Table *table_ptr = _server->GetTable(table_id); + table_ptrs.push_back(table_ptr); + } + if (table_ptrs.empty()) { + table_ptrs.push_back(table); + } + + table->CacheShuffle(request.params(0), request.params(1), cache_threshold, + send_msg_func, _server->_shuffled_ins, table_ptrs); + return 0; +} + +int32_t BrpcPsService::GetCacheThreshold(Table *table, + const PsRequestMessage &request, + PsResponseMessage &response, + brpc::Controller *cntl) { + CHECK_TABLE_EXIST(table, request, response) + table->Flush(); + double cache_threshold = 0.0; + cache_threshold = table->GetCacheThreshold(); + if (cache_threshold < 0) { + LOG(WARNING) << "wrong threshold: " << cache_threshold; + } + std::stringstream ss; + ss << std::setprecision(15) << cache_threshold; + std::string cache_threshold_str = ss.str(); + response.set_data(cache_threshold_str); + return 0; +} + int32_t BrpcPsService::ShrinkTable(Table *table, const PsRequestMessage &request, PsResponseMessage &response, diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_server.h b/paddle/fluid/distributed/ps/service/brpc_ps_server.h index 250f465d84253..40ed652ec6be3 100644 --- a/paddle/fluid/distributed/ps/service/brpc_ps_server.h +++ b/paddle/fluid/distributed/ps/service/brpc_ps_server.h @@ -53,6 +53,12 @@ class BrpcPsServer : public PSServer { } int32_t Port(); + virtual int32_t StartS2S() override; + virtual ::std::future SendPServer2PServerMsg( + int msg_type, int to_pserver_id, const std::string &msg) override; + virtual int32_t ReceiveFromPServer(int msg_type, int pserver_id, + const std::string &msg) override; + private: virtual int32_t Initialize(); mutable std::mutex mutex_; @@ -123,6 +129,16 @@ class BrpcPsService : public PsBaseService { int32_t PushGlobalStep(Table *table, const PsRequestMessage &request, PsResponseMessage &response, brpc::Controller *cntl); + int32_t CacheShuffle(Table *table, const PsRequestMessage &request, + PsResponseMessage &response, brpc::Controller *cntl); + + int32_t SaveCacheTable(Table *table, const PsRequestMessage &request, + PsResponseMessage &response, brpc::Controller *cntl); + + int32_t GetCacheThreshold(Table *table, const PsRequestMessage &request, + PsResponseMessage &response, + brpc::Controller *cntl); + bool _is_initialize_shard_info; std::mutex _initialize_shard_mutex; std::unordered_map _service_handler_map; diff --git a/paddle/fluid/distributed/ps/service/ps_client.h b/paddle/fluid/distributed/ps/service/ps_client.h index 6f27b0eb04624..0d3d23be4e8d1 100644 --- a/paddle/fluid/distributed/ps/service/ps_client.h +++ b/paddle/fluid/distributed/ps/service/ps_client.h @@ -198,6 +198,7 @@ class PSClient { _msg_handler_map[msg_type] = handler; return 0; } + virtual int HandleClient2ClientMsg(int msg_type, int from_client_id, const std::string &msg) { auto itr = _msg_handler_map.find(msg_type); @@ -239,6 +240,46 @@ class PSClient { const float **update_values, size_t num) = 0; + // for save cache + virtual std::future CacheShuffle( + uint32_t table_id, const std::string &path, const std::string &mode, + const std::string &cache_threshold) { + VLOG(0) << "Did not implement"; + std::promise promise; + std::future fut = promise.get_future(); + promise.set_value(-1); + return fut; + } + + virtual std::future CacheShuffleMultiTable( + std::vector tables, const std::string &path, const std::string &mode, + const std::string &cache_threshold) { + VLOG(0) << "Did not implement"; + std::promise promise; + std::future fut = promise.get_future(); + promise.set_value(-1); + return fut; + } + + virtual std::future SaveCache(uint32_t table_id, + const std::string &path, + const std::string &mode) { + VLOG(0) << "Did not implement"; + std::promise promise; + std::future fut = promise.get_future(); + promise.set_value(-1); + return fut; + } + + virtual std::future GetCacheThreshold(uint32_t table_id, + double &cache_threshold) { + VLOG(0) << "Did not implement"; + std::promise promise; + std::future fut = promise.get_future(); + promise.set_value(-1); + return fut; + } + protected: virtual int32_t Initialize() = 0; size_t _client_id; diff --git a/paddle/fluid/distributed/ps/service/sendrecv.proto b/paddle/fluid/distributed/ps/service/sendrecv.proto index 580f411c28c07..46dcc2058f4b8 100755 --- a/paddle/fluid/distributed/ps/service/sendrecv.proto +++ b/paddle/fluid/distributed/ps/service/sendrecv.proto @@ -65,6 +65,8 @@ enum PsCmdID { PS_SAVE_WITH_SHARD = 44; PS_QUERY_WITH_SCOPE = 45; PS_QUERY_WITH_SHARD = 46; + // pserver2pserver cmd start from 100 + PS_S2S_MSG = 101; } message PsRequestMessage { diff --git a/paddle/fluid/distributed/ps/service/server.cc b/paddle/fluid/distributed/ps/service/server.cc index 65f7ae821cef1..a6e0f39474b06 100644 --- a/paddle/fluid/distributed/ps/service/server.cc +++ b/paddle/fluid/distributed/ps/service/server.cc @@ -67,6 +67,8 @@ int32_t PSServer::Configure( _config = config.server_param(); _rank = server_rank; _environment = &env; + _shuffled_ins = + paddle::framework::MakeChannel>(); size_t shard_num = env.GetPsServers().size(); const auto &downpour_param = _config.downpour_server_param(); diff --git a/paddle/fluid/distributed/ps/service/server.h b/paddle/fluid/distributed/ps/service/server.h index 5da819326b052..c044e82884604 100644 --- a/paddle/fluid/distributed/ps/service/server.h +++ b/paddle/fluid/distributed/ps/service/server.h @@ -89,6 +89,45 @@ class PSServer { return &_table_map; } + // for cache + virtual int32_t StartS2S() { return 0; } + + virtual ::std::future SendPServer2PServerMsg( + int msg_type, int to_pserver_id, const std::string &msg) { + LOG(FATAL) << "NotImplementError: PSServer::send_pserver2pserver_msg"; + std::promise promise; + std::future fut = promise.get_future(); + promise.set_value(-1); + return fut; + } + + typedef std::function MsgHandlerFunc; + virtual int RegistePServer2PServerMsgHandler(int msg_type, + MsgHandlerFunc handler) { + _msg_handler_map[msg_type] = handler; + return 0; + } + virtual int HandlePServer2PServerMsg(int msg_type, int from_pserver_id, + const std::string &msg) { + auto itr = _msg_handler_map.find(msg_type); + if (itr == _msg_handler_map.end()) { + if (msg_type == 101) { + return ReceiveFromPServer(msg_type, from_pserver_id, msg); + } else { + LOG(WARNING) << "unknown pserver2pserver_msg type:" << msg_type; + return -1; + } + } + return itr->second(msg_type, from_pserver_id, msg); + } + virtual int32_t ReceiveFromPServer(int msg_type, int pserver_id, + const std::string &msg) { + LOG(FATAL) << "NotImplementError::PSServer::ReceiveFromPServer"; + return -1; + } + + paddle::framework::Channel> _shuffled_ins; + protected: virtual int32_t Initialize() = 0; @@ -97,6 +136,7 @@ class PSServer { ServerParameter _config; PSEnvironment *_environment; std::unordered_map> _table_map; + std::unordered_map _msg_handler_map; protected: std::shared_ptr scope_; diff --git a/paddle/fluid/distributed/ps/table/CMakeLists.txt b/paddle/fluid/distributed/ps/table/CMakeLists.txt index bb6725b08425a..f2b9eb71f5a64 100644 --- a/paddle/fluid/distributed/ps/table/CMakeLists.txt +++ b/paddle/fluid/distributed/ps/table/CMakeLists.txt @@ -18,17 +18,12 @@ include_directories(${PADDLE_LIB_THIRD_PARTY_PATH}libmct/src/extern_libmct/libmc set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp") -set(EXTERN_DEP "") -if(WITH_HETERPS) - set(TABLE_SRC memory_dense_table.cc barrier_table.cc common_graph_table.cc) - set(EXTERN_DEP rocksdb) -else() - set(TABLE_SRC memory_dense_table.cc barrier_table.cc common_graph_table.cc) -endif() +set(TABLE_SRC memory_dense_table.cc barrier_table.cc common_graph_table.cc) +#set(EXTERN_DEP rocksdb) cc_library(common_table SRCS ${TABLE_SRC} DEPS ${TABLE_DEPS} ${RPC_DEPS} graph_edge graph_node device_context string_helper -simple_threadpool xxhash generator ${EXTERN_DEP}) +simple_threadpool xxhash generator) set_source_files_properties(tensor_accessor.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) @@ -41,13 +36,13 @@ set_source_files_properties(ctr_double_accessor.cc PROPERTIES COMPILE_FLAGS ${DI set_source_files_properties(ctr_accessor.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) set_source_files_properties(sparse_accessor.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) set_source_files_properties(memory_sparse_table.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) +set_source_files_properties(ssd_sparse_table.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) +set_source_files_properties(memory_sparse_geo_table.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) + cc_library(sparse_sgd_rule SRCS sparse_sgd_rule.cc DEPS ${TABLE_DEPS} ps_framework_proto) cc_library(ctr_accessor SRCS ctr_accessor.cc ctr_double_accessor.cc sparse_accessor.cc DEPS ${TABLE_DEPS} ps_framework_proto sparse_sgd_rule) -cc_library(memory_sparse_table SRCS memory_sparse_table.cc DEPS ps_framework_proto ${TABLE_DEPS} fs afs_wrapper ctr_accessor common_table) - -set_source_files_properties(memory_sparse_geo_table.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -cc_library(memory_sparse_geo_table SRCS memory_sparse_geo_table.cc DEPS ps_framework_proto ${TABLE_DEPS} common_table) +cc_library(sparse_table SRCS memory_sparse_table.cc ssd_sparse_table.cc memory_sparse_geo_table.cc DEPS ps_framework_proto ${TABLE_DEPS} fs afs_wrapper ctr_accessor common_table rocksdb) -cc_library(table SRCS table.cc DEPS memory_sparse_table memory_sparse_geo_table common_table tensor_accessor tensor_table ps_framework_proto string_helper device_context gflags glog boost) +cc_library(table SRCS table.cc DEPS sparse_table common_table tensor_accessor tensor_table ps_framework_proto string_helper device_context gflags glog boost) target_link_libraries(table -fopenmp) diff --git a/paddle/fluid/distributed/ps/table/accessor.h b/paddle/fluid/distributed/ps/table/accessor.h index 024af327a33af..7713c2bda295f 100644 --- a/paddle/fluid/distributed/ps/table/accessor.h +++ b/paddle/fluid/distributed/ps/table/accessor.h @@ -117,6 +117,11 @@ class ValueAccessor { virtual bool Save(float* value, int param) = 0; // update delta_score and unseen_days after save virtual void UpdateStatAfterSave(float* value, int param) {} + // 判断该value是否保存到ssd + virtual bool SaveSSD(float* value) = 0; + // + virtual bool SaveCache(float* value, int param, + double global_cache_threshold) = 0; // keys不存在时，为values生成随机值 virtual int32_t Create(float** value, size_t num) = 0; diff --git a/paddle/fluid/distributed/ps/table/common_graph_table.h b/paddle/fluid/distributed/ps/table/common_graph_table.h index 863c397b08ad2..df0d8b2d3a8ab 100644 --- a/paddle/fluid/distributed/ps/table/common_graph_table.h +++ b/paddle/fluid/distributed/ps/table/common_graph_table.h @@ -38,13 +38,13 @@ #include #include "paddle/fluid/distributed/ps/table/accessor.h" #include "paddle/fluid/distributed/ps/table/common_table.h" -#include "paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h" #include "paddle/fluid/distributed/ps/table/graph/class_macro.h" #include "paddle/fluid/distributed/ps/table/graph/graph_node.h" #include "paddle/fluid/string/string_helper.h" #include "paddle/phi/core/utils/rw_lock.h" #ifdef PADDLE_WITH_HETERPS +#include "paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h" #include "paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h" #endif namespace paddle { diff --git a/paddle/fluid/distributed/ps/table/ctr_accessor.cc b/paddle/fluid/distributed/ps/table/ctr_accessor.cc index 715abe270e52b..ef7311824faa6 100644 --- a/paddle/fluid/distributed/ps/table/ctr_accessor.cc +++ b/paddle/fluid/distributed/ps/table/ctr_accessor.cc @@ -34,6 +34,8 @@ int CtrCommonAccessor::Initialize() { common_feature_value.embedx_dim = _config.embedx_dim(); common_feature_value.embedx_sgd_dim = _embedx_sgd_rule->Dim(); _show_click_decay_rate = _config.ctr_accessor_param().show_click_decay_rate(); + _ssd_unseenday_threshold = + _config.ctr_accessor_param().ssd_unseenday_threshold(); if (_config.ctr_accessor_param().show_scale()) { _show_scale = true; @@ -77,6 +79,25 @@ bool CtrCommonAccessor::Shrink(float* value) { return false; } +bool CtrCommonAccessor::SaveCache(float* value, int param, + double global_cache_threshold) { + auto base_threshold = _config.ctr_accessor_param().base_threshold(); + auto delta_keep_days = _config.ctr_accessor_param().delta_keep_days(); + if (ShowClickScore(common_feature_value.Show(value), + common_feature_value.Click(value)) >= base_threshold && + common_feature_value.UnseenDays(value) <= delta_keep_days) { + return common_feature_value.Show(value) > global_cache_threshold; + } + return false; +} + +bool CtrCommonAccessor::SaveSSD(float* value) { + if (common_feature_value.UnseenDays(value) > _ssd_unseenday_threshold) { + return true; + } + return false; +} + bool CtrCommonAccessor::Save(float* value, int param) { auto base_threshold = _config.ctr_accessor_param().base_threshold(); auto delta_threshold = _config.ctr_accessor_param().delta_threshold(); diff --git a/paddle/fluid/distributed/ps/table/ctr_accessor.h b/paddle/fluid/distributed/ps/table/ctr_accessor.h index a599bfca7f6d2..327c4cea760eb 100644 --- a/paddle/fluid/distributed/ps/table/ctr_accessor.h +++ b/paddle/fluid/distributed/ps/table/ctr_accessor.h @@ -148,6 +148,9 @@ class CtrCommonAccessor : public ValueAccessor { // param = 1, save delta feature // param = 2, save xbox base feature bool Save(float* value, int param) override; + bool SaveCache(float* value, int param, + double global_cache_threshold) override; + bool SaveSSD(float* value) override; // update delta_score and unseen_days after save void UpdateStatAfterSave(float* value, int param) override; // keys不存在时，为values生成随机值 diff --git a/paddle/fluid/distributed/ps/table/ctr_double_accessor.cc b/paddle/fluid/distributed/ps/table/ctr_double_accessor.cc index f0d9426343d7b..4b84b7e8c36c3 100644 --- a/paddle/fluid/distributed/ps/table/ctr_double_accessor.cc +++ b/paddle/fluid/distributed/ps/table/ctr_double_accessor.cc @@ -74,25 +74,26 @@ bool CtrDoubleAccessor::Shrink(float* value) { } return false; } + bool CtrDoubleAccessor::SaveSSD(float* value) { if (CtrDoubleFeatureValue::UnseenDays(value) > _ssd_unseenday_threshold) { return true; } return false; } -// bool CtrDoubleAccessor::save_cache( -// float* value, int param, double global_cache_threshold) { -// auto base_threshold = _config.ctr_accessor_param().base_threshold(); -// auto delta_keep_days = _config.ctr_accessor_param().delta_keep_days(); -// if (ShowClickScore(CtrDoubleFeatureValue::Show(value), -// CtrDoubleFeatureValue::Click(value)) >= base_threshold -// && CtrDoubleFeatureValue::UnseenDays(value) <= -// delta_keep_days) { -// return CtrDoubleFeatureValue::Show(value) > -// global_cache_threshold; -// } -// return false; -// } + +bool CtrDoubleAccessor::SaveCache(float* value, int param, + double global_cache_threshold) { + auto base_threshold = _config.ctr_accessor_param().base_threshold(); + auto delta_keep_days = _config.ctr_accessor_param().delta_keep_days(); + if (ShowClickScore(CtrDoubleFeatureValue::Show(value), + CtrDoubleFeatureValue::Click(value)) >= base_threshold && + CtrDoubleFeatureValue::UnseenDays(value) <= delta_keep_days) { + return CtrDoubleFeatureValue::Show(value) > global_cache_threshold; + } + return false; +} + bool CtrDoubleAccessor::Save(float* value, int param) { // auto base_threshold = _config.ctr_accessor_param().base_threshold(); // auto delta_threshold = _config.ctr_accessor_param().delta_threshold(); diff --git a/paddle/fluid/distributed/ps/table/ctr_double_accessor.h b/paddle/fluid/distributed/ps/table/ctr_double_accessor.h index c58602065036f..5b781b2621c5b 100644 --- a/paddle/fluid/distributed/ps/table/ctr_double_accessor.h +++ b/paddle/fluid/distributed/ps/table/ctr_double_accessor.h @@ -167,6 +167,8 @@ class CtrDoubleAccessor : public ValueAccessor { // param = 1, save delta feature // param = 3, save all feature with time decay virtual bool Save(float* value, int param) override; + bool SaveCache(float* value, int param, + double global_cache_threshold) override; // update delta_score and unseen_days after save virtual void UpdateStatAfterSave(float* value, int param) override; // 判断该value是否保存到ssd diff --git a/paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h b/paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h index ff2271d468e39..223c8fafd26ab 100644 --- a/paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h +++ b/paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h @@ -11,9 +11,8 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - #pragma once -#ifdef PADDLE_WITH_HETERPS + #include #include #include @@ -154,6 +153,5 @@ class RocksDBHandler { std::vector _handles; rocksdb::DB* _db; }; -} -} -#endif +} // distributed +} // paddle diff --git a/paddle/fluid/distributed/ps/table/memory_sparse_table.cc b/paddle/fluid/distributed/ps/table/memory_sparse_table.cc index e6c52e0b9b0c8..ee6a801fa9183 100644 --- a/paddle/fluid/distributed/ps/table/memory_sparse_table.cc +++ b/paddle/fluid/distributed/ps/table/memory_sparse_table.cc @@ -23,14 +23,17 @@ #include "glog/logging.h" #include "paddle/fluid/platform/enforce.h" +DEFINE_bool(pserver_print_missed_key_num_every_push, false, + "pserver_print_missed_key_num_every_push"); +DEFINE_bool(pserver_create_value_when_push, true, + "pserver create value when push"); +DEFINE_bool(pserver_enable_create_feasign_randomly, false, + "pserver_enable_create_feasign_randomly"); +DEFINE_int32(pserver_table_save_max_retry, 3, "pserver_table_save_max_retry"); + namespace paddle { namespace distributed { -// TODO(zhaocaibei123): configure -bool FLAGS_pserver_create_value_when_push = true; -int FLAGS_pserver_table_save_max_retry = 3; -bool FLAGS_pserver_enable_create_feasign_randomly = false; - int32_t MemorySparseTable::Initialize() { _shards_task_pool.resize(_task_pool_size); for (int i = 0; i < _shards_task_pool.size(); ++i) { @@ -142,7 +145,7 @@ int32_t MemorySparseTable::Load(const std::string& path, LOG(ERROR) << "MemorySparseTable load failed, retry it! path:" << channel_config.path << " , retry_num=" << retry_num; } - if (retry_num > paddle::distributed::FLAGS_pserver_table_save_max_retry) { + if (retry_num > FLAGS_pserver_table_save_max_retry) { LOG(ERROR) << "MemorySparseTable load failed reach max limit!"; exit(-1); } @@ -213,7 +216,7 @@ int32_t MemorySparseTable::LoadLocalFS(const std::string& path, << file_list[file_start_idx + i] << " , retry_num=" << retry_num; } - if (retry_num > paddle::distributed::FLAGS_pserver_table_save_max_retry) { + if (retry_num > FLAGS_pserver_table_save_max_retry) { LOG(ERROR) << "MemorySparseTable load failed reach max limit!"; exit(-1); } @@ -293,7 +296,7 @@ int32_t MemorySparseTable::Save(const std::string& dirname, if (is_write_failed) { _afs_client.remove(channel_config.path); } - if (retry_num > paddle::distributed::FLAGS_pserver_table_save_max_retry) { + if (retry_num > FLAGS_pserver_table_save_max_retry) { LOG(ERROR) << "MemorySparseTable save prefix failed reach max limit!"; exit(-1); } diff --git a/paddle/fluid/distributed/ps/table/memory_sparse_table.h b/paddle/fluid/distributed/ps/table/memory_sparse_table.h index 87a73bd22fa2f..ec86239ffb161 100644 --- a/paddle/fluid/distributed/ps/table/memory_sparse_table.h +++ b/paddle/fluid/distributed/ps/table/memory_sparse_table.h @@ -62,9 +62,11 @@ class MemorySparseTable : public Table { int32_t InitializeShard() override { return 0; } int32_t InitializeValue(); - int32_t Load(const std::string& path, const std::string& param) override; + virtual int32_t Load(const std::string& path, + const std::string& param) override; - int32_t Save(const std::string& path, const std::string& param) override; + virtual int32_t Save(const std::string& path, + const std::string& param) override; int32_t LoadLocalFS(const std::string& path, const std::string& param); int32_t SaveLocalFS(const std::string& path, const std::string& param, @@ -83,7 +85,7 @@ class MemorySparseTable : public Table { int32_t PushSparse(const uint64_t* keys, const float** values, size_t num); int32_t Flush() override; - int32_t Shrink(const std::string& param) override; + virtual int32_t Shrink(const std::string& param) override; void Clear() override; void* GetShard(size_t shard_idx) override { diff --git a/paddle/fluid/distributed/ps/table/sparse_accessor.h b/paddle/fluid/distributed/ps/table/sparse_accessor.h index 5ca5d21707a2b..875904847b2ea 100644 --- a/paddle/fluid/distributed/ps/table/sparse_accessor.h +++ b/paddle/fluid/distributed/ps/table/sparse_accessor.h @@ -135,6 +135,11 @@ class SparseAccessor : public ValueAccessor { // param = 1, save delta feature // param = 2, save xbox base feature bool Save(float* value, int param) override; + + bool SaveCache(float* value, int param, double global_cache_threshold) { + return false; + } + bool SaveSSD(float* value) { return false; } // update delta_score and unseen_days after save void UpdateStatAfterSave(float* value, int param) override; // keys不存在时，为values生成随机值 diff --git a/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc b/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc new file mode 100644 index 0000000000000..b1359d1323d89 --- /dev/null +++ b/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc @@ -0,0 +1,759 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/distributed/ps/table/ssd_sparse_table.h" +#include "paddle/fluid/distributed/common/cost_timer.h" +#include "paddle/fluid/distributed/common/local_random.h" +#include "paddle/fluid/distributed/common/topk_calculator.h" +#include "paddle/fluid/framework/archive.h" +#include "paddle/utils/string/string_helper.h" + +DECLARE_bool(pserver_print_missed_key_num_every_push); +DECLARE_bool(pserver_create_value_when_push); +DECLARE_bool(pserver_enable_create_feasign_randomly); +DEFINE_bool(pserver_open_strict_check, false, "pserver_open_strict_check"); +DEFINE_string(rocksdb_path, "database", "path of sparse table rocksdb file"); +DEFINE_int32(pserver_load_batch_size, 5000, "load batch size for ssd"); + +namespace paddle { +namespace distributed { + +int32_t SSDSparseTable::Initialize() { + MemorySparseTable::Initialize(); + _db = paddle::distributed::RocksDBHandler::GetInstance(); + _db->initialize(FLAGS_rocksdb_path, _real_local_shard_num); + return 0; +} + +int32_t SSDSparseTable::InitializeShard() { return 0; } + +int32_t SSDSparseTable::PullSparse(float* pull_values, const uint64_t* keys, + size_t num) { + CostTimer timer("pserver_downpour_sparse_select_all"); + size_t value_size = _value_accesor->GetAccessorInfo().size / sizeof(float); + size_t mf_value_size = + _value_accesor->GetAccessorInfo().mf_size / sizeof(float); + size_t select_value_size = + _value_accesor->GetAccessorInfo().select_size / sizeof(float); + + { // 从table取值 or create + std::vector> tasks(_real_local_shard_num); + std::vector>> task_keys( + _real_local_shard_num); + for (size_t i = 0; i < num; ++i) { + int shard_id = (keys[i] % _sparse_table_shard_num) % _avg_local_shard_num; + task_keys[shard_id].push_back({keys[i], i}); + } + + std::atomic missed_keys{0}; + for (size_t shard_id = 0; shard_id < _real_local_shard_num; ++shard_id) { + tasks[shard_id] = + _shards_task_pool[shard_id % _shards_task_pool.size()]->enqueue( + [this, shard_id, &task_keys, value_size, mf_value_size, + select_value_size, pull_values, keys, &missed_keys]() -> int { + auto& keys = task_keys[shard_id]; + auto& local_shard = _local_shards[shard_id]; + float data_buffer[value_size]; + float* data_buffer_ptr = data_buffer; + for (int i = 0; i < keys.size(); ++i) { + uint64_t key = keys[i].first; + auto itr = local_shard.find(key); + size_t data_size = value_size - mf_value_size; + if (itr == local_shard.end()) { + // pull rocksdb + std::string tmp_string(""); + if (_db->get(shard_id, (char*)&key, sizeof(uint64_t), + tmp_string) > 0) { + ++missed_keys; + if (FLAGS_pserver_create_value_when_push) { + memset(data_buffer, 0, sizeof(float) * data_size); + } else { + auto& feature_value = local_shard[key]; + feature_value.resize(data_size); + float* data_ptr = + const_cast(feature_value.data()); + _value_accesor->Create(&data_buffer_ptr, 1); + memcpy(data_ptr, data_buffer_ptr, + data_size * sizeof(float)); + } + } else { + data_size = tmp_string.size() / sizeof(float); + memcpy(data_buffer_ptr, + paddle::string::str_to_float(tmp_string), + data_size * sizeof(float)); + // from rocksdb to mem + auto& feature_value = local_shard[key]; + feature_value.resize(data_size); + memcpy(const_cast(feature_value.data()), + data_buffer_ptr, data_size * sizeof(float)); + _db->del_data(shard_id, (char*)&key, sizeof(uint64_t)); + } + } else { + data_size = itr.value().size(); + memcpy(data_buffer_ptr, itr.value().data(), + data_size * sizeof(float)); + } + for (int mf_idx = data_size; mf_idx < value_size; ++mf_idx) { + data_buffer[mf_idx] = 0.0; + } + int pull_data_idx = keys[i].second; + float* select_data = + pull_values + pull_data_idx * select_value_size; + _value_accesor->Select(&select_data, + (const float**)&data_buffer_ptr, 1); + } + return 0; + }); + } + for (size_t i = 0; i < _real_local_shard_num; ++i) { + tasks[i].wait(); + } + if (FLAGS_pserver_print_missed_key_num_every_push) { + LOG(WARNING) << "total pull keys:" << num + << " missed_keys:" << missed_keys.load(); + } + } + return 0; +} + +int32_t SSDSparseTable::PushSparse(const uint64_t* keys, const float* values, + size_t num) { + CostTimer timer("pserver_downpour_sparse_update_all"); + // 构造value push_value的数据指针 + size_t value_col = _value_accesor->GetAccessorInfo().size / sizeof(float); + size_t mf_value_col = + _value_accesor->GetAccessorInfo().mf_size / sizeof(float); + size_t update_value_col = + _value_accesor->GetAccessorInfo().update_size / sizeof(float); + { + std::vector> tasks(_real_local_shard_num); + std::vector>> task_keys( + _real_local_shard_num); + for (size_t i = 0; i < num; ++i) { + int shard_id = (keys[i] % _sparse_table_shard_num) % _avg_local_shard_num; + task_keys[shard_id].push_back({keys[i], i}); + } + for (size_t shard_id = 0; shard_id < _real_local_shard_num; ++shard_id) { + tasks[shard_id] = + _shards_task_pool[shard_id % _shards_task_pool.size()]->enqueue( + [this, shard_id, value_col, mf_value_col, update_value_col, + values, &task_keys]() -> int { + auto& keys = task_keys[shard_id]; + auto& local_shard = _local_shards[shard_id]; + float data_buffer[value_col]; + float* data_buffer_ptr = data_buffer; + for (int i = 0; i < keys.size(); ++i) { + uint64_t key = keys[i].first; + uint64_t push_data_idx = keys[i].second; + const float* update_data = + values + push_data_idx * update_value_col; + auto itr = local_shard.find(key); + if (itr == local_shard.end()) { + if (FLAGS_pserver_enable_create_feasign_randomly && + !_value_accesor->CreateValue(1, update_data)) { + continue; + } + auto value_size = value_col - mf_value_col; + auto& feature_value = local_shard[key]; + feature_value.resize(value_size); + _value_accesor->Create(&data_buffer_ptr, 1); + memcpy(const_cast(feature_value.data()), + data_buffer_ptr, value_size * sizeof(float)); + itr = local_shard.find(key); + } + auto& feature_value = itr.value(); + float* value_data = const_cast(feature_value.data()); + size_t value_size = feature_value.size(); + + if (value_size == + value_col) { // 已拓展到最大size, 则就地update + _value_accesor->Update(&value_data, &update_data, 1); + } else { // 拷入buffer区进行update，然后再回填，不需要的mf则回填时抛弃了 + memcpy(data_buffer_ptr, value_data, + value_size * sizeof(float)); + _value_accesor->Update(&data_buffer_ptr, &update_data, 1); + if (_value_accesor->NeedExtendMF(data_buffer)) { + feature_value.resize(value_col); + value_data = const_cast(feature_value.data()); + _value_accesor->Create(&value_data, 1); + } + memcpy(value_data, data_buffer_ptr, + value_size * sizeof(float)); + } + } + return 0; + }); + } + for (size_t i = 0; i < _real_local_shard_num; ++i) { + tasks[i].wait(); + } + } + /* + //update && value 的转置 + thread_local Eigen::MatrixXf update_matrix; + float* transposed_update_data[update_value_col]; + make_matrix_with_eigen(num, update_value_col, update_matrix, + transposed_update_data); + copy_array_to_eigen(values, update_matrix); + + thread_local Eigen::MatrixXf value_matrix; + float* transposed_value_data[value_col]; + make_matrix_with_eigen(num, value_col, value_matrix, transposed_value_data); + copy_matrix_to_eigen((const float**)(value_ptrs->data()), value_matrix); + + //批量update + { + CostTimer accessor_timer("pslib_downpour_sparse_update_accessor"); + _value_accesor->update(transposed_value_data, (const + float**)transposed_update_data, num); + } + copy_eigen_to_matrix(value_matrix, value_ptrs->data()); + */ + return 0; +} + +int32_t SSDSparseTable::Shrink(const std::string& param) { + int thread_num = _real_local_shard_num < 20 ? _real_local_shard_num : 20; + omp_set_num_threads(thread_num); +#pragma omp parallel for schedule(dynamic) + for (size_t i = 0; i < _real_local_shard_num; ++i) { + uint64_t mem_count = 0; + uint64_t ssd_count = 0; + + LOG(INFO) << "SSDSparseTable begin shrink shard:" << i; + auto& shard = _local_shards[i]; + for (auto it = shard.begin(); it != shard.end();) { + if (_value_accesor->Shrink(it.value().data())) { + it = shard.erase(it); + mem_count++; + } else { + ++it; + } + } + auto* it = _db->get_iterator(i); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + if (_value_accesor->Shrink( + paddle::string::str_to_float(it->value().data()))) { + _db->del_data(i, it->key().data(), it->key().size()); + ssd_count++; + } else { + _db->put(i, it->key().data(), it->key().size(), it->value().data(), + it->value().size()); + } + } + delete it; + LOG(INFO) << "SSDSparseTable shrink success. shard:" << i << " delete MEM[" + << mem_count << "] SSD[" << ssd_count << "]"; + //_db->flush(i); + } + return 0; +} + +int32_t SSDSparseTable::UpdateTable() { + // TODO implement with multi-thread + int count = 0; + for (size_t i = 0; i < _real_local_shard_num; ++i) { + auto& shard = _local_shards[i]; + // from mem to ssd + for (auto it = shard.begin(); it != shard.end();) { + if (_value_accesor->SaveSSD(it.value().data())) { + _db->put(i, (char*)&it.key(), sizeof(uint64_t), + (char*)it.value().data(), it.value().size() * sizeof(float)); + count++; + it = shard.erase(it); + } else { + ++it; + } + } + _db->flush(i); + } + LOG(INFO) << "Table>> update count: " << count; + return 0; +} + +int64_t SSDSparseTable::LocalSize() { + int64_t local_size = 0; + for (size_t i = 0; i < _real_local_shard_num; ++i) { + local_size += _local_shards[i].size(); + } + // TODO rocksdb size + uint64_t ssd_size = 0; + // _db->get_estimate_key_num(ssd_size); + // return local_size + ssd_size; + return local_size; +} + +int32_t SSDSparseTable::Save(const std::string& path, + const std::string& param) { + if (_real_local_shard_num == 0) { + _local_show_threshold = -1; + return 0; + } + int save_param = atoi(param.c_str()); // batch_model:0 xbox:1 + // if (save_param == 5) { + // return save_patch(path, save_param); + // } + + // LOG(INFO) << "table cache rate is: " << _config.sparse_table_cache_rate(); + LOG(INFO) << "table cache rate is: " << _config.sparse_table_cache_rate(); + LOG(INFO) << "enable_sparse_table_cache: " + << _config.enable_sparse_table_cache(); + LOG(INFO) << "LocalSize: " << LocalSize(); + if (_config.enable_sparse_table_cache()) { + LOG(INFO) << "Enable sparse table cache, top n:" << _cache_tk_size; + } + _cache_tk_size = LocalSize() * _config.sparse_table_cache_rate(); + TopkCalculator tk(_real_local_shard_num, _cache_tk_size); + size_t file_start_idx = _avg_local_shard_num * _shard_idx; + std::string table_path = TableDir(path); + _afs_client.remove(paddle::string::format_string( + "%s/part-%03d-*", table_path.c_str(), _shard_idx)); + int thread_num = _real_local_shard_num < 20 ? _real_local_shard_num : 20; + + // std::atomic feasign_size; + std::atomic feasign_size_all{0}; + // feasign_size = 0; + + omp_set_num_threads(thread_num); +#pragma omp parallel for schedule(dynamic) + for (size_t i = 0; i < _real_local_shard_num; ++i) { + FsChannelConfig channel_config; + if (_config.compress_in_save() && (save_param == 0 || save_param == 3)) { + channel_config.path = paddle::string::format_string( + "%s/part-%03d-%05d.gz", table_path.c_str(), _shard_idx, + file_start_idx + i); + } else { + channel_config.path = + paddle::string::format_string("%s/part-%03d-%05d", table_path.c_str(), + _shard_idx, file_start_idx + i); + } + channel_config.converter = _value_accesor->Converter(save_param).converter; + channel_config.deconverter = + _value_accesor->Converter(save_param).deconverter; + int err_no = 0; + int retry_num = 0; + bool is_write_failed = false; + int feasign_size = 0; + auto& shard = _local_shards[i]; + do { + err_no = 0; + feasign_size = 0; + is_write_failed = false; + auto write_channel = + _afs_client.open_w(channel_config, 1024 * 1024 * 40, &err_no); + for (auto it = shard.begin(); it != shard.end(); ++it) { + if (_config.enable_sparse_table_cache() && + (save_param == 1 || save_param == 2) && + _value_accesor->Save(it.value().data(), 4)) { + // tk.push(i, it.value().data()[2]); + tk.push(i, _value_accesor->GetField(it.value().data(), "show")); + } + if (_value_accesor->Save(it.value().data(), save_param)) { + std::string format_value = _value_accesor->ParseToString( + it.value().data(), it.value().size()); + if (0 != + write_channel->write_line(paddle::string::format_string( + "%lu %s", it.key(), format_value.c_str()))) { + ++retry_num; + is_write_failed = true; + LOG(ERROR) << "SSDSparseTable save failed, retry it! path:" + << channel_config.path << ", retry_num=" << retry_num; + break; + } + ++feasign_size; + } + } + + if (err_no == -1 && !is_write_failed) { + ++retry_num; + is_write_failed = true; + LOG(ERROR) << "SSDSparseTable save failed after write, retry it! " + << "path:" << channel_config.path + << " , retry_num=" << retry_num; + } + if (is_write_failed) { + _afs_client.remove(channel_config.path); + continue; + } + + // delta and cache and revert is all in mem, base in rocksdb + if (save_param != 1) { + auto* it = _db->get_iterator(i); + for (it->SeekToFirst(); it->Valid(); it->Next()) { + bool need_save = _value_accesor->Save( + paddle::string::str_to_float(it->value().data()), save_param); + _value_accesor->UpdateStatAfterSave( + paddle::string::str_to_float(it->value().data()), save_param); + if (need_save) { + std::string format_value = _value_accesor->ParseToString( + paddle::string::str_to_float(it->value().data()), + it->value().size() / sizeof(float)); + if (0 != + write_channel->write_line(paddle::string::format_string( + "%lu %s", *((uint64_t*)const_cast(it->key().data())), + format_value.c_str()))) { + ++retry_num; + is_write_failed = true; + LOG(ERROR) << "SSDSparseTable save failed, retry it! path:" + << channel_config.path << ", retry_num=" << retry_num; + break; + } + if (save_param == 3) { + _db->put(i, it->key().data(), it->key().size(), + it->value().data(), it->value().size()); + } + ++feasign_size; + } + } + delete it; + } + + write_channel->close(); + if (err_no == -1) { + ++retry_num; + is_write_failed = true; + LOG(ERROR) << "SSDSparseTable save failed after write, retry it! " + << "path:" << channel_config.path + << " , retry_num=" << retry_num; + } + if (is_write_failed) { + _afs_client.remove(channel_config.path); + } + } while (is_write_failed); + feasign_size_all += feasign_size; + for (auto it = shard.begin(); it != shard.end(); ++it) { + _value_accesor->UpdateStatAfterSave(it.value().data(), save_param); + } + } + if (save_param == 3) { + UpdateTable(); + _cache_tk_size = LocalSize() * _config.sparse_table_cache_rate(); + LOG(INFO) << "SSDSparseTable update success."; + } + LOG(INFO) << "SSDSparseTable save success, path:" + << paddle::string::format_string("%s/%03d/part-%03d-", path.c_str(), + _config.table_id(), _shard_idx) + << " from " << file_start_idx << " to " + << file_start_idx + _real_local_shard_num - 1; + // return feasign_size_all; + _local_show_threshold = tk.top(); + LOG(INFO) << "local cache threshold: " << _local_show_threshold; + // int32 may overflow need to change return value + return 0; +} + +int64_t SSDSparseTable::CacheShuffle( + const std::string& path, const std::string& param, double cache_threshold, + std::function(int msg_type, int to_pserver_id, + std::string& msg)> + send_msg_func, + paddle::framework::Channel>& + shuffled_channel, + const std::vector& table_ptrs) { + LOG(INFO) << "cache shuffle with cache threshold: " << cache_threshold + << " param:" << param; + int save_param = atoi(param.c_str()); // batch_model:0 xbox:1 + if (!_config.enable_sparse_table_cache() || cache_threshold < 0) { + LOG(WARNING) + << "cache shuffle failed not enable table cache or cache threshold < 0 " + << _config.enable_sparse_table_cache() << " or " << cache_threshold; + // return -1; + } + int shuffle_node_num = _config.sparse_table_cache_file_num(); + LOG(INFO) << "Table>> shuffle node num is: " << shuffle_node_num; + size_t file_start_idx = _avg_local_shard_num * _shard_idx; + int thread_num = _real_local_shard_num < 20 ? _real_local_shard_num : 20; + + std::vector< + paddle::framework::ChannelWriter>> + writers(_real_local_shard_num); + std::vector>> datas( + _real_local_shard_num); + + int feasign_size = 0; + std::vector>> + tmp_channels; + for (size_t i = 0; i < _real_local_shard_num; ++i) { + tmp_channels.push_back( + paddle::framework::MakeChannel>()); + } + + omp_set_num_threads(thread_num); +#pragma omp parallel for schedule(dynamic) + for (size_t i = 0; i < _real_local_shard_num; ++i) { + paddle::framework::ChannelWriter>& writer = + writers[i]; + // std::shared_ptr>> tmp_chan = + // paddle::framework::MakeChannel>(); + writer.Reset(tmp_channels[i].get()); + + auto& shard = _local_shards[i]; + for (auto it = shard.begin(); it != shard.end(); ++it) { + if (_value_accesor->SaveCache(it.value().data(), save_param, + cache_threshold)) { + std::string format_value = + _value_accesor->ParseToString(it.value().data(), it.value().size()); + std::pair pkv(it.key(), format_value.c_str()); + writer << pkv; + ++feasign_size; + } + } + + writer.Flush(); + writer.channel()->Close(); + } + LOG(INFO) << "SSDSparseTable cache KV save success to Channel feasigh size: " + << feasign_size + << " and start sparse cache data shuffle real local shard num: " + << _real_local_shard_num; + std::vector> local_datas; + for (size_t idx_shard = 0; idx_shard < _real_local_shard_num; ++idx_shard) { + paddle::framework::ChannelWriter>& writer = + writers[idx_shard]; + auto channel = writer.channel(); + std::vector>& data = datas[idx_shard]; + std::vector ars(shuffle_node_num); + while (channel->Read(data)) { + for (auto& t : data) { + auto pserver_id = + paddle::distributed::local_random_engine()() % shuffle_node_num; + if (pserver_id != _shard_idx) { + ars[pserver_id] << t; + } else { + local_datas.emplace_back(std::move(t)); + } + } + std::vector> total_status; + std::vector send_data_size(shuffle_node_num, 0); + std::vector send_index(shuffle_node_num); + for (int i = 0; i < shuffle_node_num; ++i) { + send_index[i] = i; + } + std::random_shuffle(send_index.begin(), send_index.end()); + for (auto index = 0u; index < shuffle_node_num; ++index) { + int i = send_index[index]; + if (i == _shard_idx) { + continue; + } + if (ars[i].Length() == 0) { + continue; + } + std::string msg(ars[i].Buffer(), ars[i].Length()); + auto ret = send_msg_func(101, i, msg); + total_status.push_back(std::move(ret)); + send_data_size[i] += ars[i].Length(); + } + for (auto& t : total_status) { + t.wait(); + } + ars.clear(); + ars = std::vector(shuffle_node_num); + data = std::vector>(); + } + } + shuffled_channel->Write(std::move(local_datas)); + LOG(INFO) << "cache shuffle finished"; + return 0; +} + +int32_t SSDSparseTable::SaveCache( + const std::string& path, const std::string& param, + paddle::framework::Channel>& + shuffled_channel) { + if (_shard_idx >= _config.sparse_table_cache_file_num()) { + return 0; + } + int save_param = atoi(param.c_str()); // batch_model:0 xbox:1 + size_t file_start_idx = _avg_local_shard_num * _shard_idx; + std::string table_path = paddle::string::format_string( + "%s/%03d_cache/", path.c_str(), _config.table_id()); + _afs_client.remove(paddle::string::format_string( + "%s/part-%03d", table_path.c_str(), _shard_idx)); + uint32_t feasign_size = 0; + FsChannelConfig channel_config; + // not compress cache model + channel_config.path = paddle::string::format_string( + "%s/part-%03d", table_path.c_str(), _shard_idx); + channel_config.converter = _value_accesor->Converter(save_param).converter; + channel_config.deconverter = + _value_accesor->Converter(save_param).deconverter; + auto write_channel = _afs_client.open_w(channel_config, 1024 * 1024 * 40); + std::vector> data; + bool is_write_failed = false; + shuffled_channel->Close(); + while (shuffled_channel->Read(data)) { + for (auto& t : data) { + ++feasign_size; + if (0 != + write_channel->write_line(paddle::string::format_string( + "%lu %s", t.first, t.second.c_str()))) { + LOG(ERROR) << "Cache Table save failed, " + "path:" + << channel_config.path << ", retry it!"; + is_write_failed = true; + break; + } + } + data = std::vector>(); + } + if (is_write_failed) { + _afs_client.remove(channel_config.path); + } + write_channel->close(); + LOG(INFO) << "SSDSparseTable cache save success, feasign: " << feasign_size + << ", path: " << channel_config.path; + shuffled_channel->Open(); + return feasign_size; +} + +int32_t SSDSparseTable::Load(const std::string& path, + const std::string& param) { + return MemorySparseTable::Load(path, param); +} + +//加载path目录下数据[start_idx, end_idx) +int32_t SSDSparseTable::Load(size_t start_idx, size_t end_idx, + const std::vector& file_list, + const std::string& param) { + if (start_idx >= file_list.size()) { + return 0; + } + int load_param = atoi(param.c_str()); + size_t feature_value_size = + _value_accesor->GetAccessorInfo().size / sizeof(float); + size_t mf_value_size = + _value_accesor->GetAccessorInfo().mf_size / sizeof(float); + + end_idx = + end_idx < _sparse_table_shard_num ? end_idx : _sparse_table_shard_num; + int thread_num = (end_idx - start_idx) < 20 ? (end_idx - start_idx) : 20; + omp_set_num_threads(thread_num); +#pragma omp parallel for schedule(dynamic) + for (size_t i = start_idx; i < end_idx; ++i) { + FsChannelConfig channel_config; + channel_config.path = file_list[i]; + channel_config.converter = _value_accesor->Converter(load_param).converter; + channel_config.deconverter = + _value_accesor->Converter(load_param).deconverter; + + int retry_num = 0; + int err_no = 0; + bool is_read_failed = false; + std::vector> ssd_keys; + std::vector> ssd_values; + std::vector tmp_key; + ssd_keys.reserve(FLAGS_pserver_load_batch_size); + ssd_values.reserve(FLAGS_pserver_load_batch_size); + tmp_key.reserve(FLAGS_pserver_load_batch_size); + do { + ssd_keys.clear(); + ssd_values.clear(); + tmp_key.clear(); + err_no = 0; + is_read_failed = false; + std::string line_data; + auto read_channel = _afs_client.open_r(channel_config, 0, &err_no); + char* end = NULL; + int local_shard_id = i % _avg_local_shard_num; + auto& shard = _local_shards[local_shard_id]; + float data_buffer[FLAGS_pserver_load_batch_size * feature_value_size]; + float* data_buffer_ptr = data_buffer; + uint64_t mem_count = 0; + uint64_t ssd_count = 0; + uint64_t mem_mf_count = 0; + uint64_t ssd_mf_count = 0; + try { + while (read_channel->read_line(line_data) == 0 && + line_data.size() > 1) { + uint64_t key = std::strtoul(line_data.data(), &end, 10); + if (FLAGS_pserver_open_strict_check) { + if (key % _sparse_table_shard_num != i) { + LOG(WARNING) << "SSDSparseTable key:" << key + << " not match shard," + << " file_idx:" << i + << " shard num:" << _sparse_table_shard_num + << " file:" << channel_config.path; + continue; + } + } + int value_size = + _value_accesor->ParseFromString(++end, data_buffer_ptr); + // ssd or mem + if (_value_accesor->SaveSSD(data_buffer_ptr)) { + tmp_key.emplace_back(key); + ssd_keys.emplace_back( + std::make_pair((char*)&tmp_key.back(), sizeof(uint64_t))); + ssd_values.emplace_back(std::make_pair((char*)data_buffer_ptr, + value_size * sizeof(float))); + data_buffer_ptr += feature_value_size; + if (ssd_keys.size() == FLAGS_pserver_load_batch_size) { + _db->put_batch(local_shard_id, ssd_keys, ssd_values, + ssd_keys.size()); + ssd_keys.clear(); + ssd_values.clear(); + tmp_key.clear(); + data_buffer_ptr = data_buffer; + } + ssd_count++; + if (value_size > feature_value_size - mf_value_size) { + ssd_mf_count++; + } + } else { + auto& value = shard[key]; + value.resize(value_size); + _value_accesor->ParseFromString(end, value.data()); + mem_count++; + if (value_size > feature_value_size - mf_value_size) { + mem_mf_count++; + } + } + } + // last batch + if (ssd_keys.size() > 0) { + _db->put_batch(local_shard_id, ssd_keys, ssd_values, ssd_keys.size()); + } + read_channel->close(); + if (err_no == -1) { + ++retry_num; + is_read_failed = true; + LOG(ERROR) << "SSDSparseTable load failed after read, retry it! path:" + << channel_config.path << " , retry_num=" << retry_num; + continue; + } + + _db->flush(local_shard_id); + LOG(INFO) << "Table>> load done. ALL[" << mem_count + ssd_count + << "] MEM[" << mem_count << "] MEM_MF[" << mem_mf_count + << "] SSD[" << ssd_count << "] SSD_MF[" << ssd_mf_count + << "]."; + } catch (...) { + ++retry_num; + is_read_failed = true; + LOG(ERROR) << "SSDSparseTable load failed after read, retry it! path:" + << channel_config.path << " , retry_num=" << retry_num; + } + } while (is_read_failed); + } + LOG(INFO) << "load num:" << LocalSize(); + LOG(INFO) << "SSDSparseTable load success, path from " << file_list[start_idx] + << " to " << file_list[end_idx - 1]; + + _cache_tk_size = LocalSize() * _config.sparse_table_cache_rate(); + return 0; +} + +} // namespace distributed +} // namespace paddle diff --git a/paddle/fluid/distributed/ps/table/ssd_sparse_table.h b/paddle/fluid/distributed/ps/table/ssd_sparse_table.h new file mode 100644 index 0000000000000..2a43a27c229d1 --- /dev/null +++ b/paddle/fluid/distributed/ps/table/ssd_sparse_table.h @@ -0,0 +1,94 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "gflags/gflags.h" +#include "paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h" +#include "paddle/fluid/distributed/ps/table/memory_sparse_table.h" + +namespace paddle { +namespace distributed { + +class SSDSparseTable : public MemorySparseTable { + public: + typedef SparseTableShard shard_type; + SSDSparseTable() {} + virtual ~SSDSparseTable() {} + + int32_t Initialize() override; + int32_t InitializeShard() override; + + // exchange data + int32_t UpdateTable(); + + int32_t Pull(TableContext& context) override { + CHECK(context.value_type == Sparse); + float* pull_values = context.pull_context.values; + const PullSparseValue& pull_value = context.pull_context.pull_value; + return PullSparse(pull_values, pull_value.feasigns_, pull_value.numel_); + } + + int32_t Push(TableContext& context) override { + const uint64_t* keys = context.push_context.keys; + const float* values = context.push_context.values; + size_t num = context.num; + return PushSparse(keys, values, num); + } + + virtual int32_t PullSparse(float* pull_values, const uint64_t* keys, + size_t num); + virtual int32_t PushSparse(const uint64_t* keys, const float* values, + size_t num); + + int32_t Flush() override { return 0; } + virtual int32_t Shrink(const std::string& param) override; + virtual void Clear() override { + for (size_t i = 0; i < _real_local_shard_num; ++i) { + _local_shards[i].clear(); + } + } + + virtual int32_t Save(const std::string& path, + const std::string& param) override; + virtual int32_t SaveCache( + const std::string& path, const std::string& param, + paddle::framework::Channel>& + shuffled_channel) override; + virtual double GetCacheThreshold() override { return _local_show_threshold; } + virtual int64_t CacheShuffle( + const std::string& path, const std::string& param, double cache_threshold, + std::function(int msg_type, int to_pserver_id, + std::string& msg)> + send_msg_func, + paddle::framework::Channel>& + shuffled_channel, + const std::vector& table_ptrs) override; + //加载path目录下数据 + virtual int32_t Load(const std::string& path, + const std::string& param) override; + //加载path目录下数据[start_idx, end_idx) + virtual int32_t Load(size_t start_idx, size_t end_idx, + const std::vector& file_list, + const std::string& param); + int64_t LocalSize(); + + private: + RocksDBHandler* _db; + int64_t _cache_tk_size; + double _local_show_threshold{0.0}; +}; + +} // namespace distributed +} // namespace paddle diff --git a/paddle/fluid/distributed/ps/table/table.cc b/paddle/fluid/distributed/ps/table/table.cc index 333008482f167..5eb38d9c400b0 100644 --- a/paddle/fluid/distributed/ps/table/table.cc +++ b/paddle/fluid/distributed/ps/table/table.cc @@ -25,6 +25,7 @@ #include "paddle/fluid/distributed/ps/table/memory_sparse_geo_table.h" #include "paddle/fluid/distributed/ps/table/memory_sparse_table.h" #include "paddle/fluid/distributed/ps/table/sparse_accessor.h" +#include "paddle/fluid/distributed/ps/table/ssd_sparse_table.h" #include "paddle/fluid/distributed/ps/table/tensor_accessor.h" #include "paddle/fluid/distributed/ps/table/tensor_table.h" @@ -37,6 +38,7 @@ REGISTER_PSCORE_CLASS(Table, TensorTable); REGISTER_PSCORE_CLASS(Table, DenseTensorTable); REGISTER_PSCORE_CLASS(Table, GlobalStepTable); REGISTER_PSCORE_CLASS(Table, MemorySparseTable); +REGISTER_PSCORE_CLASS(Table, SSDSparseTable); REGISTER_PSCORE_CLASS(Table, MemorySparseGeoTable); REGISTER_PSCORE_CLASS(ValueAccessor, CommMergeAccessor); REGISTER_PSCORE_CLASS(ValueAccessor, CtrCommonAccessor); diff --git a/paddle/fluid/distributed/ps/table/table.h b/paddle/fluid/distributed/ps/table/table.h index c515e03e3fa48..48fda782d489f 100644 --- a/paddle/fluid/distributed/ps/table/table.h +++ b/paddle/fluid/distributed/ps/table/table.h @@ -24,6 +24,7 @@ #include "paddle/fluid/distributed/ps/table/accessor.h" #include "paddle/fluid/distributed/ps/table/depends/sparse_utils.h" #include "paddle/fluid/distributed/ps/table/graph/graph_node.h" +#include "paddle/fluid/framework/channel.h" #include "paddle/fluid/framework/program_desc.h" #include "paddle/fluid/framework/scope.h" #include "paddle/fluid/platform/device_context.h" @@ -107,6 +108,26 @@ class Table { // 指定保存路径 virtual int32_t Save(const std::string &path, const std::string &converter) = 0; + // for cache + virtual int32_t SaveCache( + const std::string &path, const std::string ¶m, + paddle::framework::Channel> + &shuffled_channel) { + return 0; + } + + virtual int64_t CacheShuffle( + const std::string &path, const std::string ¶m, double cache_threshold, + std::function(int msg_type, int to_pserver_id, + std::string &msg)> + send_msg_func, + paddle::framework::Channel> + &shuffled_channel, + const std::vector
&table_ptrs) { + return 0; + } + + virtual double GetCacheThreshold() { return 0.0; } virtual int32_t SetShard(size_t shard_idx, size_t shard_num) { _shard_idx = shard_idx; diff --git a/paddle/fluid/distributed/ps/table/tensor_accessor.h b/paddle/fluid/distributed/ps/table/tensor_accessor.h index 60951598482ad..fad31d5df7f47 100644 --- a/paddle/fluid/distributed/ps/table/tensor_accessor.h +++ b/paddle/fluid/distributed/ps/table/tensor_accessor.h @@ -38,6 +38,12 @@ class CommMergeAccessor : public ValueAccessor { // param作为参数用于标识save阶段，如downpour的xbox与batch_model virtual bool Save(float * /*value*/, int /*param*/); + bool SaveCache(float *value, int param, double global_cache_threshold) { + return false; + } + + bool SaveSSD(float *value) { return false; } + // keys不存在时，为values生成随机值 virtual int32_t Create(float **value, size_t num); // 从values中选取到select_values中 diff --git a/paddle/fluid/distributed/ps/wrapper/fleet.cc b/paddle/fluid/distributed/ps/wrapper/fleet.cc index 7bc50a868104a..955ba75e672d1 100644 --- a/paddle/fluid/distributed/ps/wrapper/fleet.cc +++ b/paddle/fluid/distributed/ps/wrapper/fleet.cc @@ -754,6 +754,46 @@ std::future FleetWrapper::SendClientToClientMsg( return worker_ptr_->SendClient2ClientMsg(msg_type, to_client_id, msg); } +double FleetWrapper::GetCacheThreshold(int table_id) { + double cache_threshold = 0.0; + auto ret = worker_ptr_->Flush(); + ret.wait(); + ret = worker_ptr_->GetCacheThreshold(table_id, cache_threshold); + ret.wait(); + if (cache_threshold < 0) { + LOG(ERROR) << "get cache threshold failed"; + sleep(sleep_seconds_before_fail_exit_); + exit(-1); + } + return cache_threshold; +} + +void FleetWrapper::CacheShuffle(int table_id, const std::string& path, + const int mode, const double cache_threshold) { + auto ret = worker_ptr_->CacheShuffle(table_id, path, std::to_string(mode), + std::to_string(cache_threshold)); + ret.wait(); + int32_t feasign_cnt = ret.get(); + if (feasign_cnt == -1) { + LOG(ERROR) << "cache shuffle failed"; + sleep(sleep_seconds_before_fail_exit_); + exit(-1); + } +} + +int32_t FleetWrapper::SaveCache(int table_id, const std::string& path, + const int mode) { + auto ret = worker_ptr_->SaveCache(table_id, path, std::to_string(mode)); + ret.wait(); + int32_t feasign_cnt = ret.get(); + if (feasign_cnt == -1) { + LOG(ERROR) << "table save cache failed"; + sleep(sleep_seconds_before_fail_exit_); + exit(-1); + } + return feasign_cnt; +} + std::default_random_engine& FleetWrapper::LocalRandomEngine() { struct engine_wrapper_t { std::default_random_engine engine; diff --git a/paddle/fluid/distributed/ps/wrapper/fleet.h b/paddle/fluid/distributed/ps/wrapper/fleet.h index e6ec09a12637d..ce109b63cce9c 100644 --- a/paddle/fluid/distributed/ps/wrapper/fleet.h +++ b/paddle/fluid/distributed/ps/wrapper/fleet.h @@ -259,6 +259,11 @@ class FleetWrapper { // for init worker void InitGFlag(const std::string& gflags); + double GetCacheThreshold(int table_id); + void CacheShuffle(int table_id, const std::string& path, const int mode, + const double cache_threshold); + int32_t SaveCache(int table_id, const std::string& path, const int mode); + static std::shared_ptr pserver_ptr_; static std::shared_ptr worker_ptr_; diff --git a/paddle/fluid/distributed/the_one_ps.proto b/paddle/fluid/distributed/the_one_ps.proto index 32bf9eaa5aa06..1b20aca85422c 100644 --- a/paddle/fluid/distributed/the_one_ps.proto +++ b/paddle/fluid/distributed/the_one_ps.proto @@ -116,6 +116,10 @@ message TableParameter { optional TableType type = 7; optional bool compress_in_save = 8 [ default = false ]; optional GraphParameter graph_parameter = 9; + // for cache model + optional bool enable_sparse_table_cache = 10 [ default = true ]; + optional double sparse_table_cache_rate = 11 [ default = 0.00055 ]; + optional uint32 sparse_table_cache_file_num = 12 [ default = 16 ]; } message TableAccessorParameter { diff --git a/paddle/fluid/pybind/fleet_py.cc b/paddle/fluid/pybind/fleet_py.cc index 8d8301689521b..d35419e87f3a5 100644 --- a/paddle/fluid/pybind/fleet_py.cc +++ b/paddle/fluid/pybind/fleet_py.cc @@ -78,7 +78,11 @@ void BindDistFleetWrapper(py::module* m) { .def("set_clients", &FleetWrapper::SetClients) .def("get_client_info", &FleetWrapper::GetClientsInfo) .def("create_client2client_connection", - &FleetWrapper::CreateClient2ClientConnection); + &FleetWrapper::CreateClient2ClientConnection) + .def("client_flush", &FleetWrapper::ClientFlush) + .def("get_cache_threshold", &FleetWrapper::GetCacheThreshold) + .def("cache_shuffle", &FleetWrapper::CacheShuffle) + .def("save_cache", &FleetWrapper::SaveCache); } void BindPSHost(py::module* m) { diff --git a/paddle/utils/string/string_helper.h b/paddle/utils/string/string_helper.h index a02b313ef0eba..e6cb2e90b8fa1 100644 --- a/paddle/utils/string/string_helper.h +++ b/paddle/utils/string/string_helper.h @@ -100,6 +100,14 @@ inline int str_to_float(const char* str, float* v) { return index; } +inline float* str_to_float(std::string& str) { + return (float*)const_cast(str.c_str()); +} + +inline float* str_to_float(const char* str) { + return (float*)const_cast(str); +} + // checks whether the test string is a suffix of the input string. bool ends_with(std::string const& input, std::string const& test); diff --git a/python/paddle/distributed/fleet/__init__.py b/python/paddle/distributed/fleet/__init__.py index 3186df7db581a..ef0fff8283361 100644 --- a/python/paddle/distributed/fleet/__init__.py +++ b/python/paddle/distributed/fleet/__init__.py @@ -77,6 +77,7 @@ distributed_optimizer = fleet.distributed_optimizer save_inference_model = fleet.save_inference_model save_persistables = fleet.save_persistables +save_cache_model = fleet.save_cache_model load_model = fleet.load_model minimize = fleet.minimize distributed_model = fleet.distributed_model diff --git a/python/paddle/distributed/fleet/base/fleet_base.py b/python/paddle/distributed/fleet/base/fleet_base.py index 4e975e74bdb14..a1c967ab0639c 100755 --- a/python/paddle/distributed/fleet/base/fleet_base.py +++ b/python/paddle/distributed/fleet/base/fleet_base.py @@ -869,6 +869,11 @@ def save_persistables(self, executor, dirname, main_program=None, mode=0): self._runtime_handle._save_persistables(executor, dirname, main_program, mode) + @is_non_distributed_check + @inited_runtime_handler + def save_cache_model(self, dirname, **configs): + return self._runtime_handle._save_cache_model(dirname, **configs) + def shrink(self, threshold=None): self._runtime_handle._shrink(threshold) diff --git a/python/paddle/distributed/ps/the_one_ps.py b/python/paddle/distributed/ps/the_one_ps.py index 5be739785ff44..c6df7559a22e8 100755 --- a/python/paddle/distributed/ps/the_one_ps.py +++ b/python/paddle/distributed/ps/the_one_ps.py @@ -1315,6 +1315,30 @@ def _save_inference_model(self, *args, **kwargs): def _save_persistables(self, *args, **kwargs): self._ps_inference_save_persistables(*args, **kwargs) + def _save_cache_model(self, dirname, **kwargs): + mode = kwargs.get("mode", 0) + table_id = kwargs.get("table_id", 0) + self._worker.client_flush() + fleet.util.barrier() + cache_threshold = 0.0 + + if self.role_maker._is_first_worker(): + cache_threshold = self._worker.get_cache_threshold(table_id) + #check cache threshold right or not + fleet.util.barrier() + + if self.role_maker._is_first_worker(): + self._worker.cache_shuffle(table_id, dirname, mode, cache_threshold) + + fleet.util.barrier() + + feasign_num = -1 + if self.role_maker._is_first_worker(): + feasign_num = self._worker.save_cache(table_id, dirname, mode) + + fleet.util.barrier() + return feasign_num + def _load_sparse_params(self, dirname, context, main_program, mode): distributed_varnames = get_sparse_tablenames(self.origin_main_programs, True) diff --git a/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py b/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py index 2bd397b0ef3f5..be5118f0acc18 100644 --- a/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py +++ b/python/paddle/fluid/tests/unittests/dist_fleet_ctr.py @@ -339,5 +339,9 @@ def do_dataset_training(self, fleet): if dirname: fleet.save_persistables(exe, dirname=dirname) + cache_dirname = os.getenv("SAVE_CACHE_DIRNAME", None) + if cache_dirname: + fleet.save_cache_model(cache_dirname) + if __name__ == "__main__": runtime_main(TestDistCTR2x2) diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py index 59d196fdf55e5..09d64a318d6d8 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py @@ -39,6 +39,8 @@ def check_with_place(self, "http_proxy": "", "CPU_NUM": "2", "LOG_DIRNAME": "/tmp", + "SAVE_CACHE_DIRNAME": + "/tmp/TestDistMnistAsyncInMemoryDataset2x2/cache_model", "LOG_PREFIX": self.__class__.__name__, } From 4940a5255a419caf840bf426791a820246792f67 Mon Sep 17 00:00:00 2001 From: Ruibiao Chen Date: Fri, 22 Apr 2022 19:28:52 +0800 Subject: [PATCH 36/66] Add gpudnn yaml config for some OPs (#41773) * Add gpudnn yaml config for some OPs * Add grad gpudnn config * Fix CI errors * Fix CI errors * Fix CI errors * Fix conflicts --- paddle/phi/core/kernel_factory.cc | 4 ++-- paddle/phi/core/kernel_factory.h | 2 +- python/paddle/nn/functional/pooling.py | 6 +++--- python/paddle/utils/code_gen/api.yaml | 16 ++++++++++++++++ python/paddle/utils/code_gen/api_base.py | 12 ++++++------ python/paddle/utils/code_gen/backward.yaml | 17 ++++++++++++++++- .../utils/code_gen/wrapped_infermeta_gen.py | 3 ++- 7 files changed, 46 insertions(+), 14 deletions(-) diff --git a/paddle/phi/core/kernel_factory.cc b/paddle/phi/core/kernel_factory.cc index d3fd2e0204e54..6d71c5016bda4 100644 --- a/paddle/phi/core/kernel_factory.cc +++ b/paddle/phi/core/kernel_factory.cc @@ -79,7 +79,7 @@ bool KernelFactory::HasKernel(const std::string& kernel_name, const Kernel& KernelFactory::SelectKernelOrThrowError( const std::string& kernel_name, const KernelKey& kernel_key, - bool use_cudnn) const { + bool use_gpudnn) const { auto iter = kernels_.find(kernel_name); PADDLE_ENFORCE_NE( iter, @@ -87,7 +87,7 @@ const Kernel& KernelFactory::SelectKernelOrThrowError( phi::errors::NotFound("The kernel `%s` is not registered.", kernel_name)); #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) - if (use_cudnn && kernel_key.backend() == Backend::GPU) { + if (use_gpudnn && kernel_key.backend() == Backend::GPU) { auto kernel_iter = iter->second.find( {Backend::GPUDNN, kernel_key.layout(), kernel_key.dtype()}); if (kernel_iter == iter->second.end() && diff --git a/paddle/phi/core/kernel_factory.h b/paddle/phi/core/kernel_factory.h index 812b6222cb5e2..3ac99a426319d 100644 --- a/paddle/phi/core/kernel_factory.h +++ b/paddle/phi/core/kernel_factory.h @@ -247,7 +247,7 @@ class KernelFactory { const Kernel& SelectKernelOrThrowError(const std::string& kernel_name, const KernelKey& kernel_key, - bool use_cudnn = false) const; + bool use_gpudnn = false) const; const Kernel& SelectKernelOrThrowError(const std::string& kernel_name, Backend backend, diff --git a/python/paddle/nn/functional/pooling.py b/python/paddle/nn/functional/pooling.py index b9cae4784725d..3160f04e830d2 100755 --- a/python/paddle/nn/functional/pooling.py +++ b/python/paddle/nn/functional/pooling.py @@ -1401,9 +1401,9 @@ def adaptive_avg_pool2d(x, output_size, data_format='NCHW', name=None): output_size[1] = in_w if in_dygraph_mode(): - return _C_ops.final_state_pool2d(x, output_size, [1, 1], [0, 0], False, - True, data_format, 'avg', False, True, - "EXPLICIT") + return _C_ops.final_state_pool2d_gpudnn_unused( + x, output_size, [1, 1], [0, 0], False, True, data_format, 'avg', + False, True, "EXPLICIT") if _in_legacy_dygraph(): return _C_ops.pool2d(x, 'pooling_type', 'avg', 'ksize', output_size, diff --git a/python/paddle/utils/code_gen/api.yaml b/python/paddle/utils/code_gen/api.yaml index 58b80950e5529..d401e7c5190fe 100644 --- a/python/paddle/utils/code_gen/api.yaml +++ b/python/paddle/utils/code_gen/api.yaml @@ -382,6 +382,7 @@ func : ConvTransposeInferMeta kernel : func : conv2d_transpose + use_gpudnn : true backward : conv2d_transpose_grad - api : conv3d_transpose @@ -391,6 +392,7 @@ func : ConvTransposeInferMeta kernel : func : conv3d_transpose + use_gpudnn : true backward : conv3d_transpose_grad - api : copy_to @@ -1556,8 +1558,20 @@ func : PoolInferMeta kernel : func : pool2d + use_gpudnn : true backward : pool2d_grad +# Used in adaptive_avg_pool2d API +- api : pool2d_gpudnn_unused + args : (Tensor x, int[] kernel_size, int[] strides, int[] paddings, bool ceil_mode, bool exclusive, str data_format, str pooling_type, bool global_pooling, bool adaptive, str padding_algorithm) + output : Tensor(out) + infer_meta : + func : PoolInferMeta + kernel : + func : pool2d + use_gpudnn : false + backward : pool2d_grad_gpudnn_unused + - api : pool3d args : (Tensor x, int[] kernel_size, int[] strides, int[] paddings, bool ceil_mode, bool exclusive, str data_format, str pooling_type, bool global_pooling, bool adaptive, str padding_algorithm) output : Tensor(out) @@ -1565,6 +1579,7 @@ func : PoolInferMeta kernel : func : pool3d + use_gpudnn : true backward : pool3d_grad - api : pow @@ -1923,6 +1938,7 @@ func : SoftmaxInferMeta kernel : func : softmax + use_gpudnn : true backward : softmax_grad - api : split diff --git a/python/paddle/utils/code_gen/api_base.py b/python/paddle/utils/code_gen/api_base.py index 378ead7ff20aa..717870ee01d0a 100644 --- a/python/paddle/utils/code_gen/api_base.py +++ b/python/paddle/utils/code_gen/api_base.py @@ -238,7 +238,7 @@ def parse_kernel(self, kernel_config): 'backend': None, 'layout': None, 'data_type': None, - 'use_cudnn': 'false' + 'use_gpudnn': 'false' } if 'backend' in kernel_config and len(kernel_config['backend']) > 0: kernel['backend'] = kernel_config['backend'] @@ -248,10 +248,10 @@ def parse_kernel(self, kernel_config): kernel['data_type'] = kernel_config['data_type'] if 'param' in kernel_config: kernel['param'] = kernel_config['param'] - if 'use_cudnn' in kernel_config: - kernel['use_cudnn'] = kernel_config['use_cudnn'] - if isinstance(kernel['use_cudnn'], bool): - kernel['use_cudnn'] = str(kernel['use_cudnn']).lower() + if 'use_gpudnn' in kernel_config: + kernel['use_gpudnn'] = kernel_config['use_gpudnn'] + if isinstance(kernel['use_gpudnn'], bool): + kernel['use_gpudnn'] = str(kernel['use_gpudnn']).lower() kernel['func'] = [ kernel_fn.strip() for kernel_fn in kernel_config['func'].split(',') ] @@ -729,7 +729,7 @@ def gen_dense_tensor_kernel_code(self, code_indent, inplace_flag=False): self.outputs['types'], 'SetKernelOutput', code_indent, inplace_flag) api_func_name = self.get_api_func_name() + ('_' if inplace_flag else '') cudnn_args = '' if self.kernel[ - 'use_cudnn'] == 'false' else ', ' + self.kernel['use_cudnn'] + 'use_gpudnn'] == 'false' else ', ' + self.kernel['use_gpudnn'] return f""" {code_indent} VLOG(6) << "{self.api} API kernel key: [" << kernel_backend << ", " << kernel_layout << ", "<< kernel_data_type << "]"; {code_indent} const auto& kernel = phi::KernelFactory::Instance().SelectKernelOrThrowError( diff --git a/python/paddle/utils/code_gen/backward.yaml b/python/paddle/utils/code_gen/backward.yaml index dfdc2335ae180..3b47470139b90 100644 --- a/python/paddle/utils/code_gen/backward.yaml +++ b/python/paddle/utils/code_gen/backward.yaml @@ -272,7 +272,7 @@ param: [input, filter, grad_out] kernel : func : conv2d_grad_grad - use_cudnn : true + use_gpudnn : true optional : grad_input_grad, grad_filter_grad - backward_api : conv2d_transpose_grad @@ -283,6 +283,7 @@ func : ConvTransposeGradInferMeta kernel : func : conv2d_transpose_grad + use_gpudnn : true - backward_api : conv3d_transpose_grad forward : conv3d_transpose(Tensor x, Tensor filter, int[] strides, int[] paddings, int[] output_padding, int[] output_size, str padding_algorithm, int groups, int[] dilations, str data_format) -> Tensor(out) @@ -292,6 +293,7 @@ func : ConvTransposeGradInferMeta kernel : func : conv3d_transpose_grad + use_gpudnn : true - backward_api : cos_grad forward : cos (Tensor x) -> Tensor(out) @@ -1234,6 +1236,17 @@ func : PoolGradInferMeta kernel : func : pool2d_grad + use_gpudnn : true + +- backward_api : pool2d_grad_gpudnn_unused + forward : pool2d_gpudnn_unused(Tensor x, int[] kernel_size, int[] strides, int[] paddings, bool ceil_mode, bool exclusive, str data_format, str pooling_type, bool global_pooling, bool adaptive, str padding_algorithm) -> Tensor(out) + args : (Tensor x, Tensor out, Tensor out_grad, int[] kernel_size, int[] strides, int[] paddings, bool ceil_mode, bool exclusive, str data_format, str pooling_type, bool global_pooling, bool adaptive, str padding_algorithm) + output : Tensor(x_grad) + infer_meta : + func : PoolGradInferMeta + kernel : + func : pool2d_grad + use_gpudnn : false - backward_api : pool3d_grad forward : pool3d(Tensor x, int[] kernel_size, int[] strides, int[] paddings, bool ceil_mode, bool exclusive, str data_format, str pooling_type, bool global_pooling, bool adaptive, str padding_algorithm) -> Tensor(out) @@ -1243,6 +1256,7 @@ func : PoolGradInferMeta kernel : func : pool3d_grad + use_gpudnn : true - backward_api : pow_grad forward : pow(Tensor x, Scalar s) -> Tensor(out) @@ -1578,6 +1592,7 @@ param : [out] kernel : func : softmax_grad + use_gpudnn : true - backward_api : split_grad forward : split (Tensor x, IntArray num_or_sections, Scalar axis) -> Tensor[](out) diff --git a/python/paddle/utils/code_gen/wrapped_infermeta_gen.py b/python/paddle/utils/code_gen/wrapped_infermeta_gen.py index b50db007d92e9..dd077552b7962 100644 --- a/python/paddle/utils/code_gen/wrapped_infermeta_gen.py +++ b/python/paddle/utils/code_gen/wrapped_infermeta_gen.py @@ -141,7 +141,8 @@ def generate_wrapped_infermeta_and_register(api_yaml_path, header_file_path, api_item) header_file.write(declare_code) source_file.write(defind_code) - infermeta_register_code = infermeta_register_code + register_code + if infermeta_register_code.find(register_code) == -1: + infermeta_register_code = infermeta_register_code + register_code header_file.write(namespace[1]) source_file.write(namespace[1]) From 34ac7b74c216bd02d44d9bc57b1537343adc0934 Mon Sep 17 00:00:00 2001 From: YuanRisheng Date: Fri, 22 Apr 2022 19:44:09 +0800 Subject: [PATCH 37/66] Support triple grad check of op in Eager mode (#42131) * support 3-rd order gradient * change code format --- .../fluid/tests/unittests/gradient_checker.py | 222 +++++++++++++++--- .../unittests/test_elementwise_nn_grad.py | 21 ++ 2 files changed, 204 insertions(+), 39 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/gradient_checker.py b/python/paddle/fluid/tests/unittests/gradient_checker.py index 562d52668ce5b..569d994b831b6 100644 --- a/python/paddle/fluid/tests/unittests/gradient_checker.py +++ b/python/paddle/fluid/tests/unittests/gradient_checker.py @@ -60,19 +60,6 @@ def _get_item(t, i, np_dtype): raise ValueError("Not supported data type " + str(np_dtype)) -def _get_item_for_dygraph(t, i, np_dtype): - if np_dtype == np.float16: - np_t = t.numpy().astype(np.float16) - elif np_dtype == np.float32: - np_t = t.numpy().astype(np.float32) - elif np_dtype == np.float64: - np_t = t.numpy().astype(np.float64) - else: - raise ValueError("Not supported data type " + str(np_dtype)) - np_t = np_t.flatten() - return np_t[i] - - def _set_item(t, i, e, np_dtype): if np_dtype == np.float16: np_t = np.array(t).astype(np.float16) @@ -89,22 +76,6 @@ def _set_item(t, i, e, np_dtype): raise ValueError("Not supported data type " + str(np_dtype)) -def _set_item_for_dygraph(t, i, e, np_dtype): - if np_dtype == np.float16: - np_t = t.numpy().astype(np.float16) - elif np_dtype == np.float32: - np_t = t.numpy().astype(np.float32) - elif np_dtype == np.float64: - np_t = t.numpy().astype(np.float64) - else: - raise ValueError("Not supported data type " + str(np_dtype)) - shape = np_t.shape - np_t = np_t.flatten() - np_t[i] = e - np_t = np_t.reshape(shape) - paddle.assign(np_t, t) - - def set_var_in_scope(scope, place, name, value, recursive_seq_len=None): t = scope.var(name).get_tensor() t.set(value, place) @@ -169,8 +140,6 @@ def run(): np_type = dtype_to_np_dtype(x.dtype) jacobian = [make_jacobian(x, _product(yi.shape), np_type) for yi in y] - if np_type == np.float64: - delta = 1e-5 for i in six.moves.xrange(x_size): orig = _get_item(x_t, i, np_type) x_pos = orig + delta @@ -545,7 +514,12 @@ def triple_grad_check(x, rtol=rtol) -def get_static_double_grad(x, y, x_init=None, dy_init=None, place=None): +def get_static_double_grad(x, + y, + x_init=None, + dy_init=None, + place=None, + program=None): """ Get Double Grad result of static graph. @@ -555,11 +529,14 @@ def get_static_double_grad(x, y, x_init=None, dy_init=None, place=None): x_init (numpy.array|list[numpy.array]|None): the init value for input x. dy_init (numpy.array|list[numpy.array]|None): the init value for output y. place (fluid.CPUPlace or fluid.CUDAPlace): the device. + program (Program|None): a Program with forward pass. + If None, use fluid.default_main_program(). Returns: A list of numpy array that stores second derivative result calulated by static graph. """ - program = fluid.default_main_program() + if program is None: + program = fluid.default_main_program() scope = fluid.executor.global_scope() y_grads = [] for i in six.moves.xrange(len(y)): @@ -635,7 +612,10 @@ def get_static_double_grad(x, y, x_init=None, dy_init=None, place=None): return ddx_res -def get_eager_double_grad(func, x_init=None, dy_init=None): +def get_eager_double_grad(func, + x_init=None, + dy_init=None, + return_mid_result=False): """ Get Double Grad result of dygraph. @@ -643,8 +623,13 @@ def get_eager_double_grad(func, x_init=None, dy_init=None): func: A wrapped dygraph function that its logic is equal to static program x_init (numpy.array|list[numpy.array]|None): the init value for input x. dy_init (numpy.array|list[numpy.array]|None): the init value for gradient of output. + return_mid_result (bool): A flag that controls the return content. Returns: - A list of numpy array that stores second derivative result calulated by dygraph + If 'return_mid_result' set True. + the second order derivative and the inputs of second order derivative's calculation + will be returned for higher order derivative's calculation. + If 'return_mid_result' set False. + A list of numpy array that stores second derivative result calulated by dygraph. """ inputs = [] dys = [] @@ -664,13 +649,25 @@ def get_eager_double_grad(func, x_init=None, dy_init=None): # calcluate second derivative inputs = inputs + dys ddys = [] + if return_mid_result: + create_graph = True + else: + create_graph = False + for d_input in d_inputs: d_input.stop_gradient = False ddy = paddle.ones(shape=d_input.shape, dtype=d_input.dtype) ddy.stop_gradient = False ddys.append(ddy) - dd_inputs = paddle.grad(outputs=d_inputs, inputs=inputs, grad_outputs=ddys) - return [dd_input.numpy() for dd_input in dd_inputs] + dd_inputs = paddle.grad( + outputs=d_inputs, + inputs=inputs, + grad_outputs=ddys, + create_graph=create_graph) + if return_mid_result: + return dd_inputs, inputs + ddys + else: + return [dd_input.numpy() for dd_input in dd_inputs] def double_grad_check_for_dygraph(func, @@ -682,8 +679,9 @@ def double_grad_check_for_dygraph(func, rtol=1e-3, raise_exception=True): """ - Check gradients of gradients. This function will append backward to the - program before second order gradient check. + Check second order gradients of dygraph. This function will compare the + second order gradients of dygraph and second order gradients of static graph + to validate dygraph's correctness Args: func: A wrapped dygraph function that its logic is equal to static program @@ -734,3 +732,149 @@ def fail_test(msg): 'static:%s\n eager:%s\n' \ % (static_double_grad[i].name, eager_double_grad[i].name, str(place), static_double_grad[i], eager_double_grad[i]) return fail_test(msg) + + +def get_static_triple_grad(x, + y, + x_init=None, + dy_init=None, + place=None, + program=None): + """ + Get Triple Grad result of static graph. + + Args: + x (Variable|list[Variable]): input variables to the program. + y (Variable|list[Variable]): output variables to the program. + x_init (numpy.array|list[numpy.array]|None): the init value for input x. + dy_init (numpy.array|list[numpy.array]|None): the init value for output y. + place (fluid.CPUPlace or fluid.CUDAPlace): the device. + program (Program|None): a Program with forward pass. + If None, use fluid.default_main_program(). + Returns: + A list of numpy array that stores third derivative result calulated by static graph. + """ + if program is None: + program = fluid.default_main_program() + scope = fluid.executor.global_scope() + y_grads = [] + for i in six.moves.xrange(len(y)): + yi = y[i] + dyi_name = _append_grad_suffix_(yi.name) + np_type = dtype_to_np_dtype(yi.dtype) + dy = program.global_block().create_var( + name=dyi_name, shape=yi.shape, dtype=np_type, persistable=True) + dy.stop_gradient = False + set_var_in_scope(scope, place, dyi_name, dy_init[i]) + y_grads.append(dy) + + # append first order grads + dx = fluid.gradients(y, x, y_grads) + + # y_grads are the input of first-order backward, + # so, they are also the input of second-order backward. + x += y_grads + x_init += dy_init + y = dx + + x_grads_grads_init = [] + for dxi in dx: + np_type = dtype_to_np_dtype(dxi.dtype) + value = np.ones(dxi.shape, dtype=np_type) + x_grads_grads_init.append(value) + + return get_static_double_grad( + x, y, x_init, dy_init=x_grads_grads_init, place=place, program=program) + + +def get_eager_triple_grad(func, + x_init=None, + dy_init=None, + return_mid_result=False): + """ + Get triple Grad result of dygraph. + + Args: + func: A wrapped dygraph function that its logic is equal to static program + x_init (numpy.array|list[numpy.array]|None): the init value for input x. + dy_init (numpy.array|list[numpy.array]|None): the init value for gradient of output. + return_mid_result (list[Tensor], list[Tensor]): If set True, the + Returns: + A list of numpy array that stores second derivative result calulated by dygraph + """ + dd_y, dd_x = get_eager_double_grad( + func, x_init, dy_init, return_mid_result=True) + + # calcluate third derivative + dddys = [] + for dd_yi in dd_y: + dd_yi.stop_gradient = False + dddy = paddle.ones(shape=dd_yi.shape, dtype=dd_yi.dtype) + dddy.stop_gradient = False + dddys.append(dddy) + ddd_inputs = paddle.grad(outputs=dd_y, inputs=dd_x, grad_outputs=dddys) + return [ddd_input.numpy() for ddd_input in ddd_inputs] + + +def triple_grad_check_for_dygraph(func, + x, + y, + x_init=None, + place=None, + atol=1e-5, + rtol=1e-3, + raise_exception=True): + """ + Check third order gradients of dygraph. This function will compare the + third order gradients of dygraph and third order gradients of static graph + to validate dygraph's correctness + + Args: + func: A wrapped dygraph function that its logic is equal to static program + x (Variable|list[Variable]): input variables to the program. + y (Variable|list[Variable]): output variables to the program. + x_init (numpy.array|list[numpy.array]|None): the init value for input x. + place (fluid.CPUPlace or fluid.CUDAPlace): the device. + eps (float): perturbation for finite differences. + atol (float): absolute tolerance. + rtol (float): relative tolerance. + raise_exception (bool): whether to raise an exception if + the check fails. Default is True. + """ + + def fail_test(msg): + if raise_exception: + raise RuntimeError(msg) + return False + + # check input arguments + x = _as_list(x) + for v in x: + v.stop_gradient = False + v.persistable = True + y = _as_list(y) + + y_grads_init = [] + for yi in y: + np_type = dtype_to_np_dtype(yi.dtype) + v = np.random.random(size=yi.shape).astype(np_type) + y_grads_init.append(v) + + x_init = _as_list(x_init) + + paddle.disable_static() + with _test_eager_guard(): + eager_triple_grad = get_eager_triple_grad(func, x_init, y_grads_init) + paddle.enable_static() + + static_triple_grad = get_static_triple_grad(x, y, x_init, y_grads_init, + place) + + for i in six.moves.xrange(len(static_triple_grad)): + if not np.allclose(static_triple_grad[i], eager_triple_grad[i], rtol, + atol): + msg = 'Check eager double result fail. Mismatch between static_graph double grad %s ' \ + 'and eager double grad %s on %s,\n' \ + 'static:%s\n eager:%s\n' \ + % (static_triple_grad[i].name, eager_triple_grad[i].name, str(place), static_triple_grad[i], eager_triple_grad[i]) + return fail_test(msg) diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py b/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py index c51c8098706a6..8f6f9851c7006 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_nn_grad.py @@ -17,6 +17,7 @@ import unittest import numpy as np +import paddle import paddle.fluid as fluid import paddle.fluid.layers as layers import paddle.fluid.core as core @@ -45,6 +46,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -72,6 +74,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -99,6 +102,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -126,6 +130,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -153,6 +158,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -180,6 +186,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -208,6 +215,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps, atol=1e-3) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -236,6 +244,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps, atol=1e-3) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -263,6 +272,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -290,6 +300,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -298,6 +309,9 @@ def test_grad(self): class TestElementwiseMulTripleGradCheck(unittest.TestCase): + def multiply_wrapper(self, x): + return paddle.multiply(x[0], x[1]) + @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not inlcude -1. @@ -315,8 +329,14 @@ def func(self, place): gradient_checker.triple_grad_check( [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) + gradient_checker.triple_grad_check_for_dygraph( + self.multiply_wrapper, [x, y], + out, + x_init=[x_arr, y_arr], + place=place) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) @@ -344,6 +364,7 @@ def func(self, place): [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps) def test_grad(self): + paddle.enable_static() places = [fluid.CPUPlace()] if core.is_compiled_with_cuda(): places.append(fluid.CUDAPlace(0)) From f6219dda46e920efa2c37323961a8927f39a54d8 Mon Sep 17 00:00:00 2001 From: Nyakku Shigure Date: Sat, 23 Apr 2022 07:27:54 +0800 Subject: [PATCH 38/66] reuse ConvNormActivation in some vision models (#40431) * reuse ConvNormActivation in some vision models --- python/paddle/vision/models/inceptionv3.py | 477 ++++++++++---------- python/paddle/vision/models/mobilenetv1.py | 56 +-- python/paddle/vision/models/mobilenetv2.py | 89 ++-- python/paddle/vision/models/shufflenetv2.py | 124 +++-- python/paddle/vision/ops.py | 8 +- 5 files changed, 372 insertions(+), 382 deletions(-) diff --git a/python/paddle/vision/models/inceptionv3.py b/python/paddle/vision/models/inceptionv3.py index 9e8a8b814688c..27650dbe09f04 100644 --- a/python/paddle/vision/models/inceptionv3.py +++ b/python/paddle/vision/models/inceptionv3.py @@ -19,75 +19,60 @@ import math import paddle import paddle.nn as nn -from paddle.nn import Conv2D, BatchNorm, Linear, Dropout +from paddle.nn import Linear, Dropout from paddle.nn import AdaptiveAvgPool2D, MaxPool2D, AvgPool2D from paddle.nn.initializer import Uniform from paddle.fluid.param_attr import ParamAttr from paddle.utils.download import get_weights_path_from_url +from ..ops import ConvNormActivation __all__ = [] model_urls = { "inception_v3": - ("https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/legendary_models/InceptionV3_pretrained.pdparams", - "e4d0905a818f6bb7946e881777a8a935") + ("https://paddle-hapi.bj.bcebos.com/models/inception_v3.pdparams", + "649a4547c3243e8b59c656f41fe330b8") } -class ConvBNLayer(nn.Layer): - def __init__(self, - num_channels, - num_filters, - filter_size, - stride=1, - padding=0, - groups=1, - act="relu"): - super().__init__() - self.act = act - self.conv = Conv2D( - in_channels=num_channels, - out_channels=num_filters, - kernel_size=filter_size, - stride=stride, - padding=padding, - groups=groups, - bias_attr=False) - self.bn = BatchNorm(num_filters) - self.relu = nn.ReLU() - - def forward(self, x): - x = self.conv(x) - x = self.bn(x) - if self.act: - x = self.relu(x) - return x - - class InceptionStem(nn.Layer): def __init__(self): super().__init__() - self.conv_1a_3x3 = ConvBNLayer( - num_channels=3, num_filters=32, filter_size=3, stride=2, act="relu") - self.conv_2a_3x3 = ConvBNLayer( - num_channels=32, - num_filters=32, - filter_size=3, + self.conv_1a_3x3 = ConvNormActivation( + in_channels=3, + out_channels=32, + kernel_size=3, + stride=2, + padding=0, + activation_layer=nn.ReLU) + self.conv_2a_3x3 = ConvNormActivation( + in_channels=32, + out_channels=32, + kernel_size=3, stride=1, - act="relu") - self.conv_2b_3x3 = ConvBNLayer( - num_channels=32, - num_filters=64, - filter_size=3, + padding=0, + activation_layer=nn.ReLU) + self.conv_2b_3x3 = ConvNormActivation( + in_channels=32, + out_channels=64, + kernel_size=3, padding=1, - act="relu") + activation_layer=nn.ReLU) self.max_pool = MaxPool2D(kernel_size=3, stride=2, padding=0) - self.conv_3b_1x1 = ConvBNLayer( - num_channels=64, num_filters=80, filter_size=1, act="relu") - self.conv_4a_3x3 = ConvBNLayer( - num_channels=80, num_filters=192, filter_size=3, act="relu") + self.conv_3b_1x1 = ConvNormActivation( + in_channels=64, + out_channels=80, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.conv_4a_3x3 = ConvNormActivation( + in_channels=80, + out_channels=192, + kernel_size=3, + padding=0, + activation_layer=nn.ReLU) def forward(self, x): x = self.conv_1a_3x3(x) @@ -103,47 +88,53 @@ def forward(self, x): class InceptionA(nn.Layer): def __init__(self, num_channels, pool_features): super().__init__() - self.branch1x1 = ConvBNLayer( - num_channels=num_channels, - num_filters=64, - filter_size=1, - act="relu") - self.branch5x5_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=48, - filter_size=1, - act="relu") - self.branch5x5_2 = ConvBNLayer( - num_channels=48, - num_filters=64, - filter_size=5, + self.branch1x1 = ConvNormActivation( + in_channels=num_channels, + out_channels=64, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + + self.branch5x5_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=48, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch5x5_2 = ConvNormActivation( + in_channels=48, + out_channels=64, + kernel_size=5, padding=2, - act="relu") - - self.branch3x3dbl_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=64, - filter_size=1, - act="relu") - self.branch3x3dbl_2 = ConvBNLayer( - num_channels=64, - num_filters=96, - filter_size=3, + activation_layer=nn.ReLU) + + self.branch3x3dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=64, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch3x3dbl_2 = ConvNormActivation( + in_channels=64, + out_channels=96, + kernel_size=3, padding=1, - act="relu") - self.branch3x3dbl_3 = ConvBNLayer( - num_channels=96, - num_filters=96, - filter_size=3, + activation_layer=nn.ReLU) + self.branch3x3dbl_3 = ConvNormActivation( + in_channels=96, + out_channels=96, + kernel_size=3, padding=1, - act="relu") + activation_layer=nn.ReLU) + self.branch_pool = AvgPool2D( kernel_size=3, stride=1, padding=1, exclusive=False) - self.branch_pool_conv = ConvBNLayer( - num_channels=num_channels, - num_filters=pool_features, - filter_size=1, - act="relu") + self.branch_pool_conv = ConvNormActivation( + in_channels=num_channels, + out_channels=pool_features, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) def forward(self, x): branch1x1 = self.branch1x1(x) @@ -164,29 +155,34 @@ def forward(self, x): class InceptionB(nn.Layer): def __init__(self, num_channels): super().__init__() - self.branch3x3 = ConvBNLayer( - num_channels=num_channels, - num_filters=384, - filter_size=3, + self.branch3x3 = ConvNormActivation( + in_channels=num_channels, + out_channels=384, + kernel_size=3, stride=2, - act="relu") - self.branch3x3dbl_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=64, - filter_size=1, - act="relu") - self.branch3x3dbl_2 = ConvBNLayer( - num_channels=64, - num_filters=96, - filter_size=3, + padding=0, + activation_layer=nn.ReLU) + + self.branch3x3dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=64, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch3x3dbl_2 = ConvNormActivation( + in_channels=64, + out_channels=96, + kernel_size=3, padding=1, - act="relu") - self.branch3x3dbl_3 = ConvBNLayer( - num_channels=96, - num_filters=96, - filter_size=3, + activation_layer=nn.ReLU) + self.branch3x3dbl_3 = ConvNormActivation( + in_channels=96, + out_channels=96, + kernel_size=3, stride=2, - act="relu") + padding=0, + activation_layer=nn.ReLU) + self.branch_pool = MaxPool2D(kernel_size=3, stride=2) def forward(self, x): @@ -206,70 +202,74 @@ def forward(self, x): class InceptionC(nn.Layer): def __init__(self, num_channels, channels_7x7): super().__init__() - self.branch1x1 = ConvBNLayer( - num_channels=num_channels, - num_filters=192, - filter_size=1, - act="relu") - - self.branch7x7_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=channels_7x7, - filter_size=1, + self.branch1x1 = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + + self.branch7x7_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=channels_7x7, + kernel_size=1, stride=1, - act="relu") - self.branch7x7_2 = ConvBNLayer( - num_channels=channels_7x7, - num_filters=channels_7x7, - filter_size=(1, 7), + padding=0, + activation_layer=nn.ReLU) + self.branch7x7_2 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(1, 7), stride=1, padding=(0, 3), - act="relu") - self.branch7x7_3 = ConvBNLayer( - num_channels=channels_7x7, - num_filters=192, - filter_size=(7, 1), + activation_layer=nn.ReLU) + self.branch7x7_3 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=192, + kernel_size=(7, 1), stride=1, padding=(3, 0), - act="relu") - - self.branch7x7dbl_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=channels_7x7, - filter_size=1, - act="relu") - self.branch7x7dbl_2 = ConvBNLayer( - num_channels=channels_7x7, - num_filters=channels_7x7, - filter_size=(7, 1), + activation_layer=nn.ReLU) + + self.branch7x7dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=channels_7x7, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch7x7dbl_2 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(7, 1), padding=(3, 0), - act="relu") - self.branch7x7dbl_3 = ConvBNLayer( - num_channels=channels_7x7, - num_filters=channels_7x7, - filter_size=(1, 7), + activation_layer=nn.ReLU) + self.branch7x7dbl_3 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(1, 7), padding=(0, 3), - act="relu") - self.branch7x7dbl_4 = ConvBNLayer( - num_channels=channels_7x7, - num_filters=channels_7x7, - filter_size=(7, 1), + activation_layer=nn.ReLU) + self.branch7x7dbl_4 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=channels_7x7, + kernel_size=(7, 1), padding=(3, 0), - act="relu") - self.branch7x7dbl_5 = ConvBNLayer( - num_channels=channels_7x7, - num_filters=192, - filter_size=(1, 7), + activation_layer=nn.ReLU) + self.branch7x7dbl_5 = ConvNormActivation( + in_channels=channels_7x7, + out_channels=192, + kernel_size=(1, 7), padding=(0, 3), - act="relu") + activation_layer=nn.ReLU) self.branch_pool = AvgPool2D( kernel_size=3, stride=1, padding=1, exclusive=False) - self.branch_pool_conv = ConvBNLayer( - num_channels=num_channels, - num_filters=192, - filter_size=1, - act="relu") + self.branch_pool_conv = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) def forward(self, x): branch1x1 = self.branch1x1(x) @@ -296,40 +296,46 @@ def forward(self, x): class InceptionD(nn.Layer): def __init__(self, num_channels): super().__init__() - self.branch3x3_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=192, - filter_size=1, - act="relu") - self.branch3x3_2 = ConvBNLayer( - num_channels=192, - num_filters=320, - filter_size=3, + self.branch3x3_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch3x3_2 = ConvNormActivation( + in_channels=192, + out_channels=320, + kernel_size=3, stride=2, - act="relu") - self.branch7x7x3_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=192, - filter_size=1, - act="relu") - self.branch7x7x3_2 = ConvBNLayer( - num_channels=192, - num_filters=192, - filter_size=(1, 7), + padding=0, + activation_layer=nn.ReLU) + + self.branch7x7x3_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch7x7x3_2 = ConvNormActivation( + in_channels=192, + out_channels=192, + kernel_size=(1, 7), padding=(0, 3), - act="relu") - self.branch7x7x3_3 = ConvBNLayer( - num_channels=192, - num_filters=192, - filter_size=(7, 1), + activation_layer=nn.ReLU) + self.branch7x7x3_3 = ConvNormActivation( + in_channels=192, + out_channels=192, + kernel_size=(7, 1), padding=(3, 0), - act="relu") - self.branch7x7x3_4 = ConvBNLayer( - num_channels=192, - num_filters=192, - filter_size=3, + activation_layer=nn.ReLU) + self.branch7x7x3_4 = ConvNormActivation( + in_channels=192, + out_channels=192, + kernel_size=3, stride=2, - act="relu") + padding=0, + activation_layer=nn.ReLU) + self.branch_pool = MaxPool2D(kernel_size=3, stride=2) def forward(self, x): @@ -350,59 +356,64 @@ def forward(self, x): class InceptionE(nn.Layer): def __init__(self, num_channels): super().__init__() - self.branch1x1 = ConvBNLayer( - num_channels=num_channels, - num_filters=320, - filter_size=1, - act="relu") - self.branch3x3_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=384, - filter_size=1, - act="relu") - self.branch3x3_2a = ConvBNLayer( - num_channels=384, - num_filters=384, - filter_size=(1, 3), + self.branch1x1 = ConvNormActivation( + in_channels=num_channels, + out_channels=320, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch3x3_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=384, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch3x3_2a = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(1, 3), padding=(0, 1), - act="relu") - self.branch3x3_2b = ConvBNLayer( - num_channels=384, - num_filters=384, - filter_size=(3, 1), + activation_layer=nn.ReLU) + self.branch3x3_2b = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(3, 1), padding=(1, 0), - act="relu") - - self.branch3x3dbl_1 = ConvBNLayer( - num_channels=num_channels, - num_filters=448, - filter_size=1, - act="relu") - self.branch3x3dbl_2 = ConvBNLayer( - num_channels=448, - num_filters=384, - filter_size=3, + activation_layer=nn.ReLU) + + self.branch3x3dbl_1 = ConvNormActivation( + in_channels=num_channels, + out_channels=448, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) + self.branch3x3dbl_2 = ConvNormActivation( + in_channels=448, + out_channels=384, + kernel_size=3, padding=1, - act="relu") - self.branch3x3dbl_3a = ConvBNLayer( - num_channels=384, - num_filters=384, - filter_size=(1, 3), + activation_layer=nn.ReLU) + self.branch3x3dbl_3a = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(1, 3), padding=(0, 1), - act="relu") - self.branch3x3dbl_3b = ConvBNLayer( - num_channels=384, - num_filters=384, - filter_size=(3, 1), + activation_layer=nn.ReLU) + self.branch3x3dbl_3b = ConvNormActivation( + in_channels=384, + out_channels=384, + kernel_size=(3, 1), padding=(1, 0), - act="relu") + activation_layer=nn.ReLU) + self.branch_pool = AvgPool2D( kernel_size=3, stride=1, padding=1, exclusive=False) - self.branch_pool_conv = ConvBNLayer( - num_channels=num_channels, - num_filters=192, - filter_size=1, - act="relu") + self.branch_pool_conv = ConvNormActivation( + in_channels=num_channels, + out_channels=192, + kernel_size=1, + padding=0, + activation_layer=nn.ReLU) def forward(self, x): branch1x1 = self.branch1x1(x) diff --git a/python/paddle/vision/models/mobilenetv1.py b/python/paddle/vision/models/mobilenetv1.py index 671a2cd8dfd5f..6d8d96952fab4 100644 --- a/python/paddle/vision/models/mobilenetv1.py +++ b/python/paddle/vision/models/mobilenetv1.py @@ -16,59 +16,31 @@ import paddle.nn as nn from paddle.utils.download import get_weights_path_from_url +from ..ops import ConvNormActivation __all__ = [] model_urls = { 'mobilenetv1_1.0': - ('https://paddle-hapi.bj.bcebos.com/models/mobilenet_v1_x1.0.pdparams', - '42a154c2f26f86e7457d6daded114e8c') + ('https://paddle-hapi.bj.bcebos.com/models/mobilenetv1_1.0.pdparams', + '3033ab1975b1670bef51545feb65fc45') } -class ConvBNLayer(nn.Layer): - def __init__(self, - in_channels, - out_channels, - kernel_size, - stride, - padding, - num_groups=1): - super(ConvBNLayer, self).__init__() - - self._conv = nn.Conv2D( - in_channels, - out_channels, - kernel_size, - stride=stride, - padding=padding, - groups=num_groups, - bias_attr=False) - - self._norm_layer = nn.BatchNorm2D(out_channels) - self._act = nn.ReLU() - - def forward(self, x): - x = self._conv(x) - x = self._norm_layer(x) - x = self._act(x) - return x - - class DepthwiseSeparable(nn.Layer): def __init__(self, in_channels, out_channels1, out_channels2, num_groups, stride, scale): super(DepthwiseSeparable, self).__init__() - self._depthwise_conv = ConvBNLayer( + self._depthwise_conv = ConvNormActivation( in_channels, int(out_channels1 * scale), kernel_size=3, stride=stride, padding=1, - num_groups=int(num_groups * scale)) + groups=int(num_groups * scale)) - self._pointwise_conv = ConvBNLayer( + self._pointwise_conv = ConvNormActivation( int(out_channels1 * scale), int(out_channels2 * scale), kernel_size=1, @@ -94,9 +66,15 @@ class MobileNetV1(nn.Layer): Examples: .. code-block:: python + import paddle from paddle.vision.models import MobileNetV1 model = MobileNetV1() + + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) """ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): @@ -106,7 +84,7 @@ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): self.num_classes = num_classes self.with_pool = with_pool - self.conv1 = ConvBNLayer( + self.conv1 = ConvNormActivation( in_channels=3, out_channels=int(32 * scale), kernel_size=3, @@ -257,6 +235,7 @@ def mobilenet_v1(pretrained=False, scale=1.0, **kwargs): Examples: .. code-block:: python + import paddle from paddle.vision.models import mobilenet_v1 # build model @@ -266,7 +245,12 @@ def mobilenet_v1(pretrained=False, scale=1.0, **kwargs): # model = mobilenet_v1(pretrained=True) # build mobilenet v1 with scale=0.5 - model = mobilenet_v1(scale=0.5) + model_scale = mobilenet_v1(scale=0.5) + + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) """ model = _mobilenet( 'mobilenetv1_' + str(scale), pretrained, scale=scale, **kwargs) diff --git a/python/paddle/vision/models/mobilenetv2.py b/python/paddle/vision/models/mobilenetv2.py index 6c486037c7d30..9791462610deb 100644 --- a/python/paddle/vision/models/mobilenetv2.py +++ b/python/paddle/vision/models/mobilenetv2.py @@ -17,6 +17,7 @@ from paddle.utils.download import get_weights_path_from_url from .utils import _make_divisible +from ..ops import ConvNormActivation __all__ = [] @@ -27,29 +28,6 @@ } -class ConvBNReLU(nn.Sequential): - def __init__(self, - in_planes, - out_planes, - kernel_size=3, - stride=1, - groups=1, - norm_layer=nn.BatchNorm2D): - padding = (kernel_size - 1) // 2 - - super(ConvBNReLU, self).__init__( - nn.Conv2D( - in_planes, - out_planes, - kernel_size, - stride, - padding, - groups=groups, - bias_attr=False), - norm_layer(out_planes), - nn.ReLU6()) - - class InvertedResidual(nn.Layer): def __init__(self, inp, @@ -67,15 +45,20 @@ def __init__(self, layers = [] if expand_ratio != 1: layers.append( - ConvBNReLU( - inp, hidden_dim, kernel_size=1, norm_layer=norm_layer)) + ConvNormActivation( + inp, + hidden_dim, + kernel_size=1, + norm_layer=norm_layer, + activation_layer=nn.ReLU6)) layers.extend([ - ConvBNReLU( + ConvNormActivation( hidden_dim, hidden_dim, stride=stride, groups=hidden_dim, - norm_layer=norm_layer), + norm_layer=norm_layer, + activation_layer=nn.ReLU6), nn.Conv2D( hidden_dim, oup, 1, 1, 0, bias_attr=False), norm_layer(oup), @@ -90,23 +73,30 @@ def forward(self, x): class MobileNetV2(nn.Layer): - def __init__(self, scale=1.0, num_classes=1000, with_pool=True): - """MobileNetV2 model from - `"MobileNetV2: Inverted Residuals and Linear Bottlenecks" `_. + """MobileNetV2 model from + `"MobileNetV2: Inverted Residuals and Linear Bottlenecks" `_. + + Args: + scale (float): scale of channels in each layer. Default: 1.0. + num_classes (int): output dim of last fc layer. If num_classes <=0, last fc layer + will not be defined. Default: 1000. + with_pool (bool): use pool before the last fc layer or not. Default: True. - Args: - scale (float): scale of channels in each layer. Default: 1.0. - num_classes (int): output dim of last fc layer. If num_classes <=0, last fc layer - will not be defined. Default: 1000. - with_pool (bool): use pool before the last fc layer or not. Default: True. + Examples: + .. code-block:: python + + import paddle + from paddle.vision.models import MobileNetV2 - Examples: - .. code-block:: python + model = MobileNetV2() - from paddle.vision.models import MobileNetV2 + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) + """ - model = MobileNetV2() - """ + def __init__(self, scale=1.0, num_classes=1000, with_pool=True): super(MobileNetV2, self).__init__() self.num_classes = num_classes self.with_pool = with_pool @@ -130,8 +120,12 @@ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): self.last_channel = _make_divisible(last_channel * max(1.0, scale), round_nearest) features = [ - ConvBNReLU( - 3, input_channel, stride=2, norm_layer=norm_layer) + ConvNormActivation( + 3, + input_channel, + stride=2, + norm_layer=norm_layer, + activation_layer=nn.ReLU6) ] for t, c, n, s in inverted_residual_setting: @@ -148,11 +142,12 @@ def __init__(self, scale=1.0, num_classes=1000, with_pool=True): input_channel = output_channel features.append( - ConvBNReLU( + ConvNormActivation( input_channel, self.last_channel, kernel_size=1, - norm_layer=norm_layer)) + norm_layer=norm_layer, + activation_layer=nn.ReLU6)) self.features = nn.Sequential(*features) @@ -199,6 +194,7 @@ def mobilenet_v2(pretrained=False, scale=1.0, **kwargs): Examples: .. code-block:: python + import paddle from paddle.vision.models import mobilenet_v2 # build model @@ -209,6 +205,11 @@ def mobilenet_v2(pretrained=False, scale=1.0, **kwargs): # build mobilenet v2 with scale=0.5 model = mobilenet_v2(scale=0.5) + + x = paddle.rand([1, 3, 224, 224]) + out = model(x) + + print(out.shape) """ model = _mobilenet( 'mobilenetv2_' + str(scale), pretrained, scale=scale, **kwargs) diff --git a/python/paddle/vision/models/shufflenetv2.py b/python/paddle/vision/models/shufflenetv2.py index 041f3fc749b6c..90e967ee22b35 100644 --- a/python/paddle/vision/models/shufflenetv2.py +++ b/python/paddle/vision/models/shufflenetv2.py @@ -18,37 +18,50 @@ import paddle import paddle.nn as nn -from paddle.fluid.param_attr import ParamAttr -from paddle.nn import AdaptiveAvgPool2D, BatchNorm, Conv2D, Linear, MaxPool2D +from paddle.nn import AdaptiveAvgPool2D, Linear, MaxPool2D from paddle.utils.download import get_weights_path_from_url +from ..ops import ConvNormActivation + __all__ = [] model_urls = { "shufflenet_v2_x0_25": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x0_25_pretrained.pdparams", - "e753404cbd95027759c5f56ecd6c9c4b", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_x0_25.pdparams", + "1e509b4c140eeb096bb16e214796d03b", ), "shufflenet_v2_x0_33": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x0_33_pretrained.pdparams", - "776e3cf9a4923abdfce789c45b8fe1f2", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_x0_33.pdparams", + "3d7b3ab0eaa5c0927ff1026d31b729bd", ), "shufflenet_v2_x0_5": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x0_5_pretrained.pdparams", - "e3649cf531566917e2969487d2bc6b60", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_x0_5.pdparams", + "5e5cee182a7793c4e4c73949b1a71bd4", ), "shufflenet_v2_x1_0": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x1_0_pretrained.pdparams", - "7821c348ea34e58847c43a08a4ac0bdf", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_x1_0.pdparams", + "122d42478b9e81eb49f8a9ede327b1a4", ), "shufflenet_v2_x1_5": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x1_5_pretrained.pdparams", - "93a07fa557ab2d8803550f39e5b6c391", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_x1_5.pdparams", + "faced5827380d73531d0ee027c67826d", ), "shufflenet_v2_x2_0": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_x2_0_pretrained.pdparams", - "4ab1f622fd0d341e0f84b4e057797563", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_x2_0.pdparams", + "cd3dddcd8305e7bcd8ad14d1c69a5784", ), "shufflenet_v2_swish": ( - "https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/ShuffleNetV2_swish_pretrained.pdparams", - "daff38b3df1b3748fccbb13cfdf02519", ), + "https://paddle-hapi.bj.bcebos.com/models/shufflenet_v2_swish.pdparams", + "adde0aa3b023e5b0c94a68be1c394b84", ), } +def create_activation_layer(act): + if act == "swish": + return nn.Swish + elif act == "relu": + return nn.ReLU + elif act is None: + return None + else: + raise RuntimeError( + "The activation function is not supported: {}".format(act)) + + def channel_shuffle(x, groups): batch_size, num_channels, height, width = x.shape[0:4] channels_per_group = num_channels // groups @@ -65,61 +78,37 @@ def channel_shuffle(x, groups): return x -class ConvBNLayer(nn.Layer): +class InvertedResidual(nn.Layer): def __init__(self, in_channels, out_channels, - kernel_size, stride, - padding, - groups=1, - act=None): - super(ConvBNLayer, self).__init__() - self._conv = Conv2D( - in_channels=in_channels, - out_channels=out_channels, - kernel_size=kernel_size, - stride=stride, - padding=padding, - groups=groups, - weight_attr=ParamAttr(initializer=nn.initializer.KaimingNormal()), - bias_attr=False, ) - - self._batch_norm = BatchNorm(out_channels, act=act) - - def forward(self, inputs): - x = self._conv(inputs) - x = self._batch_norm(x) - return x - - -class InvertedResidual(nn.Layer): - def __init__(self, in_channels, out_channels, stride, act="relu"): + activation_layer=nn.ReLU): super(InvertedResidual, self).__init__() - self._conv_pw = ConvBNLayer( + self._conv_pw = ConvNormActivation( in_channels=in_channels // 2, out_channels=out_channels // 2, kernel_size=1, stride=1, padding=0, groups=1, - act=act) - self._conv_dw = ConvBNLayer( + activation_layer=activation_layer) + self._conv_dw = ConvNormActivation( in_channels=out_channels // 2, out_channels=out_channels // 2, kernel_size=3, stride=stride, padding=1, groups=out_channels // 2, - act=None) - self._conv_linear = ConvBNLayer( + activation_layer=None) + self._conv_linear = ConvNormActivation( in_channels=out_channels // 2, out_channels=out_channels // 2, kernel_size=1, stride=1, padding=0, groups=1, - act=act) + activation_layer=activation_layer) def forward(self, inputs): x1, x2 = paddle.split( @@ -134,51 +123,55 @@ def forward(self, inputs): class InvertedResidualDS(nn.Layer): - def __init__(self, in_channels, out_channels, stride, act="relu"): + def __init__(self, + in_channels, + out_channels, + stride, + activation_layer=nn.ReLU): super(InvertedResidualDS, self).__init__() # branch1 - self._conv_dw_1 = ConvBNLayer( + self._conv_dw_1 = ConvNormActivation( in_channels=in_channels, out_channels=in_channels, kernel_size=3, stride=stride, padding=1, groups=in_channels, - act=None) - self._conv_linear_1 = ConvBNLayer( + activation_layer=None) + self._conv_linear_1 = ConvNormActivation( in_channels=in_channels, out_channels=out_channels // 2, kernel_size=1, stride=1, padding=0, groups=1, - act=act) + activation_layer=activation_layer) # branch2 - self._conv_pw_2 = ConvBNLayer( + self._conv_pw_2 = ConvNormActivation( in_channels=in_channels, out_channels=out_channels // 2, kernel_size=1, stride=1, padding=0, groups=1, - act=act) - self._conv_dw_2 = ConvBNLayer( + activation_layer=activation_layer) + self._conv_dw_2 = ConvNormActivation( in_channels=out_channels // 2, out_channels=out_channels // 2, kernel_size=3, stride=stride, padding=1, groups=out_channels // 2, - act=None) - self._conv_linear_2 = ConvBNLayer( + activation_layer=None) + self._conv_linear_2 = ConvNormActivation( in_channels=out_channels // 2, out_channels=out_channels // 2, kernel_size=1, stride=1, padding=0, groups=1, - act=act) + activation_layer=activation_layer) def forward(self, inputs): x1 = self._conv_dw_1(inputs) @@ -221,6 +214,7 @@ def __init__(self, scale=1.0, act="relu", num_classes=1000, with_pool=True): self.num_classes = num_classes self.with_pool = with_pool stage_repeats = [4, 8, 4] + activation_layer = create_activation_layer(act) if scale == 0.25: stage_out_channels = [-1, 24, 24, 48, 96, 512] @@ -238,13 +232,13 @@ def __init__(self, scale=1.0, act="relu", num_classes=1000, with_pool=True): raise NotImplementedError("This scale size:[" + str(scale) + "] is not implemented!") # 1. conv1 - self._conv1 = ConvBNLayer( + self._conv1 = ConvNormActivation( in_channels=3, out_channels=stage_out_channels[1], kernel_size=3, stride=2, padding=1, - act=act) + activation_layer=activation_layer) self._max_pool = MaxPool2D(kernel_size=3, stride=2, padding=1) # 2. bottleneck sequences @@ -257,7 +251,7 @@ def __init__(self, scale=1.0, act="relu", num_classes=1000, with_pool=True): in_channels=stage_out_channels[stage_id + 1], out_channels=stage_out_channels[stage_id + 2], stride=2, - act=act), + activation_layer=activation_layer), name=str(stage_id + 2) + "_" + str(i + 1)) else: block = self.add_sublayer( @@ -265,17 +259,17 @@ def __init__(self, scale=1.0, act="relu", num_classes=1000, with_pool=True): in_channels=stage_out_channels[stage_id + 2], out_channels=stage_out_channels[stage_id + 2], stride=1, - act=act), + activation_layer=activation_layer), name=str(stage_id + 2) + "_" + str(i + 1)) self._block_list.append(block) # 3. last_conv - self._last_conv = ConvBNLayer( + self._last_conv = ConvNormActivation( in_channels=stage_out_channels[-2], out_channels=stage_out_channels[-1], kernel_size=1, stride=1, padding=0, - act=act) + activation_layer=activation_layer) # 4. pool if with_pool: self._pool2d_avg = AdaptiveAvgPool2D(1) diff --git a/python/paddle/vision/ops.py b/python/paddle/vision/ops.py index 2d60fd4561480..e4dd4c797fef6 100644 --- a/python/paddle/vision/ops.py +++ b/python/paddle/vision/ops.py @@ -1335,13 +1335,13 @@ class ConvNormActivation(Sequential): Args: in_channels (int): Number of channels in the input image out_channels (int): Number of channels produced by the Convolution-Normalzation-Activation block - kernel_size: (int, optional): Size of the convolving kernel. Default: 3 - stride (int, optional): Stride of the convolution. Default: 1 - padding (int, tuple or str, optional): Padding added to all four sides of the input. Default: None, + kernel_size: (int|list|tuple, optional): Size of the convolving kernel. Default: 3 + stride (int|list|tuple, optional): Stride of the convolution. Default: 1 + padding (int|str|tuple|list, optional): Padding added to all four sides of the input. Default: None, in wich case it will calculated as ``padding = (kernel_size - 1) // 2 * dilation`` groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 norm_layer (Callable[..., paddle.nn.Layer], optional): Norm layer that will be stacked on top of the convolutiuon layer. - If ``None`` this layer wont be used. Default: ``paddle.nn.BatchNorm2d`` + If ``None`` this layer wont be used. Default: ``paddle.nn.BatchNorm2D`` activation_layer (Callable[..., paddle.nn.Layer], optional): Activation function which will be stacked on top of the normalization layer (if not ``None``), otherwise on top of the conv layer. If ``None`` this layer wont be used. Default: ``paddle.nn.ReLU`` dilation (int): Spacing between kernel elements. Default: 1 From 6700294c9354fffba55229fe60ab81016ac45cb8 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Sat, 23 Apr 2022 10:55:55 +0800 Subject: [PATCH 39/66] [Performance]Remove CudaStreamSychornize in ClipGradByGlobalNorm (#42132) --- python/paddle/fluid/clip.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/python/paddle/fluid/clip.py b/python/paddle/fluid/clip.py index 0ba980c3e9233..172929608dbde 100644 --- a/python/paddle/fluid/clip.py +++ b/python/paddle/fluid/clip.py @@ -468,10 +468,15 @@ class ClipGradByGlobalNorm(ClipGradBase): sdg.step() """ - def __init__(self, clip_norm, group_name="default_group"): + def __init__(self, + clip_norm, + group_name="default_group", + auto_skip_clip=False): super(ClipGradByGlobalNorm, self).__init__() self.clip_norm = float(clip_norm) self.group_name = group_name + assert isinstance(auto_skip_clip, bool) + self.auto_skip_clip = auto_skip_clip def __str__(self): return "Gradient Clip By GlobalNorm, global_norm=%f" % (self.clip_norm) @@ -524,14 +529,19 @@ def _dygraph_clip(self, params_grads): max_global_norm = layers.fill_constant( shape=[1], dtype=global_norm_var.dtype, value=self.clip_norm) - # only when global_norm_var > max_global_norm, grad need clip need_clip = False - if global_norm_var > max_global_norm: + if not self.auto_skip_clip: # always apply clip + need_clip = True + clip_var = layers.elementwise_div( + x=max_global_norm, + y=layers.elementwise_max( + x=global_norm_var, y=max_global_norm)) + elif global_norm_var > max_global_norm: + # only when global_norm_var > max_global_norm, grad need clip need_clip = True - - if need_clip: clip_var = layers.elementwise_div( x=max_global_norm, y=global_norm_var) + for p, g in params_grads: if g is None: continue From 1587ad07345b2d258fc150384610b0e7638f6e1f Mon Sep 17 00:00:00 2001 From: TTerror Date: Sat, 23 Apr 2022 13:22:31 +0800 Subject: [PATCH 40/66] update reduce_max for kunlun, *test=kunlun (#42116) --- paddle/fluid/operators/reduce_ops/reduce_max_op_xpu.cc | 5 ++--- paddle/fluid/platform/device/xpu/xpu2_op_list.h | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/operators/reduce_ops/reduce_max_op_xpu.cc b/paddle/fluid/operators/reduce_ops/reduce_max_op_xpu.cc index 15d672da04bec..1c1269a08dbdc 100644 --- a/paddle/fluid/operators/reduce_ops/reduce_max_op_xpu.cc +++ b/paddle/fluid/operators/reduce_ops/reduce_max_op_xpu.cc @@ -105,11 +105,10 @@ class ReduceMaxGradXPUKernel : public framework::OpKernel { " wrong value[%d %s].", r, XPUAPIErrorMsg[r])); // step 2. comparse out_brocast and x - r = xpu::elementwise_equal(dev_ctx.x_context(), x_data, brocast1, equal, - x->numel()); + r = xpu::equal(dev_ctx.x_context(), x_data, brocast1, equal, x->numel()); PADDLE_ENFORCE_EQ( r == xpu::Error_t::SUCCESS, true, - platform::errors::External("XPU elementwise_equal in reduce_max_grad " + platform::errors::External("XPU equal in reduce_max_grad " "op return wrong value[%d %s].", r, XPUAPIErrorMsg[r])); // step 3. get x_grad diff --git a/paddle/fluid/platform/device/xpu/xpu2_op_list.h b/paddle/fluid/platform/device/xpu/xpu2_op_list.h index 357644b62d3ed..583014b6f4773 100644 --- a/paddle/fluid/platform/device/xpu/xpu2_op_list.h +++ b/paddle/fluid/platform/device/xpu/xpu2_op_list.h @@ -57,6 +57,9 @@ XPUOpMap& get_kl2_ops() { pOpKernelType(vartype::BOOL, XPUPlace()), pOpKernelType(vartype::INT64, XPUPlace()), pOpKernelType(vartype::INT32, XPUPlace())})}, + {"check_finite_and_unscale", + XPUKernelSet({pOpKernelType(vartype::FP32, XPUPlace()), + pOpKernelType(vartype::FP16, XPUPlace())})}, {"clip", XPUKernelSet({pOpKernelType(vartype::FP32, XPUPlace())})}, {"concat_grad", XPUKernelSet({pOpKernelType(vartype::FP32, XPUPlace()), pOpKernelType(vartype::FP16, XPUPlace())})}, From 79ac8870ec71c99188f6d487ba74922cf90468a5 Mon Sep 17 00:00:00 2001 From: Aurelius84 Date: Sat, 23 Apr 2022 13:36:52 +0800 Subject: [PATCH 41/66] [Performance]Set ShapeKernel with ALL_BACKEND and ALL_LAYOUT (#42138) * [Performance]Set ShapeKernel with ALL_BACKEND and ALL_LAYOUT * [Performance]Set ShapeKernel with ALL_BACKEND and ALL_LAYOUT --- paddle/phi/kernels/shape_kernel.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/paddle/phi/kernels/shape_kernel.cc b/paddle/phi/kernels/shape_kernel.cc index dd26a7edc9cdd..f87b5014c1207 100644 --- a/paddle/phi/kernels/shape_kernel.cc +++ b/paddle/phi/kernels/shape_kernel.cc @@ -63,5 +63,7 @@ PD_REGISTER_KERNEL(shape, double, phi::dtype::complex, phi::dtype::complex, - phi::dtype::float16) {} + phi::dtype::float16) { + kernel->InputAt(0).SetBackend(phi::Backend::ALL_BACKEND); +} #endif From c56fffb43ea2116eaebe46803a2c481ab0bde7fe Mon Sep 17 00:00:00 2001 From: zyfncg Date: Sat, 23 Apr 2022 16:56:24 +0800 Subject: [PATCH 42/66] optimize performance of dygraph (#42137) --- paddle/fluid/framework/infershape_utils.cc | 9 +- paddle/fluid/framework/operator.cc | 12 +- paddle/fluid/imperative/prepared_operator.cc | 36 +++- .../dialect/phi/pass/phi_op_convert_pass.cc | 2 +- paddle/phi/core/compat/op_utils.h | 19 +- paddle/phi/tests/ops/test_op_signature.cc | 188 +++++++++++------- 6 files changed, 166 insertions(+), 100 deletions(-) diff --git a/paddle/fluid/framework/infershape_utils.cc b/paddle/fluid/framework/infershape_utils.cc index 68ee68fdd076a..6deebe93dcc62 100644 --- a/paddle/fluid/framework/infershape_utils.cc +++ b/paddle/fluid/framework/infershape_utils.cc @@ -402,12 +402,11 @@ std::vector CompatInferMetaContext::MutableOutputBetween( CompatInferMetaContext BuildInferMetaContext(InferShapeContext* ctx, const std::string& op_type) { // 1. get kernel args - auto arg_map_fn = phi::OpUtilsMap::Instance().GetArgumentMappingFn(op_type); - PADDLE_ENFORCE_NOT_NULL( - arg_map_fn, platform::errors::NotFound( - "The ArgumentMappingFn of %s op is not found.", op_type)); + auto* arg_map_fn = phi::OpUtilsMap::Instance().GetArgumentMappingFn(op_type); InferShapeArgumentMappingContext arg_map_context(*ctx); - auto signature = arg_map_fn(arg_map_context); + KernelSignature signature = + arg_map_fn ? (*arg_map_fn)(arg_map_context) + : phi::DefaultKernelSignatureMap::Instance().Get(op_type); VLOG(3) << "BuildInferMetaContext: op kernel signature - " << signature; // 2. build infermeta context diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index 0c35786394a43..39097a787c44c 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -2117,8 +2117,16 @@ KernelSignature OperatorWithKernel::GetExpectedPhiKernelArgs( const ExecutionContext& ctx) const { ExecutionArgumentMappingContext arg_mapping_ctx(ctx); if (arg_map_fn_ == nullptr) { - arg_map_fn_.reset(new phi::ArgumentMappingFn( - phi::OpUtilsMap::Instance().GetArgumentMappingFn(Type()))); + auto* arg_map_fn = phi::OpUtilsMap::Instance().GetArgumentMappingFn(type_); + if (arg_map_fn) { + arg_map_fn_.reset(new phi::ArgumentMappingFn(*arg_map_fn)); + } else { + auto func = + [this](const phi::ArgumentMappingContext& ctx) -> KernelSignature { + return phi::DefaultKernelSignatureMap::Instance().Get(type_); + }; + arg_map_fn_.reset(new phi::ArgumentMappingFn(func)); + } } return (*arg_map_fn_)(arg_mapping_ctx); } diff --git a/paddle/fluid/imperative/prepared_operator.cc b/paddle/fluid/imperative/prepared_operator.cc index cef7417ea4195..fdeda8aa9701a 100644 --- a/paddle/fluid/imperative/prepared_operator.cc +++ b/paddle/fluid/imperative/prepared_operator.cc @@ -37,6 +37,8 @@ namespace paddle { namespace imperative { static const phi::Kernel empty_kernel; +static const framework::RuntimeContext empty_ctx({}, {}); +static const framework::Scope empty_scope; const std::shared_ptr& GetVariableWrapper( const std::shared_ptr& var) { @@ -138,8 +140,6 @@ PreparedOp PrepareImpl(const NameVarMap& ins, platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance(); auto* dev_ctx = pool.Get(place); - framework::RuntimeContext ctx({}, {}); - #ifdef PADDLE_WITH_MKLDNN // MKLDNN variant of code reads attributes in some of GetKernelTypeForVar and // GetKernelType functions, so we need to copy the attributes there. @@ -158,7 +158,7 @@ PreparedOp PrepareImpl(const NameVarMap& ins, // 1. get expected kernel key auto dygraph_exe_ctx = DygraphExecutionContext( - op, framework::Scope(), *dev_ctx, ctx, ins, outs, attrs, default_attrs); + op, empty_scope, *dev_ctx, empty_ctx, ins, outs, attrs, default_attrs); auto expected_kernel_key = op.GetExpectedKernelType(dygraph_exe_ctx); framework::KernelSignature pt_kernel_signature; @@ -172,11 +172,26 @@ PreparedOp PrepareImpl(const NameVarMap& ins, paddle::platform::is_in_xpu_black_list(op.Type()); #endif - if (phi::KernelFactory::Instance().HasCompatiblePhiKernel(op.Type())) { - pt_kernel_signature = - std::move(op.GetExpectedPhiKernelArgs(dygraph_exe_ctx)); - VLOG(6) << pt_kernel_signature; + bool has_phi_kernel = false; + + const auto* arg_map_fn = + phi::OpUtilsMap::Instance().GetArgumentMappingFn(op.Type()); + if (arg_map_fn) { + has_phi_kernel = true; + pt_kernel_signature = (*arg_map_fn)( + framework::ExecutionArgumentMappingContext(dygraph_exe_ctx)); + } else { + const auto* kernel_sig = + phi::DefaultKernelSignatureMap::Instance().GetNullable(op.Type()); + if (kernel_sig) { + has_phi_kernel = true; + pt_kernel_signature = *kernel_sig; + } + } + + if (has_phi_kernel) { + VLOG(6) << pt_kernel_signature; pt_kernel_name = pt_kernel_signature.name; // NOTE(Liu-xiandong): The register kernel used KP have library_type[KP], // But the default library_type is Plain, so we need to modify the @@ -231,7 +246,7 @@ PreparedOp PrepareImpl(const NameVarMap& ins, dev_ctx = pool.Get(expected_kernel_key.place_); } - return PreparedOp(op, ctx, expected_kernel_key, + return PreparedOp(op, empty_ctx, expected_kernel_key, std::move(pt_kernel_signature), pt_kernel, dev_ctx); } else { VLOG(6) << "Dynamic mode ChoosePhiKernel - kernel `" << pt_kernel_name @@ -280,7 +295,7 @@ PreparedOp PrepareImpl(const NameVarMap& ins, << " | kernel key: " << pt_cpu_kernel_key << " | kernel: " << pt_cpu_kernel; auto* cpu_ctx = pool.Get(paddle::platform::CPUPlace()); - return PreparedOp(op, ctx, expected_kernel_key, + return PreparedOp(op, empty_ctx, expected_kernel_key, std::move(pt_kernel_signature), pt_cpu_kernel, cpu_ctx); } @@ -373,7 +388,8 @@ PreparedOp PrepareImpl(const NameVarMap& ins, dev_ctx = pool.Get(expected_kernel_key.place_); } - return PreparedOp(op, ctx, expected_kernel_key, kernel_iter->second, dev_ctx); + return PreparedOp(op, empty_ctx, expected_kernel_key, kernel_iter->second, + dev_ctx); } PreparedOp PreparedOp::Prepare(const NameVarMap& ins, diff --git a/paddle/infrt/dialect/phi/pass/phi_op_convert_pass.cc b/paddle/infrt/dialect/phi/pass/phi_op_convert_pass.cc index 76a4b84d06f21..862c9ae4ee5af 100644 --- a/paddle/infrt/dialect/phi/pass/phi_op_convert_pass.cc +++ b/paddle/infrt/dialect/phi/pass/phi_op_convert_pass.cc @@ -193,7 +193,7 @@ void PhiOpConvertPass::convertStage() { op->replaceAllUsesWith(kernel_op.getResults()); } else { ::phi::KernelSignature kernel_sign = - ::phi::OpUtilsMap::Instance().GetArgumentMappingFn(op_name)( + (*::phi::OpUtilsMap::Instance().GetArgumentMappingFn(op_name))( infrt::ProtoArgumentMappingContext(op)); VLOG(3) << "IncompatiblePhiKernel: op(" << op_name << "), kernel(" << kernel_sign.name << ")"; diff --git a/paddle/phi/core/compat/op_utils.h b/paddle/phi/core/compat/op_utils.h index 9c926fa871b67..bd19d403c9406 100644 --- a/paddle/phi/core/compat/op_utils.h +++ b/paddle/phi/core/compat/op_utils.h @@ -86,6 +86,14 @@ class DefaultKernelSignatureMap { return it->second; } + const KernelSignature* GetNullable(const std::string& op_type) const { + auto it = map_.find(op_type); + if (it != map_.end()) { + return &it->second; + } + return nullptr; + } + void Insert(std::string op_type, KernelSignature signature) { PADDLE_ENFORCE_NE( Has(op_type), @@ -148,16 +156,13 @@ class OpUtilsMap { } } - ArgumentMappingFn GetArgumentMappingFn(const std::string& op_type) const { + const ArgumentMappingFn* GetArgumentMappingFn( + const std::string& op_type) const { auto it = arg_mapping_fn_map_.find(op_type); if (it == arg_mapping_fn_map_.end()) { - auto func = - [&op_type](const ArgumentMappingContext& ctx) -> KernelSignature { - return DefaultKernelSignatureMap::Instance().Get(op_type); - }; - return func; + return nullptr; } else { - return it->second; + return &it->second; } } diff --git a/paddle/phi/tests/ops/test_op_signature.cc b/paddle/phi/tests/ops/test_op_signature.cc index 6c9f36a5e573f..4379dfd7cc4af 100644 --- a/paddle/phi/tests/ops/test_op_signature.cc +++ b/paddle/phi/tests/ops/test_op_signature.cc @@ -30,8 +30,8 @@ namespace tests { TEST(ARG_MAP, fill_constant) { TestArgumentMappingContext arg_case1( {"ShapeTensor", "ValueTensor"}, {}, {}, {}, {"Out"}); - auto signature1 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case1); + auto signature1 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case1); ASSERT_EQ(signature1.name, "full_sr"); TestArgumentMappingContext arg_case2( @@ -40,8 +40,8 @@ TEST(ARG_MAP, fill_constant) { {{"str_value", paddle::any{std::string{"10"}}}}, {}, {"Out"}); - auto signature2 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case2); + auto signature2 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case2); ASSERT_EQ(signature2.name, "full_sr"); TestArgumentMappingContext arg_case3( @@ -50,14 +50,14 @@ TEST(ARG_MAP, fill_constant) { {{"value", paddle::any{0}}, {"str_value", paddle::any{std::string{""}}}}, {}, {"Out"}); - auto signature3 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case3); + auto signature3 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case3); ASSERT_EQ(signature3.name, "full_sr"); TestArgumentMappingContext arg_case4( {"ShapeTensorList", "ValueTensor"}, {}, {}, {}, {"Out"}); - auto signature4 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case4); + auto signature4 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case4); ASSERT_EQ(signature4.name, "full_sr"); TestArgumentMappingContext arg_case5( @@ -66,8 +66,8 @@ TEST(ARG_MAP, fill_constant) { {{"str_value", paddle::any{std::string{"10"}}}}, {}, {"Out"}); - auto signature5 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case5); + auto signature5 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case5); ASSERT_EQ(signature5.name, "full_sr"); TestArgumentMappingContext arg_case6( @@ -76,8 +76,8 @@ TEST(ARG_MAP, fill_constant) { {{"value", paddle::any{0}}, {"str_value", paddle::any{std::string{""}}}}, {}, {"Out"}); - auto signature6 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case6); + auto signature6 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case6); ASSERT_EQ(signature6.name, "full_sr"); TestArgumentMappingContext arg_case7( @@ -86,8 +86,8 @@ TEST(ARG_MAP, fill_constant) { {{"shape", paddle::any{std::vector{2, 3}}}}, {}, {"Out"}); - auto signature7 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case7); + auto signature7 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case7); ASSERT_EQ(signature7.name, "full_sr"); TestArgumentMappingContext arg_case8( @@ -98,8 +98,8 @@ TEST(ARG_MAP, fill_constant) { {"str_value", paddle::any{std::string{""}}}}, {}, {"Out"}); - auto signature8 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case8); + auto signature8 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case8); ASSERT_EQ(signature8.name, "full_sr"); TestArgumentMappingContext arg_case9( @@ -109,8 +109,8 @@ TEST(ARG_MAP, fill_constant) { {"str_value", paddle::any{std::string{"10"}}}}, {}, {"Out"}); - auto signature9 = - OpUtilsMap::Instance().GetArgumentMappingFn("fill_constant")(arg_case9); + auto signature9 = (*OpUtilsMap::Instance().GetArgumentMappingFn( + "fill_constant"))(arg_case9); ASSERT_EQ(signature9.name, "full_sr"); } @@ -122,7 +122,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case) + .name, "set_value"); TestArgumentMappingContext arg_case1( @@ -132,7 +133,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case1).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case1) + .name, "set_value"); TestArgumentMappingContext arg_case2( @@ -142,7 +144,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case2).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case2) + .name, "set_value"); TestArgumentMappingContext arg_case3( @@ -152,7 +155,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case3).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case3) + .name, "set_value"); TestArgumentMappingContext arg_case4( @@ -162,7 +166,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case4).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case4) + .name, "set_value"); TestArgumentMappingContext arg_case5( @@ -172,7 +177,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case5).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case5) + .name, "set_value_with_tensor"); TestArgumentMappingContext arg_case6( @@ -182,7 +188,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case6).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case6) + .name, "set_value"); TestArgumentMappingContext arg_case7( @@ -192,7 +199,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case7).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case7) + .name, "set_value"); TestArgumentMappingContext arg_case8( @@ -202,7 +210,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case8).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case8) + .name, "set_value"); TestArgumentMappingContext arg_case9( @@ -212,7 +221,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case9).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case9) + .name, "set_value"); TestArgumentMappingContext arg_case10( @@ -222,7 +232,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case10).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case10) + .name, "set_value_with_tensor"); TestArgumentMappingContext arg_case11( @@ -232,7 +243,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case11).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case11) + .name, "set_value"); TestArgumentMappingContext arg_case12( @@ -242,7 +254,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case12).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case12) + .name, "set_value"); TestArgumentMappingContext arg_case13( @@ -252,7 +265,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case13).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case13) + .name, "set_value"); TestArgumentMappingContext arg_case14( @@ -262,13 +276,15 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case14).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case14) + .name, "set_value"); TestArgumentMappingContext arg_case15( {"Input", "StartsTensorList", "ValueTensor"}, {}, {}, {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case15).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case15) + .name, "set_value_with_tensor"); TestArgumentMappingContext arg_case16( @@ -278,7 +294,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case16).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case16) + .name, "set_value"); TestArgumentMappingContext arg_case17( @@ -288,7 +305,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case17).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case17) + .name, "set_value"); TestArgumentMappingContext arg_case18( @@ -298,7 +316,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case18).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case18) + .name, "set_value"); TestArgumentMappingContext arg_case19( @@ -308,7 +327,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case19).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case19) + .name, "set_value"); TestArgumentMappingContext arg_case20( @@ -318,7 +338,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case20).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case20) + .name, "set_value"); TestArgumentMappingContext arg_case21( @@ -328,7 +349,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case21).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case21) + .name, "set_value_with_tensor"); TestArgumentMappingContext arg_case22( @@ -338,7 +360,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case22).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case22) + .name, "set_value"); TestArgumentMappingContext arg_case23( @@ -348,7 +371,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case23).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case23) + .name, "set_value"); TestArgumentMappingContext arg_case24( @@ -358,7 +382,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case24).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case24) + .name, "set_value"); TestArgumentMappingContext arg_case25( @@ -368,13 +393,15 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case25).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case25) + .name, "set_value"); TestArgumentMappingContext arg_case26( {"Input", "EndsTensorList", "ValueTensor"}, {}, {}, {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case26).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case26) + .name, "set_value_with_tensor"); TestArgumentMappingContext arg_case27( @@ -384,7 +411,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case27).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case27) + .name, "set_value"); TestArgumentMappingContext arg_case28( @@ -394,7 +422,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case28).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case28) + .name, "set_value"); TestArgumentMappingContext arg_case29( @@ -404,7 +433,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case29).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case29) + .name, "set_value"); TestArgumentMappingContext arg_case30( @@ -414,7 +444,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case30).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case30) + .name, "set_value"); TestArgumentMappingContext arg_case31( @@ -424,13 +455,15 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case31).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case31) + .name, "set_value"); TestArgumentMappingContext arg_case32( {"Input", "StepsTensorList", "ValueTensor"}, {}, {}, {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case32).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case32) + .name, "set_value_with_tensor"); TestArgumentMappingContext arg_case33( @@ -440,7 +473,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case33).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case33) + .name, "set_value"); TestArgumentMappingContext arg_case34( @@ -450,7 +484,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case34).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case34) + .name, "set_value"); TestArgumentMappingContext arg_case35( @@ -460,7 +495,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case35).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case35) + .name, "set_value"); TestArgumentMappingContext arg_case36( @@ -470,7 +506,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case36).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case36) + .name, "set_value"); TestArgumentMappingContext arg_case37( @@ -480,7 +517,8 @@ TEST(ARG_MAP, set_value) { {"Out"}, {}); ASSERT_EQ( - OpUtilsMap::Instance().GetArgumentMappingFn("set_value")(arg_case37).name, + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value"))(arg_case37) + .name, "set_value"); } @@ -491,10 +529,10 @@ TEST(ARG_MAP, set_value_grad) { {}, {"Input@GRAD", "ValueTensor@GRAD"}, {}); - ASSERT_EQ(OpUtilsMap::Instance() - .GetArgumentMappingFn("set_value_grad")(arg_case) - .name, - "set_value_grad"); + ASSERT_EQ( + (*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))(arg_case) + .name, + "set_value_grad"); TestArgumentMappingContext arg_case1( {"Out@GRAD", "StartsTensorList", "StepsTensorList"}, @@ -502,8 +540,8 @@ TEST(ARG_MAP, set_value_grad) { {}, {"Input@GRAD", "ValueTensor@GRAD"}, {}); - ASSERT_EQ(OpUtilsMap::Instance() - .GetArgumentMappingFn("set_value_grad")(arg_case1) + ASSERT_EQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))( + arg_case1) .name, "set_value_grad"); @@ -512,8 +550,8 @@ TEST(ARG_MAP, set_value_grad) { {}, {"Input@GRAD", "ValueTensor@GRAD"}, {}); - ASSERT_EQ(OpUtilsMap::Instance() - .GetArgumentMappingFn("set_value_grad")(arg_case2) + ASSERT_EQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))( + arg_case2) .name, "set_value_grad"); @@ -523,8 +561,8 @@ TEST(ARG_MAP, set_value_grad) { {}, {"Input@GRAD", "ValueTensor@GRAD"}, {}); - ASSERT_EQ(OpUtilsMap::Instance() - .GetArgumentMappingFn("set_value_grad")(arg_case3) + ASSERT_EQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))( + arg_case3) .name, "set_value_grad"); @@ -533,8 +571,8 @@ TEST(ARG_MAP, set_value_grad) { {}, {"Input@GRAD", "ValueTensor@GRAD"}, {}); - ASSERT_EQ(OpUtilsMap::Instance() - .GetArgumentMappingFn("set_value_grad")(arg_case4) + ASSERT_EQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))( + arg_case4) .name, "set_value_grad"); @@ -543,8 +581,8 @@ TEST(ARG_MAP, set_value_grad) { {}, {"Input@GRAD", "ValueTensor@GRAD"}, {}); - ASSERT_EQ(OpUtilsMap::Instance() - .GetArgumentMappingFn("set_value_grad")(arg_case5) + ASSERT_EQ((*OpUtilsMap::Instance().GetArgumentMappingFn("set_value_grad"))( + arg_case5) .name, "set_value_grad"); } @@ -558,7 +596,7 @@ TEST(ARG_MAP, allclose) { {"Out"}, {}); auto signature1 = - OpUtilsMap::Instance().GetArgumentMappingFn("allclose")(arg_case1); + (*OpUtilsMap::Instance().GetArgumentMappingFn("allclose"))(arg_case1); ASSERT_EQ(signature1.name, "allclose"); ASSERT_EQ(signature1.attr_names[0], "Rtol"); @@ -570,7 +608,7 @@ TEST(ARG_MAP, allclose) { {"Out"}, {}); auto signature2 = - OpUtilsMap::Instance().GetArgumentMappingFn("allclose")(arg_case2); + (*OpUtilsMap::Instance().GetArgumentMappingFn("allclose"))(arg_case2); ASSERT_EQ(signature2.name, "allclose"); ASSERT_EQ(signature2.attr_names[1], "Atol"); } @@ -578,18 +616,18 @@ TEST(ARG_MAP, allclose) { TEST(ARG_MAP, reshape) { TestArgumentMappingContext arg_case1({"X", "ShapeTensor"}, {}, {}, {"Out"}); auto signature1 = - OpUtilsMap::Instance().GetArgumentMappingFn("reshape2")(arg_case1); + (*OpUtilsMap::Instance().GetArgumentMappingFn("reshape2"))(arg_case1); ASSERT_EQ(signature1.name, "reshape"); TestArgumentMappingContext arg_case2({"X", "Shape"}, {}, {}, {"Out"}); auto signature2 = - OpUtilsMap::Instance().GetArgumentMappingFn("reshape2")(arg_case2); + (*OpUtilsMap::Instance().GetArgumentMappingFn("reshape2"))(arg_case2); ASSERT_EQ(signature2.name, "reshape"); TestArgumentMappingContext arg_case3( {"X"}, {}, {{"shape", paddle::any(std::vector({1, 2}))}}, {"Out"}); auto signature3 = - OpUtilsMap::Instance().GetArgumentMappingFn("reshape2")(arg_case3); + (*OpUtilsMap::Instance().GetArgumentMappingFn("reshape2"))(arg_case3); ASSERT_EQ(signature3.name, "reshape"); } From 532c3b4ca32b8d6624673ee829e5c4c87654a5ea Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Sun, 24 Apr 2022 10:09:35 +0800 Subject: [PATCH 43/66] refine optest logic for bfloat16 (#42151) --- python/paddle/fluid/tests/unittests/op_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/paddle/fluid/tests/unittests/op_test.py b/python/paddle/fluid/tests/unittests/op_test.py index cfe0d4e32ef7a..738ed90b12e65 100644 --- a/python/paddle/fluid/tests/unittests/op_test.py +++ b/python/paddle/fluid/tests/unittests/op_test.py @@ -1506,6 +1506,12 @@ def find_actual_value(self, name): return imperative_actual, imperative_actual_t def convert_uint16_to_float_ifneed(self, actual_np, expect_np): + if actual_np.dtype == np.uint16 and expect_np.dtype in [ + np.float32, np.float64 + ]: + self.rtol = 1.e-2 + else: + self.rtol = 1.e-5 if self.op_test.is_bfloat16_op(): if actual_np.dtype == np.uint16: actual_np = convert_uint16_to_float(actual_np) From b1c6378da874017a02051e72ca82600142fbba78 Mon Sep 17 00:00:00 2001 From: tianshuo78520a <707759223@qq.com> Date: Sun, 24 Apr 2022 10:43:53 +0800 Subject: [PATCH 44/66] Update Mac cmake version >=3.15 (#41456) * Update Mac cmake version >=3.15 * notest;read test1 notest;read test2 notest;read test3 * fix inference link error * fix inference link error * fix windows link error * fix cmake_policy * fix build big size --- CMakeLists.txt | 2 +- cmake/external/boost.cmake | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b0680a782cf7f..e7d16ecfd7002 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,7 @@ if(APPLE AND WITH_ARM) cmake_minimum_required(VERSION 3.19.2) cmake_policy(VERSION 3.19.2) else(APPLE AND WITH_ARM) - cmake_minimum_required(VERSION 3.10) + cmake_minimum_required(VERSION 3.15) cmake_policy(VERSION 3.10) endif(APPLE AND WITH_ARM) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") diff --git a/cmake/external/boost.cmake b/cmake/external/boost.cmake index 69eb62bfdc654..e47b608341bee 100644 --- a/cmake/external/boost.cmake +++ b/cmake/external/boost.cmake @@ -32,7 +32,6 @@ set(BOOST_URL "http://paddlepaddledeps.bj.bcebos.com/${BOOST_TAR}.tar.gz" CACH MESSAGE(STATUS "BOOST_VERSION: ${BOOST_VER}, BOOST_URL: ${BOOST_URL}") set(BOOST_PREFIX_DIR ${THIRD_PARTY_PATH}/boost) - set(BOOST_INCLUDE_DIR "${THIRD_PARTY_PATH}/boost/src/extern_boost" CACHE PATH "boost include directory." FORCE) set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM 1) include_directories(${BOOST_INCLUDE_DIR}) From 79f717d6b5b859b7aff5d1221a026cf8ee2e50ee Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Sun, 24 Apr 2022 10:53:03 +0800 Subject: [PATCH 45/66] Add paddle::variant and replace paddle::any (#42139) * add variant and replace any * split attribute --- paddle/fluid/framework/custom_operator.cc | 1 + .../framework/new_executor/interpretercore.cc | 1 + .../new_executor/interpretercore_util.cc | 1 + paddle/fluid/framework/operator.cc | 1 + paddle/fluid/framework/operator.h | 5 +- paddle/fluid/imperative/prepared_operator.h | 1 + paddle/phi/core/attribute.h | 50 + paddle/phi/core/kernel_context.cc | 32 +- paddle/phi/core/kernel_context.h | 15 +- paddle/phi/core/kernel_registry.h | 10 + paddle/phi/core/kernel_utils.h | 39 +- paddle/phi/core/type_defs.h | 31 + paddle/phi/tests/core/test_custom_kernel.cc | 5 - paddle/utils/variant.h | 2829 +++++++++++++++++ 14 files changed, 2993 insertions(+), 28 deletions(-) create mode 100644 paddle/phi/core/attribute.h create mode 100644 paddle/utils/variant.h diff --git a/paddle/fluid/framework/custom_operator.cc b/paddle/fluid/framework/custom_operator.cc index 3f28b2e8c7398..65c41e19ac423 100644 --- a/paddle/fluid/framework/custom_operator.cc +++ b/paddle/fluid/framework/custom_operator.cc @@ -39,6 +39,7 @@ limitations under the License. */ #include "paddle/phi/api/all.h" #include "paddle/phi/api/lib/utils/tensor_utils.h" #include "paddle/phi/core/compat/convert_utils.h" +#include "paddle/phi/core/tensor_utils.h" #include "paddle/utils/any.h" namespace paddle { diff --git a/paddle/fluid/framework/new_executor/interpretercore.cc b/paddle/fluid/framework/new_executor/interpretercore.cc index a4fcf0773f623..6735406aacde7 100644 --- a/paddle/fluid/framework/new_executor/interpretercore.cc +++ b/paddle/fluid/framework/new_executor/interpretercore.cc @@ -22,6 +22,7 @@ #include "paddle/fluid/framework/operator.h" #include "paddle/fluid/platform/os_info.h" #include "paddle/fluid/platform/profiler/event_tracing.h" +#include "paddle/phi/core/kernel_context.h" #ifdef PADDLE_WITH_MKLDNN #include "paddle/fluid/platform/mkldnn_helper.h" #endif diff --git a/paddle/fluid/framework/new_executor/interpretercore_util.cc b/paddle/fluid/framework/new_executor/interpretercore_util.cc index afddcb580b9d8..71893d661ed6b 100644 --- a/paddle/fluid/framework/new_executor/interpretercore_util.cc +++ b/paddle/fluid/framework/new_executor/interpretercore_util.cc @@ -19,6 +19,7 @@ #include "paddle/fluid/operators/controlflow/conditional_block_op_helper.h" #include "paddle/fluid/operators/controlflow/recurrent_op_helper.h" #include "paddle/fluid/operators/controlflow/while_op_helper.h" +#include "paddle/phi/core/kernel_context.h" #include "paddle/phi/core/kernel_factory.h" #ifdef PADDLE_WITH_MKLDNN diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index 39097a787c44c..da082f5d26f3b 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -35,6 +35,7 @@ limitations under the License. */ #include "paddle/fluid/platform/profiler/event_tracing.h" #include "paddle/phi/common/int_array.h" #include "paddle/phi/common/scalar.h" +#include "paddle/phi/core/kernel_context.h" #include "paddle/phi/core/kernel_factory.h" #include "paddle/phi/ops/compat/signatures.h" diff --git a/paddle/fluid/framework/operator.h b/paddle/fluid/framework/operator.h index f0887eb919c30..d85e81250563f 100644 --- a/paddle/fluid/framework/operator.h +++ b/paddle/fluid/framework/operator.h @@ -43,7 +43,6 @@ limitations under the License. */ #include "paddle/fluid/framework/convert_utils.h" #include "paddle/phi/core/compat/arg_map_context.h" #include "paddle/phi/core/compat/op_utils.h" -#include "paddle/phi/core/kernel_context.h" #include "paddle/phi/core/kernel_factory.h" namespace paddle { @@ -55,6 +54,10 @@ class Variable; } // namespace framework } // namespace paddle +namespace phi { +class KernelContext; +} + DECLARE_int32(inner_op_parallelism); namespace paddle { diff --git a/paddle/fluid/imperative/prepared_operator.h b/paddle/fluid/imperative/prepared_operator.h index 754b553bd192f..0e75775e91783 100644 --- a/paddle/fluid/imperative/prepared_operator.h +++ b/paddle/fluid/imperative/prepared_operator.h @@ -31,6 +31,7 @@ #include "paddle/fluid/framework/convert_utils.h" #include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/core/kernel_context.h" #include "paddle/phi/core/selected_rows.h" DECLARE_bool(use_mkldnn); diff --git a/paddle/phi/core/attribute.h b/paddle/phi/core/attribute.h new file mode 100644 index 0000000000000..d1b2920335576 --- /dev/null +++ b/paddle/phi/core/attribute.h @@ -0,0 +1,50 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "paddle/phi/common/data_type.h" +#include "paddle/phi/common/int_array.h" +#include "paddle/phi/common/layout.h" +#include "paddle/phi/common/scalar.h" +#include "paddle/utils/variant.h" + +namespace phi { + +class Place; + +// NOTE: Add needed type in the future +using Attribute = paddle::variant, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector, + Scalar, + std::vector, + IntArray, + DataType, + DataLayout, + Place>; + +} // namespace phi diff --git a/paddle/phi/core/kernel_context.cc b/paddle/phi/core/kernel_context.cc index cf862cbde18f9..9935a5bf5cd9f 100644 --- a/paddle/phi/core/kernel_context.cc +++ b/paddle/phi/core/kernel_context.cc @@ -73,7 +73,7 @@ void KernelContext::EmplaceBackOutputsWithoutSetRange( std::make_move_iterator(outputs.end())); } -void KernelContext::EmplaceBackAttr(paddle::any attr) { +void KernelContext::EmplaceBackAttr(Attribute attr) { attrs_.emplace_back(std::move(attr)); } @@ -113,4 +113,34 @@ const std::pair& KernelContext::OutputRangeAt(size_t idx) const { return output_range_.at(idx); } +template +const AttrType& KernelContext::AttrAt(size_t idx) const { + try { + return paddle::get(attrs_.at(idx)); + } catch (paddle::bad_variant_access const& ex) { + PADDLE_THROW(phi::errors::InvalidArgument( + "Attribute cast error in Op Kernel Context.")); + } +} + +template const bool& KernelContext::AttrAt(size_t idx) const; +template const int& KernelContext::AttrAt(size_t idx) const; +template const int64_t& KernelContext::AttrAt(size_t idx) const; +template const float& KernelContext::AttrAt(size_t idx) const; +template const double& KernelContext::AttrAt(size_t idx) const; +template const std::string& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt( + size_t idx) const; +template const Scalar& KernelContext::AttrAt(size_t idx) const; +template const std::vector& KernelContext::AttrAt(size_t idx) const; +template const IntArray& KernelContext::AttrAt(size_t idx) const; +template const DataType& KernelContext::AttrAt(size_t idx) const; +template const DataLayout& KernelContext::AttrAt(size_t idx) const; +template const Place& KernelContext::AttrAt(size_t idx) const; + } // namespace phi diff --git a/paddle/phi/core/kernel_context.h b/paddle/phi/core/kernel_context.h index ab4e044e62537..9e5660d9dc534 100644 --- a/paddle/phi/core/kernel_context.h +++ b/paddle/phi/core/kernel_context.h @@ -17,11 +17,11 @@ #include #include +#include "paddle/phi/core/attribute.h" #include "paddle/phi/core/device_context.h" #include "paddle/phi/core/enforce.h" #include "paddle/phi/core/tensor_base.h" #include "paddle/phi/core/tensor_utils.h" -#include "paddle/utils/any.h" #include "paddle/utils/optional.h" #include "paddle/utils/small_vector.h" @@ -64,7 +64,7 @@ class KernelContext { void EmplaceBackOutputsWithoutSetRange( paddle::SmallVector outputs); - void EmplaceBackAttr(paddle::any attr); + void EmplaceBackAttr(Attribute attr); const std::pair& InputRangeAt(size_t idx) const; @@ -128,14 +128,7 @@ class KernelContext { } template - AttrType AttrAt(size_t idx) const { - try { - return paddle::any_cast(attrs_.at(idx)); - } catch (paddle::bad_any_cast&) { - PADDLE_THROW(phi::errors::InvalidArgument( - "Attribute cast error in Op Kernel Context.")); - } - } + const AttrType& AttrAt(size_t idx) const; size_t InputsSize() const { return inputs_.size(); } size_t OutputsSize() const { return outputs_.size(); } @@ -146,7 +139,7 @@ class KernelContext { paddle::SmallVector inputs_; paddle::SmallVector outputs_; - paddle::SmallVector attrs_; + paddle::SmallVector attrs_; paddle::SmallVector> input_range_; paddle::SmallVector> output_range_; diff --git a/paddle/phi/core/kernel_registry.h b/paddle/phi/core/kernel_registry.h index b18fd9e05f92f..356ab58f40726 100644 --- a/paddle/phi/core/kernel_registry.h +++ b/paddle/phi/core/kernel_registry.h @@ -105,6 +105,11 @@ struct KernelArgsParseFunctor { default_tensor_layout, default_key.dtype(), arg_type); + } else if (arg_type == std::type_index(typeid(const StringTensor&))) { + args_def->AppendInput(default_key.backend(), + default_tensor_layout, + default_key.dtype(), + arg_type); } else if (arg_type == std::type_index(typeid(const SparseCooTensor&))) { args_def->AppendInput(default_key.backend(), default_tensor_layout, @@ -153,6 +158,11 @@ struct KernelArgsParseFunctor { default_tensor_layout, default_key.dtype(), arg_type); + } else if (arg_type == std::type_index(typeid(StringTensor*))) { + args_def->AppendOutput(default_key.backend(), + default_tensor_layout, + default_key.dtype(), + arg_type); } else { // Attribute deal with // TODO(chenweihang): now here allow any types of attribute, maybe diff --git a/paddle/phi/core/kernel_utils.h b/paddle/phi/core/kernel_utils.h index 55574ea03ab4a..ddc58f512bf14 100644 --- a/paddle/phi/core/kernel_utils.h +++ b/paddle/phi/core/kernel_utils.h @@ -168,6 +168,24 @@ namespace phi { } \ } +#define PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(attr_type) \ + template \ + struct KernelCallHelper { \ + template \ + static void Compute(KernelContext* ctx, PreviousArgs&... pargs) { \ + static_assert(out_idx == 0, \ + "Kernel's Attributes should appear before Outputs."); \ + const attr_type& arg = ctx->AttrAt(attr_idx); \ + KernelCallHelper:: \ + template Compute( \ + ctx, pargs..., arg); \ + } \ + } + #define PD_SPECIALIZE_KernelCallHelper_FOR_OUTPUT(tensor_type) \ template \ struct KernelCallHelper { \ @@ -270,19 +288,20 @@ struct KernelImpl { PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(int); PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(int64_t); PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(phi::dtype::float16); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const Scalar&); PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(DataType); PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(DataLayout); PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(Place); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const IntArray&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::string&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_KernelCallHelper_FOR_ATTRIBUTE(const std::vector&); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::string); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(Scalar); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(IntArray); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF( + std::vector); + PD_SPECIALIZE_KernelCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); /* Output Helpers */ diff --git a/paddle/phi/core/type_defs.h b/paddle/phi/core/type_defs.h index a1e7836088389..e3cbf2cedd077 100644 --- a/paddle/phi/core/type_defs.h +++ b/paddle/phi/core/type_defs.h @@ -15,9 +15,40 @@ #pragma once #include +#include +#include + +#include "paddle/phi/common/data_type.h" +#include "paddle/phi/common/int_array.h" +#include "paddle/phi/common/layout.h" +#include "paddle/phi/common/scalar.h" + +#include "paddle/utils/variant.h" namespace phi { +class Place; + +// NOTE: Add needed type in the future +using Attribute = paddle::variant, + std::vector, + std::vector, + std::vector, + std::vector, + std::vector, + Scalar, + std::vector, + IntArray, + DataType, + DataLayout, + Place>; + class Kernel; class KernelKey; class KernelArgsDef; diff --git a/paddle/phi/tests/core/test_custom_kernel.cc b/paddle/phi/tests/core/test_custom_kernel.cc index 07530f70b7ab5..2a5b8ec8fa000 100644 --- a/paddle/phi/tests/core/test_custom_kernel.cc +++ b/paddle/phi/tests/core/test_custom_kernel.cc @@ -49,7 +49,6 @@ void FakeDot(const Context& dev_ctx, float fake_attr_float, double fake_attr_double, int64_t fake_attr_int64, - phi::dtype::float16 fake_attr_f16, phi::DataType fake_attr_dtype, const phi::Scalar& fake_attr_scalar, const phi::IntArray& fake_attr_int_array, @@ -64,7 +63,6 @@ void FakeDot(const Context& dev_ctx, std::cout << "fake_attr_float: " << fake_attr_float << std::endl; std::cout << "fake_attr_double: " << fake_attr_double << std::endl; std::cout << "fake_attr_int64: " << fake_attr_int64 << std::endl; - std::cout << "fake_attr_f16: " << fake_attr_f16 << std::endl; std::cout << "fake_attr_dtype: " << fake_attr_dtype << std::endl; std::cout << "fake_attr_int64_vec: " << fake_attr_int64_vec.size() << std::endl; @@ -78,7 +76,6 @@ void FakeDot(const Context& dev_ctx, assert(fake_attr_float == 2); assert(fake_attr_double == 3); assert(fake_attr_int64 == 4); - assert(fake_attr_f16 == phi::dtype::float16(5)); assert(fake_attr_dtype == phi::DataType::UINT32); assert(fake_attr_int64_vec.size() == 0); assert(fake_attr_int_vec.size() == 0); @@ -248,7 +245,6 @@ TEST(CustomKernel, custom_kernel_dot) { float fake_attr_float = 2.0; double fake_attr_double = 3.0; int64_t fake_attr_int64 = 4; - phi::dtype::float16 fake_attr_f16 = phi::dtype::float16(5); phi::DataType fake_attr_dtype = phi::DataType::UINT32; paddle::framework::LoDTensor tmp_tensor; tmp_tensor.mutable_data({1}, phi::TransToPhiPlace(backend)); @@ -262,7 +258,6 @@ TEST(CustomKernel, custom_kernel_dot) { kernel_context.EmplaceBackAttr(fake_attr_float); kernel_context.EmplaceBackAttr(fake_attr_double); kernel_context.EmplaceBackAttr(fake_attr_int64); - kernel_context.EmplaceBackAttr(fake_attr_f16); kernel_context.EmplaceBackAttr(fake_attr_dtype); kernel_context.EmplaceBackAttr(fake_attr_scalar); kernel_context.EmplaceBackAttr(fake_attr_int_array); diff --git a/paddle/utils/variant.h b/paddle/utils/variant.h new file mode 100644 index 0000000000000..b856fa8f7a1d7 --- /dev/null +++ b/paddle/utils/variant.h @@ -0,0 +1,2829 @@ +// Copy from +// https://github.com/mpark/variant/blob/single-header/v1.4.0/variant.hpp +// Modify the following points: +// 1. modify namespace mpark to namespace paddle + +// MPark.Variant +// +// Copyright Michael Park, 2015-2017 +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.md or copy at +// http://boost.org/LICENSE_1_0.txt) + +#pragma once + +/* + variant synopsis + +namespace std { + + // 20.7.2, class template variant + template + class variant { + public: + + // 20.7.2.1, constructors + constexpr variant() noexcept(see below); + variant(const variant&); + variant(variant&&) noexcept(see below); + + template constexpr variant(T&&) noexcept(see below); + + template + constexpr explicit variant(in_place_type_t, Args&&...); + + template + constexpr explicit variant( + in_place_type_t, initializer_list, Args&&...); + + template + constexpr explicit variant(in_place_index_t, Args&&...); + + template + constexpr explicit variant( + in_place_index_t, initializer_list, Args&&...); + + // 20.7.2.2, destructor + ~variant(); + + // 20.7.2.3, assignment + variant& operator=(const variant&); + variant& operator=(variant&&) noexcept(see below); + + template variant& operator=(T&&) noexcept(see below); + + // 20.7.2.4, modifiers + template + T& emplace(Args&&...); + + template + T& emplace(initializer_list, Args&&...); + + template + variant_alternative& emplace(Args&&...); + + template + variant_alternative& emplace(initializer_list, Args&&...); + + // 20.7.2.5, value status + constexpr bool valueless_by_exception() const noexcept; + constexpr size_t index() const noexcept; + + // 20.7.2.6, swap + void swap(variant&) noexcept(see below); + }; + + // 20.7.3, variant helper classes + template struct variant_size; // undefined + + template + constexpr size_t variant_size_v = variant_size::value; + + template struct variant_size; + template struct variant_size; + template struct variant_size; + + template + struct variant_size>; + + template struct variant_alternative; // undefined + + template + using variant_alternative_t = typename variant_alternative::type; + + template struct variant_alternative; + template struct variant_alternative; + template struct variant_alternative; + + template + struct variant_alternative>; + + constexpr size_t variant_npos = -1; + + // 20.7.4, value access + template + constexpr bool holds_alternative(const variant&) noexcept; + + template + constexpr variant_alternative_t>& + get(variant&); + + template + constexpr variant_alternative_t>&& + get(variant&&); + + template + constexpr variant_alternative_t> const& + get(const variant&); + + template + constexpr variant_alternative_t> const&& + get(const variant&&); + + template + constexpr T& get(variant&); + + template + constexpr T&& get(variant&&); + + template + constexpr const T& get(const variant&); + + template + constexpr const T&& get(const variant&&); + + template + constexpr add_pointer_t>> + get_if(variant*) noexcept; + + template + constexpr add_pointer_t>> + get_if(const variant*) noexcept; + + template + constexpr add_pointer_t + get_if(variant*) noexcept; + + template + constexpr add_pointer_t + get_if(const variant*) noexcept; + + // 20.7.5, relational operators + template + constexpr bool operator==(const variant&, const variant&); + + template + constexpr bool operator!=(const variant&, const variant&); + + template + constexpr bool operator<(const variant&, const variant&); + + template + constexpr bool operator>(const variant&, const variant&); + + template + constexpr bool operator<=(const variant&, const variant&); + + template + constexpr bool operator>=(const variant&, const variant&); + + // 20.7.6, visitation + template + constexpr see below visit(Visitor&&, Variants&&...); + + // 20.7.7, class monostate + struct monostate; + + // 20.7.8, monostate relational operators + constexpr bool operator<(monostate, monostate) noexcept; + constexpr bool operator>(monostate, monostate) noexcept; + constexpr bool operator<=(monostate, monostate) noexcept; + constexpr bool operator>=(monostate, monostate) noexcept; + constexpr bool operator==(monostate, monostate) noexcept; + constexpr bool operator!=(monostate, monostate) noexcept; + + // 20.7.9, specialized algorithms + template + void swap(variant&, variant&) noexcept(see below); + + // 20.7.10, class bad_variant_access + class bad_variant_access; + + // 20.7.11, hash support + template struct hash; + template struct hash>; + template <> struct hash; + +} // namespace std + +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +// MPark.Variant +// +// Copyright Michael Park, 2015-2017 +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.md or copy at +// http://boost.org/LICENSE_1_0.txt) + +#ifndef MPARK_CONFIG_HPP +#define MPARK_CONFIG_HPP + +// MSVC 2015 Update 3. +#if __cplusplus < 201103L && (!defined(_MSC_VER) || _MSC_FULL_VER < 190024210) +#error "MPark.Variant requires C++11 support." +#endif + +#ifndef __has_attribute +#define __has_attribute(x) 0 +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +#ifndef __has_include +#define __has_include(x) 0 +#endif + +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + +#if __has_attribute(always_inline) || defined(__GNUC__) +#define MPARK_ALWAYS_INLINE __attribute__((__always_inline__)) inline +#elif defined(_MSC_VER) +#define MPARK_ALWAYS_INLINE __forceinline +#else +#define MPARK_ALWAYS_INLINE inline +#endif + +#if __has_builtin(__builtin_addressof) || \ + (defined(__GNUC__) && __GNUC__ >= 7) || defined(_MSC_VER) +#define MPARK_BUILTIN_ADDRESSOF +#endif + +#if __has_builtin(__builtin_unreachable) || defined(__GNUC__) +#define MPARK_BUILTIN_UNREACHABLE __builtin_unreachable() +#elif defined(_MSC_VER) +#define MPARK_BUILTIN_UNREACHABLE __assume(false) +#else +#define MPARK_BUILTIN_UNREACHABLE +#endif + +#if __has_builtin(__type_pack_element) +#define MPARK_TYPE_PACK_ELEMENT +#endif + +#if defined(__cpp_constexpr) && __cpp_constexpr >= 200704 && \ + !(defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 9) +#define MPARK_CPP11_CONSTEXPR +#endif + +#if defined(__cpp_constexpr) && __cpp_constexpr >= 201304 +#define MPARK_CPP14_CONSTEXPR +#endif + +#if __has_feature(cxx_exceptions) || defined(__cpp_exceptions) || \ + (defined(_MSC_VER) && defined(_CPPUNWIND)) +#define MPARK_EXCEPTIONS +#endif + +#if defined(__cpp_generic_lambdas) || defined(_MSC_VER) +#define MPARK_GENERIC_LAMBDAS +#endif + +#if defined(__cpp_lib_integer_sequence) +#define MPARK_INTEGER_SEQUENCE +#endif + +#if defined(__cpp_return_type_deduction) || defined(_MSC_VER) +#define MPARK_RETURN_TYPE_DEDUCTION +#endif + +#if defined(__cpp_lib_transparent_operators) || defined(_MSC_VER) +#define MPARK_TRANSPARENT_OPERATORS +#endif + +#if defined(__cpp_variable_templates) || defined(_MSC_VER) +#define MPARK_VARIABLE_TEMPLATES +#endif + +#if !defined(__GLIBCXX__) || __has_include() // >= libstdc++-5 +#define MPARK_TRIVIALITY_TYPE_TRAITS +#define MPARK_INCOMPLETE_TYPE_TRAITS +#endif + +#endif // MPARK_CONFIG_HPP + +// MPark.Variant +// +// Copyright Michael Park, 2015-2017 +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.md or copy at +// http://boost.org/LICENSE_1_0.txt) + +#ifndef MPARK_IN_PLACE_HPP +#define MPARK_IN_PLACE_HPP + +#include + +namespace paddle { + +struct in_place_t { + explicit in_place_t() = default; +}; + +template +struct in_place_index_t { + explicit in_place_index_t() = default; +}; + +template +struct in_place_type_t { + explicit in_place_type_t() = default; +}; + +#ifdef MPARK_VARIABLE_TEMPLATES +constexpr in_place_t in_place{}; + +template +constexpr in_place_index_t in_place_index{}; + +template +constexpr in_place_type_t in_place_type{}; +#endif + +} // namespace paddle + +#endif // MPARK_IN_PLACE_HPP + +// MPark.Variant +// +// Copyright Michael Park, 2015-2017 +// +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.md or copy at +// http://boost.org/LICENSE_1_0.txt) + +#ifndef MPARK_LIB_HPP +#define MPARK_LIB_HPP + +#include +#include +#include +#include + +#define MPARK_RETURN(...) \ + noexcept(noexcept(__VA_ARGS__))->decltype(__VA_ARGS__) { return __VA_ARGS__; } + +namespace paddle { +namespace lib { +template +struct identity { + using type = T; +}; + +inline namespace cpp14 { +template +struct array { + constexpr const T &operator[](std::size_t index) const { return data[index]; } + + T data[N == 0 ? 1 : N]; +}; + +template +using add_pointer_t = typename std::add_pointer::type; + +template +using common_type_t = typename std::common_type::type; + +template +using decay_t = typename std::decay::type; + +template +using enable_if_t = typename std::enable_if::type; + +template +using remove_const_t = typename std::remove_const::type; + +template +using remove_reference_t = typename std::remove_reference::type; + +template +inline constexpr T &&forward(remove_reference_t &t) noexcept { + return static_cast(t); +} + +template +inline constexpr T &&forward(remove_reference_t &&t) noexcept { + static_assert(!std::is_lvalue_reference::value, + "can not forward an rvalue as an lvalue"); + return static_cast(t); +} + +template +inline constexpr remove_reference_t &&move(T &&t) noexcept { + return static_cast &&>(t); +} + +#ifdef MPARK_INTEGER_SEQUENCE +using std::integer_sequence; +using std::index_sequence; +using std::make_index_sequence; +using std::index_sequence_for; +#else +template +struct integer_sequence { + using value_type = T; + static constexpr std::size_t size() noexcept { return sizeof...(Is); } +}; + +template +using index_sequence = integer_sequence; + +template +struct make_index_sequence_concat; + +template +struct make_index_sequence_concat, + index_sequence> + : identity> {}; + +template +struct make_index_sequence_impl; + +template +using make_index_sequence = typename make_index_sequence_impl::type; + +template +struct make_index_sequence_impl + : make_index_sequence_concat, + make_index_sequence> {}; + +template <> +struct make_index_sequence_impl<0> : identity> {}; + +template <> +struct make_index_sequence_impl<1> : identity> {}; + +template +using index_sequence_for = make_index_sequence; +#endif + +// +#ifdef MPARK_TRANSPARENT_OPERATORS +using equal_to = std::equal_to<>; +#else +struct equal_to { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + MPARK_RETURN(lib::forward(lhs) == lib::forward(rhs)) +}; +#endif + +#ifdef MPARK_TRANSPARENT_OPERATORS +using not_equal_to = std::not_equal_to<>; +#else +struct not_equal_to { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + MPARK_RETURN(lib::forward(lhs) != lib::forward(rhs)) +}; +#endif + +#ifdef MPARK_TRANSPARENT_OPERATORS +using less = std::less<>; +#else +struct less { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + MPARK_RETURN(lib::forward(lhs) < lib::forward(rhs)) +}; +#endif + +#ifdef MPARK_TRANSPARENT_OPERATORS +using greater = std::greater<>; +#else +struct greater { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + MPARK_RETURN(lib::forward(lhs) > lib::forward(rhs)) +}; +#endif + +#ifdef MPARK_TRANSPARENT_OPERATORS +using less_equal = std::less_equal<>; +#else +struct less_equal { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + MPARK_RETURN(lib::forward(lhs) <= lib::forward(rhs)) +}; +#endif + +#ifdef MPARK_TRANSPARENT_OPERATORS +using greater_equal = std::greater_equal<>; +#else +struct greater_equal { + template + inline constexpr auto operator()(Lhs &&lhs, Rhs &&rhs) const + MPARK_RETURN(lib::forward(lhs) >= lib::forward(rhs)) +}; +#endif +} // namespace cpp14 + +inline namespace cpp17 { +// +template +using bool_constant = std::integral_constant; + +template +struct voider : identity {}; + +template +using void_t = typename voider::type; + +namespace detail { +namespace swappable { + +using std::swap; + +template +struct is_swappable { + private: + template (), std::declval()))> + inline static std::true_type test(int); + + template + inline static std::false_type test(...); + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +template +struct is_nothrow_swappable { + static constexpr bool value = + noexcept(swap(std::declval(), std::declval())); +}; + +template +struct is_nothrow_swappable : std::false_type {}; + +} // namespace swappable +} // namespace detail + +using detail::swappable::is_swappable; + +template +using is_nothrow_swappable = + detail::swappable::is_nothrow_swappable::value, T>; + +// +namespace detail { + +template +struct is_reference_wrapper : std::false_type {}; + +template +struct is_reference_wrapper> : std::true_type {}; + +template +struct Invoke; + +template <> +struct Invoke { + template + inline static constexpr auto invoke(R T::*pmf, Arg &&arg, Args &&... args) + MPARK_RETURN((lib::forward(arg).*pmf)(lib::forward(args)...)) +}; + +template <> +struct Invoke { + template + inline static constexpr auto invoke(R T::*pmf, Arg &&arg, Args &&... args) + MPARK_RETURN((lib::forward(arg).get().* + pmf)(lib::forward(args)...)) +}; + +template <> +struct Invoke { + template + inline static constexpr auto invoke(R T::*pmf, Arg &&arg, Args &&... args) + MPARK_RETURN(((*lib::forward(arg)).* + pmf)(lib::forward(args)...)) +}; + +template <> +struct Invoke { + template + inline static constexpr auto invoke(R T::*pmo, Arg &&arg) + MPARK_RETURN(lib::forward(arg).*pmo) +}; + +template <> +struct Invoke { + template + inline static constexpr auto invoke(R T::*pmo, Arg &&arg) + MPARK_RETURN(lib::forward(arg).get().*pmo) +}; + +template <> +struct Invoke { + template + inline static constexpr auto invoke(R T::*pmo, Arg &&arg) + MPARK_RETURN((*lib::forward(arg)).*pmo) +}; + +template +inline constexpr auto invoke(R T::*f, Arg &&arg, Args &&... args) MPARK_RETURN( + Invoke::value, + (std::is_base_of>::value + ? 0 + : is_reference_wrapper>::value ? 1 : 2)>:: + invoke(f, lib::forward(arg), lib::forward(args)...)) + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4100) +#endif + template + inline constexpr auto invoke(F &&f, Args &&... args) + MPARK_RETURN(lib::forward(f)(lib::forward(args)...)) +#ifdef _MSC_VER +#pragma warning(pop) +#endif +} // namespace detail + +template +inline constexpr auto invoke(F &&f, Args &&... args) + MPARK_RETURN(detail::invoke(lib::forward(f), + lib::forward(args)...)) + + namespace detail { + template + struct invoke_result {}; + + template + struct invoke_result< + void_t(), std::declval()...))>, + F, + Args...> : identity(), + std::declval()...))> {}; + +} // namespace detail + +template +using invoke_result = detail::invoke_result; + +template +using invoke_result_t = typename invoke_result::type; + +namespace detail { + +template +struct is_invocable : std::false_type {}; + +template +struct is_invocable>, F, Args...> + : std::true_type {}; + +template +struct is_invocable_r : std::false_type {}; + +template +struct is_invocable_r>, R, F, Args...> + : std::is_convertible, R> {}; + +} // namespace detail + +template +using is_invocable = detail::is_invocable; + +template +using is_invocable_r = detail::is_invocable_r; + +namespace detail { + +template +struct is_nothrow_invocable { + static constexpr bool value = + noexcept(lib::invoke(std::declval(), std::declval()...)); +}; + +template +struct is_nothrow_invocable : std::false_type {}; + +template +struct is_nothrow_invocable_r { + private: + inline static R impl() { + return lib::invoke(std::declval(), std::declval()...); + } + + public: + static constexpr bool value = noexcept(impl()); +}; + +template +struct is_nothrow_invocable_r : std::false_type {}; + +} // namespace detail + +template +using is_nothrow_invocable = + detail::is_nothrow_invocable::value, F, Args...>; + +template +using is_nothrow_invocable_r = detail:: + is_nothrow_invocable_r::value, R, F, Args...>; + +// +#ifdef MPARK_BUILTIN_ADDRESSOF +template +inline constexpr T *addressof(T &arg) noexcept { + return __builtin_addressof(arg); +} +#else +namespace detail { + +namespace has_addressof_impl { + +struct fail; + +template +inline fail operator&(T &&); + +template +inline static constexpr bool impl() { + return (std::is_class::value || std::is_union::value) && + !std::is_same()), fail>::value; +} + +} // namespace has_addressof_impl + +template +using has_addressof = bool_constant()>; + +template +inline constexpr T *addressof(T &arg, std::true_type) noexcept { + return std::addressof(arg); +} + +template +inline constexpr T *addressof(T &arg, std::false_type) noexcept { + return &arg; +} + +} // namespace detail + +template +inline constexpr T *addressof(T &arg) noexcept { + return detail::addressof(arg, detail::has_addressof{}); +} +#endif + +template +inline constexpr T *addressof(const T &&) = delete; + +} // namespace cpp17 + +template +struct remove_all_extents : identity {}; + +template +struct remove_all_extents> : remove_all_extents {}; + +template +using remove_all_extents_t = typename remove_all_extents::type; + +template +using size_constant = std::integral_constant; + +template +struct indexed_type : size_constant { + using type = T; +}; + +template +using all = std::is_same, + integer_sequence>; + +#ifdef MPARK_TYPE_PACK_ELEMENT +template +using type_pack_element_t = __type_pack_element; +#else +template +struct type_pack_element_impl { + private: + template + struct set; + + template + struct set> : indexed_type... {}; + + template + inline static std::enable_if impl(indexed_type); + + inline static std::enable_if impl(...); + + public: + using type = decltype(impl(set>{})); +}; + +template +using type_pack_element = typename type_pack_element_impl::type; + +template +using type_pack_element_t = typename type_pack_element::type; +#endif + +#ifdef MPARK_TRIVIALITY_TYPE_TRAITS +using std::is_trivially_copy_constructible; +using std::is_trivially_move_constructible; +using std::is_trivially_copy_assignable; +using std::is_trivially_move_assignable; +#else +template +struct is_trivially_copy_constructible + : bool_constant::value &&__has_trivial_copy( + T)> {}; + +template +struct is_trivially_move_constructible : bool_constant<__is_trivial(T)> {}; + +template +struct is_trivially_copy_assignable + : bool_constant::value &&__has_trivial_assign( + T)> {}; + +template +struct is_trivially_move_assignable : bool_constant<__is_trivial(T)> {}; +#endif + +template +struct dependent_type : T {}; + +template +struct push_back; + +template +using push_back_t = typename push_back::type; + +template +struct push_back, J> { + using type = index_sequence; +}; + +} // namespace lib +} // namespace paddle + +#undef MPARK_RETURN + +#endif // MPARK_LIB_HPP + +namespace paddle { + +#ifdef MPARK_RETURN_TYPE_DEDUCTION + +#define AUTO auto +#define AUTO_RETURN(...) \ + { return __VA_ARGS__; } + +#define AUTO_REFREF auto && +#define AUTO_REFREF_RETURN(...) \ + { return __VA_ARGS__; } + +#define DECLTYPE_AUTO decltype(auto) +#define DECLTYPE_AUTO_RETURN(...) \ + { return __VA_ARGS__; } + +#else + +#define AUTO auto +#define AUTO_RETURN(...) \ + ->lib::decay_t { return __VA_ARGS__; } + +#define AUTO_REFREF auto +#define AUTO_REFREF_RETURN(...) \ + ->decltype((__VA_ARGS__)) { \ + static_assert(std::is_reference::value, ""); \ + return __VA_ARGS__; \ + } + +#define DECLTYPE_AUTO auto +#define DECLTYPE_AUTO_RETURN(...) \ + ->decltype(__VA_ARGS__) { return __VA_ARGS__; } + +#endif + +class bad_variant_access : public std::exception { + public: + virtual const char *what() const noexcept override { + return "bad_variant_access"; + } +}; + +[[noreturn]] inline void throw_bad_variant_access() { +#ifdef MPARK_EXCEPTIONS + throw bad_variant_access{}; +#else + std::terminate(); + MPARK_BUILTIN_UNREACHABLE; +#endif +} + +template +class variant; + +template +struct variant_size; + +#ifdef MPARK_VARIABLE_TEMPLATES +template +constexpr std::size_t variant_size_v = variant_size::value; +#endif + +template +struct variant_size : variant_size {}; + +template +struct variant_size : variant_size {}; + +template +struct variant_size : variant_size {}; + +template +struct variant_size> : lib::size_constant {}; + +template +struct variant_alternative; + +template +using variant_alternative_t = typename variant_alternative::type; + +template +struct variant_alternative + : std::add_const> {}; + +template +struct variant_alternative + : std::add_volatile> {}; + +template +struct variant_alternative + : std::add_cv> {}; + +template +struct variant_alternative> { + static_assert(I < sizeof...(Ts), + "index out of bounds in `std::variant_alternative<>`"); + using type = lib::type_pack_element_t; +}; + +constexpr std::size_t variant_npos = static_cast(-1); + +namespace detail { + +constexpr std::size_t not_found = static_cast(-1); +constexpr std::size_t ambiguous = static_cast(-2); + +#ifdef MPARK_CPP14_CONSTEXPR +template +inline constexpr std::size_t find_index() { + constexpr lib::array matches = { + {std::is_same::value...}}; + std::size_t result = not_found; + for (std::size_t i = 0; i < sizeof...(Ts); ++i) { + if (matches[i]) { + if (result != not_found) { + return ambiguous; + } + result = i; + } + } + return result; +} +#else +inline constexpr std::size_t find_index_impl(std::size_t result, std::size_t) { + return result; +} + +template +inline constexpr std::size_t find_index_impl(std::size_t result, + std::size_t idx, + bool b, + Bs... bs) { + return b ? (result != not_found ? ambiguous + : find_index_impl(idx, idx + 1, bs...)) + : find_index_impl(result, idx + 1, bs...); +} + +template +inline constexpr std::size_t find_index() { + return find_index_impl(not_found, 0, std::is_same::value...); +} +#endif + +template +using find_index_sfinae_impl = + lib::enable_if_t>; + +template +using find_index_sfinae = find_index_sfinae_impl()>; + +template +struct find_index_checked_impl : lib::size_constant { + static_assert(I != not_found, "the specified type is not found."); + static_assert(I != ambiguous, "the specified type is ambiguous."); +}; + +template +using find_index_checked = find_index_checked_impl()>; + +struct valueless_t {}; + +enum class Trait { TriviallyAvailable, Available, Unavailable }; + +template class IsTriviallyAvailable, + template class IsAvailable> +inline constexpr Trait trait() { + return IsTriviallyAvailable::value + ? Trait::TriviallyAvailable + : IsAvailable::value ? Trait::Available : Trait::Unavailable; +} + +#ifdef MPARK_CPP14_CONSTEXPR +template +inline constexpr Trait common_trait(Traits... traits_) { + Trait result = Trait::TriviallyAvailable; + lib::array traits = {{traits_...}}; + for (std::size_t i = 0; i < sizeof...(Traits); ++i) { + Trait t = traits[i]; + if (static_cast(t) > static_cast(result)) { + result = t; + } + } + return result; +} +#else +inline constexpr Trait common_trait_impl(Trait result) { return result; } + +template +inline constexpr Trait common_trait_impl(Trait result, Trait t, Traits... ts) { + return static_cast(t) > static_cast(result) + ? common_trait_impl(t, ts...) + : common_trait_impl(result, ts...); +} + +template +inline constexpr Trait common_trait(Traits... ts) { + return common_trait_impl(Trait::TriviallyAvailable, ts...); +} +#endif + +template +struct traits { + static constexpr Trait copy_constructible_trait = + common_trait(trait()...); + + static constexpr Trait move_constructible_trait = + common_trait(trait()...); + + static constexpr Trait copy_assignable_trait = + common_trait(copy_constructible_trait, + trait()...); + + static constexpr Trait move_assignable_trait = + common_trait(move_constructible_trait, + trait()...); + + static constexpr Trait destructible_trait = common_trait( + trait()...); +}; + +namespace access { + +struct recursive_union { +#ifdef MPARK_RETURN_TYPE_DEDUCTION + template + inline static constexpr auto &&get_alt(V &&v, in_place_index_t<0>) { + return lib::forward(v).head_; + } + + template + inline static constexpr auto &&get_alt(V &&v, in_place_index_t) { + return get_alt(lib::forward(v).tail_, in_place_index_t{}); + } +#else + template + struct get_alt_impl { + template + inline constexpr AUTO_REFREF operator()(V &&v) const + AUTO_REFREF_RETURN(get_alt_impl{}(lib::forward(v).tail_)) + }; + + template + struct get_alt_impl<0, Dummy> { + template + inline constexpr AUTO_REFREF operator()(V &&v) const + AUTO_REFREF_RETURN(lib::forward(v).head_) + }; + + template + inline static constexpr AUTO_REFREF get_alt(V &&v, in_place_index_t) + AUTO_REFREF_RETURN(get_alt_impl{}(lib::forward(v))) +#endif +}; + +struct base { + template + inline static constexpr AUTO_REFREF get_alt(V &&v) +#ifdef _MSC_VER + AUTO_REFREF_RETURN(recursive_union::get_alt(lib::forward(v).data_, + in_place_index_t{})) +#else + AUTO_REFREF_RETURN(recursive_union::get_alt(data(lib::forward(v)), + in_place_index_t{})) +#endif +}; + +struct variant { + template + inline static constexpr AUTO_REFREF get_alt(V &&v) + AUTO_REFREF_RETURN(base::get_alt(lib::forward(v).impl_)) +}; + +} // namespace access + +namespace visitation { + +#if defined(MPARK_CPP14_CONSTEXPR) && !defined(_MSC_VER) +#define MPARK_VARIANT_SWITCH_VISIT +#endif + +struct base { + template + using dispatch_result_t = + decltype(lib::invoke(std::declval(), + access::base::get_alt<0>(std::declval())...)); + + template + struct expected { + template + inline static constexpr bool but_got() { + return std::is_same::value; + } + }; + + template + struct visit_return_type_check { + static_assert(expected::template but_got(), + "`visit` requires the visitor to have a single return type"); + + template + inline static constexpr DECLTYPE_AUTO invoke(Visitor &&visitor, + Alts &&... alts) + DECLTYPE_AUTO_RETURN(lib::invoke(lib::forward(visitor), + lib::forward(alts)...)) + }; + +#ifdef MPARK_VARIANT_SWITCH_VISIT + template + struct dispatcher; + + template + struct dispatcher { + template + MPARK_ALWAYS_INLINE static constexpr R dispatch(F &&, + typename ITs::type &&..., + Vs &&...) { + MPARK_BUILTIN_UNREACHABLE; + } + + template + MPARK_ALWAYS_INLINE static constexpr R dispatch_case(F &&, Vs &&...) { + MPARK_BUILTIN_UNREACHABLE; + } + + template + MPARK_ALWAYS_INLINE static constexpr R dispatch_at(std::size_t, + F &&, + Vs &&...) { + MPARK_BUILTIN_UNREACHABLE; + } + }; + + template + struct dispatcher { + template + MPARK_ALWAYS_INLINE static constexpr R dispatch( + F &&f, typename ITs::type &&... visited_vs) { + using Expected = R; + using Actual = decltype( + lib::invoke(lib::forward(f), + access::base::get_alt( + lib::forward(visited_vs))...)); + return visit_return_type_check::invoke( + lib::forward(f), + access::base::get_alt( + lib::forward(visited_vs))...); + } + + template + MPARK_ALWAYS_INLINE static constexpr R dispatch( + F &&f, typename ITs::type &&... visited_vs, V &&v, Vs &&... vs) { +#define MPARK_DISPATCH(I) \ + dispatcher<(I < lib::decay_t::size()), \ + R, \ + ITs..., \ + lib::indexed_type>:: \ + template dispatch<0>(lib::forward(f), \ + lib::forward(visited_vs)..., \ + lib::forward(v), \ + lib::forward(vs)...) + +#define MPARK_DEFAULT(I) \ + dispatcher<(I < lib::decay_t::size()), R, ITs...>::template dispatch( \ + lib::forward(f), \ + lib::forward(visited_vs)..., \ + lib::forward(v), \ + lib::forward(vs)...) + + switch (v.index()) { + case B + 0: + return MPARK_DISPATCH(B + 0); + case B + 1: + return MPARK_DISPATCH(B + 1); + case B + 2: + return MPARK_DISPATCH(B + 2); + case B + 3: + return MPARK_DISPATCH(B + 3); + case B + 4: + return MPARK_DISPATCH(B + 4); + case B + 5: + return MPARK_DISPATCH(B + 5); + case B + 6: + return MPARK_DISPATCH(B + 6); + case B + 7: + return MPARK_DISPATCH(B + 7); + case B + 8: + return MPARK_DISPATCH(B + 8); + case B + 9: + return MPARK_DISPATCH(B + 9); + case B + 10: + return MPARK_DISPATCH(B + 10); + case B + 11: + return MPARK_DISPATCH(B + 11); + case B + 12: + return MPARK_DISPATCH(B + 12); + case B + 13: + return MPARK_DISPATCH(B + 13); + case B + 14: + return MPARK_DISPATCH(B + 14); + case B + 15: + return MPARK_DISPATCH(B + 15); + case B + 16: + return MPARK_DISPATCH(B + 16); + case B + 17: + return MPARK_DISPATCH(B + 17); + case B + 18: + return MPARK_DISPATCH(B + 18); + case B + 19: + return MPARK_DISPATCH(B + 19); + case B + 20: + return MPARK_DISPATCH(B + 20); + case B + 21: + return MPARK_DISPATCH(B + 21); + case B + 22: + return MPARK_DISPATCH(B + 22); + case B + 23: + return MPARK_DISPATCH(B + 23); + case B + 24: + return MPARK_DISPATCH(B + 24); + case B + 25: + return MPARK_DISPATCH(B + 25); + case B + 26: + return MPARK_DISPATCH(B + 26); + case B + 27: + return MPARK_DISPATCH(B + 27); + case B + 28: + return MPARK_DISPATCH(B + 28); + case B + 29: + return MPARK_DISPATCH(B + 29); + case B + 30: + return MPARK_DISPATCH(B + 30); + case B + 31: + return MPARK_DISPATCH(B + 31); + default: + return MPARK_DEFAULT(B + 32); + } + +#undef MPARK_DEFAULT +#undef MPARK_DISPATCH + } + + template + MPARK_ALWAYS_INLINE static constexpr R dispatch_case(F &&f, Vs &&... vs) { + using Expected = R; + using Actual = decltype( + lib::invoke(lib::forward(f), + access::base::get_alt(lib::forward(vs))...)); + return visit_return_type_check::invoke( + lib::forward(f), + access::base::get_alt(lib::forward(vs))...); + } + + template + MPARK_ALWAYS_INLINE static constexpr R dispatch_at(std::size_t index, + F &&f, + V &&v, + Vs &&... vs) { + static_assert(lib::all<(lib::decay_t::size() == + lib::decay_t::size())...>::value, + "all of the variants must be the same size."); +#define MPARK_DISPATCH_AT(I) \ + dispatcher<(I < lib::decay_t::size()), R>::template dispatch_case( \ + lib::forward(f), lib::forward(v), lib::forward(vs)...) + +#define MPARK_DEFAULT(I) \ + dispatcher<(I < lib::decay_t::size()), R>::template dispatch_at( \ + index, lib::forward(f), lib::forward(v), lib::forward(vs)...) + + switch (index) { + case B + 0: + return MPARK_DISPATCH_AT(B + 0); + case B + 1: + return MPARK_DISPATCH_AT(B + 1); + case B + 2: + return MPARK_DISPATCH_AT(B + 2); + case B + 3: + return MPARK_DISPATCH_AT(B + 3); + case B + 4: + return MPARK_DISPATCH_AT(B + 4); + case B + 5: + return MPARK_DISPATCH_AT(B + 5); + case B + 6: + return MPARK_DISPATCH_AT(B + 6); + case B + 7: + return MPARK_DISPATCH_AT(B + 7); + case B + 8: + return MPARK_DISPATCH_AT(B + 8); + case B + 9: + return MPARK_DISPATCH_AT(B + 9); + case B + 10: + return MPARK_DISPATCH_AT(B + 10); + case B + 11: + return MPARK_DISPATCH_AT(B + 11); + case B + 12: + return MPARK_DISPATCH_AT(B + 12); + case B + 13: + return MPARK_DISPATCH_AT(B + 13); + case B + 14: + return MPARK_DISPATCH_AT(B + 14); + case B + 15: + return MPARK_DISPATCH_AT(B + 15); + case B + 16: + return MPARK_DISPATCH_AT(B + 16); + case B + 17: + return MPARK_DISPATCH_AT(B + 17); + case B + 18: + return MPARK_DISPATCH_AT(B + 18); + case B + 19: + return MPARK_DISPATCH_AT(B + 19); + case B + 20: + return MPARK_DISPATCH_AT(B + 20); + case B + 21: + return MPARK_DISPATCH_AT(B + 21); + case B + 22: + return MPARK_DISPATCH_AT(B + 22); + case B + 23: + return MPARK_DISPATCH_AT(B + 23); + case B + 24: + return MPARK_DISPATCH_AT(B + 24); + case B + 25: + return MPARK_DISPATCH_AT(B + 25); + case B + 26: + return MPARK_DISPATCH_AT(B + 26); + case B + 27: + return MPARK_DISPATCH_AT(B + 27); + case B + 28: + return MPARK_DISPATCH_AT(B + 28); + case B + 29: + return MPARK_DISPATCH_AT(B + 29); + case B + 30: + return MPARK_DISPATCH_AT(B + 30); + case B + 31: + return MPARK_DISPATCH_AT(B + 31); + default: + return MPARK_DEFAULT(B + 32); + } + +#undef MPARK_DEFAULT +#undef MPARK_DISPATCH_AT + } + }; +#else + template + inline static constexpr const T &at(const T &elem) noexcept { + return elem; + } + + template + inline static constexpr const lib::remove_all_extents_t &at( + const lib::array &elems, std::size_t i, Is... is) noexcept { + return at(elems[i], is...); + } + + template + inline static constexpr lib::array, sizeof...(Fs) + 1> + make_farray(F &&f, Fs &&... fs) { + return {{lib::forward(f), lib::forward(fs)...}}; + } + + template + struct make_fmatrix_impl { + template + inline static constexpr dispatch_result_t dispatch(F &&f, + Vs &&... vs) { + using Expected = dispatch_result_t; + using Actual = decltype( + lib::invoke(lib::forward(f), + access::base::get_alt(lib::forward(vs))...)); + return visit_return_type_check::invoke( + lib::forward(f), + access::base::get_alt(lib::forward(vs))...); + } + +#ifdef MPARK_RETURN_TYPE_DEDUCTION + template + inline static constexpr auto impl(lib::index_sequence) { + return &dispatch; + } + + template + inline static constexpr auto impl(Is, + lib::index_sequence, + Ls... ls) { + return make_farray(impl(lib::push_back_t{}, ls...)...); + } +#else + template + struct impl; + + template + struct impl> { + inline constexpr AUTO operator()() const AUTO_RETURN(&dispatch) + }; + + template + struct impl, Ls...> { + inline constexpr AUTO operator()() const + AUTO_RETURN(make_farray(impl, Ls...>{}()...)) + }; +#endif + }; + +#ifdef MPARK_RETURN_TYPE_DEDUCTION + template + inline static constexpr auto make_fmatrix() { + return make_fmatrix_impl::impl( + lib::index_sequence<>{}, + lib::make_index_sequence::size()>{}...); + } +#else + template + inline static constexpr AUTO make_fmatrix() + AUTO_RETURN(typename make_fmatrix_impl::template impl< + lib::index_sequence<>, + lib::make_index_sequence::size()>...>{}()) +#endif + + template + struct make_fdiagonal_impl { + template + inline static constexpr dispatch_result_t dispatch(F &&f, + Vs &&... vs) { + using Expected = dispatch_result_t; + using Actual = decltype( + lib::invoke(lib::forward(f), + access::base::get_alt(lib::forward(vs))...)); + return visit_return_type_check::invoke( + lib::forward(f), + access::base::get_alt(lib::forward(vs))...); + } + + template + inline static constexpr AUTO impl(lib::index_sequence) + AUTO_RETURN(make_farray(&dispatch...)) + }; + + template + inline static constexpr auto make_fdiagonal() + -> decltype(make_fdiagonal_impl::impl( + lib::make_index_sequence::size()>{})) { + static_assert(lib::all<(lib::decay_t::size() == + lib::decay_t::size())...>::value, + "all of the variants must be the same size."); + return make_fdiagonal_impl::impl( + lib::make_index_sequence::size()>{}); + } +#endif +}; + +#if !defined(MPARK_VARIANT_SWITCH_VISIT) && \ + (!defined(_MSC_VER) || _MSC_VER >= 1910) +template +using fmatrix_t = decltype(base::make_fmatrix()); + +template +struct fmatrix { + static constexpr fmatrix_t value = base::make_fmatrix(); +}; + +template +constexpr fmatrix_t fmatrix::value; + +template +using fdiagonal_t = decltype(base::make_fdiagonal()); + +template +struct fdiagonal { + static constexpr fdiagonal_t value = + base::make_fdiagonal(); +}; + +template +constexpr fdiagonal_t fdiagonal::value; +#endif + +struct alt { + template + inline static constexpr DECLTYPE_AUTO visit_alt(Visitor &&visitor, + Vs &&... vs) +#ifdef MPARK_VARIANT_SWITCH_VISIT + DECLTYPE_AUTO_RETURN( + base::dispatcher(vs)))...>>:: + template dispatch<0>(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#elif !defined(_MSC_VER) || _MSC_VER >= 1910 + DECLTYPE_AUTO_RETURN( + base::at(fmatrix(vs)))...>::value, + vs.index()...)(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#else + DECLTYPE_AUTO_RETURN(base::at( + base::make_fmatrix(vs)))...>(), + vs.index()...)(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#endif + + template + inline static constexpr DECLTYPE_AUTO + visit_alt_at(std::size_t index, Visitor &&visitor, Vs &&... vs) +#ifdef MPARK_VARIANT_SWITCH_VISIT + DECLTYPE_AUTO_RETURN( + base::dispatcher< + true, + base::dispatch_result_t< + Visitor, + decltype(as_base(lib::forward(vs)))...>>:: + template dispatch_at<0>(index, + lib::forward(visitor), + as_base(lib::forward(vs))...)) +#elif !defined(_MSC_VER) || _MSC_VER >= 1910 + DECLTYPE_AUTO_RETURN(base::at( + fdiagonal(vs)))...>::value, + index)(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#else + DECLTYPE_AUTO_RETURN( + base::at(base::make_fdiagonal< + Visitor &&, + decltype(as_base(lib::forward(vs)))...>(), + index)(lib::forward(visitor), + as_base(lib::forward(vs))...)) +#endif +}; + +struct variant { + private: + template + struct visitor { + template + inline static constexpr bool does_not_handle() { + return lib::is_invocable::value; + } + }; + + template + struct visit_exhaustiveness_check { + static_assert(visitor::template does_not_handle(), + "`visit` requires the visitor to be exhaustive."); + + inline static constexpr DECLTYPE_AUTO invoke(Visitor &&visitor, + Values &&... values) + DECLTYPE_AUTO_RETURN(lib::invoke(lib::forward(visitor), + lib::forward(values)...)) + }; + + template + struct value_visitor { + Visitor &&visitor_; + + template + inline constexpr DECLTYPE_AUTO operator()(Alts &&... alts) const + DECLTYPE_AUTO_RETURN(visit_exhaustiveness_check< + Visitor, + decltype((lib::forward(alts).value))...>:: + invoke(lib::forward(visitor_), + lib::forward(alts).value...)) + }; + + template + inline static constexpr AUTO make_value_visitor(Visitor &&visitor) + AUTO_RETURN(value_visitor{lib::forward(visitor)}) + + public + : template + inline static constexpr DECLTYPE_AUTO + visit_alt(Visitor &&visitor, Vs &&... vs) + DECLTYPE_AUTO_RETURN(alt::visit_alt(lib::forward(visitor), + lib::forward(vs).impl_...)) + + template + inline static constexpr DECLTYPE_AUTO + visit_alt_at(std::size_t index, Visitor &&visitor, Vs &&... vs) + DECLTYPE_AUTO_RETURN( + alt::visit_alt_at(index, + lib::forward(visitor), + lib::forward(vs).impl_...)) + + template + inline static constexpr DECLTYPE_AUTO + visit_value(Visitor &&visitor, Vs &&... vs) DECLTYPE_AUTO_RETURN( + visit_alt(make_value_visitor(lib::forward(visitor)), + lib::forward(vs)...)) + + template + inline static constexpr DECLTYPE_AUTO + visit_value_at(std::size_t index, Visitor &&visitor, Vs &&... vs) + DECLTYPE_AUTO_RETURN( + visit_alt_at(index, + make_value_visitor(lib::forward(visitor)), + lib::forward(vs)...)) +}; + +} // namespace visitation + +template +struct alt { + using value_type = T; + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4244) +#endif + template + inline explicit constexpr alt(in_place_t, Args &&... args) + : value(lib::forward(args)...) {} +#ifdef _MSC_VER +#pragma warning(pop) +#endif + + T value; +}; + +template +union recursive_union; + +template +union recursive_union {}; + +#define MPARK_VARIANT_RECURSIVE_UNION(destructible_trait, destructor) \ + template \ + union recursive_union { \ + public: \ + inline explicit constexpr recursive_union(valueless_t) noexcept \ + : dummy_{} {} \ + \ + template \ + inline explicit constexpr recursive_union(in_place_index_t<0>, \ + Args &&... args) \ + : head_(in_place_t{}, lib::forward(args)...) {} \ + \ + template \ + inline explicit constexpr recursive_union(in_place_index_t, \ + Args &&... args) \ + : tail_(in_place_index_t{}, lib::forward(args)...) {} \ + \ + recursive_union(const recursive_union &) = default; \ + recursive_union(recursive_union &&) = default; \ + \ + destructor \ + \ + recursive_union & \ + operator=(const recursive_union &) = default; \ + recursive_union &operator=(recursive_union &&) = default; \ + \ + private: \ + char dummy_; \ + alt head_; \ + recursive_union tail_; \ + \ + friend struct access::recursive_union; \ + } + +MPARK_VARIANT_RECURSIVE_UNION(Trait::TriviallyAvailable, + ~recursive_union() = default;); +MPARK_VARIANT_RECURSIVE_UNION(Trait::Available, ~recursive_union(){}); +MPARK_VARIANT_RECURSIVE_UNION(Trait::Unavailable, ~recursive_union() = delete;); + +#undef MPARK_VARIANT_RECURSIVE_UNION + +using index_t = unsigned int; + +template +class base { + public: + inline explicit constexpr base(valueless_t tag) noexcept + : data_(tag), + index_(static_cast(-1)) {} + + template + inline explicit constexpr base(in_place_index_t, Args &&... args) + : data_(in_place_index_t{}, lib::forward(args)...), index_(I) {} + + inline constexpr bool valueless_by_exception() const noexcept { + return index_ == static_cast(-1); + } + + inline constexpr std::size_t index() const noexcept { + return valueless_by_exception() ? variant_npos : index_; + } + + protected: + using data_t = recursive_union; + + friend inline constexpr base &as_base(base &b) { return b; } + friend inline constexpr const base &as_base(const base &b) { return b; } + friend inline constexpr base &&as_base(base &&b) { return lib::move(b); } + friend inline constexpr const base &&as_base(const base &&b) { + return lib::move(b); + } + + friend inline constexpr data_t &data(base &b) { return b.data_; } + friend inline constexpr const data_t &data(const base &b) { return b.data_; } + friend inline constexpr data_t &&data(base &&b) { return lib::move(b).data_; } + friend inline constexpr const data_t &&data(const base &&b) { + return lib::move(b).data_; + } + + inline static constexpr std::size_t size() { return sizeof...(Ts); } + + data_t data_; + index_t index_; + + friend struct access::base; + friend struct visitation::base; +}; + +struct dtor { +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4100) +#endif + template + inline void operator()(Alt &alt) const noexcept { + alt.~Alt(); + } +#ifdef _MSC_VER +#pragma warning(pop) +#endif +}; + +#if !defined(_MSC_VER) || _MSC_VER >= 1910 +#define MPARK_INHERITING_CTOR(type, base) using base::base; +#else +#define MPARK_INHERITING_CTOR(type, base) \ + template \ + inline explicit constexpr type(Args &&... args) \ + : base(lib::forward(args)...) {} +#endif + +template +class destructor; + +#define MPARK_VARIANT_DESTRUCTOR(destructible_trait, definition, destroy) \ + template \ + class destructor, destructible_trait> \ + : public base { \ + using super = base; \ + \ + public: \ + MPARK_INHERITING_CTOR(destructor, super) \ + using super::operator=; \ + \ + destructor(const destructor &) = default; \ + destructor(destructor &&) = default; \ + definition destructor &operator=(const destructor &) = default; \ + destructor &operator=(destructor &&) = default; \ + \ + protected: \ + destroy \ + } + +MPARK_VARIANT_DESTRUCTOR(Trait::TriviallyAvailable, ~destructor() = default; + , inline void destroy() noexcept { + this->index_ = static_cast(-1); + }); + +MPARK_VARIANT_DESTRUCTOR(Trait::Available, + ~destructor() { destroy(); }, + inline void destroy() noexcept { + if (!this->valueless_by_exception()) { + visitation::alt::visit_alt(dtor{}, *this); + } + this->index_ = static_cast(-1); + }); + +MPARK_VARIANT_DESTRUCTOR(Trait::Unavailable, ~destructor() = delete; + , inline void destroy() noexcept = delete;); + +#undef MPARK_VARIANT_DESTRUCTOR + +template +class constructor : public destructor { + using super = destructor; + + public: + MPARK_INHERITING_CTOR(constructor, super) + using super::operator=; + + protected: +#ifndef MPARK_GENERIC_LAMBDAS + struct ctor { + template + inline void operator()(LhsAlt &lhs_alt, RhsAlt &&rhs_alt) const { + constructor::construct_alt(lhs_alt, lib::forward(rhs_alt).value); + } + }; +#endif + + template + inline static T &construct_alt(alt &a, Args &&... args) { + auto *result = ::new (static_cast(lib::addressof(a))) + alt(in_place_t{}, lib::forward(args)...); + return result->value; + } + + template + inline static void generic_construct(constructor &lhs, Rhs &&rhs) { + lhs.destroy(); + if (!rhs.valueless_by_exception()) { + visitation::alt::visit_alt_at( + rhs.index(), +#ifdef MPARK_GENERIC_LAMBDAS + [](auto &lhs_alt, auto &&rhs_alt) { + constructor::construct_alt( + lhs_alt, lib::forward(rhs_alt).value); + } +#else + ctor {} +#endif + , + lhs, + lib::forward(rhs)); + lhs.index_ = rhs.index_; + } + } +}; + +template +class move_constructor; + +#define MPARK_VARIANT_MOVE_CONSTRUCTOR(move_constructible_trait, definition) \ + template \ + class move_constructor, move_constructible_trait> \ + : public constructor> { \ + using super = constructor>; \ + \ + public: \ + MPARK_INHERITING_CTOR(move_constructor, super) \ + using super::operator=; \ + \ + move_constructor(const move_constructor &) = default; \ + definition ~move_constructor() = default; \ + move_constructor &operator=(const move_constructor &) = default; \ + move_constructor &operator=(move_constructor &&) = default; \ + } + +MPARK_VARIANT_MOVE_CONSTRUCTOR( + Trait::TriviallyAvailable, + move_constructor(move_constructor &&that) = default;); + +MPARK_VARIANT_MOVE_CONSTRUCTOR( + Trait::Available, + move_constructor(move_constructor &&that) noexcept( + lib::all::value...>::value) + : move_constructor(valueless_t{}) { + this->generic_construct(*this, lib::move(that)); + }); + +MPARK_VARIANT_MOVE_CONSTRUCTOR(Trait::Unavailable, + move_constructor(move_constructor &&) = delete;); + +#undef MPARK_VARIANT_MOVE_CONSTRUCTOR + +template +class copy_constructor; + +#define MPARK_VARIANT_COPY_CONSTRUCTOR(copy_constructible_trait, definition) \ + template \ + class copy_constructor, copy_constructible_trait> \ + : public move_constructor> { \ + using super = move_constructor>; \ + \ + public: \ + MPARK_INHERITING_CTOR(copy_constructor, super) \ + using super::operator=; \ + \ + definition copy_constructor(copy_constructor &&) = default; \ + ~copy_constructor() = default; \ + copy_constructor &operator=(const copy_constructor &) = default; \ + copy_constructor &operator=(copy_constructor &&) = default; \ + } + +MPARK_VARIANT_COPY_CONSTRUCTOR( + Trait::TriviallyAvailable, + copy_constructor(const copy_constructor &that) = default;); + +MPARK_VARIANT_COPY_CONSTRUCTOR(Trait::Available, + copy_constructor(const copy_constructor &that) + : copy_constructor(valueless_t{}) { + this->generic_construct(*this, that); + }); + +MPARK_VARIANT_COPY_CONSTRUCTOR( + Trait::Unavailable, copy_constructor(const copy_constructor &) = delete;); + +#undef MPARK_VARIANT_COPY_CONSTRUCTOR + +template +class assignment : public copy_constructor { + using super = copy_constructor; + + public: + MPARK_INHERITING_CTOR(assignment, super) + using super::operator=; + + template + inline /* auto & */ auto emplace(Args &&... args) + -> decltype(this->construct_alt(access::base::get_alt(*this), + lib::forward(args)...)) { + this->destroy(); + auto &result = this->construct_alt(access::base::get_alt(*this), + lib::forward(args)...); + this->index_ = I; + return result; + } + + protected: +#ifndef MPARK_GENERIC_LAMBDAS + template + struct assigner { + template + inline void operator()(ThisAlt &this_alt, ThatAlt &&that_alt) const { + self->assign_alt(this_alt, lib::forward(that_alt).value); + } + assignment *self; + }; +#endif + + template + inline void assign_alt(alt &a, Arg &&arg) { + if (this->index() == I) { +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4244) +#endif + a.value = lib::forward(arg); +#ifdef _MSC_VER +#pragma warning(pop) +#endif + } else { + struct { + void operator()(std::true_type) const { + this_->emplace(lib::forward(arg_)); + } + void operator()(std::false_type) const { + this_->emplace(T(lib::forward(arg_))); + } + assignment *this_; + Arg &&arg_; + } impl{this, lib::forward(arg)}; + impl(lib::bool_constant < std::is_nothrow_constructible::value || + !std::is_nothrow_move_constructible::value > {}); + } + } + + template + inline void generic_assign(That &&that) { + if (this->valueless_by_exception() && that.valueless_by_exception()) { + // do nothing. + } else if (that.valueless_by_exception()) { + this->destroy(); + } else { + visitation::alt::visit_alt_at( + that.index(), +#ifdef MPARK_GENERIC_LAMBDAS + [this](auto &this_alt, auto &&that_alt) { + this->assign_alt(this_alt, + lib::forward(that_alt).value); + } +#else + assigner { this } +#endif + , + *this, + lib::forward(that)); + } + } +}; + +template +class move_assignment; + +#define MPARK_VARIANT_MOVE_ASSIGNMENT(move_assignable_trait, definition) \ + template \ + class move_assignment, move_assignable_trait> \ + : public assignment> { \ + using super = assignment>; \ + \ + public: \ + MPARK_INHERITING_CTOR(move_assignment, super) \ + using super::operator=; \ + \ + move_assignment(const move_assignment &) = default; \ + move_assignment(move_assignment &&) = default; \ + ~move_assignment() = default; \ + move_assignment &operator=(const move_assignment &) = default; \ + definition \ + } + +MPARK_VARIANT_MOVE_ASSIGNMENT( + Trait::TriviallyAvailable, + move_assignment &operator=(move_assignment &&that) = default;); + +MPARK_VARIANT_MOVE_ASSIGNMENT( + Trait::Available, + move_assignment & + operator=(move_assignment &&that) noexcept( + lib::all<(std::is_nothrow_move_constructible::value && + std::is_nothrow_move_assignable::value)...>::value) { + this->generic_assign(lib::move(that)); + return *this; + }); + +MPARK_VARIANT_MOVE_ASSIGNMENT( + Trait::Unavailable, + move_assignment &operator=(move_assignment &&) = delete;); + +#undef MPARK_VARIANT_MOVE_ASSIGNMENT + +template +class copy_assignment; + +#define MPARK_VARIANT_COPY_ASSIGNMENT(copy_assignable_trait, definition) \ + template \ + class copy_assignment, copy_assignable_trait> \ + : public move_assignment> { \ + using super = move_assignment>; \ + \ + public: \ + MPARK_INHERITING_CTOR(copy_assignment, super) \ + using super::operator=; \ + \ + copy_assignment(const copy_assignment &) = default; \ + copy_assignment(copy_assignment &&) = default; \ + ~copy_assignment() = default; \ + definition copy_assignment &operator=(copy_assignment &&) = default; \ + } + +MPARK_VARIANT_COPY_ASSIGNMENT( + Trait::TriviallyAvailable, + copy_assignment &operator=(const copy_assignment &that) = default;); + +MPARK_VARIANT_COPY_ASSIGNMENT( + Trait::Available, copy_assignment &operator=(const copy_assignment &that) { + this->generic_assign(that); + return *this; + }); + +MPARK_VARIANT_COPY_ASSIGNMENT( + Trait::Unavailable, + copy_assignment &operator=(const copy_assignment &) = delete;); + +#undef MPARK_VARIANT_COPY_ASSIGNMENT + +template +class impl : public copy_assignment> { + using super = copy_assignment>; + + public: + MPARK_INHERITING_CTOR(impl, super) + using super::operator=; + + template + inline void assign(Arg &&arg) { + this->assign_alt(access::base::get_alt(*this), lib::forward(arg)); + } + + inline void swap(impl &that) { + if (this->valueless_by_exception() && that.valueless_by_exception()) { + // do nothing. + } else if (this->index() == that.index()) { + visitation::alt::visit_alt_at(this->index(), +#ifdef MPARK_GENERIC_LAMBDAS + [](auto &this_alt, auto &that_alt) { + using std::swap; + swap(this_alt.value, that_alt.value); + } +#else + swapper {} +#endif + , + *this, + that); + } else { + impl *lhs = this; + impl *rhs = lib::addressof(that); + if (lhs->move_nothrow() && !rhs->move_nothrow()) { + std::swap(lhs, rhs); + } + impl tmp(lib::move(*rhs)); +#ifdef MPARK_EXCEPTIONS + // EXTENSION: When the move construction of `lhs` into `rhs` throws + // and `tmp` is nothrow move constructible then we move `tmp` back + // into `rhs` and provide the strong exception safety guarantee. + try { + this->generic_construct(*rhs, lib::move(*lhs)); + } catch (...) { + if (tmp.move_nothrow()) { + this->generic_construct(*rhs, lib::move(tmp)); + } + throw; + } +#else + this->generic_construct(*rhs, lib::move(*lhs)); +#endif + this->generic_construct(*lhs, lib::move(tmp)); + } + } + + private: +#ifndef MPARK_GENERIC_LAMBDAS + struct swapper { + template + inline void operator()(ThisAlt &this_alt, ThatAlt &that_alt) const { + using std::swap; + swap(this_alt.value, that_alt.value); + } + }; +#endif + + inline constexpr bool move_nothrow() const { + return this->valueless_by_exception() || + lib::array{{std::is_nothrow_move_constructible< + Ts>::value...}}[this->index()]; + } +}; + +#undef MPARK_INHERITING_CTOR + +template +struct overload_leaf { + using F = lib::size_constant (*)(T); + operator F() const { return nullptr; } +}; + +template +struct overload_impl { + private: + template + struct impl; + + template + struct impl> : overload_leaf... {}; + + public: + using type = impl>; +}; + +template +using overload = typename overload_impl::type; + +template +using best_match = lib::invoke_result_t, T &&>; + +template +struct is_in_place_index : std::false_type {}; + +template +struct is_in_place_index> : std::true_type {}; + +template +struct is_in_place_type : std::false_type {}; + +template +struct is_in_place_type> : std::true_type {}; + +} // detail + +template +class variant { + static_assert(0 < sizeof...(Ts), + "variant must consist of at least one alternative."); + + static_assert(lib::all::value...>::value, + "variant can not have an array type as an alternative."); + + static_assert(lib::all::value...>::value, + "variant can not have a reference type as an alternative."); + + static_assert(lib::all::value...>::value, + "variant can not have a void type as an alternative."); + + public: + template < + typename Front = lib::type_pack_element_t<0, Ts...>, + lib::enable_if_t::value, int> = 0> + inline constexpr variant() noexcept( + std::is_nothrow_default_constructible::value) + : impl_(in_place_index_t<0>{}) {} + + variant(const variant &) = default; + variant(variant &&) = default; + + template < + typename Arg, + typename Decayed = lib::decay_t, + lib::enable_if_t::value, int> = 0, + lib::enable_if_t::value, int> = 0, + lib::enable_if_t::value, int> = 0, + std::size_t I = detail::best_match::value, + typename T = lib::type_pack_element_t, + lib::enable_if_t::value, int> = 0> + inline constexpr variant(Arg &&arg) noexcept( + std::is_nothrow_constructible::value) + : impl_(in_place_index_t{}, lib::forward(arg)) {} + + template , + lib::enable_if_t::value, int> = 0> + inline explicit constexpr variant( + in_place_index_t, + Args + &&... args) noexcept(std::is_nothrow_constructible::value) + : impl_(in_place_index_t{}, lib::forward(args)...) {} + + template < + std::size_t I, + typename Up, + typename... Args, + typename T = lib::type_pack_element_t, + lib::enable_if_t< + std::is_constructible &, Args...>::value, + int> = 0> + inline explicit constexpr variant( + in_place_index_t, + std::initializer_list il, + Args &&... args) noexcept(std:: + is_nothrow_constructible< + T, + std::initializer_list &, + Args...>::value) + : impl_(in_place_index_t{}, il, lib::forward(args)...) {} + + template ::value, + lib::enable_if_t::value, int> = 0> + inline explicit constexpr variant( + in_place_type_t, + Args + &&... args) noexcept(std::is_nothrow_constructible::value) + : impl_(in_place_index_t{}, lib::forward(args)...) {} + + template < + typename T, + typename Up, + typename... Args, + std::size_t I = detail::find_index_sfinae::value, + lib::enable_if_t< + std::is_constructible &, Args...>::value, + int> = 0> + inline explicit constexpr variant( + in_place_type_t, + std::initializer_list il, + Args &&... args) noexcept(std:: + is_nothrow_constructible< + T, + std::initializer_list &, + Args...>::value) + : impl_(in_place_index_t{}, il, lib::forward(args)...) {} + + ~variant() = default; + + variant &operator=(const variant &) = default; + variant &operator=(variant &&) = default; + + template , variant>::value, + int> = 0, + std::size_t I = detail::best_match::value, + typename T = lib::type_pack_element_t, + lib::enable_if_t<(std::is_assignable::value && + std::is_constructible::value), + int> = 0> + inline variant &operator=(Arg &&arg) noexcept( + (std::is_nothrow_assignable::value && + std::is_nothrow_constructible::value)) { + impl_.template assign(lib::forward(arg)); + return *this; + } + + template , + lib::enable_if_t::value, int> = 0> + inline T &emplace(Args &&... args) { + return impl_.template emplace(lib::forward(args)...); + } + + template < + std::size_t I, + typename Up, + typename... Args, + typename T = lib::type_pack_element_t, + lib::enable_if_t< + std::is_constructible &, Args...>::value, + int> = 0> + inline T &emplace(std::initializer_list il, Args &&... args) { + return impl_.template emplace(il, lib::forward(args)...); + } + + template ::value, + lib::enable_if_t::value, int> = 0> + inline T &emplace(Args &&... args) { + return impl_.template emplace(lib::forward(args)...); + } + + template < + typename T, + typename Up, + typename... Args, + std::size_t I = detail::find_index_sfinae::value, + lib::enable_if_t< + std::is_constructible &, Args...>::value, + int> = 0> + inline T &emplace(std::initializer_list il, Args &&... args) { + return impl_.template emplace(il, lib::forward(args)...); + } + + inline constexpr bool valueless_by_exception() const noexcept { + return impl_.valueless_by_exception(); + } + + inline constexpr std::size_t index() const noexcept { return impl_.index(); } + + template , + Dummy>::value && + lib::dependent_type, + Dummy>::value)...>::value, + int> = 0> + inline void swap(variant &that) noexcept( + lib::all<(std::is_nothrow_move_constructible::value && + lib::is_nothrow_swappable::value)...>::value) { + impl_.swap(that.impl_); + } + + private: + detail::impl impl_; + + friend struct detail::access::variant; + friend struct detail::visitation::variant; +}; + +template +inline constexpr bool holds_alternative(const variant &v) noexcept { + return v.index() == I; +} + +template +inline constexpr bool holds_alternative(const variant &v) noexcept { + return holds_alternative::value>(v); +} + +namespace detail { +template +struct generic_get_impl { + constexpr generic_get_impl(int) noexcept {} + + constexpr AUTO_REFREF operator()(V &&v) const + AUTO_REFREF_RETURN(access::variant::get_alt(lib::forward(v)).value) +}; + +template +inline constexpr AUTO_REFREF generic_get(V &&v) + AUTO_REFREF_RETURN(generic_get_impl(holds_alternative(v) + ? 0 + : (throw_bad_variant_access(), + 0))(lib::forward(v))) +} // namespace detail + +template +inline constexpr variant_alternative_t> &get( + variant &v) { + return detail::generic_get(v); +} + +template +inline constexpr variant_alternative_t> &&get( + variant &&v) { + return detail::generic_get(lib::move(v)); +} + +template +inline constexpr const variant_alternative_t> &get( + const variant &v) { + return detail::generic_get(v); +} + +template +inline constexpr const variant_alternative_t> &&get( + const variant &&v) { + return detail::generic_get(lib::move(v)); +} + +template +inline constexpr T &get(variant &v) { + return get::value>(v); +} + +template +inline constexpr T &&get(variant &&v) { + return get::value>(lib::move(v)); +} + +template +inline constexpr const T &get(const variant &v) { + return get::value>(v); +} + +template +inline constexpr const T &&get(const variant &&v) { + return get::value>(lib::move(v)); +} + +namespace detail { + +template +inline constexpr /* auto * */ AUTO generic_get_if(V *v) noexcept AUTO_RETURN( + v &&holds_alternative(*v) + ? lib::addressof(access::variant::get_alt(*v).value) + : nullptr) + +} // namespace detail + +template +inline constexpr lib::add_pointer_t>> +get_if(variant *v) noexcept { + return detail::generic_get_if(v); +} + +template +inline constexpr lib::add_pointer_t< + const variant_alternative_t>> +get_if(const variant *v) noexcept { + return detail::generic_get_if(v); +} + +template +inline constexpr lib::add_pointer_t get_if(variant *v) noexcept { + return get_if::value>(v); +} + +template +inline constexpr lib::add_pointer_t get_if( + const variant *v) noexcept { + return get_if::value>(v); +} + +namespace detail { +template +struct convert_to_bool { + template + inline constexpr bool operator()(Lhs &&lhs, Rhs &&rhs) const { + static_assert( + std::is_convertible, bool>::value, + "relational operators must return a type" + " implicitly convertible to bool"); + return lib::invoke(RelOp{}, lib::forward(lhs), lib::forward(rhs)); + } +}; +} // namespace detail + +template +inline constexpr bool operator==(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using equal_to = detail::convert_to_bool; +#ifdef MPARK_CPP14_CONSTEXPR + if (lhs.index() != rhs.index()) return false; + if (lhs.valueless_by_exception()) return true; + return variant::visit_value_at(lhs.index(), equal_to{}, lhs, rhs); +#else + return lhs.index() == rhs.index() && + (lhs.valueless_by_exception() || + variant::visit_value_at(lhs.index(), equal_to{}, lhs, rhs)); +#endif +} + +template +inline constexpr bool operator!=(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using not_equal_to = detail::convert_to_bool; +#ifdef MPARK_CPP14_CONSTEXPR + if (lhs.index() != rhs.index()) return true; + if (lhs.valueless_by_exception()) return false; + return variant::visit_value_at(lhs.index(), not_equal_to{}, lhs, rhs); +#else + return lhs.index() != rhs.index() || + (!lhs.valueless_by_exception() && + variant::visit_value_at(lhs.index(), not_equal_to{}, lhs, rhs)); +#endif +} + +template +inline constexpr bool operator<(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using less = detail::convert_to_bool; +#ifdef MPARK_CPP14_CONSTEXPR + if (rhs.valueless_by_exception()) return false; + if (lhs.valueless_by_exception()) return true; + if (lhs.index() < rhs.index()) return true; + if (lhs.index() > rhs.index()) return false; + return variant::visit_value_at(lhs.index(), less{}, lhs, rhs); +#else + return !rhs.valueless_by_exception() && + (lhs.valueless_by_exception() || lhs.index() < rhs.index() || + (lhs.index() == rhs.index() && + variant::visit_value_at(lhs.index(), less{}, lhs, rhs))); +#endif +} + +template +inline constexpr bool operator>(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using greater = detail::convert_to_bool; +#ifdef MPARK_CPP14_CONSTEXPR + if (lhs.valueless_by_exception()) return false; + if (rhs.valueless_by_exception()) return true; + if (lhs.index() > rhs.index()) return true; + if (lhs.index() < rhs.index()) return false; + return variant::visit_value_at(lhs.index(), greater{}, lhs, rhs); +#else + return !lhs.valueless_by_exception() && + (rhs.valueless_by_exception() || lhs.index() > rhs.index() || + (lhs.index() == rhs.index() && + variant::visit_value_at(lhs.index(), greater{}, lhs, rhs))); +#endif +} + +template +inline constexpr bool operator<=(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using less_equal = detail::convert_to_bool; +#ifdef MPARK_CPP14_CONSTEXPR + if (lhs.valueless_by_exception()) return true; + if (rhs.valueless_by_exception()) return false; + if (lhs.index() < rhs.index()) return true; + if (lhs.index() > rhs.index()) return false; + return variant::visit_value_at(lhs.index(), less_equal{}, lhs, rhs); +#else + return lhs.valueless_by_exception() || + (!rhs.valueless_by_exception() && + (lhs.index() < rhs.index() || + (lhs.index() == rhs.index() && + variant::visit_value_at(lhs.index(), less_equal{}, lhs, rhs)))); +#endif +} + +template +inline constexpr bool operator>=(const variant &lhs, + const variant &rhs) { + using detail::visitation::variant; + using greater_equal = detail::convert_to_bool; +#ifdef MPARK_CPP14_CONSTEXPR + if (rhs.valueless_by_exception()) return true; + if (lhs.valueless_by_exception()) return false; + if (lhs.index() > rhs.index()) return true; + if (lhs.index() < rhs.index()) return false; + return variant::visit_value_at(lhs.index(), greater_equal{}, lhs, rhs); +#else + return rhs.valueless_by_exception() || + (!lhs.valueless_by_exception() && + (lhs.index() > rhs.index() || + (lhs.index() == rhs.index() && + variant::visit_value_at(lhs.index(), greater_equal{}, lhs, rhs)))); +#endif +} + +struct monostate {}; + +inline constexpr bool operator<(monostate, monostate) noexcept { return false; } + +inline constexpr bool operator>(monostate, monostate) noexcept { return false; } + +inline constexpr bool operator<=(monostate, monostate) noexcept { return true; } + +inline constexpr bool operator>=(monostate, monostate) noexcept { return true; } + +inline constexpr bool operator==(monostate, monostate) noexcept { return true; } + +inline constexpr bool operator!=(monostate, monostate) noexcept { + return false; +} + +#ifdef MPARK_CPP14_CONSTEXPR +namespace detail { + +inline constexpr bool all(std::initializer_list bs) { + for (bool b : bs) { + if (!b) { + return false; + } + } + return true; +} + +} // namespace detail + +template +inline constexpr decltype(auto) visit(Visitor &&visitor, Vs &&... vs) { + return (detail::all({!vs.valueless_by_exception()...}) + ? (void)0 + : throw_bad_variant_access()), + detail::visitation::variant::visit_value( + lib::forward(visitor), lib::forward(vs)...); +} +#else +namespace detail { + +template +inline constexpr bool all_impl(const lib::array &bs, std::size_t idx) { + return idx >= N || (bs[idx] && all_impl(bs, idx + 1)); +} + +template +inline constexpr bool all(const lib::array &bs) { + return all_impl(bs, 0); +} + +} // namespace detail + +template +inline constexpr DECLTYPE_AUTO visit(Visitor &&visitor, Vs &&... vs) + DECLTYPE_AUTO_RETURN( + (detail::all(lib::array{ + {!vs.valueless_by_exception()...}}) + ? (void)0 + : throw_bad_variant_access()), + detail::visitation::variant::visit_value(lib::forward(visitor), + lib::forward(vs)...)) +#endif + +template +inline auto swap(variant &lhs, + variant &rhs) noexcept(noexcept(lhs.swap(rhs))) + -> decltype(lhs.swap(rhs)) { + lhs.swap(rhs); +} + +namespace detail { + +template +using enabled_type = T; + +namespace hash { + +template +constexpr bool meets_requirements() noexcept { + return std::is_copy_constructible::value && + std::is_move_constructible::value && + lib::is_invocable_r::value; +} + +template +constexpr bool is_enabled() noexcept { + using H = std::hash; + return meets_requirements() && + std::is_default_constructible::value && + std::is_copy_assignable::value && std::is_move_assignable::value; +} + +} // namespace hash + +} // namespace detail + +#undef AUTO +#undef AUTO_RETURN + +#undef AUTO_REFREF +#undef AUTO_REFREF_RETURN + +#undef DECLTYPE_AUTO +#undef DECLTYPE_AUTO_RETURN + +} // namespace paddle + +namespace std { + +template +struct hash, + paddle::lib::enable_if_t>()...>::value>>> { + using argument_type = paddle::variant; + using result_type = std::size_t; + + inline result_type operator()(const argument_type &v) const { + using paddle::detail::visitation::variant; + std::size_t result = + v.valueless_by_exception() + ? 299792458 // Random value chosen by the universe upon creation + : variant::visit_alt( +#ifdef MPARK_GENERIC_LAMBDAS + [](const auto &alt) { + using alt_type = paddle::lib::decay_t; + using value_type = paddle::lib::remove_const_t< + typename alt_type::value_type>; + return hash{}(alt.value); + } +#else + hasher {} +#endif + , + v); + return hash_combine(result, hash{}(v.index())); + } + + private: +#ifndef MPARK_GENERIC_LAMBDAS + struct hasher { + template + inline std::size_t operator()(const Alt &alt) const { + using alt_type = paddle::lib::decay_t; + using value_type = + paddle::lib::remove_const_t; + return hash{}(alt.value); + } + }; +#endif + + static std::size_t hash_combine(std::size_t lhs, std::size_t rhs) { + return lhs ^= rhs + 0x9e3779b9 + (lhs << 6) + (lhs >> 2); + } +}; + +template <> +struct hash { + using argument_type = paddle::monostate; + using result_type = std::size_t; + + inline result_type operator()(const argument_type &) const noexcept { + return 66740831; // return a fundamentally attractive random value. + } +}; + +} // namespace std From d6b6692435b4d6ebedfd5bd01fc0baaafacbd660 Mon Sep 17 00:00:00 2001 From: pangyoki Date: Sun, 24 Apr 2022 11:52:12 +0800 Subject: [PATCH 46/66] disable unittest failed in eager CI in temporary (#42101) * test=py3-eager * test=py3-eager * test=py3-eager --- .../fluid/tests/custom_op/test_custom_tanh_double_grad.py | 3 ++- .../fluid/tests/unittests/check_nan_inf_base_dygraph.py | 2 ++ .../fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py | 3 +++ .../tests/unittests/dygraph_to_static/test_spec_names.py | 2 ++ python/paddle/fluid/tests/unittests/test_bfgs.py | 3 +++ python/paddle/fluid/tests/unittests/test_diff_op.py | 2 ++ python/paddle/fluid/tests/unittests/test_dropout_op.py | 4 +++- python/paddle/fluid/tests/unittests/test_eigh_op.py | 2 ++ .../paddle/fluid/tests/unittests/test_faster_tokenizer_op.py | 3 ++- .../fluid/tests/unittests/test_label_smooth_functional.py | 2 ++ python/paddle/fluid/tests/unittests/test_lbfgs.py | 3 +++ python/paddle/fluid/tests/unittests/test_nan_inf.py | 2 ++ .../tests/unittests/test_nn_functional_embedding_dygraph.py | 2 ++ .../unittests/test_tensor_scalar_type_promotion_dynamic.py | 2 ++ 14 files changed, 32 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py b/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py index 1127108c361ad..5664c00d74f89 100644 --- a/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py +++ b/python/paddle/fluid/tests/custom_op/test_custom_tanh_double_grad.py @@ -21,7 +21,8 @@ from paddle.utils.cpp_extension import load, get_build_directory from paddle.utils.cpp_extension.extension_utils import run_cmd from utils import paddle_includes, extra_cc_args, extra_nvcc_args -from paddle.fluid.framework import _test_eager_guard +from paddle.fluid.framework import _test_eager_guard, _enable_legacy_dygraph +_enable_legacy_dygraph() # Because Windows don't use docker, the shared lib already exists in the # cache dir, it will not be compiled again unless the shared lib is removed. diff --git a/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py b/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py index 08bab306df1b1..f4217d11f2d9b 100644 --- a/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py +++ b/python/paddle/fluid/tests/unittests/check_nan_inf_base_dygraph.py @@ -25,6 +25,8 @@ import paddle import paddle.nn as nn +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() np.random.seed(0) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py index 872d419ff8928..ab836b088b09f 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_simnet_v2.py @@ -20,6 +20,9 @@ from simnet_dygraph_model_v2 import BOW, HingeLoss +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() + SEED = 102 random.seed(SEED) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py index 361fcbf9c73f5..bafc4707c4ad9 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_spec_names.py @@ -16,6 +16,8 @@ from paddle.nn import Layer import numpy as np import unittest +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() class Net(Layer): diff --git a/python/paddle/fluid/tests/unittests/test_bfgs.py b/python/paddle/fluid/tests/unittests/test_bfgs.py index c89f7205f0818..4bf6de3eee510 100644 --- a/python/paddle/fluid/tests/unittests/test_bfgs.py +++ b/python/paddle/fluid/tests/unittests/test_bfgs.py @@ -21,6 +21,9 @@ from paddle.incubate.optimizer.functional.bfgs import minimize_bfgs +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() + np.random.seed(123) diff --git a/python/paddle/fluid/tests/unittests/test_diff_op.py b/python/paddle/fluid/tests/unittests/test_diff_op.py index 4a96827bd7c3c..99a46bfd9584d 100644 --- a/python/paddle/fluid/tests/unittests/test_diff_op.py +++ b/python/paddle/fluid/tests/unittests/test_diff_op.py @@ -19,6 +19,8 @@ import paddle.fluid as fluid import paddle.fluid.layers as layers import paddle.fluid.core as core +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() class TestDiffOp(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_dropout_op.py b/python/paddle/fluid/tests/unittests/test_dropout_op.py index 3aca428ac77af..20abeaec7268c 100644 --- a/python/paddle/fluid/tests/unittests/test_dropout_op.py +++ b/python/paddle/fluid/tests/unittests/test_dropout_op.py @@ -22,7 +22,8 @@ import paddle.static as static import paddle.fluid as fluid from paddle.fluid import Program, program_guard -from paddle.fluid.framework import _test_eager_guard +from paddle.fluid.framework import _test_eager_guard, _enable_legacy_dygraph +_enable_legacy_dygraph() import os from paddle import _C_ops @@ -951,6 +952,7 @@ def cal_grad_downscale_in_infer(self, mask): return mask.astype("float32") def test_backward_downscale_in_infer(self): + _enable_legacy_dygraph() for place in self.places: with fluid.dygraph.guard(place): diff --git a/python/paddle/fluid/tests/unittests/test_eigh_op.py b/python/paddle/fluid/tests/unittests/test_eigh_op.py index 2abbcc98a6b7e..9c9cd883313a2 100644 --- a/python/paddle/fluid/tests/unittests/test_eigh_op.py +++ b/python/paddle/fluid/tests/unittests/test_eigh_op.py @@ -19,6 +19,8 @@ import paddle from op_test import OpTest from gradient_checker import grad_check +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() def valid_eigh_result(A, eigh_value, eigh_vector, uplo): diff --git a/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py b/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py index 190345958e0e5..87c4656cfa809 100755 --- a/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py +++ b/python/paddle/fluid/tests/unittests/test_faster_tokenizer_op.py @@ -22,7 +22,8 @@ import paddle import paddle.nn as nn from paddle.dataset.common import DATA_HOME -from paddle.fluid.framework import core, _non_static_mode +from paddle.fluid.framework import core, _non_static_mode, _enable_legacy_dygraph +_enable_legacy_dygraph() from paddle.fluid.layer_helper import LayerHelper from paddle import _C_ops diff --git a/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py b/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py index 54f5e64fda4b6..83c8ced79b1e8 100644 --- a/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py +++ b/python/paddle/fluid/tests/unittests/test_label_smooth_functional.py @@ -19,6 +19,8 @@ import paddle.nn.functional as F import paddle.fluid.initializer as I import unittest +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() class LabelSmoothTestCase(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_lbfgs.py b/python/paddle/fluid/tests/unittests/test_lbfgs.py index bb3818747601f..2cad4822b28b1 100644 --- a/python/paddle/fluid/tests/unittests/test_lbfgs.py +++ b/python/paddle/fluid/tests/unittests/test_lbfgs.py @@ -21,6 +21,9 @@ from paddle.incubate.optimizer.functional.lbfgs import minimize_lbfgs +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() + np.random.seed(123) diff --git a/python/paddle/fluid/tests/unittests/test_nan_inf.py b/python/paddle/fluid/tests/unittests/test_nan_inf.py index 84559048a2b8a..9b11f6711afc1 100644 --- a/python/paddle/fluid/tests/unittests/test_nan_inf.py +++ b/python/paddle/fluid/tests/unittests/test_nan_inf.py @@ -20,6 +20,8 @@ import sys import subprocess import paddle +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() paddle.enable_static() diff --git a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py index acff7daadeb33..e50424126e53e 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py +++ b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py @@ -19,6 +19,8 @@ import paddle import paddle.nn as nn import numpy as np +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() paddle.disable_static() diff --git a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py index 5f2dfbdd99e16..c5e3cb29e0c20 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py @@ -18,6 +18,8 @@ import numpy as np import paddle +from paddle.fluid.framework import _enable_legacy_dygraph +_enable_legacy_dygraph() # Support types are ref from `paddle.tensor.math` # - Related paddle dtypes: From 0e0f7da65df393b75ac69aaf95c4e212be68f678 Mon Sep 17 00:00:00 2001 From: seemingwang Date: Sun, 24 Apr 2022 14:09:47 +0800 Subject: [PATCH 47/66] combine graph_table and feature_table in graph_engine (#42134) * extract sub-graph * graph-engine merging * fix * fix * fix heter-ps config * test performance * test performance * test performance * test * test * update bfs * change cmake * test * test gpu speed * gpu_graph_engine optimization * add dsm sample method * add graph_neighbor_sample_v2 * Add graph_neighbor_sample_v2 * fix for loop * add cpu sample interface * fix kernel judgement * add ssd layer to graph_engine * fix allocation * fix syntax error * fix syntax error * fix pscore class * fix * change index settings * recover test * recover test * fix spelling * recover * fix * move cudamemcpy after cuda stream sync * fix linking problem * remove comment * add cpu test * test * add cpu test * change comment * combine feature table and graph table * test * test * pybind * test * test * test * test * pybind * pybind * fix cmake * pybind * fix * fix * add pybind * add pybind Co-authored-by: DesmonDay <908660116@qq.com> --- .../ps/service/graph_brpc_client.cc | 107 +--- .../ps/service/graph_brpc_client.h | 27 +- .../ps/service/graph_brpc_server.cc | 192 +++---- .../ps/service/ps_service/graph_py_service.cc | 365 ++++++++----- .../ps/service/ps_service/graph_py_service.h | 52 +- .../ps/table/common_graph_table.cc | 481 ++++++++---------- .../distributed/ps/table/common_graph_table.h | 71 +-- .../distributed/test/graph_node_split_test.cc | 56 +- .../fluid/distributed/test/graph_node_test.cc | 436 ++++++++-------- paddle/fluid/distributed/the_one_ps.proto | 20 +- .../fleet/heter_ps/.CMakeLists.txt.swp | Bin 0 -> 12288 bytes .../framework/fleet/heter_ps/CMakeLists.txt | 1 + .../framework/fleet/heter_ps/gpu_graph_node.h | 15 +- .../fleet/heter_ps/graph_gpu_ps_table.h | 3 + .../fleet/heter_ps/graph_gpu_ps_table_inl.h | 245 ++++++++- .../fleet/heter_ps/graph_gpu_wrapper.cu | 268 ++++++++++ .../fleet/heter_ps/graph_gpu_wrapper.h | 50 ++ .../framework/fleet/heter_ps/heter_comm_inl.h | 2 + .../fleet/heter_ps/test_cpu_query.cu | 87 +++- .../fleet/heter_ps/test_sample_rate.cu | 33 +- paddle/fluid/pybind/CMakeLists.txt | 3 + paddle/fluid/pybind/fleet_py.cc | 32 +- paddle/fluid/pybind/fleet_py.h | 4 + paddle/fluid/pybind/pybind.cc | 4 + 24 files changed, 1618 insertions(+), 936 deletions(-) create mode 100644 paddle/fluid/framework/fleet/heter_ps/.CMakeLists.txt.swp create mode 100644 paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.cu create mode 100644 paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.h diff --git a/paddle/fluid/distributed/ps/service/graph_brpc_client.cc b/paddle/fluid/distributed/ps/service/graph_brpc_client.cc index 827a643ee50d6..c1df490669dbe 100644 --- a/paddle/fluid/distributed/ps/service/graph_brpc_client.cc +++ b/paddle/fluid/distributed/ps/service/graph_brpc_client.cc @@ -53,7 +53,7 @@ int GraphBrpcClient::get_server_index_by_id(int64_t id) { } std::future GraphBrpcClient::get_node_feat( - const uint32_t &table_id, const std::vector &node_ids, + const uint32_t &table_id, int idx_, const std::vector &node_ids, const std::vector &feature_names, std::vector> &res) { std::vector request2server; @@ -124,9 +124,11 @@ std::future GraphBrpcClient::get_node_feat( int server_index = request2server[request_idx]; closure->request(request_idx)->set_cmd_id(PS_GRAPH_GET_NODE_FEAT); closure->request(request_idx)->set_table_id(table_id); + closure->request(request_idx)->set_client_id(_client_id); size_t node_num = node_id_buckets[request_idx].size(); + closure->request(request_idx)->add_params((char *)&idx_, sizeof(int)); closure->request(request_idx) ->add_params((char *)node_id_buckets[request_idx].data(), sizeof(int64_t) * node_num); @@ -144,7 +146,8 @@ std::future GraphBrpcClient::get_node_feat( return fut; } -std::future GraphBrpcClient::clear_nodes(uint32_t table_id) { +std::future GraphBrpcClient::clear_nodes(uint32_t table_id, + int type_id, int idx_) { DownpourBrpcClosure *closure = new DownpourBrpcClosure( server_size, [&, server_size = this->server_size ](void *done) { int ret = 0; @@ -167,7 +170,8 @@ std::future GraphBrpcClient::clear_nodes(uint32_t table_id) { closure->request(server_index)->set_cmd_id(PS_GRAPH_CLEAR); closure->request(server_index)->set_table_id(table_id); closure->request(server_index)->set_client_id(_client_id); - + closure->request(server_index)->add_params((char *)&type_id, sizeof(int)); + closure->request(server_index)->add_params((char *)&idx_, sizeof(int)); GraphPsService_Stub rpc_stub = getServiceStub(GetCmdChannel(server_index)); closure->cntl(server_index)->set_log_id(butil::gettimeofday_ms()); rpc_stub.service(closure->cntl(server_index), @@ -177,7 +181,7 @@ std::future GraphBrpcClient::clear_nodes(uint32_t table_id) { return fut; } std::future GraphBrpcClient::add_graph_node( - uint32_t table_id, std::vector &node_id_list, + uint32_t table_id, int idx_, std::vector &node_id_list, std::vector &is_weighted_list) { std::vector> request_bucket; std::vector> is_weighted_bucket; @@ -225,6 +229,7 @@ std::future GraphBrpcClient::add_graph_node( closure->request(request_idx)->set_table_id(table_id); closure->request(request_idx)->set_client_id(_client_id); size_t node_num = request_bucket[request_idx].size(); + closure->request(request_idx)->add_params((char *)&idx_, sizeof(int)); closure->request(request_idx) ->add_params((char *)request_bucket[request_idx].data(), sizeof(int64_t) * node_num); @@ -245,7 +250,7 @@ std::future GraphBrpcClient::add_graph_node( return fut; } std::future GraphBrpcClient::remove_graph_node( - uint32_t table_id, std::vector &node_id_list) { + uint32_t table_id, int idx_, std::vector &node_id_list) { std::vector> request_bucket; std::vector server_index_arr; std::vector index_mapping(server_size, -1); @@ -286,6 +291,7 @@ std::future GraphBrpcClient::remove_graph_node( closure->request(request_idx)->set_client_id(_client_id); size_t node_num = request_bucket[request_idx].size(); + closure->request(request_idx)->add_params((char *)&idx_, sizeof(int)); closure->request(request_idx) ->add_params((char *)request_bucket[request_idx].data(), sizeof(int64_t) * node_num); @@ -299,7 +305,7 @@ std::future GraphBrpcClient::remove_graph_node( } // char* &buffer,int &actual_size std::future GraphBrpcClient::batch_sample_neighbors( - uint32_t table_id, std::vector node_ids, int sample_size, + uint32_t table_id, int idx_, std::vector node_ids, int sample_size, // std::vector>> &res, std::vector> &res, std::vector> &res_weight, bool need_weight, @@ -353,6 +359,7 @@ std::future GraphBrpcClient::batch_sample_neighbors( closure->request(0)->set_cmd_id(PS_GRAPH_SAMPLE_NODES_FROM_ONE_SERVER); closure->request(0)->set_table_id(table_id); closure->request(0)->set_client_id(_client_id); + closure->request(0)->add_params((char *)&idx_, sizeof(int)); closure->request(0)->add_params((char *)node_ids.data(), sizeof(int64_t) * node_ids.size()); closure->request(0)->add_params((char *)&sample_size, sizeof(int)); @@ -452,6 +459,7 @@ std::future GraphBrpcClient::batch_sample_neighbors( closure->request(request_idx)->set_client_id(_client_id); size_t node_num = node_id_buckets[request_idx].size(); + closure->request(request_idx)->add_params((char *)&idx_, sizeof(int)); closure->request(request_idx) ->add_params((char *)node_id_buckets[request_idx].data(), sizeof(int64_t) * node_num); @@ -469,7 +477,7 @@ std::future GraphBrpcClient::batch_sample_neighbors( return fut; } std::future GraphBrpcClient::random_sample_nodes( - uint32_t table_id, int server_index, int sample_size, + uint32_t table_id, int type_id, int idx_, int server_index, int sample_size, std::vector &ids) { DownpourBrpcClosure *closure = new DownpourBrpcClosure(1, [&](void *done) { int ret = 0; @@ -498,6 +506,8 @@ std::future GraphBrpcClient::random_sample_nodes( closure->request(0)->set_cmd_id(PS_GRAPH_SAMPLE_NODES); closure->request(0)->set_table_id(table_id); closure->request(0)->set_client_id(_client_id); + closure->request(0)->add_params((char *)&type_id, sizeof(int)); + closure->request(0)->add_params((char *)&idx_, sizeof(int)); closure->request(0)->add_params((char *)&sample_size, sizeof(int)); ; // PsService_Stub rpc_stub(GetCmdChannel(server_index)); @@ -508,83 +518,9 @@ std::future GraphBrpcClient::random_sample_nodes( return fut; } -std::future GraphBrpcClient::load_graph_split_config( - uint32_t table_id, std::string path) { - DownpourBrpcClosure *closure = new DownpourBrpcClosure( - server_size, [&, server_size = this->server_size ](void *done) { - int ret = 0; - auto *closure = (DownpourBrpcClosure *)done; - size_t fail_num = 0; - for (size_t request_idx = 0; request_idx < server_size; ++request_idx) { - if (closure->check_response(request_idx, - PS_GRAPH_LOAD_GRAPH_SPLIT_CONFIG) != 0) { - ++fail_num; - break; - } - } - ret = fail_num == 0 ? 0 : -1; - closure->set_promise_value(ret); - }); - auto promise = std::make_shared>(); - closure->add_promise(promise); - std::future fut = promise->get_future(); - for (size_t i = 0; i < server_size; i++) { - int server_index = i; - closure->request(server_index) - ->set_cmd_id(PS_GRAPH_LOAD_GRAPH_SPLIT_CONFIG); - closure->request(server_index)->set_table_id(table_id); - closure->request(server_index)->set_client_id(_client_id); - closure->request(server_index)->add_params(path); - GraphPsService_Stub rpc_stub = getServiceStub(GetCmdChannel(server_index)); - closure->cntl(server_index)->set_log_id(butil::gettimeofday_ms()); - rpc_stub.service(closure->cntl(server_index), - closure->request(server_index), - closure->response(server_index), closure); - } - return fut; -} -std::future GraphBrpcClient::use_neighbors_sample_cache( - uint32_t table_id, size_t total_size_limit, size_t ttl) { - DownpourBrpcClosure *closure = new DownpourBrpcClosure( - server_size, [&, server_size = this->server_size ](void *done) { - int ret = 0; - auto *closure = (DownpourBrpcClosure *)done; - size_t fail_num = 0; - for (size_t request_idx = 0; request_idx < server_size; ++request_idx) { - if (closure->check_response( - request_idx, PS_GRAPH_USE_NEIGHBORS_SAMPLE_CACHE) != 0) { - ++fail_num; - break; - } - } - ret = fail_num == 0 ? 0 : -1; - closure->set_promise_value(ret); - }); - auto promise = std::make_shared>(); - closure->add_promise(promise); - size_t size_limit = total_size_limit / server_size + - (total_size_limit % server_size != 0 ? 1 : 0); - std::future fut = promise->get_future(); - for (size_t i = 0; i < server_size; i++) { - int server_index = i; - closure->request(server_index) - ->set_cmd_id(PS_GRAPH_USE_NEIGHBORS_SAMPLE_CACHE); - closure->request(server_index)->set_table_id(table_id); - closure->request(server_index)->set_client_id(_client_id); - closure->request(server_index) - ->add_params((char *)&size_limit, sizeof(size_t)); - closure->request(server_index)->add_params((char *)&ttl, sizeof(size_t)); - GraphPsService_Stub rpc_stub = getServiceStub(GetCmdChannel(server_index)); - closure->cntl(server_index)->set_log_id(butil::gettimeofday_ms()); - rpc_stub.service(closure->cntl(server_index), - closure->request(server_index), - closure->response(server_index), closure); - } - return fut; -} std::future GraphBrpcClient::pull_graph_list( - uint32_t table_id, int server_index, int start, int size, int step, - std::vector &res) { + uint32_t table_id, int type_id, int idx_, int server_index, int start, + int size, int step, std::vector &res) { DownpourBrpcClosure *closure = new DownpourBrpcClosure(1, [&](void *done) { int ret = 0; auto *closure = (DownpourBrpcClosure *)done; @@ -613,6 +549,8 @@ std::future GraphBrpcClient::pull_graph_list( closure->request(0)->set_cmd_id(PS_PULL_GRAPH_LIST); closure->request(0)->set_table_id(table_id); closure->request(0)->set_client_id(_client_id); + closure->request(0)->add_params((char *)&type_id, sizeof(int)); + closure->request(0)->add_params((char *)&idx_, sizeof(int)); closure->request(0)->add_params((char *)&start, sizeof(int)); closure->request(0)->add_params((char *)&size, sizeof(int)); closure->request(0)->add_params((char *)&step, sizeof(int)); @@ -625,7 +563,7 @@ std::future GraphBrpcClient::pull_graph_list( } std::future GraphBrpcClient::set_node_feat( - const uint32_t &table_id, const std::vector &node_ids, + const uint32_t &table_id, int idx_, const std::vector &node_ids, const std::vector &feature_names, const std::vector> &features) { std::vector request2server; @@ -686,6 +624,7 @@ std::future GraphBrpcClient::set_node_feat( closure->request(request_idx)->set_client_id(_client_id); size_t node_num = node_id_buckets[request_idx].size(); + closure->request(request_idx)->add_params((char *)&idx_, sizeof(int)); closure->request(request_idx) ->add_params((char *)node_id_buckets[request_idx].data(), sizeof(int64_t) * node_num); diff --git a/paddle/fluid/distributed/ps/service/graph_brpc_client.h b/paddle/fluid/distributed/ps/service/graph_brpc_client.h index d1d3c95260df4..51f14bc57cde0 100644 --- a/paddle/fluid/distributed/ps/service/graph_brpc_client.h +++ b/paddle/fluid/distributed/ps/service/graph_brpc_client.h @@ -63,40 +63,37 @@ class GraphBrpcClient : public BrpcPsClient { virtual ~GraphBrpcClient() {} // given a batch of nodes, sample graph_neighbors for each of them virtual std::future batch_sample_neighbors( - uint32_t table_id, std::vector node_ids, int sample_size, - std::vector>& res, + uint32_t table_id, int idx, std::vector node_ids, + int sample_size, std::vector>& res, std::vector>& res_weight, bool need_weight, int server_index = -1); - virtual std::future pull_graph_list(uint32_t table_id, - int server_index, int start, - int size, int step, + virtual std::future pull_graph_list(uint32_t table_id, int type_id, + int idx, int server_index, + int start, int size, int step, std::vector& res); virtual std::future random_sample_nodes(uint32_t table_id, + int type_id, int idx, int server_index, int sample_size, std::vector& ids); virtual std::future get_node_feat( - const uint32_t& table_id, const std::vector& node_ids, + const uint32_t& table_id, int idx, const std::vector& node_ids, const std::vector& feature_names, std::vector>& res); virtual std::future set_node_feat( - const uint32_t& table_id, const std::vector& node_ids, + const uint32_t& table_id, int idx, const std::vector& node_ids, const std::vector& feature_names, const std::vector>& features); - virtual std::future clear_nodes(uint32_t table_id); + virtual std::future clear_nodes(uint32_t table_id, int type_id, + int idx); virtual std::future add_graph_node( - uint32_t table_id, std::vector& node_id_list, + uint32_t table_id, int idx, std::vector& node_id_list, std::vector& is_weighted_list); - virtual std::future use_neighbors_sample_cache(uint32_t table_id, - size_t size_limit, - size_t ttl); - virtual std::future load_graph_split_config(uint32_t table_id, - std::string path); virtual std::future remove_graph_node( - uint32_t table_id, std::vector& node_id_list); + uint32_t table_id, int idx_, std::vector& node_id_list); virtual int32_t Initialize(); int get_shard_num() { return shard_num; } void set_shard_num(int shard_num) { this->shard_num = shard_num; } diff --git a/paddle/fluid/distributed/ps/service/graph_brpc_server.cc b/paddle/fluid/distributed/ps/service/graph_brpc_server.cc index 21e590997b178..8ff12265269b2 100644 --- a/paddle/fluid/distributed/ps/service/graph_brpc_server.cc +++ b/paddle/fluid/distributed/ps/service/graph_brpc_server.cc @@ -124,7 +124,9 @@ int32_t GraphBrpcService::clear_nodes(Table *table, const PsRequestMessage &request, PsResponseMessage &response, brpc::Controller *cntl) { - ((GraphTable *)table)->clear_nodes(); + int type_id = *(int *)(request.params(0).c_str()); + int idx_ = *(int *)(request.params(1).c_str()); + ((GraphTable *)table)->clear_nodes(type_id, idx_); return 0; } @@ -133,25 +135,34 @@ int32_t GraphBrpcService::add_graph_node(Table *table, PsResponseMessage &response, brpc::Controller *cntl) { CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 1) { - set_response_code( - response, -1, - "graph_get_node_feat request requires at least 2 arguments"); + if (request.params_size() < 2) { + set_response_code(response, -1, + "add_graph_node request requires at least 2 arguments"); return 0; } - size_t node_num = request.params(0).size() / sizeof(int64_t); - int64_t *node_data = (int64_t *)(request.params(0).c_str()); + int idx_ = *(int *)(request.params(0).c_str()); + size_t node_num = request.params(1).size() / sizeof(int64_t); + int64_t *node_data = (int64_t *)(request.params(1).c_str()); + // size_t node_num = request.params(0).size() / sizeof(int64_t); + // int64_t *node_data = (int64_t *)(request.params(0).c_str()); std::vector node_ids(node_data, node_data + node_num); std::vector is_weighted_list; - if (request.params_size() == 2) { - size_t weight_list_size = request.params(1).size() / sizeof(bool); - bool *is_weighted_buffer = (bool *)(request.params(1).c_str()); + if (request.params_size() == 3) { + size_t weight_list_size = request.params(2).size() / sizeof(bool); + bool *is_weighted_buffer = (bool *)(request.params(2).c_str()); is_weighted_list = std::vector(is_weighted_buffer, is_weighted_buffer + weight_list_size); } + // if (request.params_size() == 2) { + // size_t weight_list_size = request.params(1).size() / sizeof(bool); + // bool *is_weighted_buffer = (bool *)(request.params(1).c_str()); + // is_weighted_list = std::vector(is_weighted_buffer, + // is_weighted_buffer + + // weight_list_size); + // } - ((GraphTable *)table)->add_graph_node(node_ids, is_weighted_list); + ((GraphTable *)table)->add_graph_node(idx_, node_ids, is_weighted_list); return 0; } int32_t GraphBrpcService::remove_graph_node(Table *table, @@ -159,17 +170,20 @@ int32_t GraphBrpcService::remove_graph_node(Table *table, PsResponseMessage &response, brpc::Controller *cntl) { CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 1) { + if (request.params_size() < 2) { set_response_code( response, -1, - "graph_get_node_feat request requires at least 1 argument"); + "remove_graph_node request requires at least 2 arguments"); return 0; } - size_t node_num = request.params(0).size() / sizeof(int64_t); - int64_t *node_data = (int64_t *)(request.params(0).c_str()); + int idx_ = *(int *)(request.params(0).c_str()); + size_t node_num = request.params(1).size() / sizeof(int64_t); + int64_t *node_data = (int64_t *)(request.params(1).c_str()); + // size_t node_num = request.params(0).size() / sizeof(int64_t); + // int64_t *node_data = (int64_t *)(request.params(0).c_str()); std::vector node_ids(node_data, node_data + node_num); - ((GraphTable *)table)->remove_graph_node(node_ids); + ((GraphTable *)table)->remove_graph_node(idx_, node_ids); return 0; } int32_t GraphBrpcServer::Port() { return _server.listen_address().port; } @@ -201,10 +215,10 @@ int32_t GraphBrpcService::Initialize() { &GraphBrpcService::graph_set_node_feat; _service_handler_map[PS_GRAPH_SAMPLE_NODES_FROM_ONE_SERVER] = &GraphBrpcService::sample_neighbors_across_multi_servers; - _service_handler_map[PS_GRAPH_USE_NEIGHBORS_SAMPLE_CACHE] = - &GraphBrpcService::use_neighbors_sample_cache; - _service_handler_map[PS_GRAPH_LOAD_GRAPH_SPLIT_CONFIG] = - &GraphBrpcService::load_graph_split_config; + // _service_handler_map[PS_GRAPH_USE_NEIGHBORS_SAMPLE_CACHE] = + // &GraphBrpcService::use_neighbors_sample_cache; + // _service_handler_map[PS_GRAPH_LOAD_GRAPH_SPLIT_CONFIG] = + // &GraphBrpcService::load_graph_split_config; // shard初始化,server启动后才可从env获取到server_list的shard信息 InitializeShardInfo(); @@ -360,18 +374,24 @@ int32_t GraphBrpcService::pull_graph_list(Table *table, PsResponseMessage &response, brpc::Controller *cntl) { CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 3) { + if (request.params_size() < 5) { set_response_code(response, -1, - "pull_graph_list request requires at least 3 arguments"); + "pull_graph_list request requires at least 5 arguments"); return 0; } - int start = *(int *)(request.params(0).c_str()); - int size = *(int *)(request.params(1).c_str()); - int step = *(int *)(request.params(2).c_str()); + int type_id = *(int *)(request.params(0).c_str()); + int idx = *(int *)(request.params(1).c_str()); + int start = *(int *)(request.params(2).c_str()); + int size = *(int *)(request.params(3).c_str()); + int step = *(int *)(request.params(4).c_str()); + // int start = *(int *)(request.params(0).c_str()); + // int size = *(int *)(request.params(1).c_str()); + // int step = *(int *)(request.params(2).c_str()); std::unique_ptr buffer; int actual_size; ((GraphTable *)table) - ->pull_graph_list(start, size, buffer, actual_size, false, step); + ->pull_graph_list(type_id, idx, start, size, buffer, actual_size, false, + step); cntl->response_attachment().append(buffer.get(), actual_size); return 0; } @@ -379,21 +399,26 @@ int32_t GraphBrpcService::graph_random_sample_neighbors( Table *table, const PsRequestMessage &request, PsResponseMessage &response, brpc::Controller *cntl) { CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 3) { + if (request.params_size() < 4) { set_response_code( response, -1, "graph_random_sample_neighbors request requires at least 3 arguments"); return 0; } - size_t node_num = request.params(0).size() / sizeof(int64_t); - int64_t *node_data = (int64_t *)(request.params(0).c_str()); - int sample_size = *(int64_t *)(request.params(1).c_str()); - bool need_weight = *(bool *)(request.params(2).c_str()); + int idx_ = *(int *)(request.params(0).c_str()); + size_t node_num = request.params(1).size() / sizeof(int64_t); + int64_t *node_data = (int64_t *)(request.params(1).c_str()); + int sample_size = *(int64_t *)(request.params(2).c_str()); + bool need_weight = *(bool *)(request.params(3).c_str()); + // size_t node_num = request.params(0).size() / sizeof(int64_t); + // int64_t *node_data = (int64_t *)(request.params(0).c_str()); + // int sample_size = *(int64_t *)(request.params(1).c_str()); + // bool need_weight = *(bool *)(request.params(2).c_str()); std::vector> buffers(node_num); std::vector actual_sizes(node_num, 0); ((GraphTable *)table) - ->random_sample_neighbors(node_data, sample_size, buffers, actual_sizes, - need_weight); + ->random_sample_neighbors(idx_, node_data, sample_size, buffers, + actual_sizes, need_weight); cntl->response_attachment().append(&node_num, sizeof(size_t)); cntl->response_attachment().append(actual_sizes.data(), @@ -406,10 +431,14 @@ int32_t GraphBrpcService::graph_random_sample_neighbors( int32_t GraphBrpcService::graph_random_sample_nodes( Table *table, const PsRequestMessage &request, PsResponseMessage &response, brpc::Controller *cntl) { - size_t size = *(int64_t *)(request.params(0).c_str()); + int type_id = *(int *)(request.params(0).c_str()); + int idx_ = *(int *)(request.params(1).c_str()); + size_t size = *(int64_t *)(request.params(2).c_str()); + // size_t size = *(int64_t *)(request.params(0).c_str()); std::unique_ptr buffer; int actual_size; - if (((GraphTable *)table)->random_sample_nodes(size, buffer, actual_size) == + if (((GraphTable *)table) + ->random_sample_nodes(type_id, idx_, size, buffer, actual_size) == 0) { cntl->response_attachment().append(buffer.get(), actual_size); } else @@ -423,23 +452,26 @@ int32_t GraphBrpcService::graph_get_node_feat(Table *table, PsResponseMessage &response, brpc::Controller *cntl) { CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 2) { + if (request.params_size() < 3) { set_response_code( response, -1, - "graph_get_node_feat request requires at least 2 arguments"); + "graph_get_node_feat request requires at least 3 arguments"); return 0; } - size_t node_num = request.params(0).size() / sizeof(int64_t); - int64_t *node_data = (int64_t *)(request.params(0).c_str()); + int idx_ = *(int *)(request.params(0).c_str()); + size_t node_num = request.params(1).size() / sizeof(int64_t); + int64_t *node_data = (int64_t *)(request.params(1).c_str()); + // size_t node_num = request.params(0).size() / sizeof(int64_t); + // int64_t *node_data = (int64_t *)(request.params(0).c_str()); std::vector node_ids(node_data, node_data + node_num); std::vector feature_names = - paddle::string::split_string(request.params(1), "\t"); + paddle::string::split_string(request.params(2), "\t"); std::vector> feature( feature_names.size(), std::vector(node_num)); - ((GraphTable *)table)->get_node_feat(node_ids, feature_names, feature); + ((GraphTable *)table)->get_node_feat(idx_, node_ids, feature_names, feature); for (size_t feat_idx = 0; feat_idx < feature_names.size(); ++feat_idx) { for (size_t node_idx = 0; node_idx < node_num; ++node_idx) { @@ -457,17 +489,25 @@ int32_t GraphBrpcService::sample_neighbors_across_multi_servers( brpc::Controller *cntl) { // sleep(5); CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 3) { + if (request.params_size() < 4) { set_response_code(response, -1, "sample_neighbors_across_multi_servers request requires " - "at least 3 arguments"); + "at least 4 arguments"); return 0; } - size_t node_num = request.params(0).size() / sizeof(int64_t), + + int idx_ = *(int *)(request.params(0).c_str()); + size_t node_num = request.params(1).size() / sizeof(int64_t), size_of_size_t = sizeof(size_t); - int64_t *node_data = (int64_t *)(request.params(0).c_str()); - int sample_size = *(int64_t *)(request.params(1).c_str()); - bool need_weight = *(int64_t *)(request.params(2).c_str()); + int64_t *node_data = (int64_t *)(request.params(1).c_str()); + int sample_size = *(int64_t *)(request.params(2).c_str()); + bool need_weight = *(int64_t *)(request.params(3).c_str()); + + // size_t node_num = request.params(0).size() / sizeof(int64_t), + // size_of_size_t = sizeof(size_t); + // int64_t *node_data = (int64_t *)(request.params(0).c_str()); + // int sample_size = *(int64_t *)(request.params(1).c_str()); + // bool need_weight = *(int64_t *)(request.params(2).c_str()); // std::vector res = ((GraphTable // *)table).filter_out_non_exist_nodes(node_data, sample_size); std::vector request2server; @@ -580,6 +620,8 @@ int32_t GraphBrpcService::sample_neighbors_across_multi_servers( closure->request(request_idx)->set_client_id(rank); size_t node_num = node_id_buckets[request_idx].size(); + closure->request(request_idx)->add_params((char *)&idx_, sizeof(int)); + closure->request(request_idx) ->add_params((char *)node_id_buckets[request_idx].data(), sizeof(int64_t) * node_num); @@ -597,9 +639,9 @@ int32_t GraphBrpcService::sample_neighbors_across_multi_servers( } if (server2request[rank] != -1) { ((GraphTable *)table) - ->random_sample_neighbors(node_id_buckets.back().data(), sample_size, - local_buffers, local_actual_sizes, - need_weight); + ->random_sample_neighbors(idx_, node_id_buckets.back().data(), + sample_size, local_buffers, + local_actual_sizes, need_weight); } local_promise.get()->set_value(0); if (remote_call_num == 0) func(closure); @@ -611,23 +653,31 @@ int32_t GraphBrpcService::graph_set_node_feat(Table *table, PsResponseMessage &response, brpc::Controller *cntl) { CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 3) { + if (request.params_size() < 4) { set_response_code( response, -1, "graph_set_node_feat request requires at least 3 arguments"); return 0; } - size_t node_num = request.params(0).size() / sizeof(int64_t); - int64_t *node_data = (int64_t *)(request.params(0).c_str()); + int idx_ = *(int *)(request.params(0).c_str()); + + // size_t node_num = request.params(0).size() / sizeof(int64_t); + // int64_t *node_data = (int64_t *)(request.params(0).c_str()); + size_t node_num = request.params(1).size() / sizeof(int64_t); + int64_t *node_data = (int64_t *)(request.params(1).c_str()); std::vector node_ids(node_data, node_data + node_num); + // std::vector feature_names = + // paddle::string::split_string(request.params(1), "\t"); + std::vector feature_names = - paddle::string::split_string(request.params(1), "\t"); + paddle::string::split_string(request.params(2), "\t"); std::vector> features( feature_names.size(), std::vector(node_num)); - const char *buffer = request.params(2).c_str(); + // const char *buffer = request.params(2).c_str(); + const char *buffer = request.params(3).c_str(); for (size_t feat_idx = 0; feat_idx < feature_names.size(); ++feat_idx) { for (size_t node_idx = 0; node_idx < node_num; ++node_idx) { @@ -639,40 +689,10 @@ int32_t GraphBrpcService::graph_set_node_feat(Table *table, } } - ((GraphTable *)table)->set_node_feat(node_ids, feature_names, features); + ((GraphTable *)table)->set_node_feat(idx_, node_ids, feature_names, features); return 0; } -int32_t GraphBrpcService::use_neighbors_sample_cache( - Table *table, const PsRequestMessage &request, PsResponseMessage &response, - brpc::Controller *cntl) { - CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 2) { - set_response_code(response, -1, - "use_neighbors_sample_cache request requires at least 2 " - "arguments[cache_size, ttl]"); - return 0; - } - size_t size_limit = *(size_t *)(request.params(0).c_str()); - size_t ttl = *(size_t *)(request.params(1).c_str()); - ((GraphTable *)table)->make_neighbor_sample_cache(size_limit, ttl); - return 0; -} - -int32_t GraphBrpcService::load_graph_split_config( - Table *table, const PsRequestMessage &request, PsResponseMessage &response, - brpc::Controller *cntl) { - CHECK_TABLE_EXIST(table, request, response) - if (request.params_size() < 1) { - set_response_code(response, -1, - "load_graph_split_configrequest requires at least 1 " - "argument1[file_path]"); - return 0; - } - ((GraphTable *)table)->load_graph_split_config(request.params(0)); - return 0; -} - } // namespace distributed } // namespace paddle diff --git a/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.cc b/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.cc index 92dfeb6818a28..ced51b8cbe383 100644 --- a/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.cc +++ b/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.cc @@ -35,35 +35,71 @@ std::vector GraphPyService::split(std::string& str, void GraphPyService::add_table_feat_conf(std::string table_name, std::string feat_name, std::string feat_dtype, - int32_t feat_shape) { - if (this->table_id_map.count(table_name)) { - this->table_feat_conf_table_name.push_back(table_name); - this->table_feat_conf_feat_name.push_back(feat_name); - this->table_feat_conf_feat_dtype.push_back(feat_dtype); - this->table_feat_conf_feat_shape.push_back(feat_shape); + int feat_shape) { + if (feature_to_id.find(table_name) != feature_to_id.end()) { + int idx = feature_to_id[table_name]; + VLOG(0) << "for table name" << table_name << " idx = " << idx; + if (table_feat_mapping[idx].find(feat_name) == + table_feat_mapping[idx].end()) { + VLOG(0) << "for table name not found,make a new one"; + int res = (int)table_feat_mapping[idx].size(); + table_feat_mapping[idx][feat_name] = res; + VLOG(0) << "seq id = " << table_feat_mapping[idx][feat_name]; + } + int feat_idx = table_feat_mapping[idx][feat_name]; + VLOG(0) << "table_name " << table_name << " mapping id " << idx; + VLOG(0) << " feat name " << feat_name << " feat id" << feat_idx; + if (feat_idx < table_feat_conf_feat_name[idx].size()) { + // overide + table_feat_conf_feat_name[idx][feat_idx] = feat_name; + table_feat_conf_feat_dtype[idx][feat_idx] = feat_dtype; + table_feat_conf_feat_shape[idx][feat_idx] = feat_shape; + } else { + // new + table_feat_conf_feat_name[idx].push_back(feat_name); + table_feat_conf_feat_dtype[idx].push_back(feat_dtype); + table_feat_conf_feat_shape[idx].push_back(feat_shape); + } } + VLOG(0) << "add conf over"; } -void add_graph_node(std::vector node_ids, +void add_graph_node(std::string name, std::vector node_ids, std::vector weight_list) {} -void remove_graph_node(std::vector node_ids) {} +void remove_graph_node(std::string name, std::vector node_ids) {} void GraphPyService::set_up(std::string ips_str, int shard_num, std::vector node_types, std::vector edge_types) { set_shard_num(shard_num); set_num_node_types(node_types.size()); - - for (size_t table_id = 0; table_id < node_types.size(); table_id++) { - this->table_id_map[node_types[table_id]] = this->table_id_map.size(); - } + /* + int num_node_types; + std::unordered_map edge_idx, feature_idx; + std::vector> table_feat_mapping; + std::vector> table_feat_conf_feat_name; + std::vector> table_feat_conf_feat_dtype; + std::vector> table_feat_conf_feat_shape; + */ + id_to_edge = edge_types; for (size_t table_id = 0; table_id < edge_types.size(); table_id++) { - this->table_id_map[edge_types[table_id]] = this->table_id_map.size(); + int res = (int)edge_to_id.size(); + edge_to_id[edge_types[table_id]] = res; + } + id_to_feature = node_types; + for (size_t table_id = 0; table_id < node_types.size(); table_id++) { + int res = (int)feature_to_id.size(); + feature_to_id[node_types[table_id]] = res; } + table_feat_mapping.resize(node_types.size()); + this->table_feat_conf_feat_name.resize(node_types.size()); + this->table_feat_conf_feat_dtype.resize(node_types.size()); + this->table_feat_conf_feat_shape.resize(node_types.size()); std::istringstream stream(ips_str); std::string ip; server_size = 0; std::vector ips_list = split(ips_str, ';'); int index = 0; + VLOG(0) << "start to build server"; for (auto ips : ips_list) { auto ip_and_port = split(ips, ':'); server_list.push_back(ip_and_port[0]); @@ -73,6 +109,7 @@ void GraphPyService::set_up(std::string ips_str, int shard_num, host_sign_list.push_back(ph_host.SerializeToString()); index++; } + VLOG(0) << "build server done"; } void GraphPyClient::start_client() { std::map> dense_regions; @@ -130,30 +167,29 @@ ::paddle::distributed::PSParameter GraphPyServer::GetServerProto() { server_service_proto->set_start_server_port(0); server_service_proto->set_server_thread_num(12); - for (auto& tuple : this->table_id_map) { - VLOG(0) << " make a new table " << tuple.second; - ::paddle::distributed::TableParameter* sparse_table_proto = - downpour_server_proto->add_downpour_table_param(); - std::vector feat_name; - std::vector feat_dtype; - std::vector feat_shape; - for (size_t i = 0; i < this->table_feat_conf_table_name.size(); i++) { - if (tuple.first == table_feat_conf_table_name[i]) { - feat_name.push_back(table_feat_conf_feat_name[i]); - feat_dtype.push_back(table_feat_conf_feat_dtype[i]); - feat_shape.push_back(table_feat_conf_feat_shape[i]); - } - } - std::string table_type; - if (tuple.second < this->num_node_types) { - table_type = "node"; - } else { - table_type = "edge"; - } + // for (auto& tuple : this->table_id_map) { + // VLOG(0) << " make a new table " << tuple.second; + ::paddle::distributed::TableParameter* sparse_table_proto = + downpour_server_proto->add_downpour_table_param(); + // std::vector feat_name; + // std::vector feat_dtype; + // std::vector feat_shape; + // for (size_t i = 0; i < this->table_feat_conf_table_name.size(); i++) { + // if (tuple.first == table_feat_conf_table_name[i]) { + // feat_name.push_back(table_feat_conf_feat_name[i]); + // feat_dtype.push_back(table_feat_conf_feat_dtype[i]); + // feat_shape.push_back(table_feat_conf_feat_shape[i]); + // } + // } + // std::string table_type; + // if (tuple.second < this->num_node_types) { + // table_type = "node"; + // } else { + // table_type = "edge"; + // } - GetDownpourSparseTableProto(sparse_table_proto, tuple.second, tuple.first, - table_type, feat_name, feat_dtype, feat_shape); - } + GetDownpourSparseTableProto(sparse_table_proto); + //} return server_fleet_desc; } @@ -166,31 +202,29 @@ ::paddle::distributed::PSParameter GraphPyClient::GetWorkerProto() { ::paddle::distributed::DownpourWorkerParameter* downpour_worker_proto = worker_proto->mutable_downpour_worker_param(); - for (auto& tuple : this->table_id_map) { - VLOG(0) << " make a new table " << tuple.second; - ::paddle::distributed::TableParameter* worker_sparse_table_proto = - downpour_worker_proto->add_downpour_table_param(); - std::vector feat_name; - std::vector feat_dtype; - std::vector feat_shape; - for (size_t i = 0; i < this->table_feat_conf_table_name.size(); i++) { - if (tuple.first == table_feat_conf_table_name[i]) { - feat_name.push_back(table_feat_conf_feat_name[i]); - feat_dtype.push_back(table_feat_conf_feat_dtype[i]); - feat_shape.push_back(table_feat_conf_feat_shape[i]); - } - } - std::string table_type; - if (tuple.second < this->num_node_types) { - table_type = "node"; - } else { - table_type = "edge"; - } + // for (auto& tuple : this->table_id_map) { + // VLOG(0) << " make a new table " << tuple.second; + ::paddle::distributed::TableParameter* worker_sparse_table_proto = + downpour_worker_proto->add_downpour_table_param(); + // std::vector feat_name; + // std::vector feat_dtype; + // std::vector feat_shape; + // for (size_t i = 0; i < this->table_feat_conf_table_name.size(); i++) { + // if (tuple.first == table_feat_conf_table_name[i]) { + // feat_name.push_back(table_feat_conf_feat_name[i]); + // feat_dtype.push_back(table_feat_conf_feat_dtype[i]); + // feat_shape.push_back(table_feat_conf_feat_shape[i]); + // } + // } + // std::string table_type; + // if (tuple.second < this->num_node_types) { + // table_type = "node"; + // } else { + // table_type = "edge"; + // } - GetDownpourSparseTableProto(worker_sparse_table_proto, tuple.second, - tuple.first, table_type, feat_name, feat_dtype, - feat_shape); - } + GetDownpourSparseTableProto(worker_sparse_table_proto); + //} ::paddle::distributed::ServerParameter* server_proto = worker_fleet_desc.mutable_server_param(); @@ -204,30 +238,29 @@ ::paddle::distributed::PSParameter GraphPyClient::GetWorkerProto() { server_service_proto->set_start_server_port(0); server_service_proto->set_server_thread_num(12); - for (auto& tuple : this->table_id_map) { - VLOG(0) << " make a new table " << tuple.second; - ::paddle::distributed::TableParameter* sparse_table_proto = - downpour_server_proto->add_downpour_table_param(); - std::vector feat_name; - std::vector feat_dtype; - std::vector feat_shape; - for (size_t i = 0; i < this->table_feat_conf_table_name.size(); i++) { - if (tuple.first == table_feat_conf_table_name[i]) { - feat_name.push_back(table_feat_conf_feat_name[i]); - feat_dtype.push_back(table_feat_conf_feat_dtype[i]); - feat_shape.push_back(table_feat_conf_feat_shape[i]); - } - } - std::string table_type; - if (tuple.second < this->num_node_types) { - table_type = "node"; - } else { - table_type = "edge"; - } + // for (auto& tuple : this->table_id_map) { + // VLOG(0) << " make a new table " << tuple.second; + ::paddle::distributed::TableParameter* sparse_table_proto = + downpour_server_proto->add_downpour_table_param(); + // std::vector feat_name; + // std::vector feat_dtype; + // std::vector feat_shape; + // for (size_t i = 0; i < this->table_feat_conf_table_name.size(); i++) { + // if (tuple.first == table_feat_conf_table_name[i]) { + // feat_name.push_back(table_feat_conf_feat_name[i]); + // feat_dtype.push_back(table_feat_conf_feat_dtype[i]); + // feat_shape.push_back(table_feat_conf_feat_shape[i]); + // } + // } + // std::string table_type; + // if (tuple.second < this->num_node_types) { + // table_type = "node"; + // } else { + // table_type = "edge"; + // } - GetDownpourSparseTableProto(sparse_table_proto, tuple.second, tuple.first, - table_type, feat_name, feat_dtype, feat_shape); - } + GetDownpourSparseTableProto(sparse_table_proto); + //} return worker_fleet_desc; } @@ -237,57 +270,88 @@ void GraphPyClient::load_edge_file(std::string name, std::string filepath, std::string params = "e"; if (reverse) { // 'e<' means load edges from $2 to $1 - params += "<"; + params += "<" + name; } else { // 'e>' means load edges from $1 to $2 - params += ">"; + params += ">" + name; } - if (this->table_id_map.count(name)) { - VLOG(0) << "loadding data with type " << name << " from " << filepath; - uint32_t table_id = this->table_id_map[name]; - auto status = - get_ps_client()->Load(table_id, std::string(filepath), params); + if (edge_to_id.find(name) != edge_to_id.end()) { + auto status = get_ps_client()->Load(0, std::string(filepath), params); status.wait(); } + // if (this->table_id_map.count(name)) { + // VLOG(0) << "loadding data with type " << name << " from " << filepath; + // uint32_t table_id = this->table_id_map[name]; + // auto status = + // get_ps_client()->Load(table_id, std::string(filepath), params); + // status.wait(); + // } } void GraphPyClient::clear_nodes(std::string name) { - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = get_ps_client()->clear_nodes(table_id); + if (edge_to_id.find(name) != edge_to_id.end()) { + int idx = edge_to_id[name]; + auto status = get_ps_client()->clear_nodes(0, 0, idx); + status.wait(); + } else if (feature_to_id.find(name) != feature_to_id.end()) { + int idx = feature_to_id[name]; + auto status = get_ps_client()->clear_nodes(0, 1, idx); status.wait(); } + + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = get_ps_client()->clear_nodes(table_id); + // status.wait(); + // } } void GraphPyClient::add_graph_node(std::string name, std::vector& node_ids, std::vector& weight_list) { - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = + // get_ps_client()->add_graph_node(table_id, node_ids, weight_list); + // status.wait(); + // } + if (edge_to_id.find(name) != edge_to_id.end()) { + int idx = edge_to_id[name]; auto status = - get_ps_client()->add_graph_node(table_id, node_ids, weight_list); + get_ps_client()->add_graph_node(0, idx, node_ids, weight_list); status.wait(); } } void GraphPyClient::remove_graph_node(std::string name, std::vector& node_ids) { - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = get_ps_client()->remove_graph_node(table_id, node_ids); + if (edge_to_id.find(name) != edge_to_id.end()) { + int idx = edge_to_id[name]; + auto status = get_ps_client()->remove_graph_node(0, idx, node_ids); status.wait(); } + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = get_ps_client()->remove_graph_node(table_id, node_ids); + // status.wait(); + // } } void GraphPyClient::load_node_file(std::string name, std::string filepath) { // 'n' means load nodes and 'node_type' follows + std::string params = "n" + name; - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = - get_ps_client()->Load(table_id, std::string(filepath), params); + + if (feature_to_id.find(name) != feature_to_id.end()) { + auto status = get_ps_client()->Load(0, std::string(filepath), params); status.wait(); } + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = + // get_ps_client()->Load(table_id, std::string(filepath), params); + // status.wait(); + // } } std::pair>, std::vector> @@ -297,12 +361,18 @@ GraphPyClient::batch_sample_neighbors(std::string name, bool return_edges) { std::vector> v; std::vector> v1; - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = worker_ptr->batch_sample_neighbors( - table_id, node_ids, sample_size, v, v1, return_weight); + if (edge_to_id.find(name) != edge_to_id.end()) { + int idx = edge_to_id[name]; + auto status = get_ps_client()->batch_sample_neighbors( + 0, idx, node_ids, sample_size, v, v1, return_weight); status.wait(); } + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = worker_ptr->batch_sample_neighbors( + // table_id, node_ids, sample_size, v, v1, return_weight); + // status.wait(); + // } // res.first[0]: neighbors (nodes) // res.first[1]: slice index @@ -331,54 +401,70 @@ GraphPyClient::batch_sample_neighbors(std::string name, return res; } -void GraphPyClient::use_neighbors_sample_cache(std::string name, - size_t total_size_limit, - size_t ttl) { - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = - worker_ptr->use_neighbors_sample_cache(table_id, total_size_limit, ttl); - status.wait(); - } -} std::vector GraphPyClient::random_sample_nodes(std::string name, int server_index, int sample_size) { std::vector v; - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = - worker_ptr->random_sample_nodes(table_id, server_index, sample_size, v); + if (feature_to_id.find(name) != feature_to_id.end()) { + int idx = feature_to_id[name]; + auto status = get_ps_client()->random_sample_nodes(0, 1, idx, server_index, + sample_size, v); + status.wait(); + } else if (edge_to_id.find(name) != edge_to_id.end()) { + int idx = edge_to_id[name]; + auto status = get_ps_client()->random_sample_nodes(0, 0, idx, server_index, + sample_size, v); status.wait(); } + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = + // worker_ptr->random_sample_nodes(table_id, server_index, sample_size, + // v); + // status.wait(); + // } return v; } // (name, dtype, ndarray) std::vector> GraphPyClient::get_node_feat( - std::string node_type, std::vector node_ids, + std::string name, std::vector node_ids, std::vector feature_names) { std::vector> v( feature_names.size(), std::vector(node_ids.size())); - if (this->table_id_map.count(node_type)) { - uint32_t table_id = this->table_id_map[node_type]; + if (feature_to_id.find(name) != feature_to_id.end()) { + int idx = feature_to_id[name]; auto status = - worker_ptr->get_node_feat(table_id, node_ids, feature_names, v); + get_ps_client()->get_node_feat(0, idx, node_ids, feature_names, v); status.wait(); } + // if (this->table_id_map.count(node_type)) { + // uint32_t table_id = this->table_id_map[node_type]; + // auto status = + // worker_ptr->get_node_feat(table_id, node_ids, feature_names, v); + // status.wait(); + // } return v; } void GraphPyClient::set_node_feat( - std::string node_type, std::vector node_ids, + std::string name, std::vector node_ids, std::vector feature_names, const std::vector> features) { - if (this->table_id_map.count(node_type)) { - uint32_t table_id = this->table_id_map[node_type]; - auto status = - worker_ptr->set_node_feat(table_id, node_ids, feature_names, features); + if (feature_to_id.find(name) != feature_to_id.end()) { + int idx = feature_to_id[name]; + auto status = get_ps_client()->set_node_feat(0, idx, node_ids, + feature_names, features); status.wait(); } + + // if (this->table_id_map.count(node_type)) { + // uint32_t table_id = this->table_id_map[node_type]; + // auto status = + // worker_ptr->set_node_feat(table_id, node_ids, feature_names, + // features); + // status.wait(); + // } return; } @@ -387,10 +473,21 @@ std::vector GraphPyClient::pull_graph_list(std::string name, int start, int size, int step) { std::vector res; - if (this->table_id_map.count(name)) { - uint32_t table_id = this->table_id_map[name]; - auto status = worker_ptr->pull_graph_list(table_id, server_index, start, - size, step, res); + // if (this->table_id_map.count(name)) { + // uint32_t table_id = this->table_id_map[name]; + // auto status = worker_ptr->pull_graph_list(table_id, server_index, start, + // size, step, res); + // status.wait(); + // } + if (feature_to_id.find(name) != feature_to_id.end()) { + int idx = feature_to_id[name]; + auto status = get_ps_client()->pull_graph_list(0, 1, idx, server_index, + start, size, step, res); + status.wait(); + } else if (edge_to_id.find(name) != edge_to_id.end()) { + int idx = edge_to_id[name]; + auto status = get_ps_client()->pull_graph_list(0, 0, idx, server_index, + start, size, step, res); status.wait(); } return res; diff --git a/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h b/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h index 19f34dad80745..55beb9b3932a6 100644 --- a/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h +++ b/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h @@ -49,21 +49,19 @@ class GraphPyService { std::vector server_list, port_list, host_sign_list; int server_size, shard_num; int num_node_types; - std::unordered_map table_id_map; - std::vector table_feat_conf_table_name; - std::vector table_feat_conf_feat_name; - std::vector table_feat_conf_feat_dtype; - std::vector table_feat_conf_feat_shape; + std::unordered_map edge_to_id, feature_to_id; + std::vector id_to_feature, id_to_edge; + std::vector> table_feat_mapping; + std::vector> table_feat_conf_feat_name; + std::vector> table_feat_conf_feat_dtype; + std::vector> table_feat_conf_feat_shape; public: int get_shard_num() { return shard_num; } void set_shard_num(int shard_num) { this->shard_num = shard_num; } void GetDownpourSparseTableProto( - ::paddle::distributed::TableParameter* sparse_table_proto, - uint32_t table_id, std::string table_name, std::string table_type, - std::vector feat_name, std::vector feat_dtype, - std::vector feat_shape) { - sparse_table_proto->set_table_id(table_id); + ::paddle::distributed::TableParameter* sparse_table_proto) { + sparse_table_proto->set_table_id(0); sparse_table_proto->set_table_class("GraphTable"); sparse_table_proto->set_shard_num(shard_num); sparse_table_proto->set_type(::paddle::distributed::PS_SPARSE_TABLE); @@ -76,14 +74,26 @@ class GraphPyService { ::paddle::distributed::GraphParameter* graph_proto = sparse_table_proto->mutable_graph_parameter(); - ::paddle::distributed::GraphFeature* graph_feature = - graph_proto->mutable_graph_feature(); + // ::paddle::distributed::GraphFeature* graph_feature = + // graph_proto->mutable_graph_feature(); graph_proto->set_task_pool_size(24); - graph_proto->set_table_name(table_name); - graph_proto->set_table_type(table_type); + graph_proto->set_table_name("cpu_graph_table"); graph_proto->set_use_cache(false); + for (int i = 0; i < id_to_edge.size(); i++) + graph_proto->add_edge_types(id_to_edge[i]); + for (int i = 0; i < id_to_feature.size(); i++) { + graph_proto->add_node_types(id_to_feature[i]); + auto feat_node = id_to_feature[i]; + ::paddle::distributed::GraphFeature* g_f = + graph_proto->add_graph_feature(); + for (int x = 0; x < table_feat_conf_feat_name[i].size(); x++) { + g_f->add_name(table_feat_conf_feat_name[i][x]); + g_f->add_dtype(table_feat_conf_feat_dtype[i][x]); + g_f->add_shape(table_feat_conf_feat_shape[i][x]); + } + } // Set GraphTable Parameter // common_proto->set_table_name(table_name); // common_proto->set_name(table_type); @@ -93,11 +103,11 @@ class GraphPyService { // common_proto->add_attributes(feat_name[i]); // } - for (size_t i = 0; i < feat_name.size(); i++) { - graph_feature->add_dtype(feat_dtype[i]); - graph_feature->add_shape(feat_shape[i]); - graph_feature->add_name(feat_name[i]); - } + // for (size_t i = 0; i < feat_name.size(); i++) { + // graph_feature->add_dtype(feat_dtype[i]); + // graph_feature->add_shape(feat_shape[i]); + // graph_feature->add_name(feat_name[i]); + // } accessor_proto->set_accessor_class("CommMergeAccessor"); } @@ -172,10 +182,8 @@ class GraphPyClient : public GraphPyService { std::vector random_sample_nodes(std::string name, int server_index, int sample_size); std::vector> get_node_feat( - std::string node_type, std::vector node_ids, + std::string name, std::vector node_ids, std::vector feature_names); - void use_neighbors_sample_cache(std::string name, size_t total_size_limit, - size_t ttl); void set_node_feat(std::string node_type, std::vector node_ids, std::vector feature_names, const std::vector> features); diff --git a/paddle/fluid/distributed/ps/table/common_graph_table.cc b/paddle/fluid/distributed/ps/table/common_graph_table.cc index d7ceb4a18ea19..a9cd0021c8578 100644 --- a/paddle/fluid/distributed/ps/table/common_graph_table.cc +++ b/paddle/fluid/distributed/ps/table/common_graph_table.cc @@ -29,7 +29,7 @@ namespace distributed { #ifdef PADDLE_WITH_HETERPS paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph( - std::vector ids) { + int idx, std::vector ids) { std::vector> bags(task_pool_size_); for (auto x : ids) { int location = x % shard_num % task_pool_size_; @@ -43,7 +43,7 @@ paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph( tasks.push_back(_shards_task_pool[i]->enqueue([&, i, this]() -> int { paddle::framework::GpuPsGraphNode x; for (int j = 0; j < (int)bags[i].size(); j++) { - Node *v = find_node(bags[i][j]); + Node *v = find_node(0, idx, bags[i][j]); x.node_id = bags[i][j]; if (v == NULL) { x.neighbor_size = 0; @@ -85,22 +85,32 @@ paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph( } return res; } -int32_t GraphTable::add_node_to_ssd(int64_t src_id, char *data, int len) { - if (_db != NULL) - _db->put(src_id % shard_num % task_pool_size_, (char *)&src_id, - sizeof(uint64_t), (char *)data, sizeof(int64_t) * len); +int32_t GraphTable::add_node_to_ssd(int type_id, int idx, int64_t src_id, + char *data, int len) { + if (_db != NULL) { + char ch[sizeof(int) * 2 + sizeof(int64_t)]; + memcpy(ch, &type_id, sizeof(int)); + memcpy(ch + sizeof(int), &idx, sizeof(int)); + memcpy(ch + sizeof(int) * 2, &src_id, sizeof(int64_t)); + _db->put(src_id % shard_num % task_pool_size_, ch, + sizeof(int) * 2 + sizeof(int64_t), (char *)data, len); + } return 0; } char *GraphTable::random_sample_neighbor_from_ssd( - int64_t id, int sample_size, const std::shared_ptr rng, - int &actual_size) { + int idx, int64_t id, int sample_size, + const std::shared_ptr rng, int &actual_size) { if (_db == NULL) { actual_size = 0; return NULL; } std::string str; - if (_db->get(id % shard_num % task_pool_size_, (char *)&id, sizeof(uint64_t), - str) == 0) { + char ch[sizeof(int) * 2 + sizeof(int64_t)]; + memset(ch, 0, sizeof(int)); + memcpy(ch + sizeof(int), &idx, sizeof(int)); + memcpy(ch + sizeof(int) * 2, &id, sizeof(int64_t)); + if (_db->get(id % shard_num % task_pool_size_, ch, sizeof(uint64_t), str) == + 0) { int64_t *data = ((int64_t *)str.c_str()); int n = str.size() / sizeof(int64_t); std::unordered_map m; @@ -423,20 +433,20 @@ std::vector GraphShard::get_batch(int start, int end, int step) { size_t GraphShard::get_size() { return bucket.size(); } -int32_t GraphTable::add_comm_edge(int64_t src_id, int64_t dst_id) { +int32_t GraphTable::add_comm_edge(int idx, int64_t src_id, int64_t dst_id) { size_t src_shard_id = src_id % shard_num; if (src_shard_id >= shard_end || src_shard_id < shard_start) { return -1; } size_t index = src_shard_id - shard_start; - VLOG(0) << "index add edge " << src_id << " " << dst_id; - shards[index]->add_graph_node(src_id)->build_edges(false); - shards[index]->add_neighbor(src_id, dst_id, 1.0); + edge_shards[idx][index]->add_graph_node(src_id)->build_edges(false); + edge_shards[idx][index]->add_neighbor(src_id, dst_id, 1.0); return 0; } -int32_t GraphTable::add_graph_node(std::vector &id_list, +int32_t GraphTable::add_graph_node(int idx, std::vector &id_list, std::vector &is_weight_list) { + auto &shards = edge_shards[idx]; size_t node_size = id_list.size(); std::vector>> batch(task_pool_size_); for (size_t i = 0; i < node_size; i++) { @@ -450,19 +460,20 @@ int32_t GraphTable::add_graph_node(std::vector &id_list, std::vector> tasks; for (size_t i = 0; i < batch.size(); ++i) { if (!batch[i].size()) continue; - tasks.push_back(_shards_task_pool[i]->enqueue([&batch, i, this]() -> int { - for (auto &p : batch[i]) { - size_t index = p.first % this->shard_num - this->shard_start; - this->shards[index]->add_graph_node(p.first)->build_edges(p.second); - } - return 0; - })); + tasks.push_back( + _shards_task_pool[i]->enqueue([&shards, &batch, i, this]() -> int { + for (auto &p : batch[i]) { + size_t index = p.first % this->shard_num - this->shard_start; + shards[index]->add_graph_node(p.first)->build_edges(p.second); + } + return 0; + })); } for (size_t i = 0; i < tasks.size(); i++) tasks[i].get(); return 0; } -int32_t GraphTable::remove_graph_node(std::vector &id_list) { +int32_t GraphTable::remove_graph_node(int idx, std::vector &id_list) { size_t node_size = id_list.size(); std::vector> batch(task_pool_size_); for (size_t i = 0; i < node_size; i++) { @@ -470,16 +481,18 @@ int32_t GraphTable::remove_graph_node(std::vector &id_list) { if (shard_id >= shard_end || shard_id < shard_start) continue; batch[get_thread_pool_index(id_list[i])].push_back(id_list[i]); } + auto &shards = edge_shards[idx]; std::vector> tasks; for (size_t i = 0; i < batch.size(); ++i) { if (!batch[i].size()) continue; - tasks.push_back(_shards_task_pool[i]->enqueue([&batch, i, this]() -> int { - for (auto &p : batch[i]) { - size_t index = p % this->shard_num - this->shard_start; - this->shards[index]->delete_node(p); - } - return 0; - })); + tasks.push_back( + _shards_task_pool[i]->enqueue([&shards, &batch, i, this]() -> int { + for (auto &p : batch[i]) { + size_t index = p % this->shard_num - this->shard_start; + shards[index]->delete_node(p); + } + return 0; + })); } for (size_t i = 0; i < tasks.size(); i++) tasks[i].get(); return 0; @@ -541,30 +554,19 @@ Node *GraphShard::find_node(int64_t id) { } GraphTable::~GraphTable() { - for (auto p : shards) { - delete p; - } - for (auto p : extra_shards) { - delete p; + for (int i = 0; i < (int)edge_shards.size(); i++) { + for (auto p : edge_shards[i]) { + delete p; + } + edge_shards[i].clear(); } - shards.clear(); - extra_shards.clear(); -} -int32_t GraphTable::load_graph_split_config(const std::string &path) { - VLOG(4) << "in server side load graph split config\n"; - std::ifstream file(path); - std::string line; - while (std::getline(file, line)) { - auto values = paddle::string::split_string(line, "\t"); - if (values.size() < 2) continue; - size_t index = (size_t)std::stoi(values[0]); - if (index != _shard_idx) continue; - auto dst_id = std::stoull(values[1]); - extra_nodes.insert(dst_id); - } - if (extra_nodes.size() != 0) use_duplicate_nodes = true; - return 0; + for (int i = 0; i < (int)feature_shards.size(); i++) { + for (auto p : feature_shards[i]) { + delete p; + } + feature_shards[i].clear(); + } } int32_t GraphTable::Load(const std::string &path, const std::string ¶m) { @@ -572,7 +574,8 @@ int32_t GraphTable::Load(const std::string &path, const std::string ¶m) { bool load_node = (param[0] == 'n'); if (load_edge) { bool reverse_edge = (param[1] == '<'); - return this->load_edges(path, reverse_edge); + std::string edge_type = param.substr(2); + return this->load_edges(path, reverse_edge, edge_type); } if (load_node) { std::string node_type = param.substr(1); @@ -582,9 +585,11 @@ int32_t GraphTable::Load(const std::string &path, const std::string ¶m) { } int32_t GraphTable::get_nodes_ids_by_ranges( - std::vector> ranges, std::vector &res) { + int type_id, int idx, std::vector> ranges, + std::vector &res) { int start = 0, end, index = 0, total_size = 0; res.clear(); + auto &shards = type_id == 0 ? edge_shards[idx] : feature_shards[idx]; std::vector>> tasks; for (size_t i = 0; i < shards.size() && index < (int)ranges.size(); i++) { end = total_size + shards[i]->get_size(); @@ -601,7 +606,7 @@ int32_t GraphTable::get_nodes_ids_by_ranges( first -= total_size; second -= total_size; tasks.push_back(_shards_task_pool[i % task_pool_size_]->enqueue( - [this, first, second, i]() -> std::vector { + [&shards, this, first, second, i]() -> std::vector { return shards[i]->get_ids_by_range(first, second); })); } @@ -622,6 +627,18 @@ int32_t GraphTable::load_nodes(const std::string &path, std::string node_type) { auto paths = paddle::string::split_string(path, ";"); int64_t count = 0; int64_t valid_count = 0; + int idx = 0; + if (node_type == "") { + VLOG(0) << "node_type not specified, loading edges to " << id_to_feature[0] + << " part"; + } else { + if (feature_to_id.find(node_type) == feature_to_id.end()) { + VLOG(0) << "node_type " << node_type + << " is not defined, nothing will be loaded"; + return 0; + } + idx = feature_to_id[node_type]; + } for (auto path : paths) { std::ifstream file(path); std::string line; @@ -650,12 +667,12 @@ int32_t GraphTable::load_nodes(const std::string &path, std::string node_type) { size_t index = shard_id - shard_start; - auto node = shards[index]->add_feature_node(id); - - node->set_feature_size(feat_name.size()); + // auto node = shards[index]->add_feature_node(id); + auto node = feature_shards[idx][index]->add_feature_node(id); + node->set_feature_size(feat_name[idx].size()); for (size_t slice = 2; slice < values.size(); slice++) { - auto feat = this->parse_feature(values[slice]); + auto feat = this->parse_feature(idx, values[slice]); if (feat.first >= 0) { node->set_feature(feat.first, feat.second); } else { @@ -672,16 +689,37 @@ int32_t GraphTable::load_nodes(const std::string &path, std::string node_type) { return 0; } -int32_t GraphTable::load_edges(const std::string &path, bool reverse_edge) { +int32_t GraphTable::build_sampler(int idx, std::string sample_type) { + for (auto &shard : edge_shards[idx]) { + auto bucket = shard->get_bucket(); + for (size_t i = 0; i < bucket.size(); i++) { + bucket[i]->build_sampler(sample_type); + } + } + return 0; +} +int32_t GraphTable::load_edges(const std::string &path, bool reverse_edge, + const std::string &edge_type) { // #ifdef PADDLE_WITH_HETERPS // if (gpups_mode) pthread_rwlock_rdlock(rw_lock.get()); // #endif + int idx = 0; + if (edge_type == "") { + VLOG(0) << "edge_type not specified, loading edges to " << id_to_edge[0] + << " part"; + } else { + if (edge_to_id.find(edge_type) == edge_to_id.end()) { + VLOG(0) << "edge_type " << edge_type + << " is not defined, nothing will be loaded"; + return 0; + } + idx = edge_to_id[edge_type]; + } auto paths = paddle::string::split_string(path, ";"); int64_t count = 0; std::string sample_type = "random"; bool is_weighted = false; int valid_count = 0; - int extra_alloc_index = 0; for (auto path : paths) { std::ifstream file(path); std::string line; @@ -704,195 +742,68 @@ int32_t GraphTable::load_edges(const std::string &path, bool reverse_edge) { size_t src_shard_id = src_id % shard_num; if (src_shard_id >= shard_end || src_shard_id < shard_start) { - if (use_duplicate_nodes == false || - extra_nodes.find(src_id) == extra_nodes.end()) { - VLOG(4) << "will not load " << src_id << " from " << path - << ", please check id distribution"; - continue; - } - int index; - if (extra_nodes_to_thread_index.find(src_id) != - extra_nodes_to_thread_index.end()) { - index = extra_nodes_to_thread_index[src_id]; - } else { - index = extra_alloc_index++; - extra_alloc_index %= task_pool_size_; - extra_nodes_to_thread_index[src_id] = index; - } - extra_shards[index]->add_graph_node(src_id)->build_edges(is_weighted); - extra_shards[index]->add_neighbor(src_id, dst_id, weight); - valid_count++; + VLOG(4) << "will not load " << src_id << " from " << path + << ", please check id distribution"; continue; } + if (count % 1000000 == 0) { VLOG(0) << count << " edges are loaded from filepath"; VLOG(0) << line; } size_t index = src_shard_id - shard_start; - shards[index]->add_graph_node(src_id)->build_edges(is_weighted); - shards[index]->add_neighbor(src_id, dst_id, weight); + edge_shards[idx][index]->add_graph_node(src_id)->build_edges(is_weighted); + edge_shards[idx][index]->add_neighbor(src_id, dst_id, weight); valid_count++; } } VLOG(0) << valid_count << "/" << count << " edges are loaded successfully in " << path; - std::vector used(task_pool_size_, 0); // Build Sampler j - for (auto &shard : shards) { - auto bucket = shard->get_bucket(); - for (size_t i = 0; i < bucket.size(); i++) { - bucket[i]->build_sampler(sample_type); - used[get_thread_pool_index(bucket[i]->get_id())]++; - } - } - /*----------------------- - relocate the duplicate nodes to make them distributed evenly among threads. -*/ - if (!use_duplicate_nodes) { - // #ifdef PADDLE_WITH_HETERPS - // if (gpups_mode) pthread_rwlock_unlock(rw_lock.get()); - // #endif - - return 0; - } - for (auto &shard : extra_shards) { + for (auto &shard : edge_shards[idx]) { auto bucket = shard->get_bucket(); for (size_t i = 0; i < bucket.size(); i++) { bucket[i]->build_sampler(sample_type); } } - int size = extra_nodes_to_thread_index.size(); - if (size == 0) return 0; - std::vector index; - for (int i = 0; i < (int)used.size(); i++) index.push_back(i); - sort(index.begin(), index.end(), - [&](int &a, int &b) { return used[a] < used[b]; }); - std::vector alloc(index.size(), 0), has_alloc(index.size(), 0); - int t = 1, aim = 0, mod = 0; - for (; t < (int)used.size(); t++) { - if ((used[index[t]] - used[index[t - 1]]) * t >= size) { - break; - } else { - size -= (used[index[t]] - used[index[t - 1]]) * t; - } - } - aim = used[index[t - 1]] + size / t; - mod = size % t; - for (int x = t - 1; x >= 0; x--) { - alloc[index[x]] = aim; - if (t - x <= mod) alloc[index[x]]++; - alloc[index[x]] -= used[index[x]]; - } - std::vector vec[index.size()]; - for (auto p : extra_nodes_to_thread_index) { - has_alloc[p.second]++; - vec[p.second].push_back(p.first); - } - sort(index.begin(), index.end(), [&](int &a, int &b) { - return has_alloc[a] - alloc[a] < has_alloc[b] - alloc[b]; - }); - int left = 0, right = (int)index.size() - 1; - while (left < right) { - if (has_alloc[index[right]] - alloc[index[right]] == 0) break; - int x = std::min(alloc[index[left]] - has_alloc[index[left]], - has_alloc[index[right]] - alloc[index[right]]); - has_alloc[index[left]] += x; - has_alloc[index[right]] -= x; - int64_t id; - while (x--) { - id = vec[index[right]].back(); - vec[index[right]].pop_back(); - extra_nodes_to_thread_index[id] = index[left]; - vec[index[left]].push_back(id); - } - if (has_alloc[index[right]] - alloc[index[right]] == 0) right--; - if (alloc[index[left]] - has_alloc[index[left]] == 0) left++; - } - std::vector extra_shards_copy; - for (int i = 0; i < task_pool_size_; ++i) { - extra_shards_copy.push_back(new GraphShard()); - } - for (auto &shard : extra_shards) { - auto &bucket = shard->get_bucket(); - auto &node_location = shard->get_node_location(); - while (bucket.size()) { - Node *temp = bucket.back(); - bucket.pop_back(); - node_location.erase(temp->get_id()); - extra_shards_copy[extra_nodes_to_thread_index[temp->get_id()]] - ->add_graph_node(temp); - } - } - for (int i = 0; i < task_pool_size_; ++i) { - delete extra_shards[i]; - extra_shards[i] = extra_shards_copy[i]; - } - // #ifdef PADDLE_WITH_HETERPS - // if (gpups_mode) pthread_rwlock_unlock(rw_lock.get()); - // #endif return 0; } -Node *GraphTable::find_node(int64_t id) { +Node *GraphTable::find_node(int type_id, int idx, int64_t id) { size_t shard_id = id % shard_num; if (shard_id >= shard_end || shard_id < shard_start) { - if (use_duplicate_nodes == false || extra_nodes_to_thread_index.size() == 0) - return nullptr; - auto iter = extra_nodes_to_thread_index.find(id); - if (iter == extra_nodes_to_thread_index.end()) - return nullptr; - else { - return extra_shards[iter->second]->find_node(id); - } + return nullptr; } size_t index = shard_id - shard_start; - Node *node = shards[index]->find_node(id); + auto &search_shards = type_id == 0 ? edge_shards[idx] : feature_shards[idx]; + Node *node = search_shards[index]->find_node(id); return node; } uint32_t GraphTable::get_thread_pool_index(int64_t node_id) { - if (use_duplicate_nodes == false || extra_nodes_to_thread_index.size() == 0) - return node_id % shard_num % shard_num_per_server % task_pool_size_; - size_t src_shard_id = node_id % shard_num; - if (src_shard_id >= shard_end || src_shard_id < shard_start) { - auto iter = extra_nodes_to_thread_index.find(node_id); - if (iter != extra_nodes_to_thread_index.end()) { - return iter->second; - } - } - return src_shard_id % shard_num_per_server % task_pool_size_; + return node_id % shard_num % shard_num_per_server % task_pool_size_; } uint32_t GraphTable::get_thread_pool_index_by_shard_index(int64_t shard_index) { return shard_index % shard_num_per_server % task_pool_size_; } -int32_t GraphTable::clear_nodes() { - std::vector> tasks; - for (size_t i = 0; i < shards.size(); i++) { - tasks.push_back( - _shards_task_pool[i % task_pool_size_]->enqueue([this, i]() -> int { - this->shards[i]->clear(); - return 0; - })); - } - for (size_t i = 0; i < extra_shards.size(); i++) { - tasks.push_back(_shards_task_pool[i]->enqueue([this, i]() -> int { - this->extra_shards[i]->clear(); - return 0; - })); +int32_t GraphTable::clear_nodes(int type_id, int idx) { + auto &search_shards = type_id == 0 ? edge_shards[idx] : feature_shards[idx]; + for (int i = 0; i < search_shards.size(); i++) { + search_shards[i]->clear(); } - for (size_t i = 0; i < tasks.size(); i++) tasks[i].get(); return 0; } -int32_t GraphTable::random_sample_nodes(int sample_size, +int32_t GraphTable::random_sample_nodes(int type_id, int idx, int sample_size, std::unique_ptr &buffer, int &actual_size) { int total_size = 0; + auto &shards = type_id == 0 ? edge_shards[idx] : feature_shards[idx]; for (int i = 0; i < (int)shards.size(); i++) { total_size += shards[i]->get_size(); } @@ -947,7 +858,7 @@ int32_t GraphTable::random_sample_nodes(int sample_size, } for (auto &pair : first_half) second_half.push_back(pair); std::vector res; - get_nodes_ids_by_ranges(second_half, res); + get_nodes_ids_by_ranges(type_id, idx, second_half, res); actual_size = res.size() * sizeof(int64_t); buffer.reset(new char[actual_size]); char *pointer = buffer.get(); @@ -955,7 +866,7 @@ int32_t GraphTable::random_sample_nodes(int sample_size, return 0; } int32_t GraphTable::random_sample_neighbors( - int64_t *node_ids, int sample_size, + int idx, int64_t *node_ids, int sample_size, std::vector> &buffers, std::vector &actual_sizes, bool need_weight) { size_t node_num = buffers.size(); @@ -964,11 +875,12 @@ int32_t GraphTable::random_sample_neighbors( std::vector> seq_id(task_pool_size_); std::vector> id_list(task_pool_size_); size_t index; - for (size_t idx = 0; idx < node_num; ++idx) { - index = get_thread_pool_index(node_ids[idx]); - seq_id[index].emplace_back(idx); - id_list[index].emplace_back(node_ids[idx], sample_size, need_weight); + for (size_t idy = 0; idy < node_num; ++idy) { + index = get_thread_pool_index(node_ids[idy]); + seq_id[index].emplace_back(idy); + id_list[index].emplace_back(idx, node_ids[idy], sample_size, need_weight); } + for (int i = 0; i < (int)seq_id.size(); i++) { if (seq_id[i].size() == 0) continue; tasks.push_back(_shards_task_pool[i]->enqueue([&, i, this]() -> int { @@ -987,20 +899,20 @@ int32_t GraphTable::random_sample_neighbors( for (size_t k = 0; k < id_list[i].size(); k++) { if (index < (int)r.size() && r[index].first.node_key == id_list[i][k].node_key) { - idx = seq_id[i][k]; - actual_sizes[idx] = r[index].second.actual_size; - buffers[idx] = r[index].second.buffer; + int idy = seq_id[i][k]; + actual_sizes[idy] = r[index].second.actual_size; + buffers[idy] = r[index].second.buffer; index++; } else { node_id = id_list[i][k].node_key; - Node *node = find_node(node_id); - idx = seq_id[i][k]; - int &actual_size = actual_sizes[idx]; + Node *node = find_node(0, idx, node_id); + int idy = seq_id[i][k]; + int &actual_size = actual_sizes[idy]; if (node == nullptr) { #ifdef PADDLE_WITH_HETERPS if (search_level == 2) { char *buffer_addr = random_sample_neighbor_from_ssd( - node_id, sample_size, rng, actual_size); + idx, node_id, sample_size, rng, actual_size); if (actual_size != 0) { std::shared_ptr &buffer = buffers[idx]; buffer.reset(buffer_addr, char_del); @@ -1011,7 +923,7 @@ int32_t GraphTable::random_sample_neighbors( actual_size = 0; continue; } - std::shared_ptr &buffer = buffers[idx]; + std::shared_ptr &buffer = buffers[idy]; std::vector res = node->sample_k(sample_size, rng); actual_size = res.size() * (need_weight ? (Node::id_size + Node::weight_size) @@ -1021,7 +933,7 @@ int32_t GraphTable::random_sample_neighbors( float weight; char *buffer_addr = new char[actual_size]; if (response == LRUResponse::ok) { - sample_keys.emplace_back(node_id, sample_size, need_weight); + sample_keys.emplace_back(idx, node_id, sample_size, need_weight); sample_res.emplace_back(actual_size, buffer_addr); buffer = sample_res.back().buffer; } else { @@ -1052,16 +964,16 @@ int32_t GraphTable::random_sample_neighbors( return 0; } -int32_t GraphTable::get_node_feat(const std::vector &node_ids, +int32_t GraphTable::get_node_feat(int idx, const std::vector &node_ids, const std::vector &feature_names, std::vector> &res) { size_t node_num = node_ids.size(); std::vector> tasks; - for (size_t idx = 0; idx < node_num; ++idx) { - int64_t node_id = node_ids[idx]; + for (size_t idy = 0; idy < node_num; ++idy) { + int64_t node_id = node_ids[idy]; tasks.push_back(_shards_task_pool[get_thread_pool_index(node_id)]->enqueue( - [&, idx, node_id]() -> int { - Node *node = find_node(node_id); + [&, idx, idy, node_id]() -> int { + Node *node = find_node(1, idx, node_id); if (node == nullptr) { return 0; @@ -1069,59 +981,61 @@ int32_t GraphTable::get_node_feat(const std::vector &node_ids, for (int feat_idx = 0; feat_idx < (int)feature_names.size(); ++feat_idx) { const std::string &feature_name = feature_names[feat_idx]; - if (feat_id_map.find(feature_name) != feat_id_map.end()) { + if (feat_id_map[idx].find(feature_name) != feat_id_map[idx].end()) { // res[feat_idx][idx] = // node->get_feature(feat_id_map[feature_name]); - auto feat = node->get_feature(feat_id_map[feature_name]); - res[feat_idx][idx] = feat; + auto feat = node->get_feature(feat_id_map[idx][feature_name]); + res[feat_idx][idy] = feat; } } return 0; })); } - for (size_t idx = 0; idx < node_num; ++idx) { - tasks[idx].get(); + for (size_t idy = 0; idy < node_num; ++idy) { + tasks[idy].get(); } return 0; } int32_t GraphTable::set_node_feat( - const std::vector &node_ids, + int idx, const std::vector &node_ids, const std::vector &feature_names, const std::vector> &res) { size_t node_num = node_ids.size(); std::vector> tasks; - for (size_t idx = 0; idx < node_num; ++idx) { - int64_t node_id = node_ids[idx]; + for (size_t idy = 0; idy < node_num; ++idy) { + int64_t node_id = node_ids[idy]; tasks.push_back(_shards_task_pool[get_thread_pool_index(node_id)]->enqueue( - [&, idx, node_id]() -> int { + [&, idx, idy, node_id]() -> int { size_t index = node_id % this->shard_num - this->shard_start; - auto node = shards[index]->add_feature_node(node_id); - node->set_feature_size(this->feat_name.size()); + auto node = feature_shards[idx][index]->add_feature_node(node_id); + node->set_feature_size(this->feat_name[idx].size()); for (int feat_idx = 0; feat_idx < (int)feature_names.size(); ++feat_idx) { const std::string &feature_name = feature_names[feat_idx]; - if (feat_id_map.find(feature_name) != feat_id_map.end()) { - node->set_feature(feat_id_map[feature_name], res[feat_idx][idx]); + if (feat_id_map[idx].find(feature_name) != feat_id_map[idx].end()) { + node->set_feature(feat_id_map[idx][feature_name], + res[feat_idx][idy]); } } return 0; })); } - for (size_t idx = 0; idx < node_num; ++idx) { - tasks[idx].get(); + for (size_t idy = 0; idy < node_num; ++idy) { + tasks[idy].get(); } return 0; } std::pair GraphTable::parse_feature( - std::string feat_str) { + int idx, std::string feat_str) { // Return (feat_id, btyes) if name are in this->feat_name, else return (-1, // "") auto fields = paddle::string::split_string(feat_str, " "); - if (this->feat_id_map.count(fields[0])) { - int32_t id = this->feat_id_map[fields[0]]; - std::string dtype = this->feat_dtype[id]; + if (feat_id_map[idx].count(fields[0])) { + // if (this->feat_id_map.count(fields[0])) { + int32_t id = this->feat_id_map[idx][fields[0]]; + std::string dtype = this->feat_dtype[idx][id]; std::vector values(fields.begin() + 1, fields.end()); if (dtype == "feasign") { return std::make_pair( @@ -1146,15 +1060,17 @@ std::pair GraphTable::parse_feature( return std::make_pair(-1, ""); } -int32_t GraphTable::pull_graph_list(int start, int total_size, +int32_t GraphTable::pull_graph_list(int type_id, int idx, int start, + int total_size, std::unique_ptr &buffer, int &actual_size, bool need_feature, int step) { if (start < 0) start = 0; int size = 0, cur_size; + auto &search_shards = type_id == 0 ? edge_shards[idx] : feature_shards[idx]; std::vector>> tasks; - for (size_t i = 0; i < shards.size() && total_size > 0; i++) { - cur_size = shards[i]->get_size(); + for (size_t i = 0; i < search_shards.size() && total_size > 0; i++) { + cur_size = search_shards[i]->get_size(); if (size + cur_size <= start) { size += cur_size; continue; @@ -1162,8 +1078,9 @@ int32_t GraphTable::pull_graph_list(int start, int total_size, int count = std::min(1 + (size + cur_size - start - 1) / step, total_size); int end = start + (count - 1) * step + 1; tasks.push_back(_shards_task_pool[i % task_pool_size_]->enqueue( - [this, i, start, end, step, size]() -> std::vector { - return this->shards[i]->get_batch(start - size, end - size, step); + [&search_shards, this, i, start, end, step, + size]() -> std::vector { + return search_shards[i]->get_batch(start - size, end - size, step); })); start += count * step; total_size -= count; @@ -1250,6 +1167,41 @@ int32_t GraphTable::Initialize(const GraphParameter &graph) { _shards_task_rng_pool.push_back(paddle::framework::GetCPURandomEngine(0)); } auto graph_feature = graph.graph_feature(); + auto node_types = graph.node_types(); + auto edge_types = graph.edge_types(); + VLOG(0) << "got " << edge_types.size() << "edge types in total"; + feat_id_map.resize(node_types.size()); + for (int k = 0; k < edge_types.size(); k++) { + VLOG(0) << "in initialize: get a edge_type " << edge_types[k]; + edge_to_id[edge_types[k]] = k; + id_to_edge.push_back(edge_types[k]); + } + feat_name.resize(node_types.size()); + feat_shape.resize(node_types.size()); + feat_dtype.resize(node_types.size()); + VLOG(0) << "got " << node_types.size() << "node types in total"; + for (int k = 0; k < node_types.size(); k++) { + feature_to_id[node_types[k]] = k; + auto node_type = node_types[k]; + auto feature = graph_feature[k]; + id_to_feature.push_back(node_type); + int feat_conf_size = static_cast(feature.name().size()); + + for (int i = 0; i < feat_conf_size; i++) { + // auto &f_name = common.attributes()[i]; + // auto &f_shape = common.dims()[i]; + // auto &f_dtype = common.params()[i]; + auto &f_name = feature.name()[i]; + auto &f_shape = feature.shape()[i]; + auto &f_dtype = feature.dtype()[i]; + feat_name[k].push_back(f_name); + feat_shape[k].push_back(f_shape); + feat_dtype[k].push_back(f_dtype); + feat_id_map[k][f_name] = i; + VLOG(0) << "init graph table feat conf name:" << f_name + << " shape:" << f_shape << " dtype:" << f_dtype; + } + } // this->table_name = common.table_name(); // this->table_type = common.name(); this->table_name = graph.table_name(); @@ -1257,21 +1209,7 @@ int32_t GraphTable::Initialize(const GraphParameter &graph) { VLOG(0) << " init graph table type " << this->table_type << " table name " << this->table_name; // int feat_conf_size = static_cast(common.attributes().size()); - int feat_conf_size = static_cast(graph_feature.name().size()); - for (int i = 0; i < feat_conf_size; i++) { - // auto &f_name = common.attributes()[i]; - // auto &f_shape = common.dims()[i]; - // auto &f_dtype = common.params()[i]; - auto &f_name = graph_feature.name()[i]; - auto &f_shape = graph_feature.shape()[i]; - auto &f_dtype = graph_feature.dtype()[i]; - this->feat_name.push_back(f_name); - this->feat_shape.push_back(f_shape); - this->feat_dtype.push_back(f_dtype); - this->feat_id_map[f_name] = i; - VLOG(0) << "init graph table feat conf name:" << f_name - << " shape:" << f_shape << " dtype:" << f_dtype; - } + // int feat_conf_size = static_cast(graph_feature.name().size()); VLOG(0) << "in init graph table shard num = " << shard_num << " shard_idx" << _shard_idx; shard_num_per_server = sparse_local_shard_num(shard_num, server_num); @@ -1279,12 +1217,17 @@ int32_t GraphTable::Initialize(const GraphParameter &graph) { shard_end = shard_start + shard_num_per_server; VLOG(0) << "in init graph table shard idx = " << _shard_idx << " shard_start " << shard_start << " shard_end " << shard_end; - for (size_t i = 0; i < shard_num_per_server; i++) { - shards.push_back(new GraphShard()); + edge_shards.resize(id_to_edge.size()); + for (int k = 0; k < (int)edge_shards.size(); k++) { + for (size_t i = 0; i < shard_num_per_server; i++) { + edge_shards[k].push_back(new GraphShard()); + } } - use_duplicate_nodes = false; - for (int i = 0; i < task_pool_size_; i++) { - extra_shards.push_back(new GraphShard()); + feature_shards.resize(id_to_feature.size()); + for (int k = 0; k < (int)feature_shards.size(); k++) { + for (size_t i = 0; i < shard_num_per_server; i++) { + feature_shards[k].push_back(new GraphShard()); + } } return 0; diff --git a/paddle/fluid/distributed/ps/table/common_graph_table.h b/paddle/fluid/distributed/ps/table/common_graph_table.h index df0d8b2d3a8ab..059bcb09a0a6e 100644 --- a/paddle/fluid/distributed/ps/table/common_graph_table.h +++ b/paddle/fluid/distributed/ps/table/common_graph_table.h @@ -83,16 +83,20 @@ class GraphShard { enum LRUResponse { ok = 0, blocked = 1, err = 2 }; struct SampleKey { + int idx; int64_t node_key; size_t sample_size; bool is_weighted; - SampleKey(int64_t _node_key, size_t _sample_size, bool _is_weighted) - : node_key(_node_key), - sample_size(_sample_size), - is_weighted(_is_weighted) {} + SampleKey(int _idx, int64_t _node_key, size_t _sample_size, + bool _is_weighted) { + idx = _idx; + node_key = _node_key; + sample_size = _sample_size; + is_weighted = _is_weighted; + } bool operator==(const SampleKey &s) const { - return node_key == s.node_key && sample_size == s.sample_size && - is_weighted == s.is_weighted; + return idx == s.idx && node_key == s.node_key && + sample_size == s.sample_size && is_weighted == s.is_weighted; } }; @@ -435,44 +439,46 @@ class GraphTable : public Table { return (key % shard_num) / sparse_local_shard_num(shard_num, server_num); } - virtual int32_t pull_graph_list(int start, int size, + virtual int32_t pull_graph_list(int type_id, int idx, int start, int size, std::unique_ptr &buffer, int &actual_size, bool need_feature, int step); virtual int32_t random_sample_neighbors( - int64_t *node_ids, int sample_size, + int idx, int64_t *node_ids, int sample_size, std::vector> &buffers, std::vector &actual_sizes, bool need_weight); - int32_t random_sample_nodes(int sample_size, std::unique_ptr &buffers, + int32_t random_sample_nodes(int type_id, int idx, int sample_size, + std::unique_ptr &buffers, int &actual_sizes); virtual int32_t get_nodes_ids_by_ranges( - std::vector> ranges, std::vector &res); + int type_id, int idx, std::vector> ranges, + std::vector &res); virtual int32_t Initialize() { return 0; } virtual int32_t Initialize(const TableParameter &config, const FsClientParameter &fs_config); virtual int32_t Initialize(const GraphParameter &config); int32_t Load(const std::string &path, const std::string ¶m); - int32_t load_graph_split_config(const std::string &path); - int32_t load_edges(const std::string &path, bool reverse); + int32_t load_edges(const std::string &path, bool reverse, + const std::string &edge_type); int32_t load_nodes(const std::string &path, std::string node_type); - int32_t add_graph_node(std::vector &id_list, + int32_t add_graph_node(int idx, std::vector &id_list, std::vector &is_weight_list); - int32_t remove_graph_node(std::vector &id_list); + int32_t remove_graph_node(int idx, std::vector &id_list); int32_t get_server_index_by_id(int64_t id); - Node *find_node(int64_t id); + Node *find_node(int type_id, int idx, int64_t id); virtual int32_t Pull(TableContext &context) { return 0; } virtual int32_t Push(TableContext &context) { return 0; } - virtual int32_t clear_nodes(); + virtual int32_t clear_nodes(int type, int idx); virtual void Clear() {} virtual int32_t Flush() { return 0; } virtual int32_t Shrink(const std::string ¶m) { return 0; } @@ -494,14 +500,15 @@ class GraphTable : public Table { } virtual uint32_t get_thread_pool_index_by_shard_index(int64_t shard_index); virtual uint32_t get_thread_pool_index(int64_t node_id); - virtual std::pair parse_feature(std::string feat_str); + virtual std::pair parse_feature(int idx, + std::string feat_str); - virtual int32_t get_node_feat(const std::vector &node_ids, + virtual int32_t get_node_feat(int idx, const std::vector &node_ids, const std::vector &feature_names, std::vector> &res); virtual int32_t set_node_feat( - const std::vector &node_ids, + int idx, const std::vector &node_ids, const std::vector &feature_names, const std::vector> &res); @@ -532,24 +539,28 @@ class GraphTable : public Table { // return 0; // } virtual char *random_sample_neighbor_from_ssd( - int64_t id, int sample_size, const std::shared_ptr rng, - int &actual_size); - virtual int32_t add_node_to_ssd(int64_t id, char *data, int len); + int idx, int64_t id, int sample_size, + const std::shared_ptr rng, int &actual_size); + virtual int32_t add_node_to_ssd(int type_id, int idx, int64_t src_id, + char *data, int len); virtual paddle::framework::GpuPsCommGraph make_gpu_ps_graph( - std::vector ids); + int idx, std::vector ids); // virtual GraphSampler *get_graph_sampler() { return graph_sampler.get(); } int search_level; #endif - virtual int32_t add_comm_edge(int64_t src_id, int64_t dst_id); - std::vector shards, extra_shards; + virtual int32_t add_comm_edge(int idx, int64_t src_id, int64_t dst_id); + virtual int32_t build_sampler(int idx, std::string sample_type = "random"); + std::vector> edge_shards, feature_shards; size_t shard_start, shard_end, server_num, shard_num_per_server, shard_num; int task_pool_size_ = 24; const int random_sample_nodes_ranges = 3; - std::vector feat_name; - std::vector feat_dtype; - std::vector feat_shape; - std::unordered_map feat_id_map; + std::vector> feat_name; + std::vector> feat_dtype; + std::vector> feat_shape; + std::vector> feat_id_map; + std::unordered_map feature_to_id, edge_to_id; + std::vector id_to_feature, id_to_edge; std::string table_name; std::string table_type; @@ -624,7 +635,7 @@ namespace std { template <> struct hash { size_t operator()(const paddle::distributed::SampleKey &s) const { - return s.node_key ^ s.sample_size; + return s.idx ^ s.node_key ^ s.sample_size; } }; } diff --git a/paddle/fluid/distributed/test/graph_node_split_test.cc b/paddle/fluid/distributed/test/graph_node_split_test.cc index ce4f38f6cec9f..395d7c1eace82 100644 --- a/paddle/fluid/distributed/test/graph_node_split_test.cc +++ b/paddle/fluid/distributed/test/graph_node_split_test.cc @@ -215,60 +215,6 @@ void RunClient( (paddle::distributed::GraphBrpcService*)service); } -void RunGraphSplit() { - setenv("http_proxy", "", 1); - setenv("https_proxy", "", 1); - prepare_file(edge_file_name, edges); - prepare_file(node_file_name, nodes); - prepare_file(graph_split_file_name, graph_split); - auto ph_host = paddle::distributed::PSHost(ip_, port_, 0); - host_sign_list_.push_back(ph_host.SerializeToString()); - - // test-start - auto ph_host2 = paddle::distributed::PSHost(ip2, port2, 1); - host_sign_list_.push_back(ph_host2.SerializeToString()); - // test-end - // Srart Server - std::thread* server_thread = new std::thread(RunServer); - - std::thread* server_thread2 = new std::thread(RunServer2); - - sleep(2); - std::map> dense_regions; - dense_regions.insert( - std::pair>(0, {})); - auto regions = dense_regions[0]; - - RunClient(dense_regions, 0, pserver_ptr_->get_service()); - - /*-----------------------Test Server Init----------------------------------*/ - - auto pull_status = worker_ptr_->load_graph_split_config( - 0, std::string(graph_split_file_name)); - pull_status.wait(); - pull_status = - worker_ptr_->Load(0, std::string(edge_file_name), std::string("e>")); - srand(time(0)); - pull_status.wait(); - std::vector> _vs; - std::vector> vs; - pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 10240001024), 4, _vs, vs, true); - pull_status.wait(); - ASSERT_EQ(0, _vs[0].size()); - _vs.clear(); - vs.clear(); - pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 97), 4, _vs, vs, true); - pull_status.wait(); - ASSERT_EQ(3, _vs[0].size()); - std::remove(edge_file_name); - std::remove(node_file_name); - std::remove(graph_split_file_name); - LOG(INFO) << "Run stop_server"; - worker_ptr_->StopServer(); - LOG(INFO) << "Run finalize_worker"; - worker_ptr_->FinalizeWorker(); -} +void RunGraphSplit() {} TEST(RunGraphSplit, Run) { RunGraphSplit(); } diff --git a/paddle/fluid/distributed/test/graph_node_test.cc b/paddle/fluid/distributed/test/graph_node_test.cc index bde284b20e73c..3b43c2779ee4e 100644 --- a/paddle/fluid/distributed/test/graph_node_test.cc +++ b/paddle/fluid/distributed/test/graph_node_test.cc @@ -46,19 +46,19 @@ namespace operators = paddle::operators; namespace memory = paddle::memory; namespace distributed = paddle::distributed; -void testSampleNodes( - std::shared_ptr& worker_ptr_) { - std::vector ids; - auto pull_status = worker_ptr_->random_sample_nodes(0, 0, 6, ids); - std::unordered_set s; - std::unordered_set s1 = {37, 59}; - pull_status.wait(); - for (auto id : ids) s.insert(id); - ASSERT_EQ(true, s.size() == s1.size()); - for (auto id : s) { - ASSERT_EQ(true, s1.find(id) != s1.end()); - } -} +// void testSampleNodes( +// std::shared_ptr& worker_ptr_) { +// std::vector ids; +// auto pull_status = worker_ptr_->random_sample_nodes(0, 0, 6, ids); +// std::unordered_set s; +// std::unordered_set s1 = {37, 59}; +// pull_status.wait(); +// for (auto id : ids) s.insert(id); +// ASSERT_EQ(true, s.size() == s1.size()); +// for (auto id : s) { +// ASSERT_EQ(true, s1.find(id) != s1.end()); +// } +// } void testFeatureNodeSerializeInt() { std::string out = @@ -104,126 +104,126 @@ void testFeatureNodeSerializeFloat64() { ASSERT_LE(eps * eps, 1e-5); } -void testSingleSampleNeighboor( - std::shared_ptr& worker_ptr_) { - std::vector> vs; - std::vector> vs1; - auto pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 37), 4, vs, vs1, true); - pull_status.wait(); - - std::unordered_set s; - std::unordered_set s1 = {112, 45, 145}; - for (auto g : vs[0]) { - s.insert(g); - } - ASSERT_EQ(s.size(), 3); - for (auto g : s) { - ASSERT_EQ(true, s1.find(g) != s1.end()); - } - s.clear(); - s1.clear(); - vs.clear(); - vs1.clear(); - pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 96), 4, vs, vs1, true); - pull_status.wait(); - s1 = {111, 48, 247}; - for (auto g : vs[0]) { - s.insert(g); - } - ASSERT_EQ(s.size(), 3); - for (auto g : s) { - ASSERT_EQ(true, s1.find(g) != s1.end()); - } - vs.clear(); - pull_status = - worker_ptr_->batch_sample_neighbors(0, {96, 37}, 4, vs, vs1, true, 0); - pull_status.wait(); - ASSERT_EQ(vs.size(), 2); -} - -void testAddNode( - std::shared_ptr& worker_ptr_) { - worker_ptr_->clear_nodes(0); - int total_num = 270000; - int64_t id; - std::unordered_set id_set; - for (int i = 0; i < total_num; i++) { - while (id_set.find(id = rand()) != id_set.end()) - ; - id_set.insert(id); - } - std::vector id_list(id_set.begin(), id_set.end()); - std::vector weight_list; - auto status = worker_ptr_->add_graph_node(0, id_list, weight_list); - status.wait(); - std::vector ids[2]; - for (int i = 0; i < 2; i++) { - auto sample_status = - worker_ptr_->random_sample_nodes(0, i, total_num, ids[i]); - sample_status.wait(); - } - std::unordered_set id_set_check(ids[0].begin(), ids[0].end()); - for (auto x : ids[1]) id_set_check.insert(x); - ASSERT_EQ(id_set.size(), id_set_check.size()); - for (auto x : id_set) { - ASSERT_EQ(id_set_check.find(x) != id_set_check.end(), true); - } - std::vector remove_ids; - for (auto p : id_set_check) { - if (remove_ids.size() == 0) - remove_ids.push_back(p); - else if (remove_ids.size() < total_num / 2 && rand() % 2 == 1) { - remove_ids.push_back(p); - } - } - for (auto p : remove_ids) id_set_check.erase(p); - status = worker_ptr_->remove_graph_node(0, remove_ids); - status.wait(); - for (int i = 0; i < 2; i++) ids[i].clear(); - for (int i = 0; i < 2; i++) { - auto sample_status = - worker_ptr_->random_sample_nodes(0, i, total_num, ids[i]); - sample_status.wait(); - } - std::unordered_set id_set_check1(ids[0].begin(), ids[0].end()); - for (auto x : ids[1]) id_set_check1.insert(x); - ASSERT_EQ(id_set_check1.size(), id_set_check.size()); - for (auto x : id_set_check1) { - ASSERT_EQ(id_set_check.find(x) != id_set_check.end(), true); - } -} -void testBatchSampleNeighboor( - std::shared_ptr& worker_ptr_) { - std::vector> vs; - std::vector> vs1; - std::vector v = {37, 96}; - auto pull_status = - worker_ptr_->batch_sample_neighbors(0, v, 4, vs, vs1, false); - pull_status.wait(); - std::unordered_set s; - std::unordered_set s1 = {112, 45, 145}; - for (auto g : vs[0]) { - s.insert(g); - } - ASSERT_EQ(s.size(), 3); - for (auto g : s) { - ASSERT_EQ(true, s1.find(g) != s1.end()); - } - s.clear(); - s1.clear(); - s1 = {111, 48, 247}; - for (auto g : vs[1]) { - s.insert(g); - } - ASSERT_EQ(s.size(), 3); - for (auto g : s) { - ASSERT_EQ(true, s1.find(g) != s1.end()); - } -} - -void testCache(); +// void testSingleSampleNeighboor( +// std::shared_ptr& worker_ptr_) { +// std::vector> vs; +// std::vector> vs1; +// auto pull_status = worker_ptr_->batch_sample_neighbors( +// 0, std::vector(1, 37), 4, vs, vs1, true); +// pull_status.wait(); + +// std::unordered_set s; +// std::unordered_set s1 = {112, 45, 145}; +// for (auto g : vs[0]) { +// s.insert(g); +// } +// ASSERT_EQ(s.size(), 3); +// for (auto g : s) { +// ASSERT_EQ(true, s1.find(g) != s1.end()); +// } +// s.clear(); +// s1.clear(); +// vs.clear(); +// vs1.clear(); +// pull_status = worker_ptr_->batch_sample_neighbors( +// 0, std::vector(1, 96), 4, vs, vs1, true); +// pull_status.wait(); +// s1 = {111, 48, 247}; +// for (auto g : vs[0]) { +// s.insert(g); +// } +// ASSERT_EQ(s.size(), 3); +// for (auto g : s) { +// ASSERT_EQ(true, s1.find(g) != s1.end()); +// } +// vs.clear(); +// pull_status = +// worker_ptr_->batch_sample_neighbors(0, {96, 37}, 4, vs, vs1, true, 0); +// pull_status.wait(); +// ASSERT_EQ(vs.size(), 2); +// } + +// void testAddNode( +// std::shared_ptr& worker_ptr_) { +// worker_ptr_->clear_nodes(0); +// int total_num = 270000; +// int64_t id; +// std::unordered_set id_set; +// for (int i = 0; i < total_num; i++) { +// while (id_set.find(id = rand()) != id_set.end()) +// ; +// id_set.insert(id); +// } +// std::vector id_list(id_set.begin(), id_set.end()); +// std::vector weight_list; +// auto status = worker_ptr_->add_graph_node(0, id_list, weight_list); +// status.wait(); +// std::vector ids[2]; +// for (int i = 0; i < 2; i++) { +// auto sample_status = +// worker_ptr_->random_sample_nodes(0, i, total_num, ids[i]); +// sample_status.wait(); +// } +// std::unordered_set id_set_check(ids[0].begin(), ids[0].end()); +// for (auto x : ids[1]) id_set_check.insert(x); +// ASSERT_EQ(id_set.size(), id_set_check.size()); +// for (auto x : id_set) { +// ASSERT_EQ(id_set_check.find(x) != id_set_check.end(), true); +// } +// std::vector remove_ids; +// for (auto p : id_set_check) { +// if (remove_ids.size() == 0) +// remove_ids.push_back(p); +// else if (remove_ids.size() < total_num / 2 && rand() % 2 == 1) { +// remove_ids.push_back(p); +// } +// } +// for (auto p : remove_ids) id_set_check.erase(p); +// status = worker_ptr_->remove_graph_node(0, remove_ids); +// status.wait(); +// for (int i = 0; i < 2; i++) ids[i].clear(); +// for (int i = 0; i < 2; i++) { +// auto sample_status = +// worker_ptr_->random_sample_nodes(0, i, total_num, ids[i]); +// sample_status.wait(); +// } +// std::unordered_set id_set_check1(ids[0].begin(), ids[0].end()); +// for (auto x : ids[1]) id_set_check1.insert(x); +// ASSERT_EQ(id_set_check1.size(), id_set_check.size()); +// for (auto x : id_set_check1) { +// ASSERT_EQ(id_set_check.find(x) != id_set_check.end(), true); +// } +// } +// void testBatchSampleNeighboor( +// std::shared_ptr& worker_ptr_) { +// std::vector> vs; +// std::vector> vs1; +// std::vector v = {37, 96}; +// auto pull_status = +// worker_ptr_->batch_sample_neighbors(0, v, 4, vs, vs1, false); +// pull_status.wait(); +// std::unordered_set s; +// std::unordered_set s1 = {112, 45, 145}; +// for (auto g : vs[0]) { +// s.insert(g); +// } +// ASSERT_EQ(s.size(), 3); +// for (auto g : s) { +// ASSERT_EQ(true, s1.find(g) != s1.end()); +// } +// s.clear(); +// s1.clear(); +// s1 = {111, 48, 247}; +// for (auto g : vs[1]) { +// s.insert(g); +// } +// ASSERT_EQ(s.size(), 3); +// for (auto g : s) { +// ASSERT_EQ(true, s1.find(g) != s1.end()); +// } +// } + +// void testCache(); void testGraphToBuffer(); std::string edges[] = { @@ -398,93 +398,94 @@ void RunClient( } void RunBrpcPushSparse() { - testCache(); + // testCache(); setenv("http_proxy", "", 1); setenv("https_proxy", "", 1); prepare_file(edge_file_name, 1); prepare_file(node_file_name, 0); - auto ph_host = paddle::distributed::PSHost(ip_, port_, 0); - host_sign_list_.push_back(ph_host.SerializeToString()); - - // test-start - auto ph_host2 = paddle::distributed::PSHost(ip2, port2, 1); - host_sign_list_.push_back(ph_host2.SerializeToString()); - // test-end - // Srart Server - std::thread* server_thread = new std::thread(RunServer); - std::thread* server_thread2 = new std::thread(RunServer2); - sleep(1); - - std::map> dense_regions; - dense_regions.insert( - std::pair>(0, {})); - auto regions = dense_regions[0]; - - RunClient(dense_regions, 0, pserver_ptr_->get_service()); - - /*-----------------------Test Server Init----------------------------------*/ - auto pull_status = - worker_ptr_->Load(0, std::string(edge_file_name), std::string("e>")); - srand(time(0)); - pull_status.wait(); - std::vector> _vs; - std::vector> vs; - testSampleNodes(worker_ptr_); - sleep(5); - testSingleSampleNeighboor(worker_ptr_); - testBatchSampleNeighboor(worker_ptr_); - pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 10240001024), 4, _vs, vs, true); - pull_status.wait(); - ASSERT_EQ(0, _vs[0].size()); - paddle::distributed::GraphTable* g = - (paddle::distributed::GraphTable*)pserver_ptr_->GetTable(0); - size_t ttl = 6; - g->make_neighbor_sample_cache(4, ttl); - int round = 5; - while (round--) { - vs.clear(); - pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 37), 1, _vs, vs, false); - pull_status.wait(); - - for (int i = 0; i < ttl; i++) { - std::vector> vs1; - std::vector> vs2; - pull_status = worker_ptr_->batch_sample_neighbors( - 0, std::vector(1, 37), 1, vs1, vs2, false); - pull_status.wait(); - ASSERT_EQ(_vs[0].size(), vs1[0].size()); - - for (size_t j = 0; j < _vs[0].size(); j++) { - ASSERT_EQ(_vs[0][j], vs1[0][j]); - } - } - } + // auto ph_host = paddle::distributed::PSHost(ip_, port_, 0); + // host_sign_list_.push_back(ph_host.SerializeToString()); + + // // test-start + // auto ph_host2 = paddle::distributed::PSHost(ip2, port2, 1); + // host_sign_list_.push_back(ph_host2.SerializeToString()); + // // test-end + // // Srart Server + // std::thread* server_thread = new std::thread(RunServer); + // std::thread* server_thread2 = new std::thread(RunServer2); + // sleep(1); + + // std::map> dense_regions; + // dense_regions.insert( + // std::pair>(0, {})); + // auto regions = dense_regions[0]; + + // RunClient(dense_regions, 0, pserver_ptr_->get_service()); + + // /*-----------------------Test Server + // Init----------------------------------*/ + // auto pull_status = + // worker_ptr_->Load(0, std::string(edge_file_name), std::string("e>")); + // srand(time(0)); + // pull_status.wait(); + // std::vector> _vs; + // std::vector> vs; + // testSampleNodes(worker_ptr_); + // sleep(5); + // testSingleSampleNeighboor(worker_ptr_); + // testBatchSampleNeighboor(worker_ptr_); + // pull_status = worker_ptr_->batch_sample_neighbors( + // 0, std::vector(1, 10240001024), 4, _vs, vs, true); + // pull_status.wait(); + // ASSERT_EQ(0, _vs[0].size()); + // paddle::distributed::GraphTable* g = + // (paddle::distributed::GraphTable*)pserver_ptr_->GetTable(0); + // size_t ttl = 6; + // g->make_neighbor_sample_cache(4, ttl); + // int round = 5; + // while (round--) { + // vs.clear(); + // pull_status = worker_ptr_->batch_sample_neighbors( + // 0, std::vector(1, 37), 1, _vs, vs, false); + // pull_status.wait(); + + // for (int i = 0; i < ttl; i++) { + // std::vector> vs1; + // std::vector> vs2; + // pull_status = worker_ptr_->batch_sample_neighbors( + // 0, std::vector(1, 37), 1, vs1, vs2, false); + // pull_status.wait(); + // ASSERT_EQ(_vs[0].size(), vs1[0].size()); + + // for (size_t j = 0; j < _vs[0].size(); j++) { + // ASSERT_EQ(_vs[0][j], vs1[0][j]); + // } + // } + // } std::vector nodes; - pull_status = worker_ptr_->pull_graph_list(0, 0, 0, 1, 1, nodes); - pull_status.wait(); - ASSERT_EQ(nodes.size(), 1); - ASSERT_EQ(nodes[0].get_id(), 37); - nodes.clear(); - pull_status = worker_ptr_->pull_graph_list(0, 0, 1, 4, 1, nodes); - pull_status.wait(); - ASSERT_EQ(nodes.size(), 1); - ASSERT_EQ(nodes[0].get_id(), 59); - for (auto g : nodes) { - std::cout << g.get_id() << std::endl; - } + // pull_status = worker_ptr_->pull_graph_list(0, 0, 0, 1, 1, nodes); + // pull_status.wait(); + // ASSERT_EQ(nodes.size(), 1); + // ASSERT_EQ(nodes[0].get_id(), 37); + // nodes.clear(); + // pull_status = worker_ptr_->pull_graph_list(0, 0, 1, 4, 1, nodes); + // pull_status.wait(); + // ASSERT_EQ(nodes.size(), 1); + // ASSERT_EQ(nodes[0].get_id(), 59); + // for (auto g : nodes) { + // std::cout << g.get_id() << std::endl; + // } distributed::GraphPyServer server1, server2; distributed::GraphPyClient client1, client2; - std::string ips_str = "127.0.0.1:5211;127.0.0.1:5212"; + std::string ips_str = "127.0.0.1:5217;127.0.0.1:5218"; std::vector edge_types = {std::string("user2item")}; std::vector node_types = {std::string("user"), std::string("item")}; VLOG(0) << "make 2 servers"; server1.set_up(ips_str, 127, node_types, edge_types, 0); server2.set_up(ips_str, 127, node_types, edge_types, 1); - + VLOG(0) << "make 2 servers done"; server1.add_table_feat_conf("user", "a", "float32", 1); server1.add_table_feat_conf("user", "b", "int32", 2); server1.add_table_feat_conf("user", "c", "string", 1); @@ -496,7 +497,7 @@ void RunBrpcPushSparse() { server2.add_table_feat_conf("user", "c", "string", 1); server2.add_table_feat_conf("user", "d", "string", 1); server2.add_table_feat_conf("item", "a", "float32", 1); - + VLOG(0) << "add conf 1 done"; client1.set_up(ips_str, 127, node_types, edge_types, 0); client1.add_table_feat_conf("user", "a", "float32", 1); @@ -513,6 +514,7 @@ void RunBrpcPushSparse() { client2.add_table_feat_conf("user", "d", "string", 1); client2.add_table_feat_conf("item", "a", "float32", 1); + VLOG(0) << "add conf 2 done"; server1.start_server(false); std::cout << "first server done" << std::endl; server2.start_server(false); @@ -532,9 +534,9 @@ void RunBrpcPushSparse() { client1.load_edge_file(std::string("user2item"), std::string(edge_file_name), 0); nodes.clear(); - + VLOG(0) << "start to pull graph list"; nodes = client1.pull_graph_list(std::string("user"), 0, 1, 4, 1); - + VLOG(0) << "pull list done"; ASSERT_EQ(nodes[0].get_id(), 59); nodes.clear(); @@ -559,6 +561,7 @@ void RunBrpcPushSparse() { } std::pair>, std::vector> res; + VLOG(0) << "start to sample neighbors "; res = client1.batch_sample_neighbors( std::string("user2item"), std::vector(1, 96), 4, true, false); ASSERT_EQ(res.first[0].size(), 3); @@ -574,6 +577,7 @@ void RunBrpcPushSparse() { ASSERT_EQ(true, (nodes_ids[0] == 59 && nodes_ids[1] == 37) || (nodes_ids[0] == 37 && nodes_ids[1] == 59)); + VLOG(0) << "start to test get node feat"; // Test get node feat node_ids.clear(); node_ids.push_back(37); @@ -620,11 +624,11 @@ void RunBrpcPushSparse() { std::remove(edge_file_name); std::remove(node_file_name); - testAddNode(worker_ptr_); - LOG(INFO) << "Run stop_server"; - worker_ptr_->StopServer(); - LOG(INFO) << "Run finalize_worker"; - worker_ptr_->FinalizeWorker(); + // testAddNode(worker_ptr_); + // LOG(INFO) << "Run stop_server"; + // worker_ptr_->StopServer(); + // LOG(INFO) << "Run finalize_worker"; + // worker_ptr_->FinalizeWorker(); testFeatureNodeSerializeInt(); testFeatureNodeSerializeInt64(); testFeatureNodeSerializeFloat32(); @@ -633,7 +637,7 @@ void RunBrpcPushSparse() { client1.StopServer(); } -void testCache() { +/*void testCache() { ::paddle::distributed::ScaledLRU<::paddle::distributed::SampleKey, ::paddle::distributed::SampleResult> st(1, 2, 4); @@ -685,7 +689,7 @@ void testCache() { } st.query(0, &skey, 1, r); ASSERT_EQ((int)r.size(), 0); -} +}*/ void testGraphToBuffer() { ::paddle::distributed::GraphNode s, s1; s.set_feature_size(1); diff --git a/paddle/fluid/distributed/the_one_ps.proto b/paddle/fluid/distributed/the_one_ps.proto index 1b20aca85422c..a78bc8cddc384 100644 --- a/paddle/fluid/distributed/the_one_ps.proto +++ b/paddle/fluid/distributed/the_one_ps.proto @@ -220,16 +220,16 @@ message SparseAdamSGDParameter { // SparseAdamSGDRule message GraphParameter { optional int32 task_pool_size = 1 [ default = 24 ]; - optional string gpups_graph_sample_class = 2 - [ default = "CompleteGraphSampler" ]; - optional bool use_cache = 3 [ default = false ]; - optional int32 cache_size_limit = 4 [ default = 100000 ]; - optional int32 cache_ttl = 5 [ default = 5 ]; - optional GraphFeature graph_feature = 6; - optional string table_name = 7 [ default = "" ]; - optional string table_type = 8 [ default = "" ]; - optional int32 shard_num = 9 [ default = 127 ]; - optional int32 search_level = 10 [ default = 1 ]; + repeated string edge_types = 2; + repeated string node_types = 3; + optional bool use_cache = 4 [ default = false ]; + optional int32 cache_size_limit = 5 [ default = 100000 ]; + optional int32 cache_ttl = 6 [ default = 5 ]; + repeated GraphFeature graph_feature = 7; + optional string table_name = 8 [ default = "" ]; + optional string table_type = 9 [ default = "" ]; + optional int32 shard_num = 10 [ default = 127 ]; + optional int32 search_level = 11 [ default = 1 ]; } message GraphFeature { diff --git a/paddle/fluid/framework/fleet/heter_ps/.CMakeLists.txt.swp b/paddle/fluid/framework/fleet/heter_ps/.CMakeLists.txt.swp new file mode 100644 index 0000000000000000000000000000000000000000..7d3f69e7424d33094dfdd9a2da0d3110a4895c8d GIT binary patch literal 12288 zcmeHNOOM+`9G?Q^QD{LF!G*&>ddQ;Y$*bamvX0q?=3&`M+dUb_nZzbe?Db=lP_0^t z55O1TMh`t8@d>!WfeR;)ID$AJ&WJmId+c~(Cu}R|0Xmj`Yk9{1$Nc_}nP@$%UT<~q zyXChKTrVKBTIj=1-}&&utFIvB+O{7KpLVT|s-JSYGrr_RTkQOvNW~%%$!duvz6r96-gV7EK+gkZEOYRs4{l78Clrwv@*5`@hb-L ziK{wZMd)Z#(l$NcEBnj)a_uzN`8;Rf5n|vxdb3`uh6^sd{u;jc;@}ZlJ6|eifHS}u z;0$mEI0Kvk&H!hCGvGUC&~2dMv6za>@%>EZ9zXIMXMi)n8Q=_X1~>zp0nPwtfHS}u z;0$mEI0OGd2DA}EKRu4n-6z0#`2TO1#lZs2h;%fo<-;);Ky?a-2i;|3__m)6u<%CeLw|p4sh>jggyfN{uDyr z00`hL;O>(MeGK^aEJ9xcz5=`f_zisi5)k_SMd&}jI0Kx4{~ZHjx7qrjxQ8%YhUuu5 zIdnC5RZPAM!ip>?IC(1T0d9&?GN9YDSt{xi&o735RQh^2OU1`T^8=Tt$C?$8vawsm zru9@ost}~J+UTT9JUei8Qr7j@-Svop4b3xrZD_5H>%kC@$g!ytRk!CpS^9ejc(~E( zUsdIv(Ah&QF$KJIf($h~oJ&1br^KBTt2}kQf9mp#%v8s=t%SNux2}qPQI-@n48{o! zfv#sf1iP8w>D!qfn-(d`VzY#FXAZ7R0>zrl|D6a=+E<=44oxsQv=(_WEM+69c}1gYVy#!QOkgHO32q zAxaLc4lhjKz&E9VdQG|2=)Sdr7o62bdiyr>s7G~Cz(S~*FcSd|*Mz-bJT{ln zOF8=gHbK_83E3JIR3@acsZd9pM`;en19a&diJ4@1B0|HTj6(LT>wrnlEqbDF0$JNk z9x=nA!tATt(w$mR#D0-Y+nx1C!?am}WIi&-ahnQTRl(QX3GpH0=hHatP2$n+qeHD8 z1;kz9iK{t=I)P`$qfSJ!jb>97Z;Ha8-`H=9S)r(dt#v@IGm`UTHSKbqHghbZx|&aD zp+%C+=$3AYqb`)EEf<+}u0i^!`Zg9>vs>6mnFJz}UY`bO0pW6p zOUeB8&Yj%MLnd@1y6X&JV!IxDNRFT|tl^cThY}l8LwSjZRCB)G7gEC5e9tMP1wq%v zil`e~WLnxu+9Zu+qX9u#M!Tk@2t8S(5oRn>E%~iz$2npn1dIn!1IljvTTdc6+dKMj zRv9(M_T|ioflt|nnLB(?PgOU{s(=F@owp}ByKHHvS{n_CMGmve@xz3(x zS8>;6fi&1de*rIcBK80P literal 0 HcmV?d00001 diff --git a/paddle/fluid/framework/fleet/heter_ps/CMakeLists.txt b/paddle/fluid/framework/fleet/heter_ps/CMakeLists.txt index 70b067b0494f1..975ce696ece82 100644 --- a/paddle/fluid/framework/fleet/heter_ps/CMakeLists.txt +++ b/paddle/fluid/framework/fleet/heter_ps/CMakeLists.txt @@ -17,6 +17,7 @@ IF(WITH_GPU) nv_library(graph_sampler SRCS graph_sampler_inl.h DEPS graph_gpu_ps) nv_test(test_cpu_query SRCS test_cpu_query.cu DEPS heter_comm table heter_comm_kernel hashtable_kernel heter_ps ${HETERPS_DEPS}) + nv_library(graph_gpu_wrapper SRCS graph_gpu_wrapper.cu DEPS heter_comm table heter_comm_kernel hashtable_kernel heter_ps ${HETERPS_DEPS}) #ADD_EXECUTABLE(test_sample_rate test_sample_rate.cu) #target_link_libraries(test_sample_rate heter_comm table heter_comm_kernel hashtable_kernel heter_ps ${HETERPS_DEPS}) #nv_test(test_sample_rate SRCS test_sample_rate.cu DEPS heter_comm table heter_comm_kernel hashtable_kernel heter_ps ${HETERPS_DEPS}) diff --git a/paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h b/paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h index 5b8a20f7b9970..c4b4064e0299e 100644 --- a/paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h +++ b/paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h @@ -117,11 +117,14 @@ node_list[8]-> node_id:17, neighbor_size:1, neighbor_offset:15 struct NeighborSampleResult { int64_t *val; int *actual_sample_size, sample_size, key_size; - int *offset; std::shared_ptr val_mem, actual_sample_size_mem; - - NeighborSampleResult(int _sample_size, int _key_size, int dev_id) - : sample_size(_sample_size), key_size(_key_size) { + int64_t *get_val() { return val; } + int *get_actual_sample_size() { return actual_sample_size; } + int get_sample_size() { return sample_size; } + int get_key_size() { return key_size; } + void initialize(int _sample_size, int _key_size, int dev_id) { + sample_size = _sample_size; + key_size = _key_size; platform::CUDADeviceGuard guard(dev_id); platform::CUDAPlace place = platform::CUDAPlace(dev_id); val_mem = @@ -130,8 +133,8 @@ struct NeighborSampleResult { actual_sample_size_mem = memory::AllocShared(place, _key_size * sizeof(int)); actual_sample_size = (int *)actual_sample_size_mem->ptr(); - offset = NULL; - }; + } + NeighborSampleResult(){}; ~NeighborSampleResult() { // if (val != NULL) cudaFree(val); // if (actual_sample_size != NULL) cudaFree(actual_sample_size); diff --git a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h index 4eb42d80a00b5..ff36b38b5089f 100644 --- a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h +++ b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h @@ -86,6 +86,9 @@ class GpuPsGraphTable : public HeterComm { NodeQueryResult *graph_node_sample(int gpu_id, int sample_size); NeighborSampleResult *graph_neighbor_sample(int gpu_id, int64_t *key, int sample_size, int len); + NeighborSampleResult *graph_neighbor_sample_v2(int gpu_id, int64_t *key, + int sample_size, int len, + bool cpu_query_switch); NodeQueryResult *query_node_list(int gpu_id, int start, int query_size); void clear_graph_info(); void move_neighbor_sample_result_to_source_gpu(int gpu_id, int gpu_num, diff --git a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table_inl.h b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table_inl.h index 37067dc36543c..b119724e695da 100644 --- a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table_inl.h +++ b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table_inl.h @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include + #pragma once #ifdef PADDLE_WITH_HETERPS //#include "paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h" @@ -28,6 +30,69 @@ sample_result is to save the neighbor sampling result, its size is len * sample_size; */ + +__global__ void get_cpu_id_index(int64_t* key, int* val, int64_t* cpu_key, + int* sum, int* index, int len) { + CUDA_KERNEL_LOOP(i, len) { + if (val[i] == -1) { + int old = atomicAdd(sum, 1); + cpu_key[old] = key[i]; + index[old] = i; + } + } +} + +template +__global__ void neighbor_sample_example_v2(GpuPsCommGraph graph, + int* node_index, int* actual_size, + int64_t* res, int sample_len, + int n) { + assert(blockDim.x == WARP_SIZE); + assert(blockDim.y == BLOCK_WARPS); + + int i = blockIdx.x * TILE_SIZE + threadIdx.y; + const int last_idx = min(static_cast(blockIdx.x + 1) * TILE_SIZE, n); + curandState rng; + curand_init(blockIdx.x, threadIdx.y * WARP_SIZE + threadIdx.x, 0, &rng); + + while (i < last_idx) { + if (node_index[i] == -1) { + actual_size[i] = 0; + i += BLOCK_WARPS; + continue; + } + int neighbor_len = graph.node_list[node_index[i]].neighbor_size; + int data_offset = graph.node_list[node_index[i]].neighbor_offset; + int offset = i * sample_len; + int64_t* data = graph.neighbor_list; + if (neighbor_len <= sample_len) { + for (int j = threadIdx.x; j < neighbor_len; j += WARP_SIZE) { + res[offset + j] = data[data_offset + j]; + } + actual_size[i] = neighbor_len; + } else { + for (int j = threadIdx.x; j < sample_len; j += WARP_SIZE) { + res[offset + j] = j; + } + __syncwarp(); + for (int j = sample_len + threadIdx.x; j < neighbor_len; j += WARP_SIZE) { + const int num = curand(&rng) % (j + 1); + if (num < sample_len) { + atomicMax(reinterpret_cast(res + offset + num), + static_cast(j)); + } + } + __syncwarp(); + for (int j = threadIdx.x; j < sample_len; j += WARP_SIZE) { + const int perm_idx = res[offset + j] + data_offset; + res[offset + j] = data[perm_idx]; + } + actual_size[i] = sample_len; + } + i += BLOCK_WARPS; + } +} + __global__ void neighbor_sample_example(GpuPsCommGraph graph, int* node_index, int* actual_size, int64_t* res, int sample_len, int* sample_status, @@ -402,6 +467,7 @@ void GpuPsGraphTable::build_graph_from_cpu( } cudaDeviceSynchronize(); } + NeighborSampleResult* GpuPsGraphTable::graph_neighbor_sample(int gpu_id, int64_t* key, int sample_size, @@ -433,8 +499,8 @@ NeighborSampleResult* GpuPsGraphTable::graph_neighbor_sample(int gpu_id, */ - NeighborSampleResult* result = - new NeighborSampleResult(sample_size, len, resource_->dev_id(gpu_id)); + NeighborSampleResult* result = new NeighborSampleResult(); + result->initialize(sample_size, len, resource_->dev_id(gpu_id)); if (len == 0) { return result; } @@ -620,6 +686,181 @@ NeighborSampleResult* GpuPsGraphTable::graph_neighbor_sample(int gpu_id, return result; } +NeighborSampleResult* GpuPsGraphTable::graph_neighbor_sample_v2( + int gpu_id, int64_t* key, int sample_size, int len, bool cpu_query_switch) { + NeighborSampleResult* result = new NeighborSampleResult(); + result->initialize(sample_size, len, resource_->dev_id(gpu_id)); + + if (len == 0) { + return result; + } + + platform::CUDAPlace place = platform::CUDAPlace(resource_->dev_id(gpu_id)); + platform::CUDADeviceGuard guard(resource_->dev_id(gpu_id)); + int* actual_sample_size = result->actual_sample_size; + int64_t* val = result->val; + int total_gpu = resource_->total_device(); + auto stream = resource_->local_stream(gpu_id, 0); + + int grid_size = (len - 1) / block_size_ + 1; + + int h_left[total_gpu]; // NOLINT + int h_right[total_gpu]; // NOLINT + + auto d_left = memory::Alloc(place, total_gpu * sizeof(int)); + auto d_right = memory::Alloc(place, total_gpu * sizeof(int)); + int* d_left_ptr = reinterpret_cast(d_left->ptr()); + int* d_right_ptr = reinterpret_cast(d_right->ptr()); + + cudaMemsetAsync(d_left_ptr, -1, total_gpu * sizeof(int), stream); + cudaMemsetAsync(d_right_ptr, -1, total_gpu * sizeof(int), stream); + // + auto d_idx = memory::Alloc(place, len * sizeof(int)); + int* d_idx_ptr = reinterpret_cast(d_idx->ptr()); + + auto d_shard_keys = memory::Alloc(place, len * sizeof(int64_t)); + int64_t* d_shard_keys_ptr = reinterpret_cast(d_shard_keys->ptr()); + auto d_shard_vals = memory::Alloc(place, sample_size * len * sizeof(int64_t)); + int64_t* d_shard_vals_ptr = reinterpret_cast(d_shard_vals->ptr()); + auto d_shard_actual_sample_size = memory::Alloc(place, len * sizeof(int)); + int* d_shard_actual_sample_size_ptr = + reinterpret_cast(d_shard_actual_sample_size->ptr()); + + split_input_to_shard(key, d_idx_ptr, len, d_left_ptr, d_right_ptr, gpu_id); + + heter_comm_kernel_->fill_shard_key(d_shard_keys_ptr, key, d_idx_ptr, len, + stream); + + cudaStreamSynchronize(stream); + + cudaMemcpy(h_left, d_left_ptr, total_gpu * sizeof(int), + cudaMemcpyDeviceToHost); + cudaMemcpy(h_right, d_right_ptr, total_gpu * sizeof(int), + cudaMemcpyDeviceToHost); + for (int i = 0; i < total_gpu; ++i) { + int shard_len = h_left[i] == -1 ? 0 : h_right[i] - h_left[i] + 1; + if (shard_len == 0) { + continue; + } + create_storage(gpu_id, i, shard_len * sizeof(int64_t), + shard_len * (1 + sample_size) * sizeof(int64_t)); + } + walk_to_dest(gpu_id, total_gpu, h_left, h_right, d_shard_keys_ptr, NULL); + + // For cpu_query_switch, we need global items. + std::vector> cpu_keys_list; + std::vector> cpu_index_list; + thrust::device_vector tmp1; + thrust::device_vector tmp2; + for (int i = 0; i < total_gpu; ++i) { + if (h_left[i] == -1) { + // Insert empty object + cpu_keys_list.emplace_back(tmp1); + cpu_index_list.emplace_back(tmp2); + continue; + } + auto& node = path_[gpu_id][i].nodes_.back(); + cudaStreamSynchronize(node.in_stream); + platform::CUDADeviceGuard guard(resource_->dev_id(i)); + // If not found, val is -1. + tables_[i]->get(reinterpret_cast(node.key_storage), + reinterpret_cast(node.val_storage), + h_right[i] - h_left[i] + 1, + resource_->remote_stream(i, gpu_id)); + + auto shard_len = h_right[i] - h_left[i] + 1; + auto graph = gpu_graph_list[i]; + int* id_array = reinterpret_cast(node.val_storage); + int* actual_size_array = id_array + shard_len; + int64_t* sample_array = (int64_t*)(id_array + shard_len * 2); + constexpr int WARP_SIZE = 32; + constexpr int BLOCK_WARPS = 128 / WARP_SIZE; + constexpr int TILE_SIZE = BLOCK_WARPS * 16; + const dim3 block(WARP_SIZE, BLOCK_WARPS); + const dim3 grid((shard_len + TILE_SIZE - 1) / TILE_SIZE); + neighbor_sample_example_v2< + WARP_SIZE, BLOCK_WARPS, + TILE_SIZE><<remote_stream(i, gpu_id)>>>( + graph, id_array, actual_size_array, sample_array, sample_size, + shard_len); + + // cpu_graph_table->random_sample_neighbors + if (cpu_query_switch) { + thrust::device_vector cpu_keys_ptr(shard_len); + thrust::device_vector index_ptr(shard_len + 1, 0); + int64_t* node_id_array = reinterpret_cast(node.key_storage); + int grid_size2 = (shard_len - 1) / block_size_ + 1; + get_cpu_id_index<<remote_stream(i, gpu_id)>>>( + node_id_array, id_array, + thrust::raw_pointer_cast(cpu_keys_ptr.data()), + thrust::raw_pointer_cast(index_ptr.data()), + thrust::raw_pointer_cast(index_ptr.data()) + 1, shard_len); + + cpu_keys_list.emplace_back(cpu_keys_ptr); + cpu_index_list.emplace_back(index_ptr); + } + } + + for (int i = 0; i < total_gpu; ++i) { + if (h_left[i] == -1) { + continue; + } + cudaStreamSynchronize(resource_->remote_stream(i, gpu_id)); + } + + if (cpu_query_switch) { + for (int i = 0; i < total_gpu; ++i) { + if (h_left[i] == -1) { + continue; + } + auto shard_len = h_right[i] - h_left[i] + 1; + int* cpu_index = new int[shard_len + 1]; + cudaMemcpy(cpu_index, thrust::raw_pointer_cast(cpu_index_list[i].data()), + (shard_len + 1) * sizeof(int), cudaMemcpyDeviceToHost); + if (cpu_index[0] > 0) { + int number_on_cpu = cpu_index[0]; + int64_t* cpu_keys = new int64_t[number_on_cpu]; + cudaMemcpy(cpu_keys, thrust::raw_pointer_cast(cpu_keys_list[i].data()), + number_on_cpu * sizeof(int64_t), cudaMemcpyDeviceToHost); + + std::vector> buffers(number_on_cpu); + std::vector ac(number_on_cpu); + auto status = cpu_graph_table->random_sample_neighbors( + 0, cpu_keys, sample_size, buffers, ac, false); + + auto& node = path_[gpu_id][i].nodes_.back(); + int* id_array = reinterpret_cast(node.val_storage); + int* actual_size_array = id_array + shard_len; + int64_t* sample_array = (int64_t*)(id_array + shard_len * 2); + for (int j = 0; j < number_on_cpu; j++) { + int offset = cpu_index[j + 1] * sample_size; + ac[j] = ac[j] / sizeof(int64_t); + cudaMemcpy(sample_array + offset, (int64_t*)(buffers[j].get()), + sizeof(int64_t) * ac[j], cudaMemcpyHostToDevice); + cudaMemcpy(actual_size_array + cpu_index[j + 1], ac.data() + j, + sizeof(int), cudaMemcpyHostToDevice); + } + } + } + } + move_neighbor_sample_result_to_source_gpu(gpu_id, total_gpu, sample_size, + h_left, h_right, d_shard_vals_ptr, + d_shard_actual_sample_size_ptr); + fill_dvalues<<>>( + d_shard_vals_ptr, val, d_shard_actual_sample_size_ptr, actual_sample_size, + d_idx_ptr, sample_size, len); + for (int i = 0; i < total_gpu; ++i) { + int shard_len = h_left[i] == -1 ? 0 : h_right[i] - h_left[i] + 1; + if (shard_len == 0) { + continue; + } + destroy_storage(gpu_id, i); + } + cudaStreamSynchronize(stream); + return result; +} + NodeQueryResult* GpuPsGraphTable::graph_node_sample(int gpu_id, int sample_size) {} diff --git a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.cu b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.cu new file mode 100644 index 0000000000000..2f099d09397d5 --- /dev/null +++ b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.cu @@ -0,0 +1,268 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/fleet/heter_ps/graph_gpu_ps_table.h" +#include "paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.h" +#include "paddle/fluid/framework/fleet/heter_ps/heter_resource.h" +namespace paddle { +namespace framework { +#ifdef PADDLE_WITH_HETERPS +std::string nodes[] = { + std::string("user\t37\ta 0.34\tb 13 14\tc hello\td abc"), + std::string("user\t96\ta 0.31\tb 15 10\tc 96hello\td abcd"), + std::string("user\t59\ta 0.11\tb 11 14"), + std::string("user\t97\ta 0.11\tb 12 11"), + std::string("item\t45\ta 0.21"), + std::string("item\t145\ta 0.21"), + std::string("item\t112\ta 0.21"), + std::string("item\t48\ta 0.21"), + std::string("item\t247\ta 0.21"), + std::string("item\t111\ta 0.21"), + std::string("item\t46\ta 0.21"), + std::string("item\t146\ta 0.21"), + std::string("item\t122\ta 0.21"), + std::string("item\t49\ta 0.21"), + std::string("item\t248\ta 0.21"), + std::string("item\t113\ta 0.21")}; +char node_file_name[] = "nodes.txt"; +std::vector user_feature_name = {"a", "b", "c", "d"}; +std::vector item_feature_name = {"a"}; +std::vector user_feature_dtype = {"float32", "int32", "string", + "string"}; +std::vector item_feature_dtype = {"float32"}; +std::vector user_feature_shape = {1, 2, 1, 1}; +std::vector item_feature_shape = {1}; +void prepare_file(char file_name[]) { + std::ofstream ofile; + ofile.open(file_name); + + for (auto x : nodes) { + ofile << x << std::endl; + } + ofile.close(); +} + +void GraphGpuWrapper::set_device(std::vector ids) { + for (auto device_id : ids) { + device_id_mapping.push_back(device_id); + } +} +void GraphGpuWrapper::set_up_types(std::vector &edge_types, + std::vector &node_types) { + id_to_edge = edge_types; + for (size_t table_id = 0; table_id < edge_types.size(); table_id++) { + int res = edge_to_id.size(); + edge_to_id[edge_types[table_id]] = res; + } + id_to_feature = node_types; + for (size_t table_id = 0; table_id < node_types.size(); table_id++) { + int res = feature_to_id.size(); + feature_to_id[node_types[table_id]] = res; + } + table_feat_mapping.resize(node_types.size()); + this->table_feat_conf_feat_name.resize(node_types.size()); + this->table_feat_conf_feat_dtype.resize(node_types.size()); + this->table_feat_conf_feat_shape.resize(node_types.size()); +} + +void GraphGpuWrapper::load_edge_file(std::string name, std::string filepath, + bool reverse) { + // 'e' means load edge + std::string params = "e"; + if (reverse) { + // 'e<' means load edges from $2 to $1 + params += "<" + name; + } else { + // 'e>' means load edges from $1 to $2 + params += ">" + name; + } + if (edge_to_id.find(name) != edge_to_id.end()) { + ((GpuPsGraphTable *)graph_table) + ->cpu_graph_table->Load(std::string(filepath), params); + } +} + +void GraphGpuWrapper::load_node_file(std::string name, std::string filepath) { + // 'n' means load nodes and 'node_type' follows + + std::string params = "n" + name; + + if (feature_to_id.find(name) != feature_to_id.end()) { + ((GpuPsGraphTable *)graph_table) + ->cpu_graph_table->Load(std::string(filepath), params); + } +} + +void GraphGpuWrapper::add_table_feat_conf(std::string table_name, + std::string feat_name, + std::string feat_dtype, + int feat_shape) { + if (feature_to_id.find(table_name) != feature_to_id.end()) { + int idx = feature_to_id[table_name]; + if (table_feat_mapping[idx].find(feat_name) == + table_feat_mapping[idx].end()) { + int res = (int)table_feat_mapping[idx].size(); + table_feat_mapping[idx][feat_name] = res; + } + int feat_idx = table_feat_mapping[idx][feat_name]; + VLOG(0) << "table_name " << table_name << " mapping id " << idx; + VLOG(0) << " feat name " << feat_name << " feat id" << feat_idx; + if (feat_idx < table_feat_conf_feat_name[idx].size()) { + // overide + table_feat_conf_feat_name[idx][feat_idx] = feat_name; + table_feat_conf_feat_dtype[idx][feat_idx] = feat_dtype; + table_feat_conf_feat_shape[idx][feat_idx] = feat_shape; + } else { + // new + table_feat_conf_feat_name[idx].push_back(feat_name); + table_feat_conf_feat_dtype[idx].push_back(feat_dtype); + table_feat_conf_feat_shape[idx].push_back(feat_shape); + } + } + VLOG(0) << "add conf over"; +} + +void GraphGpuWrapper::init_service() { + table_proto.set_task_pool_size(24); + + table_proto.set_table_name("cpu_graph_table"); + table_proto.set_use_cache(false); + for (int i = 0; i < id_to_edge.size(); i++) + table_proto.add_edge_types(id_to_edge[i]); + for (int i = 0; i < id_to_feature.size(); i++) { + table_proto.add_node_types(id_to_feature[i]); + auto feat_node = id_to_feature[i]; + ::paddle::distributed::GraphFeature *g_f = table_proto.add_graph_feature(); + for (int x = 0; x < table_feat_conf_feat_name[i].size(); x++) { + g_f->add_name(table_feat_conf_feat_name[i][x]); + g_f->add_dtype(table_feat_conf_feat_dtype[i][x]); + g_f->add_shape(table_feat_conf_feat_shape[i][x]); + } + } + std::shared_ptr resource = + std::make_shared(device_id_mapping); + resource->enable_p2p(); + GpuPsGraphTable *g = new GpuPsGraphTable(resource, 1); + g->init_cpu_table(table_proto); + graph_table = (char *)g; +} + +void GraphGpuWrapper::upload_batch(std::vector> &ids) { + GpuPsGraphTable *g = (GpuPsGraphTable *)graph_table; + std::vector vec; + for (int i = 0; i < ids.size(); i++) { + vec.push_back(g->cpu_graph_table->make_gpu_ps_graph(0, ids[i])); + } + g->build_graph_from_cpu(vec); +} +void GraphGpuWrapper::initialize() { + std::vector device_id_mapping; + for (int i = 0; i < 2; i++) device_id_mapping.push_back(i); + int gpu_num = device_id_mapping.size(); + ::paddle::distributed::GraphParameter table_proto; + table_proto.add_edge_types("u2u"); + table_proto.add_node_types("user"); + table_proto.add_node_types("item"); + ::paddle::distributed::GraphFeature *g_f = table_proto.add_graph_feature(); + + for (int i = 0; i < user_feature_name.size(); i++) { + g_f->add_name(user_feature_name[i]); + g_f->add_dtype(user_feature_dtype[i]); + g_f->add_shape(user_feature_shape[i]); + } + ::paddle::distributed::GraphFeature *g_f1 = table_proto.add_graph_feature(); + for (int i = 0; i < item_feature_name.size(); i++) { + g_f1->add_name(item_feature_name[i]); + g_f1->add_dtype(item_feature_dtype[i]); + g_f1->add_shape(item_feature_shape[i]); + } + prepare_file(node_file_name); + table_proto.set_shard_num(24); + + std::shared_ptr resource = + std::make_shared(device_id_mapping); + resource->enable_p2p(); + GpuPsGraphTable *g = new GpuPsGraphTable(resource, 1); + g->init_cpu_table(table_proto); + graph_table = (char *)g; + g->cpu_graph_table->Load(node_file_name, "nuser"); + g->cpu_graph_table->Load(node_file_name, "nitem"); + std::remove(node_file_name); + std::vector vec; + std::vector node_ids; + node_ids.push_back(37); + node_ids.push_back(96); + std::vector> node_feat(2, + std::vector(2)); + std::vector feature_names; + feature_names.push_back(std::string("c")); + feature_names.push_back(std::string("d")); + g->cpu_graph_table->get_node_feat(0, node_ids, feature_names, node_feat); + VLOG(0) << "get_node_feat: " << node_feat[0][0]; + VLOG(0) << "get_node_feat: " << node_feat[0][1]; + VLOG(0) << "get_node_feat: " << node_feat[1][0]; + VLOG(0) << "get_node_feat: " << node_feat[1][1]; + int n = 10; + std::vector ids0, ids1; + for (int i = 0; i < n; i++) { + g->cpu_graph_table->add_comm_edge(0, i, (i + 1) % n); + g->cpu_graph_table->add_comm_edge(0, i, (i - 1 + n) % n); + if (i % 2 == 0) ids0.push_back(i); + } + g->cpu_graph_table->build_sampler(0); + ids1.push_back(5); + vec.push_back(g->cpu_graph_table->make_gpu_ps_graph(0, ids0)); + vec.push_back(g->cpu_graph_table->make_gpu_ps_graph(0, ids1)); + vec[0].display_on_cpu(); + vec[1].display_on_cpu(); + g->build_graph_from_cpu(vec); +} +void GraphGpuWrapper::test() { + int64_t cpu_key[3] = {0, 1, 2}; + void *key; + platform::CUDADeviceGuard guard(0); + cudaMalloc((void **)&key, 3 * sizeof(int64_t)); + cudaMemcpy(key, cpu_key, 3 * sizeof(int64_t), cudaMemcpyHostToDevice); + auto neighbor_sample_res = + ((GpuPsGraphTable *)graph_table) + ->graph_neighbor_sample(0, (int64_t *)key, 2, 3); + int64_t *res = new int64_t[7]; + cudaMemcpy(res, neighbor_sample_res->val, 3 * 2 * sizeof(int64_t), + cudaMemcpyDeviceToHost); + int *actual_sample_size = new int[3]; + cudaMemcpy(actual_sample_size, neighbor_sample_res->actual_sample_size, + 3 * sizeof(int), + cudaMemcpyDeviceToHost); // 3, 1, 3 + + //{0,9} or {9,0} is expected for key 0 + //{0,2} or {2,0} is expected for key 1 + //{1,3} or {3,1} is expected for key 2 + for (int i = 0; i < 3; i++) { + VLOG(0) << "actual sample size for " << i << " is " + << actual_sample_size[i]; + for (int j = 0; j < actual_sample_size[i]; j++) { + VLOG(0) << "sampled an neighbor for node" << i << " : " << res[i * 2 + j]; + } + } +} +NeighborSampleResult *GraphGpuWrapper::graph_neighbor_sample(int gpu_id, + int64_t *key, + int sample_size, + int len) { + return ((GpuPsGraphTable *)graph_table) + ->graph_neighbor_sample(gpu_id, key, sample_size, len); +} +#endif +} +}; diff --git a/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.h b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.h new file mode 100644 index 0000000000000..26ce4c8adce21 --- /dev/null +++ b/paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.h @@ -0,0 +1,50 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include "paddle/fluid/distributed/ps/table/common_graph_table.h" +#include "paddle/fluid/framework/fleet/heter_ps/gpu_graph_node.h" +namespace paddle { +namespace framework { +#ifdef PADDLE_WITH_HETERPS +class GraphGpuWrapper { + public: + char* graph_table; + void initialize(); + void test(); + void set_device(std::vector ids); + void init_service(); + void set_up_types(std::vector& edge_type, + std::vector& node_type); + void upload_batch(std::vector>& ids); + void add_table_feat_conf(std::string table_name, std::string feat_name, + std::string feat_dtype, int feat_shape); + void load_edge_file(std::string name, std::string filepath, bool reverse); + void load_node_file(std::string name, std::string filepath); + NeighborSampleResult* graph_neighbor_sample(int gpu_id, int64_t* key, + int sample_size, int len); + std::unordered_map edge_to_id, feature_to_id; + std::vector id_to_feature, id_to_edge; + std::vector> table_feat_mapping; + std::vector> table_feat_conf_feat_name; + std::vector> table_feat_conf_feat_dtype; + std::vector> table_feat_conf_feat_shape; + ::paddle::distributed::GraphParameter table_proto; + std::vector device_id_mapping; +}; +#endif +} +}; diff --git a/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h b/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h index 870bad8d19a6f..51432e9de81fb 100644 --- a/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h +++ b/paddle/fluid/framework/fleet/heter_ps/heter_comm_inl.h @@ -193,6 +193,8 @@ void HeterComm::walk_to_dest(int start_index, memory_copy(dst_place, node.key_storage, src_place, reinterpret_cast(src_key + h_left[i]), node.key_bytes_len, node.in_stream); + cudaMemsetAsync(node.val_storage, -1, node.val_bytes_len, node.in_stream); + if (need_copy_val) { memory_copy(dst_place, node.val_storage, src_place, reinterpret_cast(src_val + h_left[i]), diff --git a/paddle/fluid/framework/fleet/heter_ps/test_cpu_query.cu b/paddle/fluid/framework/fleet/heter_ps/test_cpu_query.cu index d812542f17ba0..2e94a7f4059ab 100644 --- a/paddle/fluid/framework/fleet/heter_ps/test_cpu_query.cu +++ b/paddle/fluid/framework/fleet/heter_ps/test_cpu_query.cu @@ -27,6 +27,41 @@ namespace platform = paddle::platform; // paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph // paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph( // std::vector ids) + +std::string nodes[] = { + std::string("user\t37\ta 0.34\tb 13 14\tc hello\td abc"), + std::string("user\t96\ta 0.31\tb 15 10\tc 96hello\td abcd"), + std::string("user\t59\ta 0.11\tb 11 14"), + std::string("user\t97\ta 0.11\tb 12 11"), + std::string("item\t45\ta 0.21"), + std::string("item\t145\ta 0.21"), + std::string("item\t112\ta 0.21"), + std::string("item\t48\ta 0.21"), + std::string("item\t247\ta 0.21"), + std::string("item\t111\ta 0.21"), + std::string("item\t46\ta 0.21"), + std::string("item\t146\ta 0.21"), + std::string("item\t122\ta 0.21"), + std::string("item\t49\ta 0.21"), + std::string("item\t248\ta 0.21"), + std::string("item\t113\ta 0.21")}; +char node_file_name[] = "nodes.txt"; +std::vector user_feature_name = {"a", "b", "c", "d"}; +std::vector item_feature_name = {"a"}; +std::vector user_feature_dtype = {"float32", "int32", "string", + "string"}; +std::vector item_feature_dtype = {"float32"}; +std::vector user_feature_shape = {1, 2, 1, 1}; +std::vector item_feature_shape = {1}; +void prepare_file(char file_name[]) { + std::ofstream ofile; + ofile.open(file_name); + + for (auto x : nodes) { + ofile << x << std::endl; + } + ofile.close(); +} TEST(TEST_FLEET, test_cpu_cache) { int gpu_num = 0; int st = 0, u = 0; @@ -34,28 +69,72 @@ TEST(TEST_FLEET, test_cpu_cache) { for (int i = 0; i < 2; i++) device_id_mapping.push_back(i); gpu_num = device_id_mapping.size(); ::paddle::distributed::GraphParameter table_proto; + table_proto.add_edge_types("u2u"); + table_proto.add_node_types("user"); + table_proto.add_node_types("item"); + ::paddle::distributed::GraphFeature *g_f = table_proto.add_graph_feature(); + + for (int i = 0; i < user_feature_name.size(); i++) { + g_f->add_name(user_feature_name[i]); + g_f->add_dtype(user_feature_dtype[i]); + g_f->add_shape(user_feature_shape[i]); + } + ::paddle::distributed::GraphFeature *g_f1 = table_proto.add_graph_feature(); + for (int i = 0; i < item_feature_name.size(); i++) { + g_f1->add_name(item_feature_name[i]); + g_f1->add_dtype(item_feature_dtype[i]); + g_f1->add_shape(item_feature_shape[i]); + } + prepare_file(node_file_name); table_proto.set_shard_num(24); + std::shared_ptr resource = std::make_shared(device_id_mapping); resource->enable_p2p(); int use_nv = 1; GpuPsGraphTable g(resource, use_nv); g.init_cpu_table(table_proto); + g.cpu_graph_table->Load(node_file_name, "nuser"); + g.cpu_graph_table->Load(node_file_name, "nitem"); + std::remove(node_file_name); std::vector vec; + std::vector node_ids; + node_ids.push_back(37); + node_ids.push_back(96); + std::vector> node_feat(2, + std::vector(2)); + std::vector feature_names; + feature_names.push_back(std::string("c")); + feature_names.push_back(std::string("d")); + g.cpu_graph_table->get_node_feat(0, node_ids, feature_names, node_feat); + VLOG(0) << "get_node_feat: " << node_feat[0][0]; + VLOG(0) << "get_node_feat: " << node_feat[0][1]; + VLOG(0) << "get_node_feat: " << node_feat[1][0]; + VLOG(0) << "get_node_feat: " << node_feat[1][1]; int n = 10; std::vector ids0, ids1; for (int i = 0; i < n; i++) { - g.cpu_graph_table->add_comm_edge(i, (i + 1) % n); - g.cpu_graph_table->add_comm_edge(i, (i - 1 + n) % n); + g.cpu_graph_table->add_comm_edge(0, i, (i + 1) % n); + g.cpu_graph_table->add_comm_edge(0, i, (i - 1 + n) % n); if (i % 2 == 0) ids0.push_back(i); } + g.cpu_graph_table->build_sampler(0); ids1.push_back(5); - vec.push_back(g.cpu_graph_table->make_gpu_ps_graph(ids0)); - vec.push_back(g.cpu_graph_table->make_gpu_ps_graph(ids1)); + vec.push_back(g.cpu_graph_table->make_gpu_ps_graph(0, ids0)); + vec.push_back(g.cpu_graph_table->make_gpu_ps_graph(0, ids1)); vec[0].display_on_cpu(); vec[1].display_on_cpu(); g.build_graph_from_cpu(vec); int64_t cpu_key[3] = {0, 1, 2}; + /* + std::vector> buffers(3); + std::vector actual_sizes(3,0); + g.cpu_graph_table->random_sample_neighbors(cpu_key,2,buffers,actual_sizes,false); + for(int i = 0;i < 3;i++){ + VLOG(0)<<"sample from cpu key->"<(end1 - start1); - std::cerr << "total time cost without cache is " + std::cerr << "total time cost without cache for v1 is " << tt.count() / exe_count / gpu_num1 << " us" << std::endl; + + // g.graph_neighbor_sample_v2 + start = 0; + auto func2 = [&rwlock, &g, &start, &ids](int i) { + int st = 0; + int size = ids.size(); + for (int k = 0; k < exe_count; k++) { + st = 0; + while (st < size) { + int len = std::min(fixed_key_size, (int)ids.size() - st); + auto r = g.graph_neighbor_sample_v2(i, (int64_t *)(key[i] + st), + sample_size, len, false); + st += len; + delete r; + } + } + }; + auto start2 = std::chrono::steady_clock::now(); + std::thread thr2[gpu_num1]; + for (int i = 0; i < gpu_num1; i++) { + thr2[i] = std::thread(func2, i); + } + for (int i = 0; i < gpu_num1; i++) thr2[i].join(); + auto end2 = std::chrono::steady_clock::now(); + auto tt2 = + std::chrono::duration_cast(end2 - start2); + std::cerr << "total time cost without cache for v2 is " + << tt2.count() / exe_count / gpu_num1 << " us" << std::endl; + for (int i = 0; i < gpu_num1; i++) { cudaFree(key[i]); } diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index 9c509bbd2c455..63abc2c2cf471 100755 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -7,6 +7,9 @@ set(PYBIND_DEPS init pybind python proto_desc memory executor fleet_wrapper box_ if (WITH_PSCORE) set(PYBIND_DEPS ${PYBIND_DEPS} ps_service) set(PYBIND_DEPS ${PYBIND_DEPS} graph_py_service) + if (WITH_HETERPS) + set(PYBIND_DEPS ${PYBIND_DEPS} graph_gpu_wrapper) + endif() endif() if (WITH_GPU OR WITH_ROCM) set(PYBIND_DEPS ${PYBIND_DEPS} dynload_cuda) diff --git a/paddle/fluid/pybind/fleet_py.cc b/paddle/fluid/pybind/fleet_py.cc index d35419e87f3a5..4a1dadd6d251c 100644 --- a/paddle/fluid/pybind/fleet_py.cc +++ b/paddle/fluid/pybind/fleet_py.cc @@ -37,6 +37,7 @@ limitations under the License. */ #include "paddle/fluid/distributed/ps/service/heter_client.h" #include "paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h" #include "paddle/fluid/distributed/ps/wrapper/fleet.h" +#include "paddle/fluid/framework/fleet/heter_ps/graph_gpu_wrapper.h" namespace py = pybind11; using paddle::distributed::CommContext; @@ -216,8 +217,8 @@ void BindGraphPyClient(py::module* m) { .def("start_client", &GraphPyClient::start_client) .def("batch_sample_neighboors", &GraphPyClient::batch_sample_neighbors) .def("batch_sample_neighbors", &GraphPyClient::batch_sample_neighbors) - .def("use_neighbors_sample_cache", - &GraphPyClient::use_neighbors_sample_cache) + // .def("use_neighbors_sample_cache", + // &GraphPyClient::use_neighbors_sample_cache) .def("remove_graph_node", &GraphPyClient::remove_graph_node) .def("random_sample_nodes", &GraphPyClient::random_sample_nodes) .def("stop_server", &GraphPyClient::StopServer) @@ -255,6 +256,10 @@ void BindGraphPyClient(py::module* m) { using paddle::distributed::TreeIndex; using paddle::distributed::IndexWrapper; using paddle::distributed::IndexNode; +#ifdef PADDLE_WITH_HETERPS +using paddle::framework::GraphGpuWrapper; +using paddle::framework::NeighborSampleResult; +#endif void BindIndexNode(py::module* m) { py::class_(*m, "IndexNode") @@ -305,6 +310,29 @@ void BindIndexWrapper(py::module* m) { .def("clear_tree", &IndexWrapper::clear_tree); } +#ifdef PADDLE_WITH_HETERPS +void BindNeighborSampleResult(py::module* m) { + py::class_(*m, "NeighborSampleResult") + .def(py::init<>()) + .def("initialize", &NeighborSampleResult::initialize); +} + +void BindGraphGpuWrapper(py::module* m) { + py::class_(*m, "GraphGpuWrapper") + .def(py::init<>()) + .def("test", &GraphGpuWrapper::test) + .def("initialize", &GraphGpuWrapper::initialize) + .def("graph_neighbor_sample", &GraphGpuWrapper::graph_neighbor_sample) + .def("set_device", &GraphGpuWrapper::set_device) + .def("init_service", &GraphGpuWrapper::init_service) + .def("set_up_types", &GraphGpuWrapper::set_up_types) + .def("add_table_feat_conf", &GraphGpuWrapper::add_table_feat_conf) + .def("load_edge_file", &GraphGpuWrapper::load_edge_file) + .def("upload_batch", &GraphGpuWrapper::upload_batch) + .def("load_node_file", &GraphGpuWrapper::load_node_file); +} +#endif + using paddle::distributed::IndexSampler; using paddle::distributed::LayerWiseSampler; diff --git a/paddle/fluid/pybind/fleet_py.h b/paddle/fluid/pybind/fleet_py.h index 206a69f5a8019..81ed25913ba1a 100644 --- a/paddle/fluid/pybind/fleet_py.h +++ b/paddle/fluid/pybind/fleet_py.h @@ -36,5 +36,9 @@ void BindIndexNode(py::module* m); void BindTreeIndex(py::module* m); void BindIndexWrapper(py::module* m); void BindIndexSampler(py::module* m); +#ifdef PADDLE_WITH_HETERPS +void BindNeighborSampleResult(py::module* m); +void BindGraphGpuWrapper(py::module* m); +#endif } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index b135af43ab174..79ed7d9a08d6a 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -4563,6 +4563,10 @@ All parameter, weight, gradient are variables in Paddle. BindTreeIndex(&m); BindIndexWrapper(&m); BindIndexSampler(&m); +#ifdef PADDLE_WITH_HETERPS + BindNeighborSampleResult(&m); + BindGraphGpuWrapper(&m); +#endif #endif } } // namespace pybind From ccafd2e577c31971358597fee4867ec3ec7e910b Mon Sep 17 00:00:00 2001 From: ronnywang <524019753@qq.com> Date: Sun, 24 Apr 2022 14:51:35 +0800 Subject: [PATCH 48/66] [CustomDevice] add eager mode support (#42034) --- paddle/fluid/pybind/eager.cc | 5 ++++- paddle/fluid/pybind/eager_utils.cc | 8 +++++++- paddle/fluid/pybind/pybind.cc | 9 ++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/paddle/fluid/pybind/eager.cc b/paddle/fluid/pybind/eager.cc index 8695928205bb0..6601c8e8e3e4d 100644 --- a/paddle/fluid/pybind/eager.cc +++ b/paddle/fluid/pybind/eager.cc @@ -146,10 +146,13 @@ void InitTensorWithNumpyValue(TensorObject* self, const py::object& array, zero_copy); } else if (platform::is_npu_place(place)) { SetTensorFromPyArray(impl_ptr, array, place, zero_copy); + } else if (platform::is_custom_place(place)) { + SetTensorFromPyArray(impl_ptr, array, place, + zero_copy); } else { PADDLE_THROW(platform::errors::InvalidArgument( "Place should be one of " - "CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/NPUPlace")); + "CPUPlace/XPUPlace/CUDAPlace/CUDAPinnedPlace/NPUPlace/CustomPlace")); } } diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc index 78db1a6f1b991..b391274843368 100644 --- a/paddle/fluid/pybind/eager_utils.cc +++ b/paddle/fluid/pybind/eager_utils.cc @@ -46,6 +46,7 @@ extern PyTypeObject* g_cpuplace_pytype; extern PyTypeObject* g_xpuplace_pytype; extern PyTypeObject* g_npuplace_pytype; extern PyTypeObject* g_cudapinnedplace_pytype; +extern PyTypeObject* g_customplace_pytype; extern PyTypeObject* g_framework_tensor_pytype; extern PyTypeObject* g_framework_lodtensorarray_pytype; extern PyTypeObject* g_custom_op_kernel_ctx_pytype; @@ -377,10 +378,15 @@ platform::Place CastPyArg2Place(PyObject* obj, ssize_t arg_pos) { } else if (PyObject_IsInstance( obj, reinterpret_cast(g_cudapinnedplace_pytype))) { place = ::pybind11::handle(obj).cast(); + } else if (PyObject_IsInstance( + obj, reinterpret_cast(g_customplace_pytype))) { + place = ::pybind11::handle(obj).cast(); } else { PADDLE_THROW(platform::errors::InvalidArgument( "argument (position %d) must be " - "one of(Place,CUDAPlace,CPUPlace,XPUPlace,NPUPlace,CUDAPinnedPlace), " + "one " + "of(Place,CUDAPlace,CPUPlace,XPUPlace,NPUPlace,CUDAPinnedPlace," + "CustomPlace), " "but got %s", arg_pos + 1, reinterpret_cast(obj->ob_type)->tp_name)); } diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 79ed7d9a08d6a..dc380f83bf71b 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -193,6 +193,7 @@ PyTypeObject *g_xpuplace_pytype = nullptr; PyTypeObject *g_npuplace_pytype = nullptr; PyTypeObject *g_cudapinnedplace_pytype = nullptr; PyTypeObject *g_mluplace_pytype = nullptr; +PyTypeObject *g_customplace_pytype = nullptr; PyTypeObject *g_framework_tensor_pytype = nullptr; PyTypeObject *g_framework_lodtensorarray_pytype = nullptr; PyTypeObject *g_custom_op_kernel_ctx_pytype = nullptr; @@ -2125,8 +2126,8 @@ All parameter, weight, gradient are variables in Paddle. #endif return devices; }); - py::class_(m, "CustomPlace", - R"DOC( + py::class_ customplace(m, "CustomPlace", + R"DOC( CustomPlace is a descriptor of a device. It represents a custom device on which a tensor will be allocated and a model will run. @@ -2135,7 +2136,9 @@ All parameter, weight, gradient are variables in Paddle. import paddle fake_cpu_place = paddle.CustomPlace("FakeCPU", 0) - )DOC") + )DOC"); + g_customplace_pytype = reinterpret_cast(customplace.ptr()); + customplace .def("__init__", [](platform::CustomPlace &self, const std::string &device_type, int dev_id) { From 2bcec75a10c3e35fb5b4d18f07606184dba28229 Mon Sep 17 00:00:00 2001 From: baoachun <962571062@qq.com> Date: Sun, 24 Apr 2022 15:31:43 +0800 Subject: [PATCH 49/66] fix FlattenContiguousRangeOpConverter out dim error (#42087) * fix FlattenContiguousRangeOpConverter out dim error * update code --- .../convert/flatten_contiguous_range_op.cc | 150 +++++++++++------- 1 file changed, 92 insertions(+), 58 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc b/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc index 706814340a0e9..e08f50833ed99 100644 --- a/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/flatten_contiguous_range_op.cc @@ -30,14 +30,17 @@ class FlattenContiguousRangeOpConverter : public OpConverter { public: void operator()(const framework::proto::OpDesc& op, const framework::Scope& scope, bool test_mode) override { + VLOG(3) << "convert a fluid flatten_contiguous_range op to tensorrt layer"; framework::OpDesc op_desc(op, nullptr); // Declare inputs auto* input = engine_->GetITensor(op_desc.Input("X")[0]); - int dims = input->getDimensions().nbDims; + const auto input_dim = input->getDimensions(); + const int dims = input_dim.nbDims; int start_axis = BOOST_GET_CONST(int, op_desc.GetAttr("start_axis")); int stop_axis = BOOST_GET_CONST(int, op_desc.GetAttr("stop_axis")); - nvinfer1::IShuffleLayer* layer = nullptr; + nvinfer1::IShuffleLayer* layer = + TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input); if (!engine_->with_dynamic_shape()) { if (start_axis < 0) start_axis += dims + 1; if (stop_axis < 0) stop_axis += dims + 1; @@ -46,7 +49,7 @@ class FlattenContiguousRangeOpConverter : public OpConverter { flatten_dim.nbDims = dims - (stop_axis - start_axis); for (int i = 0, j = 0; i < dims; ++i) { if (start_axis <= i + 1 && i + 1 <= stop_axis) { - int dim_i = input->getDimensions().d[i]; + int dim_i = input_dim.d[i]; PADDLE_ENFORCE_GT(dim_i, 0, platform::errors::InvalidArgument( "flatten_contiguous_range input dim " "should be > 0, but got %d.", @@ -56,72 +59,103 @@ class FlattenContiguousRangeOpConverter : public OpConverter { flatten_dim.d[j++] = dim_prod; } } else { - flatten_dim.d[j++] = input->getDimensions().d[i]; + flatten_dim.d[j++] = input_dim.d[i]; } } - layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input); layer->setReshapeDimensions(flatten_dim); } else { if (start_axis < 0) start_axis += dims; if (stop_axis < 0) stop_axis += dims; - auto* shape_layer = TRT_ENGINE_ADD_LAYER(engine_, Shape, *input); - auto* shape_layer_itensor = shape_layer->getOutput(0); - nvinfer1::Dims start_dim, size_dim, stride_dim; - start_dim.nbDims = 1; - size_dim.nbDims = 1; - stride_dim.nbDims = 1; - start_dim.d[0] = start_axis; - size_dim.d[0] = stop_axis - start_axis + 1; - stride_dim.d[0] = 1; - auto* slice_layer = - TRT_ENGINE_ADD_LAYER(engine_, Slice, *shape_layer_itensor, start_dim, - size_dim, stride_dim); - uint32_t reduce_dim = 1; - auto* reduce_prod_layer = TRT_ENGINE_ADD_LAYER( - engine_, Reduce, *(slice_layer->getOutput(0)), - nvinfer1::ReduceOperation::kPROD, reduce_dim, true); - - nvinfer1::ITensor* input_shape = nullptr; - if (start_axis == 0 && stop_axis == dims - 1) { - input_shape = reduce_prod_layer->getOutput(0); - } else { - std::vector itensors; - if (start_axis > 0) { - nvinfer1::Dims left_start_dim, left_size_dim, left_stride_dim; - left_start_dim.nbDims = 1; - left_size_dim.nbDims = 1; - left_stride_dim.nbDims = 1; - left_start_dim.d[0] = 0; - left_size_dim.d[0] = start_axis; - left_stride_dim.d[0] = 1; - auto* slice_layer_left = TRT_ENGINE_ADD_LAYER( - engine_, Slice, *shape_layer_itensor, left_start_dim, - left_size_dim, left_stride_dim); - itensors.push_back(slice_layer_left->getOutput(0)); + int dim_prod = 1; + int dim_negative = 0; + nvinfer1::Dims flatten_dim; + flatten_dim.nbDims = dims - (stop_axis - start_axis); + bool need_slice = false; + for (int i = 0, j = 0; i < dims; ++i) { + int dim_i = input_dim.d[i]; + if (start_axis <= i && i <= stop_axis) { + if (dim_i < 0) { + need_slice = true; + break; + } + dim_prod *= dim_i; + if (i == stop_axis) { + flatten_dim.d[j++] = dim_prod; + } + } else { + if (dim_i < 0) dim_negative++; + if (dim_negative > 1) { + need_slice = true; + break; + } + flatten_dim.d[j++] = input_dim.d[i]; } - itensors.push_back(reduce_prod_layer->getOutput(0)); - if (stop_axis < dims - 1) { - nvinfer1::Dims right_start_dim, right_size_dim, right_stride_dim; - right_start_dim.nbDims = 1; - right_size_dim.nbDims = 1; - right_stride_dim.nbDims = 1; - right_start_dim.d[0] = stop_axis + 1; - right_size_dim.d[0] = dims - stop_axis - 1; - right_stride_dim.d[0] = 1; - auto* slice_layer_right = TRT_ENGINE_ADD_LAYER( - engine_, Slice, *shape_layer_itensor, right_start_dim, - right_size_dim, right_stride_dim); - itensors.push_back(slice_layer_right->getOutput(0)); + } + + if (need_slice) { + VLOG(3) << "slice input dim when the input dimension has -1"; + auto* shape_layer = TRT_ENGINE_ADD_LAYER(engine_, Shape, *input); + auto* shape_layer_itensor = shape_layer->getOutput(0); + + nvinfer1::Dims start_dim, size_dim, stride_dim; + start_dim.nbDims = 1; + size_dim.nbDims = 1; + stride_dim.nbDims = 1; + start_dim.d[0] = start_axis; + size_dim.d[0] = stop_axis - start_axis + 1; + stride_dim.d[0] = 1; + auto* slice_layer = + TRT_ENGINE_ADD_LAYER(engine_, Slice, *shape_layer_itensor, + start_dim, size_dim, stride_dim); + uint32_t reduce_dim = 1; + auto* reduce_prod_layer = TRT_ENGINE_ADD_LAYER( + engine_, Reduce, *(slice_layer->getOutput(0)), + nvinfer1::ReduceOperation::kPROD, reduce_dim, true); + + nvinfer1::ITensor* input_shape = nullptr; + if (start_axis == 0 && stop_axis == dims - 1) { + input_shape = reduce_prod_layer->getOutput(0); + } else { + std::vector itensors; + if (start_axis > 0) { + nvinfer1::Dims left_start_dim, left_size_dim, left_stride_dim; + left_start_dim.nbDims = 1; + left_size_dim.nbDims = 1; + left_stride_dim.nbDims = 1; + left_start_dim.d[0] = 0; + left_size_dim.d[0] = start_axis; + left_stride_dim.d[0] = 1; + auto* slice_layer_left = TRT_ENGINE_ADD_LAYER( + engine_, Slice, *shape_layer_itensor, left_start_dim, + left_size_dim, left_stride_dim); + itensors.push_back(slice_layer_left->getOutput(0)); + } + itensors.push_back(reduce_prod_layer->getOutput(0)); + if (stop_axis < dims - 1) { + nvinfer1::Dims right_start_dim, right_size_dim, right_stride_dim; + right_start_dim.nbDims = 1; + right_size_dim.nbDims = 1; + right_stride_dim.nbDims = 1; + right_start_dim.d[0] = stop_axis + 1; + right_size_dim.d[0] = dims - stop_axis - 1; + right_stride_dim.d[0] = 1; + auto* slice_layer_right = TRT_ENGINE_ADD_LAYER( + engine_, Slice, *shape_layer_itensor, right_start_dim, + right_size_dim, right_stride_dim); + itensors.push_back(slice_layer_right->getOutput(0)); + } + auto* concat_layer = TRT_ENGINE_ADD_LAYER( + engine_, Concatenation, itensors.data(), itensors.size()); + concat_layer->setAxis(0); + input_shape = concat_layer->getOutput(0); } - auto* concat_layer = TRT_ENGINE_ADD_LAYER( - engine_, Concatenation, itensors.data(), itensors.size()); - concat_layer->setAxis(0); - input_shape = concat_layer->getOutput(0); + layer->setInput(1, *input_shape); + } else { + layer->setReshapeDimensions(flatten_dim); } - layer = TRT_ENGINE_ADD_LAYER(engine_, Shuffle, *input); - layer->setInput(1, *input_shape); } + auto output_name = op_desc.Output("Out")[0]; RreplenishLayerAndOutput(layer, "flatten_contiguous_range", {output_name}, test_mode); From 13190707e56688bc65ee0d6daf0f060c2f0ff981 Mon Sep 17 00:00:00 2001 From: Zhou Wei <1183042833@qq.com> Date: Sun, 24 Apr 2022 16:10:20 +0800 Subject: [PATCH 50/66] fix python3.10 compile bug on windows (#42140) --- paddle/fluid/pybind/bind_fleet_executor.h | 4 ++++ paddle/fluid/pybind/compatible.h | 4 ++++ paddle/fluid/pybind/eager_functions.cc | 6 ++++++ paddle/fluid/pybind/eager_method.cc | 6 ++++++ paddle/fluid/pybind/eager_utils.h | 5 +++++ paddle/fluid/pybind/inference_api.h | 5 +++++ paddle/fluid/pybind/io.h | 5 +++++ paddle/fluid/pybind/op_function_common.h | 5 +++++ paddle/fluid/pybind/protobuf.h | 4 ++++ python/paddle/fluid/tests/unittests/cc_imp_py_test.cc | 3 ++- 10 files changed, 46 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/pybind/bind_fleet_executor.h b/paddle/fluid/pybind/bind_fleet_executor.h index 733701fa36ba8..f9568819688e5 100644 --- a/paddle/fluid/pybind/bind_fleet_executor.h +++ b/paddle/fluid/pybind/bind_fleet_executor.h @@ -14,6 +14,10 @@ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif #include namespace paddle { diff --git a/paddle/fluid/pybind/compatible.h b/paddle/fluid/pybind/compatible.h index f9d4cf5888fee..5f7628e5f2ab9 100644 --- a/paddle/fluid/pybind/compatible.h +++ b/paddle/fluid/pybind/compatible.h @@ -14,6 +14,10 @@ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif #include namespace paddle { diff --git a/paddle/fluid/pybind/eager_functions.cc b/paddle/fluid/pybind/eager_functions.cc index 1073cdc83a428..4d7b50943d084 100644 --- a/paddle/fluid/pybind/eager_functions.cc +++ b/paddle/fluid/pybind/eager_functions.cc @@ -9,6 +9,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // disable numpy compile error + +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + #include #include diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc index 13fba2baa1d6c..e6bd1c0b52682 100644 --- a/paddle/fluid/pybind/eager_method.cc +++ b/paddle/fluid/pybind/eager_method.cc @@ -9,6 +9,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // disable numpy compile error + +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + #include #include diff --git a/paddle/fluid/pybind/eager_utils.h b/paddle/fluid/pybind/eager_utils.h index 22c41073c9dd7..c4ddb34763228 100644 --- a/paddle/fluid/pybind/eager_utils.h +++ b/paddle/fluid/pybind/eager_utils.h @@ -10,6 +10,11 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + #include #include "paddle/phi/common/backend.h" #include "paddle/phi/common/data_type.h" diff --git a/paddle/fluid/pybind/inference_api.h b/paddle/fluid/pybind/inference_api.h index c2adfbecf72ca..300d3b480e113 100644 --- a/paddle/fluid/pybind/inference_api.h +++ b/paddle/fluid/pybind/inference_api.h @@ -14,6 +14,11 @@ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + #include namespace paddle { diff --git a/paddle/fluid/pybind/io.h b/paddle/fluid/pybind/io.h index dfe3154cb95da..942c93deccf99 100644 --- a/paddle/fluid/pybind/io.h +++ b/paddle/fluid/pybind/io.h @@ -14,6 +14,11 @@ limitations under the License. */ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + #include #include "paddle/fluid/pybind/pybind_boost_headers.h" diff --git a/paddle/fluid/pybind/op_function_common.h b/paddle/fluid/pybind/op_function_common.h index debaf8fae17b7..549da39d9b891 100644 --- a/paddle/fluid/pybind/op_function_common.h +++ b/paddle/fluid/pybind/op_function_common.h @@ -14,6 +14,11 @@ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif + #include #include #include diff --git a/paddle/fluid/pybind/protobuf.h b/paddle/fluid/pybind/protobuf.h index 4c5aa9701cd5a..54b788cccba5b 100644 --- a/paddle/fluid/pybind/protobuf.h +++ b/paddle/fluid/pybind/protobuf.h @@ -13,6 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once +#if defined(_MSC_VER) +#include +typedef SSIZE_T ssize_t; +#endif #include #include diff --git a/python/paddle/fluid/tests/unittests/cc_imp_py_test.cc b/python/paddle/fluid/tests/unittests/cc_imp_py_test.cc index 8609aff1fa556..a0b9ec5f9f6d4 100644 --- a/python/paddle/fluid/tests/unittests/cc_imp_py_test.cc +++ b/python/paddle/fluid/tests/unittests/cc_imp_py_test.cc @@ -50,7 +50,8 @@ TEST(CC, IMPORT_PY) { // 3. C/C++ Run Python file std::string file_name(cwd); file_name.append("/test_install_check.py"); - FILE* fp = _Py_fopen(file_name.c_str(), "r+"); + PyObject* obj = Py_BuildValue("s", file_name.c_str()); + FILE* fp = _Py_fopen_obj(obj, "r+"); ASSERT_TRUE(fp != NULL); ASSERT_FALSE(PyRun_SimpleFile(fp, file_name.c_str())); From 3a0d7bf0d9612b8e69f71f5c352d03e50bd95065 Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Mon, 25 Apr 2022 07:48:03 +0800 Subject: [PATCH 51/66] Optimize dygraph GetExpectedKernelType perf (#42154) * opt dygraph scheduling * revert part impl --- paddle/fluid/framework/operator.cc | 47 ++++++++++++++++++--- paddle/fluid/framework/operator.h | 12 +++--- paddle/fluid/imperative/execution_context.h | 18 +++++--- paddle/fluid/operators/transpose_op.cc | 2 +- paddle/phi/core/kernel_context.h | 8 ++-- 5 files changed, 68 insertions(+), 19 deletions(-) diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index da082f5d26f3b..945b8a89848b1 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -940,7 +940,7 @@ class RuntimeInferShapeContext : public InferShapeContext { return ((op_with_kernel.kernel_type()) && (op_with_kernel.kernel_type()->data_layout_ == framework::DataLayout::kMKLDNN)); - } catch (std::bad_cast exp) { + } catch (const std::bad_cast& exp) { return false; } } @@ -1965,6 +1965,36 @@ Scope* OperatorWithKernel::PrepareData( } void OperatorWithKernel::ParseInputDataType( + const Variable* var, const std::string& name, + proto::VarType::Type* data_type) const { + if (var != nullptr) { + const Tensor* t = nullptr; + if (var->IsType()) { + t = &var->Get(); + } else if (var->IsType()) { + t = &var->Get(); + } else if (var->IsType()) { + t = &(var->Get().value()); + } else if (var->IsType()) { + auto t_arr = &var->Get(); + for (size_t j = 0; j < t_arr->size(); j++) { + if (t_arr->at(j).IsInitialized()) { + t = &(t_arr->at(j)); + } + } + } + if (t != nullptr) { + PADDLE_ENFORCE_EQ( + t->IsInitialized(), true, + platform::errors::InvalidArgument("The %s Op's Input Variable `%s` " + "contains uninitialized Tensor.", + Type(), name)); + *data_type = paddle::framework::TransToProtoVarType(t->dtype()); + } + } +} + +void OperatorWithKernel::ParseMultiInputDataType( const std::vector& vars, const std::string& name, proto::VarType::Type* data_type) const { proto::VarType::Type default_data_type = @@ -2015,9 +2045,12 @@ proto::VarType::Type OperatorWithKernel::IndicateDataType( proto::VarType::Type dafault_data_type = static_cast(-1); proto::VarType::Type data_type = dafault_data_type; - for (auto& input : ctx.InNameList()) { - const std::vector vars = ctx.MultiInputVar(input); - ParseInputDataType(vars, input, &data_type); + for (auto* name : ctx.InNameList()) { + if (ctx.InputSize(*name) == 1UL) { + ParseInputDataType(ctx.InputVar(*name), *name, &data_type); + } else { + ParseMultiInputDataType(ctx.MultiInputVar(*name), *name, &data_type); + } } PADDLE_ENFORCE_NE( data_type, dafault_data_type, @@ -2031,7 +2064,11 @@ proto::VarType::Type OperatorWithKernel::IndicateVarDataType( proto::VarType::Type dafault_data_type = static_cast(-1); proto::VarType::Type data_type = dafault_data_type; - ParseInputDataType(ctx.MultiInputVar(name), name, &data_type); + if (ctx.InputSize(name) == 1UL) { + ParseInputDataType(ctx.InputVar(name), name, &data_type); + } else { + ParseMultiInputDataType(ctx.MultiInputVar(name), name, &data_type); + } PADDLE_ENFORCE_NE( data_type, dafault_data_type, platform::errors::InvalidArgument( diff --git a/paddle/fluid/framework/operator.h b/paddle/fluid/framework/operator.h index d85e81250563f..dd21be12f4abf 100644 --- a/paddle/fluid/framework/operator.h +++ b/paddle/fluid/framework/operator.h @@ -333,12 +333,12 @@ class ExecutionContext { return it->second; } - virtual std::vector InNameList() const { - std::vector vec_temp; + virtual paddle::SmallVector InNameList() const { + paddle::SmallVector vec_temp; vec_temp.reserve(ctx_.inputs.size()); for (auto& input : ctx_.inputs) { - vec_temp.push_back(input.first); + vec_temp.push_back(&input.first); } return vec_temp; @@ -680,9 +680,11 @@ class OperatorWithKernel : public OperatorBase { // By default all input data must be same. proto::VarType::Type IndicateDataType(const ExecutionContext& ctx) const; // used for IndicateDataType - void ParseInputDataType(const std::vector& vars, - const std::string& name, + void ParseInputDataType(const Variable* vars, const std::string& name, proto::VarType::Type* data_type) const; + void ParseMultiInputDataType(const std::vector& vars, + const std::string& name, + proto::VarType::Type* data_type) const; // used for IndicateOrPromoteVarDataTypes Tensor* GetTensorFormInputSafely(const ExecutionContext& ctx, const std::string& name) const; diff --git a/paddle/fluid/imperative/execution_context.h b/paddle/fluid/imperative/execution_context.h index fbc47f81fd331..330a5a0cfa90e 100644 --- a/paddle/fluid/imperative/execution_context.h +++ b/paddle/fluid/imperative/execution_context.h @@ -117,12 +117,12 @@ class DygraphExecutionContext : public framework::ExecutionContext { return it->second; } - std::vector InNameList() const override { - std::vector vec_temp; + paddle::SmallVector InNameList() const override { + paddle::SmallVector vec_temp; vec_temp.reserve(var_map_in_.size()); for (auto& v : var_map_in_) { - vec_temp.push_back(v.first); + vec_temp.push_back(&v.first); } return vec_temp; @@ -144,11 +144,19 @@ class DygraphExecutionContext : public framework::ExecutionContext { } size_t InputSize(const std::string& name) const override { - return InputNames(name).size(); + auto it = var_map_in_.find(name); + PADDLE_ENFORCE_NE( + it, var_map_in_.end(), + platform::errors::NotFound("Can not find [%s] in Input", name)); + return it->second.size(); } size_t OutputSize(const std::string& name) const override { - return OutputNames(name).size(); + auto it = var_map_out_.find(name); + PADDLE_ENFORCE_NE( + it, var_map_out_.end(), + platform::errors::NotFound("Can not find [%s] in Output", name)); + return it->second.size(); } const Variable* InputVar(const std::string& name) const override { diff --git a/paddle/fluid/operators/transpose_op.cc b/paddle/fluid/operators/transpose_op.cc index 1a297e7238ccd..a45d32b34b983 100644 --- a/paddle/fluid/operators/transpose_op.cc +++ b/paddle/fluid/operators/transpose_op.cc @@ -90,7 +90,7 @@ class TransposeOp : public framework::OperatorWithKernel { framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext &ctx) const override { framework::LibraryType library_{framework::LibraryType::kPlain}; - std::string data_format = ctx.Attr("data_format"); + auto &data_format = ctx.Attr("data_format"); framework::DataLayout layout_ = framework::StringToDataLayout(data_format); auto data_type = OperatorWithKernel::IndicateVarDataType(ctx, "X"); #ifdef PADDLE_WITH_MKLDNN diff --git a/paddle/phi/core/kernel_context.h b/paddle/phi/core/kernel_context.h index 9e5660d9dc534..a06efb573a62f 100644 --- a/paddle/phi/core/kernel_context.h +++ b/paddle/phi/core/kernel_context.h @@ -22,6 +22,7 @@ #include "paddle/phi/core/enforce.h" #include "paddle/phi/core/tensor_base.h" #include "paddle/phi/core/tensor_utils.h" +#include "paddle/phi/core/type_defs.h" #include "paddle/utils/optional.h" #include "paddle/utils/small_vector.h" @@ -139,10 +140,11 @@ class KernelContext { paddle::SmallVector inputs_; paddle::SmallVector outputs_; - paddle::SmallVector attrs_; + paddle::SmallVector attrs_; - paddle::SmallVector> input_range_; - paddle::SmallVector> output_range_; + paddle::SmallVector, kInputSmallVectorSize> input_range_; + paddle::SmallVector, kOutputSmallVectorSize> + output_range_; }; } // namespace phi From 05739d9e418482fc34f5d21f319594f11ae68c7e Mon Sep 17 00:00:00 2001 From: tiancaishaonvjituizi <452565578@qq.com> Date: Mon, 25 Apr 2022 10:08:16 +0800 Subject: [PATCH 52/66] fix incorrect usages of std::move and other compile errors (#41045) * fix bug of std::move and others * fix an compile error in debug mode * fix wrong copy assignment operator Signed-off-by: tiancaishaonvjituizi <452565578@qq.com> * reformat Signed-off-by: tiancaishaonvjituizi <452565578@qq.com> * reformat Signed-off-by: tiancaishaonvjituizi <452565578@qq.com> * fix ArrayRef constructor following llvm * fix format * fix conflict with master --- paddle/fluid/distributed/test/ctr_accessor_test.cc | 2 +- .../ir/fusion_group/code_generator_tester.cc | 4 ---- .../ir/fusion_group/fusion_group_pass_tester.cc | 8 -------- .../framework/new_executor/interpretercore_util.cc | 4 ++-- paddle/fluid/framework/var_desc.h | 6 ++++++ paddle/fluid/inference/utils/table_printer.cc | 2 +- paddle/fluid/platform/profiler.cc | 2 +- paddle/phi/api/lib/api_gen_utils.cc | 2 +- paddle/phi/api/lib/data_transform.cc | 2 +- paddle/phi/core/compat/arg_map_context.h | 5 +++++ paddle/phi/core/utils/type_registry.h | 3 ++- paddle/utils/array_ref.h | 13 ++++++++++++- 12 files changed, 32 insertions(+), 21 deletions(-) diff --git a/paddle/fluid/distributed/test/ctr_accessor_test.cc b/paddle/fluid/distributed/test/ctr_accessor_test.cc index 258b4d3326209..ee893ff01b59e 100644 --- a/paddle/fluid/distributed/test/ctr_accessor_test.cc +++ b/paddle/fluid/distributed/test/ctr_accessor_test.cc @@ -61,7 +61,7 @@ TableAccessorParameter gen_param() { naive_param->add_weight_bounds(-10.0); naive_param->add_weight_bounds(10.0); - return std::move(param); + return param; } TEST(downpour_feature_value_accessor_test, test_shrink) { diff --git a/paddle/fluid/framework/ir/fusion_group/code_generator_tester.cc b/paddle/fluid/framework/ir/fusion_group/code_generator_tester.cc index f671e0ae7690a..7b6bbf0251001 100644 --- a/paddle/fluid/framework/ir/fusion_group/code_generator_tester.cc +++ b/paddle/fluid/framework/ir/fusion_group/code_generator_tester.cc @@ -420,11 +420,7 @@ std::unique_ptr BuildGraph(bool backward, n->Var()->SetDataType(proto_dtype); } } -#ifdef __clang__ return graph; -#else - return std::move(graph); -#endif } std::unordered_set DistilGradNodes( diff --git a/paddle/fluid/framework/ir/fusion_group/fusion_group_pass_tester.cc b/paddle/fluid/framework/ir/fusion_group/fusion_group_pass_tester.cc index d14c7e433bd08..db22c03a7d9c0 100644 --- a/paddle/fluid/framework/ir/fusion_group/fusion_group_pass_tester.cc +++ b/paddle/fluid/framework/ir/fusion_group/fusion_group_pass_tester.cc @@ -63,11 +63,7 @@ std::unique_ptr BuildElementwiseListGraph(bool backward = false) { n->Var()->SetDataType(proto::VarType::FP32); } } -#ifdef __clang__ return graph; -#else - return std::move(graph); -#endif } std::unique_ptr BuildElementwiseTreeGraph(bool backward = false) { @@ -125,11 +121,7 @@ std::unique_ptr BuildElementwiseTreeGraph(bool backward = false) { n->Var()->SetDataType(proto::VarType::FP32); } } -#ifdef __clang__ return graph; -#else - return std::move(graph); -#endif } int TestMain(std::unique_ptr graph, std::string prefix) { diff --git a/paddle/fluid/framework/new_executor/interpretercore_util.cc b/paddle/fluid/framework/new_executor/interpretercore_util.cc index 71893d661ed6b..d6de37a72c772 100644 --- a/paddle/fluid/framework/new_executor/interpretercore_util.cc +++ b/paddle/fluid/framework/new_executor/interpretercore_util.cc @@ -741,7 +741,7 @@ std::map> get_downstream_map( VLOG(6) << "downstream count: " << downstream_map_count(); VLOG(6) << "downstream_map: " << std::endl << downstream_map_to_str(); - return std::move(downstream); + return downstream; } std::map> build_op_downstream_map( @@ -995,7 +995,7 @@ std::map> build_op_downstream_map( std::ostream_iterator(oss, " ")); VLOG(10) << oss.str(); } - return std::move(get_downstream_map(op2dependences, op_happens_before)); + return get_downstream_map(op2dependences, op_happens_before); } } // namespace interpreter diff --git a/paddle/fluid/framework/var_desc.h b/paddle/fluid/framework/var_desc.h index a20ef58f9c95f..0f8c10604f39a 100644 --- a/paddle/fluid/framework/var_desc.h +++ b/paddle/fluid/framework/var_desc.h @@ -74,6 +74,12 @@ class VarDesc { : desc_(other.desc_), attrs_(other.attrs_), original_id_(other.original_id_) {} + VarDesc &operator=(const VarDesc &other) { + desc_ = other.desc_; + attrs_ = other.attrs_; + original_id_ = other.original_id_; + return *this; + } proto::VarDesc *Proto() { return &desc_; } diff --git a/paddle/fluid/inference/utils/table_printer.cc b/paddle/fluid/inference/utils/table_printer.cc index bd19320cbe647..628465c423b03 100644 --- a/paddle/fluid/inference/utils/table_printer.cc +++ b/paddle/fluid/inference/utils/table_printer.cc @@ -53,7 +53,7 @@ std::string TablePrinter::PrintTable() { AddRowDivider(ss); - return std::move(ss.str()); + return ss.str(); } TablePrinter::TablePrinter(const std::vector& header) { diff --git a/paddle/fluid/platform/profiler.cc b/paddle/fluid/platform/profiler.cc index 8fa48ffcfb158..75abf36e676d0 100644 --- a/paddle/fluid/platform/profiler.cc +++ b/paddle/fluid/platform/profiler.cc @@ -612,7 +612,7 @@ static std::map DockHostEventRecorderHostPart() { auto host_evt_sec = HostEventRecorder::GetInstance().GatherEvents(); EmulateEventPushAndPop(host_evt_sec, &thr_events); EmulateCPURecordsAdd(host_evt_sec); - return std::move(thr_events); + return thr_events; } static void DockHostEventRecorderDevicePart( diff --git a/paddle/phi/api/lib/api_gen_utils.cc b/paddle/phi/api/lib/api_gen_utils.cc index e0c910ba3d66c..a0fd42d769aac 100644 --- a/paddle/phi/api/lib/api_gen_utils.cc +++ b/paddle/phi/api/lib/api_gen_utils.cc @@ -41,7 +41,7 @@ std::unique_ptr> TensorToDenseTensor( *std::dynamic_pointer_cast(t.impl())); } - return std::move(pt_tensors); + return pt_tensors; } std::shared_ptr TensorToSelectedRows(const Tensor& tensor) { diff --git a/paddle/phi/api/lib/data_transform.cc b/paddle/phi/api/lib/data_transform.cc index 65cb37d414299..58827a98503ce 100644 --- a/paddle/phi/api/lib/data_transform.cc +++ b/paddle/phi/api/lib/data_transform.cc @@ -253,7 +253,7 @@ std::unique_ptr> PrepareData( } } - return std::move(pt_tensors); + return pt_tensors; } } // namespace experimental diff --git a/paddle/phi/core/compat/arg_map_context.h b/paddle/phi/core/compat/arg_map_context.h index 102dca48b998b..f807f268a2d33 100644 --- a/paddle/phi/core/compat/arg_map_context.h +++ b/paddle/phi/core/compat/arg_map_context.h @@ -58,6 +58,11 @@ struct KernelSignature { // TODO(chenweihang): add assign constructor to solve windows compile // problem, remove it later + KernelSignature(const KernelSignature& other) + : name(other.name), + input_names(other.input_names), + attr_names(other.attr_names), + output_names(other.output_names) {} KernelSignature& operator=(const KernelSignature& other) { name = other.name; input_names = other.input_names; diff --git a/paddle/phi/core/utils/type_registry.h b/paddle/phi/core/utils/type_registry.h index f27c3db2275c3..5b64dbd01643e 100644 --- a/paddle/phi/core/utils/type_registry.h +++ b/paddle/phi/core/utils/type_registry.h @@ -50,7 +50,8 @@ template TypeInfo TypeRegistry::RegisterType(const std::string& type) { std::lock_guard guard(mutex_); assert(name_to_id_.find(type) == name_to_id_.end()); - assert(names_.size() < std::numeric_limits::max()); + assert(names_.size() < static_cast( + std::numeric_limits::max())); int8_t id = static_cast(names_.size()); names_.emplace_back(type); name_to_id_[type] = id; diff --git a/paddle/utils/array_ref.h b/paddle/utils/array_ref.h index d2ab762bb154f..788710925936b 100644 --- a/paddle/utils/array_ref.h +++ b/paddle/utils/array_ref.h @@ -96,10 +96,21 @@ class ArrayRef { template /*implicit*/ constexpr ArrayRef(const T (&Arr)[N]) : Data(Arr), Length(N) {} - /// Construct an ArrayRef from a std::initializer_list. +/// Construct an ArrayRef from a std::initializer_list. +#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 9 +// Disable gcc's warning in this constructor as it generates an enormous +// amount +// of messages. Anyone using ArrayRef should already be aware of the fact that +// it does not do lifetime extension. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Winit-list-lifetime" +#endif /*implicit*/ ArrayRef(const std::initializer_list &Vec) : Data(Vec.begin() == Vec.end() ? (T *)nullptr : Vec.begin()), Length(Vec.size()) {} +#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 9 +#pragma GCC diagnostic pop +#endif /// Construct an ArrayRef from ArrayRef. This uses SFINAE to /// ensure that only ArrayRefs of pointers can be converted. From 1178f153a830670c48c5a9fff2966155a007214e Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Mon, 25 Apr 2022 10:12:20 +0800 Subject: [PATCH 53/66] fix variant compile error (#42203) --- paddle/phi/kernels/cpu/where_grad_kernel.cc | 2 ++ paddle/phi/kernels/cpu/where_kernel.cc | 2 ++ paddle/phi/kernels/funcs/activation_functor.h | 2 +- paddle/phi/kernels/gpu/where_grad_kernel.cu | 3 +++ paddle/phi/kernels/gpu/where_kernel.cu | 2 ++ paddle/phi/kernels/where_grad_kernel.h | 3 --- paddle/phi/kernels/where_kernel.h | 3 --- paddle/utils/variant.h | 3 ++- 8 files changed, 12 insertions(+), 8 deletions(-) diff --git a/paddle/phi/kernels/cpu/where_grad_kernel.cc b/paddle/phi/kernels/cpu/where_grad_kernel.cc index 67c8cee1038c7..a9cdbd7ad77cc 100644 --- a/paddle/phi/kernels/cpu/where_grad_kernel.cc +++ b/paddle/phi/kernels/cpu/where_grad_kernel.cc @@ -14,6 +14,8 @@ #include "paddle/phi/kernels/where_grad_kernel.h" +#include "paddle/phi/core/kernel_registry.h" + namespace phi { template diff --git a/paddle/phi/kernels/cpu/where_kernel.cc b/paddle/phi/kernels/cpu/where_kernel.cc index f624c13c26229..353d11c93c1cc 100644 --- a/paddle/phi/kernels/cpu/where_kernel.cc +++ b/paddle/phi/kernels/cpu/where_kernel.cc @@ -14,6 +14,8 @@ #include "paddle/phi/kernels/where_kernel.h" +#include "paddle/phi/core/kernel_registry.h" + namespace phi { template diff --git a/paddle/phi/kernels/funcs/activation_functor.h b/paddle/phi/kernels/funcs/activation_functor.h index 84da69ed5da02..b75477a1af982 100644 --- a/paddle/phi/kernels/funcs/activation_functor.h +++ b/paddle/phi/kernels/funcs/activation_functor.h @@ -13,6 +13,7 @@ // limitations under the License. #pragma once + #include #include #include @@ -33,7 +34,6 @@ #include "paddle/phi/common/float16.h" #include "paddle/phi/core/dense_tensor.h" #include "paddle/phi/core/enforce.h" -#include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/funcs/eigen/common.h" #include "paddle/phi/kernels/funcs/eigen/extensions.h" diff --git a/paddle/phi/kernels/gpu/where_grad_kernel.cu b/paddle/phi/kernels/gpu/where_grad_kernel.cu index f21aca80e21b3..14cc1d311321d 100644 --- a/paddle/phi/kernels/gpu/where_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/where_grad_kernel.cu @@ -14,6 +14,9 @@ #include "paddle/phi/kernels/where_grad_kernel.h" +#include "paddle/phi/backends/gpu/gpu_launch_config.h" +#include "paddle/phi/core/kernel_registry.h" + namespace phi { template diff --git a/paddle/phi/kernels/gpu/where_kernel.cu b/paddle/phi/kernels/gpu/where_kernel.cu index 03c24eea3a95a..a0be388065f4b 100644 --- a/paddle/phi/kernels/gpu/where_kernel.cu +++ b/paddle/phi/kernels/gpu/where_kernel.cu @@ -14,6 +14,8 @@ #include "paddle/phi/kernels/where_kernel.h" +#include "paddle/phi/backends/gpu/gpu_launch_config.h" +#include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/funcs/broadcast_function.h" #include "paddle/phi/kernels/funcs/elementwise_functor.h" diff --git a/paddle/phi/kernels/where_grad_kernel.h b/paddle/phi/kernels/where_grad_kernel.h index 1a3c66ee6ed84..5f596da93e9c2 100644 --- a/paddle/phi/kernels/where_grad_kernel.h +++ b/paddle/phi/kernels/where_grad_kernel.h @@ -14,10 +14,7 @@ #pragma once -#include "paddle/phi/backends/all_context.h" -#include "paddle/phi/backends/gpu/gpu_launch_config.h" #include "paddle/phi/core/dense_tensor.h" -#include "paddle/phi/core/kernel_registry.h" namespace phi { diff --git a/paddle/phi/kernels/where_kernel.h b/paddle/phi/kernels/where_kernel.h index 254271ac9c723..6348177e69764 100644 --- a/paddle/phi/kernels/where_kernel.h +++ b/paddle/phi/kernels/where_kernel.h @@ -14,10 +14,7 @@ #pragma once -#include "paddle/phi/backends/all_context.h" -#include "paddle/phi/backends/gpu/gpu_launch_config.h" #include "paddle/phi/core/dense_tensor.h" -#include "paddle/phi/core/kernel_registry.h" namespace phi { diff --git a/paddle/utils/variant.h b/paddle/utils/variant.h index b856fa8f7a1d7..a7546d094c2ff 100644 --- a/paddle/utils/variant.h +++ b/paddle/utils/variant.h @@ -2691,7 +2691,8 @@ inline constexpr bool all(std::initializer_list bs) { template inline constexpr decltype(auto) visit(Visitor &&visitor, Vs &&... vs) { - return (detail::all({!vs.valueless_by_exception()...}) + return (detail::all( + lib::array{!vs.valueless_by_exception()...}) ? (void)0 : throw_bad_variant_access()), detail::visitation::variant::visit_value( From 4a16d5c6a03df776b08ff587d01048971fb64b2e Mon Sep 17 00:00:00 2001 From: Weilong Wu Date: Mon, 25 Apr 2022 10:27:34 +0800 Subject: [PATCH 54/66] [Eager] Support numpy.ndarry in CastNumpy2Scalar (#42136) --- paddle/fluid/pybind/eager_utils.cc | 15 ++++++++++++++- python/paddle/fluid/tests/unittests/test_bfgs.py | 8 +++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc index b391274843368..d07cbd5ee21a2 100644 --- a/paddle/fluid/pybind/eager_utils.cc +++ b/paddle/fluid/pybind/eager_utils.cc @@ -1025,7 +1025,20 @@ paddle::experimental::Scalar CastNumpy2Scalar(PyObject* obj, PyTypeObject* type = obj->ob_type; auto type_name = std::string(type->tp_name); VLOG(1) << "type_name: " << type_name; - if (type_name == "numpy.float64") { + if (type_name == "numpy.ndarray" && PySequence_Check(obj)) { + PyObject* item = nullptr; + item = PySequence_GetItem(obj, 0); + if (PyObject_CheckFloatOrToFloat(&item)) { + float value = static_cast(PyFloat_AsDouble(item)); + return paddle::experimental::Scalar(value); + } else { + PADDLE_THROW(platform::errors::InvalidArgument( + "%s(): argument (position %d) is numpy.ndarry, the inner elements " + "must be " + "numpy.float32/float64 now, but got %s", + op_type, arg_pos + 1, type_name)); // NOLINT + } + } else if (type_name == "numpy.float64") { double value = CastPyArg2Double(obj, op_type, arg_pos); return paddle::experimental::Scalar(value); } else if (type_name == "numpy.float32") { diff --git a/python/paddle/fluid/tests/unittests/test_bfgs.py b/python/paddle/fluid/tests/unittests/test_bfgs.py index 4bf6de3eee510..1a12913bc72e9 100644 --- a/python/paddle/fluid/tests/unittests/test_bfgs.py +++ b/python/paddle/fluid/tests/unittests/test_bfgs.py @@ -20,6 +20,7 @@ import paddle.nn.functional as F from paddle.incubate.optimizer.functional.bfgs import minimize_bfgs +from paddle.fluid.framework import _test_eager_guard from paddle.fluid.framework import _enable_legacy_dygraph _enable_legacy_dygraph() @@ -120,7 +121,7 @@ def func(x): results = test_static_graph(func, x0, dtype='float64') self.assertTrue(np.allclose(0.8, results[2])) - def test_rosenbrock(self): + def func_rosenbrock(self): # The Rosenbrock function is a standard optimization test case. a = np.random.random(size=[1]).astype('float32') minimum = [a.item(), (a**2).item()] @@ -139,6 +140,11 @@ def func(position): results = test_dynamic_graph(func, x0) self.assertTrue(np.allclose(minimum, results[2])) + def test_rosenbrock(self): + with _test_eager_guard(): + self.func_rosenbrock() + self.func_rosenbrock() + def test_exception(self): def func(x): return paddle.dot(x, x) From 3b8f8b6cc272e226db306bc338a45d0ef316151c Mon Sep 17 00:00:00 2001 From: Weilong Wu Date: Mon, 25 Apr 2022 10:35:34 +0800 Subject: [PATCH 55/66] [Eager] Remove redundancy code, fix fp16 case (#42169) --- python/paddle/fluid/initializer.py | 1 - .../fluid/tests/unittests/test_dygraph_mnist_fp16.py | 8 +++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/initializer.py b/python/paddle/fluid/initializer.py index ba5e51c11dd65..1c8e399436625 100644 --- a/python/paddle/fluid/initializer.py +++ b/python/paddle/fluid/initializer.py @@ -353,7 +353,6 @@ def __call__(self, var, block=None): out_var = _C_ops.final_state_gaussian_random( var.shape, self._mean, self._std_dev, self._seed, out_dtype, place) - out_var._share_underline_tensor_to(var) if var.dtype in [VarDesc.VarType.FP16, VarDesc.VarType.BF16]: var_tmp = _C_ops.final_state_cast(out_var, var.dtype) diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py b/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py index 7503a9172fc21..6c2516d6c11ef 100644 --- a/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py +++ b/python/paddle/fluid/tests/unittests/test_dygraph_mnist_fp16.py @@ -19,6 +19,7 @@ import paddle.fluid as fluid from paddle.fluid.dygraph.nn import Conv2D, Pool2D, Linear +from paddle.fluid.framework import _test_eager_guard class SimpleImgConvPool(fluid.dygraph.Layer): @@ -117,7 +118,7 @@ def forward(self, inputs, label): class TestMnist(unittest.TestCase): - def test_mnist_fp16(self): + def func_mnist_fp16(self): if not fluid.is_compiled_with_cuda(): return x = np.random.randn(1, 3, 224, 224).astype("float16") @@ -129,6 +130,11 @@ def test_mnist_fp16(self): loss = model(x, y) print(loss.numpy()) + def test_mnist_fp16(self): + with _test_eager_guard(): + self.func_mnist_fp16() + self.func_mnist_fp16() + if __name__ == "__main__": unittest.main() From f4ce8a927757f42bcaf21a086a94b5208ff237df Mon Sep 17 00:00:00 2001 From: Weilong Wu Date: Mon, 25 Apr 2022 10:35:45 +0800 Subject: [PATCH 56/66] [Eager] Support div(scalar) in eager mode (#42148) * [Eager] Support div scalar in eager mode * Updated and remove debug logs * Remove list, use 'or' directly * Remove useless statement --- python/paddle/fluid/dygraph/math_op_patch.py | 7 +- ...st_tensor_scalar_type_promotion_dynamic.py | 73 ++++++++++++++++--- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/python/paddle/fluid/dygraph/math_op_patch.py b/python/paddle/fluid/dygraph/math_op_patch.py index 8ce56d5a92686..8a19be640a7ff 100644 --- a/python/paddle/fluid/dygraph/math_op_patch.py +++ b/python/paddle/fluid/dygraph/math_op_patch.py @@ -222,7 +222,9 @@ def __impl__(self, other_var): # so the calculation result here and the calculation result of numpy are # different after 6 decimal point. If necessary, we can also use float64 here. # torch's behavior here is consistent with ours - if op_type == 'elementwise_div' and self.dtype in _supported_int_dtype_: + if (op_type == "final_state_divide" or + op_type == "elementwise_div" + ) and self.dtype in _supported_int_dtype_: self = astype(self, 'float32') # here use `scale` replace `elementwise` to get better performance # but only +, -, *, / can use this method @@ -277,7 +279,8 @@ def __impl__(self, other_var): self = other_var other_var = tmp - if op_type == 'elementwise_div' and self.dtype in _supported_int_dtype_: + if (op_type == "final_state_divide" or op_type == "elementwise_div" + ) and self.dtype in _supported_int_dtype_: self = astype(self, 'float32') other_var = astype(other_var, 'float32') diff --git a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py index c5e3cb29e0c20..774d40a17c66d 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py +++ b/python/paddle/fluid/tests/unittests/test_tensor_scalar_type_promotion_dynamic.py @@ -18,8 +18,7 @@ import numpy as np import paddle -from paddle.fluid.framework import _enable_legacy_dygraph -_enable_legacy_dygraph() +from paddle.fluid.framework import _test_eager_guard # Support types are ref from `paddle.tensor.math` # - Related paddle dtypes: @@ -52,7 +51,7 @@ def check_operation(self, a, b, c, op): self.assertEqual(c_rlt.dtype, c.dtype) self.assertTrue(np.array_equal(c_rlt.numpy(), c.numpy())) - def test_tensor_add_scalar(self): + def func_tensor_add_scalar(self): # tensor(int64) + scalar(int) a = paddle.ones([2, 2, 2], dtype='int64') b = 1 @@ -83,7 +82,12 @@ def test_tensor_add_scalar(self): c = paddle.full([2, 2, 2], 2.5, dtype="float32") self.check_operation(a, b, c, '+') - def test_tensor_sub_scalar(self): + def test_tensor_add_scalar(self): + with _test_eager_guard(): + self.func_tensor_add_scalar() + self.func_tensor_add_scalar() + + def func_tensor_sub_scalar(self): # tensor(int64) - scalar(int) a = paddle.ones([2, 2, 2], dtype='int64') b = 1 @@ -114,7 +118,12 @@ def test_tensor_sub_scalar(self): c = paddle.full([2, 2, 2], 0.5, dtype="float32") self.check_operation(a, b, c, '-') - def test_scalar_sub_tensor(self): + def test_tensor_sub_scalar(self): + with _test_eager_guard(): + self.func_tensor_sub_scalar() + self.func_tensor_sub_scalar() + + def func_scalar_sub_tensor(self): # scalar(int) - tensor(int64) a = 1 b = paddle.ones([2, 2, 2], dtype='int64') @@ -145,7 +154,12 @@ def test_scalar_sub_tensor(self): c = paddle.full([2, 2, 2], -0.5, dtype="float32") self.check_operation(a, b, c, '-') - def test_tensor_mul_tensor(self): + def test_scalar_sub_tensor(self): + with _test_eager_guard(): + self.func_scalar_sub_tensor() + self.func_scalar_sub_tensor() + + def func_tensor_mul_tensor(self): # tensor(int64) * scalar(int) a = paddle.ones([2, 2, 2], dtype='int64') b = 1 @@ -176,7 +190,12 @@ def test_tensor_mul_tensor(self): c = paddle.full([2, 2, 2], 1.5, dtype="float32") self.check_operation(a, b, c, '*') - def test_tensor_div_scalar(self): + def test_tensor_mul_tensor(self): + with _test_eager_guard(): + self.func_tensor_mul_tensor() + self.func_tensor_mul_tensor() + + def func_tensor_div_scalar(self): # tensor(int64) / scalar(int) a = paddle.ones([2, 2, 2], dtype='int64') b = 2 @@ -207,7 +226,12 @@ def test_tensor_div_scalar(self): c = paddle.full([2, 2, 2], 2, dtype="float32") self.check_operation(a, b, c, '/') - def test_scalar_div_tensor(self): + def test_tensor_div_scalar(self): + with _test_eager_guard(): + self.func_tensor_div_scalar() + self.func_tensor_div_scalar() + + def func_scalar_div_tensor(self): # scalar(int) / tensor(int64) a = 1 b = paddle.full([2, 2, 2], 2, dtype='int64') @@ -232,7 +256,12 @@ def test_scalar_div_tensor(self): c = paddle.full([2, 2, 2], 2, dtype="float32") self.check_operation(a, b, c, '/') - def test_tensor_pow_scalar(self): + def test_scalar_div_tensor(self): + with _test_eager_guard(): + self.func_scalar_div_tensor() + self.func_scalar_div_tensor() + + def func_tensor_pow_scalar(self): # tensor(int64) ** scalar(int) a = paddle.full([2, 2, 2], 2, dtype='int64') b = 3 @@ -257,7 +286,12 @@ def test_tensor_pow_scalar(self): c = paddle.full([2, 2, 2], 8, dtype="float32") self.check_operation(a, b, c, '**') - def test_scalar_pow_tensor(self): + def test_tensor_pow_scalar(self): + with _test_eager_guard(): + self.func_tensor_pow_scalar() + self.func_tensor_pow_scalar() + + def func_scalar_pow_tensor(self): # scalar(int) ** tensor(int64) a = 3 b = paddle.full([2, 2, 2], 2, dtype='int64') @@ -282,15 +316,25 @@ def test_scalar_pow_tensor(self): c = paddle.full([2, 2, 2], 9, dtype="float32") self.check_operation(a, b, c, '**') + def test_scalar_pow_tensor(self): + with _test_eager_guard(): + self.func_scalar_pow_tensor() + self.func_scalar_pow_tensor() + ## TODO: floordiv op kernel doesn't support float - def test_tensor_floordiv_scalar(self): + def func_tensor_floordiv_scalar(self): # tensor(int64) // scalar(int) a = paddle.full([2, 2, 2], 3, dtype='int64') b = 2 c = paddle.full([2, 2, 2], 1, dtype="int64") self.check_operation(a, b, c, '//') - def test_tensor_mod_scalar(self): + def test_tensor_floordiv_scalar(self): + with _test_eager_guard(): + self.func_tensor_floordiv_scalar() + self.func_tensor_floordiv_scalar() + + def func_tensor_mod_scalar(self): # tensor(int64) % scalar(int) a = paddle.full([2, 2, 2], 3, dtype='int64') b = 2 @@ -315,6 +359,11 @@ def test_tensor_mod_scalar(self): c = paddle.full([2, 2, 2], 1, dtype="float32") self.check_operation(a, b, c, '%') + def test_tensor_mod_scalar(self): + with _test_eager_guard(): + self.func_tensor_mod_scalar() + self.func_tensor_mod_scalar() + if __name__ == '__main__': unittest.main() From f21824d93dcb448ce5fb443202fafa7af4182f7f Mon Sep 17 00:00:00 2001 From: Roc <30228238+sljlp@users.noreply.github.com> Date: Mon, 25 Apr 2022 10:40:27 +0800 Subject: [PATCH 57/66] fix recompute (#42128) * fix recompute * modify return --- python/paddle/incubate/distributed/models/moe/moe_layer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/incubate/distributed/models/moe/moe_layer.py b/python/paddle/incubate/distributed/models/moe/moe_layer.py index eebb635e3ead7..ba22ffee3e4d6 100644 --- a/python/paddle/incubate/distributed/models/moe/moe_layer.py +++ b/python/paddle/incubate/distributed/models/moe/moe_layer.py @@ -399,7 +399,7 @@ def forward(self, inp): def experts_fwd(x, fwd_expert_count, experts): if x.shape[0] == 0: - return paddle.empty(x.shape, x.dtype) + return x y = [] last_index = 0 assert isinstance(fwd_expert_count, np.ndarray) @@ -411,7 +411,7 @@ def experts_fwd(x, fwd_expert_count, experts): last_index = expert_count + last_index return paddle.concat(y, axis=0) - if self.recompute_interval <= 0: + if self.recompute_interval <= 0 or x.shape[0] == 0: x = experts_fwd(x, fwd_expert_count.numpy(), self.experts) else: x = _hp_recompute(experts_fwd, x, From a3a6f0cfbde0a3e441a59d12f1cc13c57208e7fc Mon Sep 17 00:00:00 2001 From: pangyoki Date: Mon, 25 Apr 2022 10:43:36 +0800 Subject: [PATCH 58/66] add LICENSE in wheel dist-info package (#42187) --- python/setup.py.in | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/setup.py.in b/python/setup.py.in index e4637444be171..0f231e34168d9 100755 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -13,6 +13,7 @@ from contextlib import contextmanager from setuptools import Command from setuptools import setup, Distribution, Extension from setuptools.command.install import install as InstallCommandBase +from setuptools.command.egg_info import egg_info class BinaryDistribution(Distribution): @@ -678,6 +679,17 @@ class InstallHeaders(Command): def get_outputs(self): return self.outfiles +class EggInfo(egg_info): + """Copy license file into `.dist-info` folder.""" + + def run(self): + # don't duplicate license into `.dist-info` when building a distribution + if not self.distribution.have_run.get('install', True): + self.mkpath(self.egg_info) + self.copy_file("@PADDLE_SOURCE_DIR@/LICENSE", self.egg_info) + + egg_info.run(self) + # we redirect setuptools log for non-windows if sys.platform != 'win32': @contextmanager @@ -733,6 +745,7 @@ with redirect_stdout(): cmdclass={ 'install_headers': InstallHeaders, 'install': InstallCommand, + 'egg_info': EggInfo, }, entry_points={ 'console_scripts': [ From c2a05a9041f7a076f28bdeb75037b4e0289137fc Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Mon, 25 Apr 2022 10:43:47 +0800 Subject: [PATCH 59/66] replace any by variant in infermeta (#42181) --- paddle/phi/core/infermeta_utils.cc | 34 +++++++++++- paddle/phi/core/infermeta_utils.h | 60 +++++++++++---------- paddle/phi/core/type_defs.h | 29 ---------- paddle/phi/infermeta/unary.cc | 8 --- paddle/phi/infermeta/unary.h | 5 -- paddle/phi/tests/core/test_meta_fn_utils.cc | 26 --------- 6 files changed, 66 insertions(+), 96 deletions(-) diff --git a/paddle/phi/core/infermeta_utils.cc b/paddle/phi/core/infermeta_utils.cc index 70f26102cbad1..8bdad9d6d2b6e 100644 --- a/paddle/phi/core/infermeta_utils.cc +++ b/paddle/phi/core/infermeta_utils.cc @@ -30,7 +30,7 @@ void InferMetaContext::EmplaceBackOutput(MetaTensor output) { outputs_.emplace_back(std::move(output)); output_range_.emplace_back(std::pair(index, index + 1)); } -void InferMetaContext::EmplaceBackAttr(paddle::any attr) { +void InferMetaContext::EmplaceBackAttr(Attribute attr) { attrs_.emplace_back(std::move(attr)); } @@ -120,6 +120,38 @@ std::vector InferMetaContext::MutableOutputBetween(size_t start, return result; } +template +const AttrType& InferMetaContext::AttrAt(size_t idx) const { + try { + return paddle::get(attrs_.at(idx)); + } catch (paddle::bad_variant_access const& e) { + PADDLE_THROW(phi::errors::InvalidArgument( + "Attribute cast error in InferMeta Context, the expected attribute " + "type is `%s`.", + std::type_index(typeid(AttrType)).name())); + } +} + +template const bool& InferMetaContext::AttrAt(size_t idx) const; +template const int& InferMetaContext::AttrAt(size_t idx) const; +template const int64_t& InferMetaContext::AttrAt(size_t idx) const; +template const float& InferMetaContext::AttrAt(size_t idx) const; +template const double& InferMetaContext::AttrAt(size_t idx) const; +template const std::string& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt( + size_t idx) const; +template const Scalar& InferMetaContext::AttrAt(size_t idx) const; +template const std::vector& InferMetaContext::AttrAt(size_t idx) const; +template const IntArray& InferMetaContext::AttrAt(size_t idx) const; +template const DataType& InferMetaContext::AttrAt(size_t idx) const; +template const DataLayout& InferMetaContext::AttrAt(size_t idx) const; +template const Place& InferMetaContext::AttrAt(size_t idx) const; + MetaFnFactory& MetaFnFactory::Instance() { static MetaFnFactory g_meta_fn_map; return g_meta_fn_map; diff --git a/paddle/phi/core/infermeta_utils.h b/paddle/phi/core/infermeta_utils.h index 699c38ebd4702..8c726bffa2fc9 100644 --- a/paddle/phi/core/infermeta_utils.h +++ b/paddle/phi/core/infermeta_utils.h @@ -21,6 +21,7 @@ limitations under the License. */ #include "paddle/phi/common/int_array.h" #include "paddle/phi/common/scalar.h" +#include "paddle/phi/core/attribute.h" #include "paddle/phi/core/enforce.h" #include "paddle/phi/core/macros.h" #include "paddle/phi/core/meta_tensor.h" @@ -41,7 +42,7 @@ class InferMetaContext { void EmplaceBackInput(MetaTensor input); void EmplaceBackOutput(MetaTensor output); - void EmplaceBackAttr(paddle::any attr); + void EmplaceBackAttr(Attribute attr); void EmplaceBackInputs( paddle::SmallVector inputs); @@ -61,17 +62,7 @@ class InferMetaContext { size_t end); template - AttrType AttrAt(size_t idx) { - try { - return paddle::any_cast(attrs_.at(idx)); - } catch (paddle::bad_any_cast& e) { - PADDLE_THROW(phi::errors::InvalidArgument( - "Attribute cast error in InferMeta Context, the expected attribute " - "type is `%s`, but actual attribute type is `%s`.", - std::type_index(typeid(AttrType)).name(), - std::type_index(attrs_.at(idx).type()).name())); - } - } + const AttrType& AttrAt(size_t idx) const; const std::pair& InputRangeAt(size_t idx) const; const std::pair& OutputRangeAt(size_t idx) const; @@ -81,7 +72,7 @@ class InferMetaContext { protected: MetaConfig config_; - paddle::SmallVector attrs_; + paddle::SmallVector attrs_; paddle::SmallVector, phi::kInputSmallVectorSize> input_range_; @@ -111,6 +102,21 @@ class InferMetaContext { } \ } +#define PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF(attr_type) \ + template \ + struct InferMetaFnCallHelper { \ + template \ + static void Call(InferMetaContext* ctx, PreviousArgs&... pargs) { \ + static_assert(out_idx == 0, \ + "InferMeta's Attributes should appear before Outputs."); \ + const attr_type& arg = ctx->AttrAt(attr_idx); \ + InferMetaFnCallHelper< \ + Tail...>::template Call(ctx, \ + pargs..., \ + arg); \ + } \ + } + template struct InferMetaTypeTag {}; @@ -201,27 +207,27 @@ struct InferMetaFnImpl { } }; - // TODO(chenweihang): support other attr type later PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(bool); PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(int); PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(int64_t); PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(float); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const std::string&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE( - const std::vector&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const std::vector&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE( - const std::vector&); PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(DataType); PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(Backend); PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(DataLayout); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const Scalar&); - PD_SPECIALIZE_InferMetaFnCallHelper_FOR_ATTRIBUTE(const IntArray&); - - // TODO(chenweihang): support vector input later + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF(std::string); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF(Scalar); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF(IntArray); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF( + std::vector); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF(std::vector); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF( + std::vector); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF( + std::vector); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF( + std::vector); + PD_SPECIALIZE_InferMetaFnCallHelper_FOR_CONST_ATTRIBUTE_REF( + std::vector); template struct InferMetaFnCallHelper { diff --git a/paddle/phi/core/type_defs.h b/paddle/phi/core/type_defs.h index e3cbf2cedd077..0af1c0af230f7 100644 --- a/paddle/phi/core/type_defs.h +++ b/paddle/phi/core/type_defs.h @@ -18,37 +18,8 @@ #include #include -#include "paddle/phi/common/data_type.h" -#include "paddle/phi/common/int_array.h" -#include "paddle/phi/common/layout.h" -#include "paddle/phi/common/scalar.h" - -#include "paddle/utils/variant.h" - namespace phi { -class Place; - -// NOTE: Add needed type in the future -using Attribute = paddle::variant, - std::vector, - std::vector, - std::vector, - std::vector, - std::vector, - Scalar, - std::vector, - IntArray, - DataType, - DataLayout, - Place>; - class Kernel; class KernelKey; class KernelArgsDef; diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index e3e1211e3ece8..e5d83a4013d30 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -228,13 +228,6 @@ void CholeskyInferMeta(const MetaTensor& x, bool upper, MetaTensor* out) { out->set_dtype(x.dtype()); } -void CopyToInferMeta(const MetaTensor& x, - Backend backend, - bool blocking, - MetaTensor* out) { - UnchangedInferMeta(x, out); -} - void CreateLikeInferMeta(const MetaTensor& x, DataType dtype, MetaTensor* out) { out->set_dims(x.dims()); out->set_dtype(dtype == DataType::UNDEFINED ? x.dtype() : dtype); @@ -3008,6 +3001,5 @@ void WhereIndexInferMeta(const MetaTensor& condition, MetaTensor* out) { } // namespace phi -PD_REGISTER_INFER_META_FN(copy_to, phi::CopyToInferMeta); PD_REGISTER_INFER_META_FN(flatten, phi::FlattenInferMeta); PD_REGISTER_INFER_META_FN(split, phi::SplitInferMeta); diff --git a/paddle/phi/infermeta/unary.h b/paddle/phi/infermeta/unary.h index ac5040388b334..70b868eeb5d8d 100644 --- a/paddle/phi/infermeta/unary.h +++ b/paddle/phi/infermeta/unary.h @@ -58,11 +58,6 @@ void CastInferMeta(const MetaTensor& x, DataType out_dtype, MetaTensor* out); void CholeskyInferMeta(const MetaTensor& x, bool upper, MetaTensor* out); -void CopyToInferMeta(const MetaTensor& x, - Backend backend, - bool blocking, - MetaTensor* out); - void CreateLikeInferMeta(const MetaTensor& x, DataType dtype, MetaTensor* out); void CumsumInferMeta(const MetaTensor& x, diff --git a/paddle/phi/tests/core/test_meta_fn_utils.cc b/paddle/phi/tests/core/test_meta_fn_utils.cc index 028b9d23352c7..07832494d50ec 100644 --- a/paddle/phi/tests/core/test_meta_fn_utils.cc +++ b/paddle/phi/tests/core/test_meta_fn_utils.cc @@ -60,32 +60,6 @@ TEST(MetaFnFactory, InferMetaFnExists) { EXPECT_EQ(dense_out1.dims()[1], dense_out2.dims()[1]); } -TEST(MetaFnFactory, CopyInferMetaFn) { - phi::DenseTensor dense_x; - dense_x.Resize({3, 4}); - - phi::MetaTensor meta_x(&dense_x); - phi::DenseTensor dense_out1; - phi::MetaTensor meta_out(&dense_out1); - phi::UnchangedInferMeta(meta_x, &meta_out); - - auto shared_meat_x = phi::MetaTensor(&dense_x); - phi::DenseTensor dense_out2; - auto shared_meta_out = phi::MetaTensor(&dense_out2); - - phi::InferMetaContext ctx; - ctx.EmplaceBackInput(shared_meat_x); - ctx.EmplaceBackAttr(Backend::CPU); - ctx.EmplaceBackAttr(false); - ctx.EmplaceBackOutput(shared_meta_out); - ctx.SetMetaConfig({/*is_runtime =*/true, /*is_run_mkldnn_kernel=*/false}); - phi::MetaFnFactory::Instance().Get("copy_to")(&ctx); - - EXPECT_EQ(dense_out1.dims().size(), dense_out2.dims().size()); - EXPECT_EQ(dense_out1.dims()[0], dense_out2.dims()[0]); - EXPECT_EQ(dense_out1.dims()[1], dense_out2.dims()[1]); -} - TEST(MetaFnFactory, SplitInferMetaFn) { phi::DenseTensor dense_x; dense_x.Resize({4, 10}); From bbaaf217b676e52159eace210c689d27a4f36948 Mon Sep 17 00:00:00 2001 From: BrilliantYuKaimin <91609464+BrilliantYuKaimin@users.noreply.github.com> Date: Mon, 25 Apr 2022 12:44:13 +0800 Subject: [PATCH 60/66] =?UTF-8?q?=E3=80=90PaddlePaddle=20Hackathon=202?= =?UTF-8?q?=E3=80=9124=E3=80=81=E4=B8=BA=20Paddle=20=E6=96=B0=E5=A2=9E=20n?= =?UTF-8?q?n.ChannelShuffle=20=E7=BB=84=E7=BD=91=20API=20(#40743)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add infermeta for ChannelShuffle * Create channel_shuffle_grad_kernel.h * Create channel_shuffle_kernel.h * Create channel_shuffle_sig.cc * Create channel_shuffle_op.cc ChannelShuffle算子的描述 * Create channel_shuffle_kernel_impl.h ChannelShuffle核函数的实现 * Create channel_shuffle_grad_kernel_impl.h ChannelShuffle反向核函数的实现 * Add kernel register of channel shuffle and grad 注册ChannelShuffle及其反向的核函数 * add nn.functional.channel_shuffle * add nn.ChannelShuffle * Create test_channel_shuffle.py * Update example of ChannelShuffle in vision.py * Update test_channel_shuffle.py * 修改channel_shuffle核函数的实现位置 * 修正代码格式 * 删除多余空格 * 完善channel_shuffle的错误检查 * Update unary.cc * Update channel_shuffle_op.cc * Update test_channel_shuffle.py * Update unary.cc * add channel_shuffle * Update test_channel_shuffle.py * Update vision.py * 调整代码格式 * Update channel_shuffle_sig.cc * 更新ChannelShuffle的文档 * 更新channel_shuffle的文档 * remove ChannelShuffleOpArgumentMapping * add ChannelShuffleGradInferMeta * Update channel_shuffle_op.cc * 调整channel_shuffle及其梯度的核函数的位置 --- paddle/fluid/operators/channel_shuffle_op.cc | 100 +++++++ paddle/phi/infermeta/backward.cc | 16 ++ paddle/phi/infermeta/backward.h | 5 + paddle/phi/infermeta/unary.cc | 46 ++++ paddle/phi/infermeta/unary.h | 5 + .../phi/kernels/channel_shuffle_grad_kernel.h | 29 ++ paddle/phi/kernels/channel_shuffle_kernel.h | 29 ++ .../cpu/channel_shuffle_grad_kernel.cc | 26 ++ .../phi/kernels/cpu/channel_shuffle_kernel.cc | 26 ++ .../gpu/channel_shuffle_grad_kernel.cu | 26 ++ .../phi/kernels/gpu/channel_shuffle_kernel.cu | 26 ++ .../impl/channel_shuffle_grad_kernel_impl.h | 58 ++++ .../impl/channel_shuffle_kernel_impl.h | 57 ++++ paddle/phi/ops/compat/channel_shuffle_sig.cc | 30 +++ .../tests/unittests/test_channel_shuffle.py | 250 ++++++++++++++++++ python/paddle/nn/__init__.py | 2 + python/paddle/nn/functional/__init__.py | 2 + python/paddle/nn/functional/vision.py | 69 +++++ python/paddle/nn/layer/__init__.py | 1 + python/paddle/nn/layer/vision.py | 73 +++++ tools/static_mode_white_list.py | 1 + 21 files changed, 877 insertions(+) create mode 100644 paddle/fluid/operators/channel_shuffle_op.cc create mode 100644 paddle/phi/kernels/channel_shuffle_grad_kernel.h create mode 100644 paddle/phi/kernels/channel_shuffle_kernel.h create mode 100644 paddle/phi/kernels/cpu/channel_shuffle_grad_kernel.cc create mode 100644 paddle/phi/kernels/cpu/channel_shuffle_kernel.cc create mode 100644 paddle/phi/kernels/gpu/channel_shuffle_grad_kernel.cu create mode 100644 paddle/phi/kernels/gpu/channel_shuffle_kernel.cu create mode 100644 paddle/phi/kernels/impl/channel_shuffle_grad_kernel_impl.h create mode 100644 paddle/phi/kernels/impl/channel_shuffle_kernel_impl.h create mode 100644 paddle/phi/ops/compat/channel_shuffle_sig.cc create mode 100644 python/paddle/fluid/tests/unittests/test_channel_shuffle.py diff --git a/paddle/fluid/operators/channel_shuffle_op.cc b/paddle/fluid/operators/channel_shuffle_op.cc new file mode 100644 index 0000000000000..74b2e04e63f70 --- /dev/null +++ b/paddle/fluid/operators/channel_shuffle_op.cc @@ -0,0 +1,100 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/infershape_utils.h" +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/framework/op_version_registry.h" +#include "paddle/phi/core/infermeta_utils.h" +#include "paddle/phi/infermeta/backward.h" +#include "paddle/phi/infermeta/unary.h" + +namespace paddle { +namespace operators { + +class ChannelShuffleOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; +}; + +class ChannelShuffleOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("X", + "(Tensor, default Tensor), " + "the input feature data of ChannelShuffleOp, the layout is " + "[N, C, H, W] or [N, H, W, C]."); + AddOutput("Out", + "(Tensor, default Tensor), the output of " + "ChannelShuffleOp. The layout is also [N, C, " + "H, W] or [N, H, W, C]."); + AddAttr("groups", "number of groups to divide channels in."); + AddAttr( + "data_format", + "An optional string from: \"NHWC\", \"NCHW\". " + "Defaults to \"NHWC\", Specify the data format of the input data.") + .SetDefault("NCHW"); + + AddComment(R"DOC( + Channel Shuffle operator + This operator divides channels in a tensor of shape :math:`(*, C, H, W)` + into :math:`g` groups and rearranges them as :math:`(*, C/g, g, H, W)` + while keeping the original tensor shape. + + Please refer to the paper: + `ShuffleNet: An Extremely Efficient Convolutional Neural Network for + Mobile Devices `_ + by Zhang et. al (2017) for more details. + + )DOC"); + } +}; + +class ChannelShuffleGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; +}; + +template +class ChannelShuffleGradOpMaker : public framework::SingleGradOpMaker { + public: + using framework::SingleGradOpMaker::SingleGradOpMaker; + + protected: + void Apply(GradOpPtr op) const override { + op->SetType("channel_shuffle_grad"); + op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out")); + op->SetAttrMap(this->Attrs()); + op->SetOutput(framework::GradVarName("X"), this->InputGrad("X")); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +DECLARE_INFER_SHAPE_FUNCTOR(channel_shuffle, ChannelShuffleInferShapeFunctor, + PD_INFER_META(phi::ChannelShuffleInferMeta)); + +REGISTER_OPERATOR(channel_shuffle, ops::ChannelShuffleOp, + ops::ChannelShuffleOpMaker, + ops::ChannelShuffleGradOpMaker, + ops::ChannelShuffleGradOpMaker, + ChannelShuffleInferShapeFunctor); + +DECLARE_INFER_SHAPE_FUNCTOR(channel_shuffle_grad, + ChannelShuffleGradInferShapeFunctor, + PD_INFER_META(phi::ChannelShuffleGradInferMeta)); + +REGISTER_OPERATOR(channel_shuffle_grad, ops::ChannelShuffleGradOp, + ChannelShuffleGradInferShapeFunctor); diff --git a/paddle/phi/infermeta/backward.cc b/paddle/phi/infermeta/backward.cc index 567f39a915c02..4a4585e00eed6 100644 --- a/paddle/phi/infermeta/backward.cc +++ b/paddle/phi/infermeta/backward.cc @@ -67,6 +67,22 @@ void BilinearTensorProductGradInferMeta(const MetaTensor& x, } } +void ChannelShuffleGradInferMeta(const MetaTensor& out_grad, + int groups, + const std::string& data_format, + MetaTensor* x_grad) { + auto do_dims = out_grad.dims(); + PADDLE_ENFORCE_EQ(do_dims.size(), + 4, + phi::errors::InvalidArgument( + "Input should be a 4-D tensor of format [N, C, H, W] " + "or [N, H, W, C], but got %u.", + do_dims.size())); + auto dx_dims = do_dims; + x_grad->set_dims(dx_dims); + x_grad->set_dtype(out_grad.dtype()); +} + void ConvTransposeGradInferMeta(const MetaTensor& x, const MetaTensor& filter, const MetaTensor& dout, diff --git a/paddle/phi/infermeta/backward.h b/paddle/phi/infermeta/backward.h index 6807438ebbb75..9db958778d597 100644 --- a/paddle/phi/infermeta/backward.h +++ b/paddle/phi/infermeta/backward.h @@ -37,6 +37,11 @@ void BilinearTensorProductGradInferMeta(const MetaTensor& x, MetaTensor* dweight, MetaTensor* dbias); +void ChannelShuffleGradInferMeta(const MetaTensor& out_grad, + int groups, + const std::string& data_format, + MetaTensor* x_grad); + void ConvTransposeGradInferMeta(const MetaTensor& x, const MetaTensor& filter, const MetaTensor& dout, diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index e5d83a4013d30..5066d0cfd16fa 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -2999,6 +2999,52 @@ void WhereIndexInferMeta(const MetaTensor& condition, MetaTensor* out) { out->set_dtype(DataType::INT64); } +void ChannelShuffleInferMeta(const MetaTensor& x, + int groups, + const std::string& data_format, + MetaTensor* out) { + auto input_dims = x.dims(); + PADDLE_ENFORCE_EQ(input_dims.size(), + 4, + phi::errors::InvalidArgument( + "Input should be a 4-D tensor of format [N, C, H, W] " + "or [N, H, W, C], but got %u.", + input_dims.size())); + PADDLE_ENFORCE_GE( + groups, + 1, + phi::errors::InvalidArgument("groups should be larger than 0.")); + PADDLE_ENFORCE_EQ(data_format == "NCHW" || data_format == "NHWC", + true, + phi::errors::InvalidArgument( + "data_format must be one of " + "NCHW and NHWC. But recevied data_format: %s", + data_format)); + + const bool channel_last = (data_format == "NHWC"); + + if (!channel_last) { + PADDLE_ENFORCE_EQ(input_dims[1] % groups, + 0, + phi::errors::InvalidArgument( + "The number of groups to divide channels in [%u] " + "should divide the number of channel [%u]", + groups, + input_dims[1])); + } else { + PADDLE_ENFORCE_EQ(input_dims[3] % groups, + 0, + phi::errors::InvalidArgument( + "The number of groups to divide channels in [%u] " + "should divide the number of channel [%u]", + groups, + input_dims[3])); + } + auto output_dims = input_dims; + out->set_dtype(x.dtype()); + out->set_dims(output_dims); +} + } // namespace phi PD_REGISTER_INFER_META_FN(flatten, phi::FlattenInferMeta); diff --git a/paddle/phi/infermeta/unary.h b/paddle/phi/infermeta/unary.h index 70b868eeb5d8d..c67eb2068d8bf 100644 --- a/paddle/phi/infermeta/unary.h +++ b/paddle/phi/infermeta/unary.h @@ -435,4 +435,9 @@ void OneHotInferMeta(const MetaTensor& x, const Scalar& depth, MetaTensor* out); void WhereIndexInferMeta(const MetaTensor& condition, MetaTensor* out); +void ChannelShuffleInferMeta(const MetaTensor& x, + int groups, + const std::string& data_format, + MetaTensor* out); + } // namespace phi diff --git a/paddle/phi/kernels/channel_shuffle_grad_kernel.h b/paddle/phi/kernels/channel_shuffle_grad_kernel.h new file mode 100644 index 0000000000000..ac89f3336bc76 --- /dev/null +++ b/paddle/phi/kernels/channel_shuffle_grad_kernel.h @@ -0,0 +1,29 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "paddle/phi/core/dense_tensor.h" + +namespace phi { + +template +void ChannelShuffleGradKernel(const Context& dev_ctx, + const DenseTensor& out_grad, + int groups, + const std::string& data_format, + DenseTensor* x_grad); + +} // namespace phi diff --git a/paddle/phi/kernels/channel_shuffle_kernel.h b/paddle/phi/kernels/channel_shuffle_kernel.h new file mode 100644 index 0000000000000..12de25606dd96 --- /dev/null +++ b/paddle/phi/kernels/channel_shuffle_kernel.h @@ -0,0 +1,29 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "paddle/phi/core/dense_tensor.h" + +namespace phi { + +template +void ChannelShuffleKernel(const Context& dev_ctx, + const DenseTensor& x, + int groups, + const std::string& data_format, + DenseTensor* out); + +} // namespace phi diff --git a/paddle/phi/kernels/cpu/channel_shuffle_grad_kernel.cc b/paddle/phi/kernels/cpu/channel_shuffle_grad_kernel.cc new file mode 100644 index 0000000000000..fcc91b2191673 --- /dev/null +++ b/paddle/phi/kernels/cpu/channel_shuffle_grad_kernel.cc @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/channel_shuffle_grad_kernel.h" +#include "paddle/phi/kernels/impl/channel_shuffle_grad_kernel_impl.h" + +#include "paddle/phi/backends/cpu/cpu_context.h" +#include "paddle/phi/core/kernel_registry.h" + +PD_REGISTER_KERNEL(channel_shuffle_grad, + CPU, + ALL_LAYOUT, + phi::ChannelShuffleGradKernel, + float, + double) {} diff --git a/paddle/phi/kernels/cpu/channel_shuffle_kernel.cc b/paddle/phi/kernels/cpu/channel_shuffle_kernel.cc new file mode 100644 index 0000000000000..95d19ec6a7746 --- /dev/null +++ b/paddle/phi/kernels/cpu/channel_shuffle_kernel.cc @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/channel_shuffle_kernel.h" +#include "paddle/phi/kernels/impl/channel_shuffle_kernel_impl.h" + +#include "paddle/phi/backends/cpu/cpu_context.h" +#include "paddle/phi/core/kernel_registry.h" + +PD_REGISTER_KERNEL(channel_shuffle, + CPU, + ALL_LAYOUT, + phi::ChannelShuffleKernel, + float, + double) {} diff --git a/paddle/phi/kernels/gpu/channel_shuffle_grad_kernel.cu b/paddle/phi/kernels/gpu/channel_shuffle_grad_kernel.cu new file mode 100644 index 0000000000000..63d3d4a554f81 --- /dev/null +++ b/paddle/phi/kernels/gpu/channel_shuffle_grad_kernel.cu @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/channel_shuffle_grad_kernel.h" +#include "paddle/phi/kernels/impl/channel_shuffle_grad_kernel_impl.h" + +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/core/kernel_registry.h" + +PD_REGISTER_KERNEL(channel_shuffle_grad, + GPU, + ALL_LAYOUT, + phi::ChannelShuffleGradKernel, + float, + double) {} diff --git a/paddle/phi/kernels/gpu/channel_shuffle_kernel.cu b/paddle/phi/kernels/gpu/channel_shuffle_kernel.cu new file mode 100644 index 0000000000000..f85cb4aafd1dc --- /dev/null +++ b/paddle/phi/kernels/gpu/channel_shuffle_kernel.cu @@ -0,0 +1,26 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/kernels/channel_shuffle_kernel.h" +#include "paddle/phi/kernels/impl/channel_shuffle_kernel_impl.h" + +#include "paddle/phi/backends/gpu/gpu_context.h" +#include "paddle/phi/core/kernel_registry.h" + +PD_REGISTER_KERNEL(channel_shuffle, + GPU, + ALL_LAYOUT, + phi::ChannelShuffleKernel, + float, + double) {} diff --git a/paddle/phi/kernels/impl/channel_shuffle_grad_kernel_impl.h b/paddle/phi/kernels/impl/channel_shuffle_grad_kernel_impl.h new file mode 100644 index 0000000000000..26bee763eca52 --- /dev/null +++ b/paddle/phi/kernels/impl/channel_shuffle_grad_kernel_impl.h @@ -0,0 +1,58 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/kernels/funcs/math_function.h" + +namespace phi { + +template +void ChannelShuffleGradKernel(const Context& dev_ctx, + const DenseTensor& out_grad, + int groups, + const std::string& data_format, + DenseTensor* x_grad) { + auto* dout = &out_grad; + auto* dx = x_grad; + dev_ctx.template Alloc(dx); + bool channel_last = (data_format == "NHWC"); + auto do_dims = dout->dims(); + auto dx_dims = dx->dims(); + + DenseTensor t(*dout); + if (!channel_last) { + t.Resize({do_dims[0], do_dims[1] / groups, groups, do_dims[2], do_dims[3]}); + } else { + t.Resize({do_dims[0], do_dims[1], do_dims[2], do_dims[3] / groups, groups}); + } + auto axis = !channel_last ? std::vector{0, 2, 1, 3, 4} + : std::vector{0, 1, 2, 4, 3}; + + DenseTensor o(*dx); + if (!channel_last) { + o.Resize({dx_dims[0], groups, dx_dims[1] / groups, dx_dims[2], dx_dims[3]}); + } else { + o.Resize({dx_dims[0], dx_dims[1], dx_dims[2], groups, dx_dims[3] / groups}); + } + phi::funcs::Transpose trans; + trans(dev_ctx, t, &o, axis); + dx->Resize(dx_dims); +} + +} // namespace phi diff --git a/paddle/phi/kernels/impl/channel_shuffle_kernel_impl.h b/paddle/phi/kernels/impl/channel_shuffle_kernel_impl.h new file mode 100644 index 0000000000000..c723cd3622af9 --- /dev/null +++ b/paddle/phi/kernels/impl/channel_shuffle_kernel_impl.h @@ -0,0 +1,57 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "paddle/phi/core/dense_tensor.h" +#include "paddle/phi/kernels/funcs/math_function.h" + +namespace phi { + +template +void ChannelShuffleKernel(const Context& dev_ctx, + const DenseTensor& x, + int groups, + const std::string& data_format, + DenseTensor* out) { + auto* in = &x; + dev_ctx.template Alloc(out); + bool channel_last = (data_format == "NHWC"); + auto in_dims = in->dims(); + auto o_dims = out->dims(); + + DenseTensor t(*in); + if (!channel_last) { + t.Resize({in_dims[0], groups, in_dims[1] / groups, in_dims[2], in_dims[3]}); + } else { + t.Resize({in_dims[0], in_dims[1], in_dims[2], groups, in_dims[3] / groups}); + } + auto axis = !channel_last ? std::vector{0, 2, 1, 3, 4} + : std::vector{0, 1, 2, 4, 3}; + + DenseTensor o(*out); + if (!channel_last) { + o.Resize({in_dims[0], in_dims[1] / groups, groups, in_dims[2], in_dims[3]}); + } else { + o.Resize({in_dims[0], in_dims[1], in_dims[2], in_dims[3] / groups, groups}); + } + phi::funcs::Transpose trans; + trans(dev_ctx, t, &o, axis); + out->Resize(o_dims); +} + +} // namespace phi diff --git a/paddle/phi/ops/compat/channel_shuffle_sig.cc b/paddle/phi/ops/compat/channel_shuffle_sig.cc new file mode 100644 index 0000000000000..ae0aa0a80b6f0 --- /dev/null +++ b/paddle/phi/ops/compat/channel_shuffle_sig.cc @@ -0,0 +1,30 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/phi/core/compat/op_utils.h" + +namespace phi { + +KernelSignature ChannelShuffleGradOpArgumentMapping( + const ArgumentMappingContext& ctx) { + return KernelSignature("channel_shuffle_grad", + {"Out@GRAD"}, + {"groups", "data_format"}, + {"X@GRAD"}); +} + +} // namespace phi + +PD_REGISTER_ARG_MAPPING_FN(channel_shuffle_grad, + phi::ChannelShuffleGradOpArgumentMapping); diff --git a/python/paddle/fluid/tests/unittests/test_channel_shuffle.py b/python/paddle/fluid/tests/unittests/test_channel_shuffle.py new file mode 100644 index 0000000000000..b4a3fc387068c --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_channel_shuffle.py @@ -0,0 +1,250 @@ +# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import numpy as np + +from op_test import OpTest +import paddle +import paddle.nn.functional as F +import paddle.fluid.core as core +import paddle.fluid as fluid + + +def channel_shuffle_np(x, groups, data_format="NCHW"): + if data_format == "NCHW": + n, c, h, w = x.shape + new_shape = (n, groups, c // groups, h, w) + npresult = np.reshape(x, new_shape) + npresult = npresult.transpose(0, 2, 1, 3, 4) + oshape = [n, c, h, w] + npresult = np.reshape(npresult, oshape) + return npresult + else: + n, h, w, c = x.shape + new_shape = (n, h, w, groups, c // groups) + npresult = np.reshape(x, new_shape) + npresult = npresult.transpose(0, 1, 2, 4, 3) + oshape = [n, h, w, c] + npresult = np.reshape(npresult, oshape) + return npresult + + +class TestChannelShuffleOp(OpTest): + def setUp(self): + self.op_type = "channel_shuffle" + self.init_data_format() + n, c, h, w = 2, 9, 4, 4 + + if self.format == "NCHW": + shape = [n, c, h, w] + if self.format == "NHWC": + shape = [n, h, w, c] + + groups = 3 + + x = np.random.random(shape).astype("float64") + npresult = channel_shuffle_np(x, groups, self.format) + + self.inputs = {'X': x} + self.outputs = {'Out': npresult} + self.attrs = {'groups': groups, "data_format": self.format} + + def init_data_format(self): + self.format = "NCHW" + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +class TestChannelLast(TestChannelShuffleOp): + def init_data_format(self): + self.format = "NHWC" + + +class TestChannelShuffleAPI(unittest.TestCase): + def setUp(self): + self.x_1_np = np.random.random([2, 9, 4, 4]).astype("float64") + self.x_2_np = np.random.random([2, 4, 4, 9]).astype("float64") + self.out_1_np = channel_shuffle_np(self.x_1_np, 3) + self.out_2_np = channel_shuffle_np(self.x_2_np, 3, "NHWC") + + def test_static_graph_functional(self): + for use_cuda in ([False, True] + if core.is_compiled_with_cuda() else [False]): + place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() + + paddle.enable_static() + x_1 = paddle.fluid.data( + name="x", shape=[2, 9, 4, 4], dtype="float64") + x_2 = paddle.fluid.data( + name="x2", shape=[2, 4, 4, 9], dtype="float64") + out_1 = F.channel_shuffle(x_1, 3) + out_2 = F.channel_shuffle(x_2, 3, "NHWC") + + exe = paddle.static.Executor(place=place) + res_1 = exe.run(fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True) + + res_2 = exe.run(fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True) + + assert np.allclose(res_1, self.out_1_np) + assert np.allclose(res_2, self.out_2_np) + + # same test between layer and functional in this op. + def test_static_graph_layer(self): + for use_cuda in ([False, True] + if core.is_compiled_with_cuda() else [False]): + place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() + + paddle.enable_static() + x_1 = paddle.fluid.data( + name="x", shape=[2, 9, 4, 4], dtype="float64") + x_2 = paddle.fluid.data( + name="x2", shape=[2, 4, 4, 9], dtype="float64") + # init instance + ps_1 = paddle.nn.ChannelShuffle(3) + ps_2 = paddle.nn.ChannelShuffle(3, "NHWC") + out_1 = ps_1(x_1) + out_2 = ps_2(x_2) + out_1_np = channel_shuffle_np(self.x_1_np, 3) + out_2_np = channel_shuffle_np(self.x_2_np, 3, "NHWC") + + exe = paddle.static.Executor(place=place) + res_1 = exe.run(fluid.default_main_program(), + feed={"x": self.x_1_np}, + fetch_list=out_1, + use_prune=True) + + res_2 = exe.run(fluid.default_main_program(), + feed={"x2": self.x_2_np}, + fetch_list=out_2, + use_prune=True) + + assert np.allclose(res_1, out_1_np) + assert np.allclose(res_2, out_2_np) + + def run_dygraph(self, groups, data_format): + + n, c, h, w = 2, 9, 4, 4 + + if data_format == "NCHW": + shape = [n, c, h, w] + if data_format == "NHWC": + shape = [n, h, w, c] + + x = np.random.random(shape).astype("float64") + + npresult = channel_shuffle_np(x, groups, data_format) + + for use_cuda in ([False, True] + if core.is_compiled_with_cuda() else [False]): + place = paddle.CUDAPlace(0) if use_cuda else paddle.CPUPlace() + + paddle.disable_static(place=place) + + channel_shuffle = paddle.nn.ChannelShuffle( + groups, data_format=data_format) + result = channel_shuffle(paddle.to_tensor(x)) + + self.assertTrue(np.allclose(result.numpy(), npresult)) + + result_functional = F.channel_shuffle( + paddle.to_tensor(x), 3, data_format) + self.assertTrue(np.allclose(result_functional.numpy(), npresult)) + + channel_shuffle_str = 'groups={}'.format(groups) + if data_format != 'NCHW': + channel_shuffle_str += ', data_format={}'.format(data_format) + self.assertEqual(channel_shuffle.extra_repr(), channel_shuffle_str) + + def test_dygraph1(self): + self.run_dygraph(3, "NCHW") + + def test_dygraph2(self): + self.run_dygraph(3, "NHWC") + + +class TestChannelShuffleError(unittest.TestCase): + def test_error_functional(self): + def error_input(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([9, 4, 4]).astype("float64") + channel_shuffle = F.channel_shuffle(paddle.to_tensor(x), 3) + + self.assertRaises(ValueError, error_input) + + def error_groups_1(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 9, 4, 4]).astype("float64") + channel_shuffle = F.channel_shuffle(paddle.to_tensor(x), 3.33) + + self.assertRaises(TypeError, error_groups_1) + + def error_groups_2(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 9, 4, 4]).astype("float64") + channel_shuffle = F.channel_shuffle(paddle.to_tensor(x), -1) + + self.assertRaises(ValueError, error_groups_2) + + def error_data_format(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 9, 4, 4]).astype("float64") + channel_shuffle = F.channel_shuffle( + paddle.to_tensor(x), 3, "WOW") + + self.assertRaises(ValueError, error_data_format) + + def test_error_layer(self): + def error_input_layer(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([9, 4, 4]).astype("float64") + cs = paddle.nn.ChannelShuffle(3) + cs(paddle.to_tensor(x)) + + self.assertRaises(ValueError, error_input_layer) + + def error_groups_layer_1(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 9, 4, 4]).astype("float64") + cs = paddle.nn.ChannelShuffle(3.33) + + self.assertRaises(TypeError, error_groups_layer_1) + + def error_groups_layer_2(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 9, 4, 4]).astype("float64") + cs = paddle.nn.ChannelShuffle(-1) + + self.assertRaises(ValueError, error_groups_layer_2) + + def error_data_format_layer(): + with paddle.fluid.dygraph.guard(): + x = np.random.random([2, 9, 4, 4]).astype("float64") + cs = paddle.nn.ChannelShuffle(3, "MEOW") + + self.assertRaises(ValueError, error_data_format_layer) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py index b4824eff007d6..70e3518a1af46 100644 --- a/python/paddle/nn/__init__.py +++ b/python/paddle/nn/__init__.py @@ -138,6 +138,7 @@ from .layer.distance import PairwiseDistance # noqa: F401 from .layer.vision import PixelShuffle # noqa: F401 +from .layer.vision import ChannelShuffle # noqa: F401 from .layer.container import LayerDict # noqa: F401 from .utils.spectral_norm_hook import spectral_norm @@ -300,6 +301,7 @@ def weight_norm(*args): 'Swish', 'Mish', 'PixelShuffle', + 'ChannelShuffle', 'ELU', 'ReLU6', 'LayerDict', diff --git a/python/paddle/nn/functional/__init__.py b/python/paddle/nn/functional/__init__.py index a24afc45a5995..58251c2890430 100644 --- a/python/paddle/nn/functional/__init__.py +++ b/python/paddle/nn/functional/__init__.py @@ -114,6 +114,7 @@ from .vision import affine_grid # noqa: F401 from .vision import grid_sample # noqa: F401 from .vision import pixel_shuffle # noqa: F401 +from .vision import channel_shuffle # noqa: F401 from .input import one_hot # noqa: F401 from .input import embedding # noqa: F401 from ...fluid.layers import gather_tree # noqa: F401 @@ -213,6 +214,7 @@ 'grid_sample', 'local_response_norm', 'pixel_shuffle', + 'channel_shuffle', 'embedding', 'gather_tree', 'one_hot', diff --git a/python/paddle/nn/functional/vision.py b/python/paddle/nn/functional/vision.py index 43c7757a8777b..07e68d71dc1f1 100644 --- a/python/paddle/nn/functional/vision.py +++ b/python/paddle/nn/functional/vision.py @@ -21,6 +21,7 @@ from paddle import _C_ops from ...device import is_compiled_with_rocm from paddle import in_dynamic_mode +from paddle.framework import _non_static_mode __all__ = [] @@ -344,3 +345,71 @@ def pixel_shuffle(x, upscale_factor, data_format="NCHW", name=None): attrs={"upscale_factor": upscale_factor, "data_format": data_format}) return out + + +def channel_shuffle(x, groups, data_format="NCHW", name=None): + """ + This API implements channel shuffle operation. + See more details in :ref:`api_nn_vision_ChannelShuffle` . + + Parameters: + x (Tensor): 4-D tensor, the data type should be float32 or float64. + groups (int): Number of groups to divide channels in. + data_format (str): The data format of the input and output data. An optional string of NCHW or NHWC. The default is NCHW. When it is NCHW, the data is stored in the order of [batch_size, input_channels, input_height, input_width]. + name (str, optional): Name for the operation (optional, default is None). Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. + + Returns: + Out (Tensor): Rearranged tensor keeping the original tensor shape. + + Examples: + .. code-block:: python + :name: channel_shuffle-example + + import paddle + import paddle.nn.functional as F + x = paddle.arange(0, 0.6, 0.1, 'float32') + x = paddle.reshape(x, [1, 6, 1, 1]) + # [[[[0. ]], + # [[0.10000000]], + # [[0.20000000]], + # [[0.30000001]], + # [[0.40000001]], + # [[0.50000000]]]] + y = F.channel_shuffle(x, 3) + # [[[[0. ]], + # [[0.20000000]], + # [[0.40000001]], + # [[0.10000000]], + # [[0.30000001]], + # [[0.50000000]]]] + """ + if len(x.shape) != 4: + raise ValueError( + "Input x should be 4D tensor, but received x with the shape of {}". + format(x.shape)) + + if not isinstance(groups, int): + raise TypeError("groups must be int type") + + if groups <= 0: + raise ValueError("groups must be positive") + + if data_format not in ["NCHW", "NHWC"]: + raise ValueError("Attr(data_format) should be 'NCHW' or 'NHWC'." + "But recevie Attr(data_format): {} ".format( + data_format)) + + if _non_static_mode(): + return _C_ops.channel_shuffle(x, "groups", groups, "data_format", + data_format) + + helper = LayerHelper("channel_shuffle", **locals()) + check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'channel_shuffle') + out = helper.create_variable_for_type_inference(dtype=x.dtype) + helper.append_op( + type="channel_shuffle", + inputs={"X": x}, + outputs={"Out": out}, + attrs={"groups": groups, + "data_format": data_format}) + return out diff --git a/python/paddle/nn/layer/__init__.py b/python/paddle/nn/layer/__init__.py index 7dd18f1fefd65..339feef8f32e6 100644 --- a/python/paddle/nn/layer/__init__.py +++ b/python/paddle/nn/layer/__init__.py @@ -88,6 +88,7 @@ from .norm import LocalResponseNorm # noqa: F401 from .vision import PixelShuffle # noqa: F401 +from .vision import ChannelShuffle # noqa: F401 from .distance import PairwiseDistance # noqa: F401 from .container import LayerDict # noqa: F401 diff --git a/python/paddle/nn/layer/vision.py b/python/paddle/nn/layer/vision.py index 0531afb4eeeeb..e775d4fcf6dfb 100644 --- a/python/paddle/nn/layer/vision.py +++ b/python/paddle/nn/layer/vision.py @@ -87,3 +87,76 @@ def extra_repr(self): if self._name is not None: main_str += ', name={}'.format(self._name) return main_str + + +class ChannelShuffle(Layer): + """ + This operator divides channels in a tensor of shape [N, C, H, W] or [N, H, W, C] into g groups, + getting a tensor with the shape of [N, g, C/g, H, W] or [N, H, W, g, C/g], and transposes them + as [N, C/g, g, H, W] or [N, H, W, g, C/g], then rearranges them to original tensor shape. This + operation can improve the interaction between channels, using features efficiently. Please + refer to the paper: `ShuffleNet: An Extremely Efficient + Convolutional Neural Network for Mobile Devices `_ . + by Zhang et. al (2017) for more details. + + Parameters: + groups (int): Number of groups to divide channels in. + data_format (str): The data format of the input and output data. An optional string of NCHW or NHWC. The default is NCHW. When it is NCHW, the data is stored in the order of [batch_size, input_channels, input_height, input_width]. + name (str, optional): Name for the operation (optional, default is None). Normally there is no need for user to set this property. For more information, please refer to :ref:`api_guide_Name`. + + Shape: + - **x**: 4-D tensor with shape of [N, C, H, W] or [N, H, W, C]. + - **out**: 4-D tensor with shape and dtype same as x. + + Examples: + .. code-block:: python + :name: ChannelShuffle-example + + import paddle + import paddle.nn as nn + x = paddle.arange(0, 0.6, 0.1, 'float32') + x = paddle.reshape(x, [1, 6, 1, 1]) + # [[[[0. ]], + # [[0.10000000]], + # [[0.20000000]], + # [[0.30000001]], + # [[0.40000001]], + # [[0.50000000]]]] + channel_shuffle = nn.ChannelShuffle(3) + y = channel_shuffle(x) + # [[[[0. ]], + # [[0.20000000]], + # [[0.40000001]], + # [[0.10000000]], + # [[0.30000001]], + # [[0.50000000]]]] + """ + + def __init__(self, groups, data_format="NCHW", name=None): + super(ChannelShuffle, self).__init__() + + if not isinstance(groups, int): + raise TypeError("groups must be int type") + + if groups <= 0: + raise ValueError("groups must be positive") + + if data_format not in ["NCHW", "NHWC"]: + raise ValueError("Data format should be 'NCHW' or 'NHWC'." + "But recevie data format: {}".format(data_format)) + + self._groups = groups + self._data_format = data_format + self._name = name + + def forward(self, x): + return functional.channel_shuffle(x, self._groups, self._data_format, + self._name) + + def extra_repr(self): + main_str = 'groups={}'.format(self._groups) + if self._data_format != 'NCHW': + main_str += ', data_format={}'.format(self._data_format) + if self._name is not None: + main_str += ', name={}'.format(self._name) + return main_str diff --git a/tools/static_mode_white_list.py b/tools/static_mode_white_list.py index 47b1ba5700e1b..5dcff12c2c87e 100755 --- a/tools/static_mode_white_list.py +++ b/tools/static_mode_white_list.py @@ -92,6 +92,7 @@ 'test_case', 'test_cast_op', 'test_center_loss', + 'test_channel_shuffle', 'test_cholesky_op', 'test_chunk_eval_op', 'test_chunk_op', From 6553a9d7a355c4e9ef04a0cd42702b1d36b46700 Mon Sep 17 00:00:00 2001 From: Ruibiao Chen Date: Mon, 25 Apr 2022 13:25:02 +0800 Subject: [PATCH 61/66] Do not reset default stream for StreamSafeCUDAAllocator (#42149) --- .../fluid/memory/allocation/allocator_facade.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/paddle/fluid/memory/allocation/allocator_facade.cc b/paddle/fluid/memory/allocation/allocator_facade.cc index e2730a1b825e9..e2649a7fd334d 100644 --- a/paddle/fluid/memory/allocation/allocator_facade.cc +++ b/paddle/fluid/memory/allocation/allocator_facade.cc @@ -415,6 +415,23 @@ class AllocatorFacadePrivate { void SetDefaultStream(const platform::CUDAPlace& place, gpuStream_t stream) { const std::shared_ptr& allocator = GetDefaultStreamSafeCUDAAllocator(place); + + // NOTE(Ruibiao): The default stream will be set when the CUDADeviceContext + // created. Normally, the DeviceContextPool is a global singleton and one + // Place only correspond to one DeviceContext. However, to support + // multi-stream scheduling, standalone executor creates two extra + // DeviceContextPools for H2D and D2H stream in StreamAnalyzer, which make + // one Place correspond to multiple DeviceContext and unexpectedly reset the + // default stream in runtime. To avoid this behavior, we do not allow + // changing default stream after initially setting. + if (allocator->GetDefaultStream() != nullptr) { + VLOG(5) << "The default stream for StreamSafeCUDAAllocator(" + << allocator.get() << ") in " << place << " has been set to " + << allocator->GetDefaultStream() + << " before, not allow to change now."; + return; + } + allocator->SetDefaultStream(stream); VLOG(8) << "Set default stream to " << stream << " for StreamSafeCUDAAllocator(" << allocator.get() << ") in " From 9a0bfece0cbc813caa9a34be66367b7d06b7d697 Mon Sep 17 00:00:00 2001 From: Feiyu Chan Date: Mon, 25 Apr 2022 15:08:34 +0800 Subject: [PATCH 62/66] remove redundant computation in Categorical.probs (#42114) --- python/paddle/distribution/categorical.py | 51 +++++++---------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/python/paddle/distribution/categorical.py b/python/paddle/distribution/categorical.py index b181a25fbcee1..97a3df490b1d0 100644 --- a/python/paddle/distribution/categorical.py +++ b/python/paddle/distribution/categorical.py @@ -115,6 +115,8 @@ def __init__(self, logits, name=None): self.logits = self._to_tensor(logits)[0] if self.dtype != convert_dtype(self.logits.dtype): self.logits = tensor.cast(self.logits, dtype=self.dtype) + dist_sum = paddle.sum(self.logits, axis=-1, keepdim=True) + self._prob = self.logits / dist_sum def sample(self, shape): """Generate samples of the specified shape. @@ -297,42 +299,21 @@ def probs(self, value): """ name = self.name + '_probs' - - dist_sum = paddle.sum(self.logits, axis=-1, keepdim=True) - prob = self.logits / dist_sum - - shape = list(prob.shape) - value_shape = list(value.shape) - if len(shape) == 1: - num_value_in_one_dist = np.prod(value_shape) - index_value = paddle.reshape(value, [num_value_in_one_dist, 1]) - index = index_value + if len(self._prob.shape) == 1: # batch_shape is empty + return paddle.gather( + self._prob, value.reshape( + [-1], name=name), name=name).reshape( + value.shape, name=name) else: - num_dist = np.prod(shape[:-1]) - num_value_in_one_dist = value_shape[-1] - prob = paddle.reshape(prob, [num_dist, shape[-1]]) - if len(value_shape) == 1: - value = nn.expand(value, [num_dist]) - value_shape = shape[:-1] + value_shape - index_value = paddle.reshape(value, [num_dist, -1, 1]) - if shape[:-1] != value_shape[:-1]: - raise ValueError( - "shape of value {} must match shape of logits {}".format( - str(value_shape[:-1]), str(shape[:-1]))) - - index_prefix = paddle.unsqueeze( - arange( - num_dist, dtype=index_value.dtype), axis=-1) - index_prefix = nn.expand(index_prefix, [1, num_value_in_one_dist]) - index_prefix = paddle.unsqueeze(index_prefix, axis=-1) - - if index_value.dtype != index_prefix.dtype: - tensor.cast(index_prefix, dtype=index_value.dtype) - index = concat([index_prefix, index_value], axis=-1) - - # value is the category index to search for the corresponding probability. - select_prob = gather_nd(prob, index) - return paddle.reshape(select_prob, value_shape, name=name) + if len(value.shape) == 1: + return paddle.take_along_axis( + self._prob, + paddle.reshape( + value, (len(self._prob.shape) - 1) * [1] + [-1], + name=name), + axis=-1) + else: + return paddle.take_along_axis(self._prob, value, axis=-1) def log_prob(self, value): """Log probabilities of the given category. Refer to ``probs`` method. From 418522648d393c1faee8a5dc32158649b4043c3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Siwek?= Date: Mon, 25 Apr 2022 09:37:47 +0200 Subject: [PATCH 63/66] Downloading data for test_analyzer_vit_ocr (#42041) * Change server URL * update config * add test to parallel UT rule * add checksum to ensure files are downloaded * change downloading target * reuse existing variable * change target directory --- paddle/fluid/inference/tests/api/CMakeLists.txt | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/inference/tests/api/CMakeLists.txt b/paddle/fluid/inference/tests/api/CMakeLists.txt index e9b8c0ce70f66..fc85f83661889 100644 --- a/paddle/fluid/inference/tests/api/CMakeLists.txt +++ b/paddle/fluid/inference/tests/api/CMakeLists.txt @@ -346,17 +346,13 @@ inference_analysis_test(test_analyzer_transformer_profile SRCS analyzer_transfor --cpu_num_threads=${CPU_NUM_THREADS_ON_CI}) # VIT-OCR -set(VIT_OCR_URL "https://paddle-qa.bj.bcebos.com/inference_model/2.1.1/ocr") -set(VIT_OCR_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/vit_ocr") +set(VIT_OCR_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/vit") if (NOT EXISTS ${VIT_OCR_INSTALL_DIR}/vit_ocr.tgz) - inference_download_and_uncompress_without_verify(${VIT_OCR_INSTALL_DIR} ${VIT_OCR_URL} vit_ocr.tgz) -endif() -if (NOT EXISTS ${VIT_OCR_INSTALL_DIR}/datavit.txt) - file(DOWNLOAD ${VIT_OCR_URL}/datavit.txt ${VIT_OCR_INSTALL_DIR}/datavit.txt) + inference_download_and_uncompress_without_verify(${VIT_OCR_INSTALL_DIR} ${INFERENCE_URL} "ocr/vit_ocr.tgz") endif() inference_analysis_test(test_analyzer_vit_ocr SRCS analyzer_vit_ocr_tester.cc EXTRA_DEPS ${INFERENCE_EXTRA_DEPS} - ARGS --infer_model=${VIT_OCR_INSTALL_DIR}/vit_ocr --infer_data=${VIT_OCR_INSTALL_DIR}/datavit.txt) + ARGS --infer_model=${VIT_OCR_INSTALL_DIR}/vit_ocr/model --infer_data=${VIT_OCR_INSTALL_DIR}/vit_ocr/datavit.txt) # ocr set(OCR_INSTALL_DIR "${INFERENCE_DEMO_INSTALL_DIR}/ocr") From 6dd9dd3976a80dca53c8a82693c6b112b33d1d4b Mon Sep 17 00:00:00 2001 From: Yilingyelu <103369238+Yilingyelu@users.noreply.github.com> Date: Mon, 25 Apr 2022 15:47:51 +0800 Subject: [PATCH 64/66] fix en docs of some Apis (gradients, scope_guard, cuda_places, name_scope, device_guard, load_program_state, scale, ParamAttr and WeightNormParamAttr) (#41604) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update scope_guard; test=document_fix * gradients; test=document_fix * gradients; test=document_fix * name_scope; test=document_fix * cpu_places; test=document_fix * WeightNormParamAttr; test=document_fix * cuda_places; test=document_fix * load_program_state; test=document_fix * device_guard; test=document_fix * device_guard; test=document_fix * ParamAttr; test=document_fix * scale; test=document_fix * scale; test=document_fix * update code example；test=document_fix Co-authored-by: Chen Long <1300851984@qq.com> --- python/paddle/fluid/backward.py | 4 ++-- python/paddle/fluid/executor.py | 2 +- python/paddle/fluid/framework.py | 15 ++++++++++----- python/paddle/fluid/io.py | 2 +- python/paddle/fluid/layers/nn.py | 4 ++-- python/paddle/fluid/param_attr.py | 27 ++++++++++++++------------- 6 files changed, 30 insertions(+), 24 deletions(-) diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py index adce805195960..c7e69753b5335 100755 --- a/python/paddle/fluid/backward.py +++ b/python/paddle/fluid/backward.py @@ -2021,7 +2021,6 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None): @framework.static_only def gradients(targets, inputs, target_gradients=None, no_grad_set=None): """ - :api_attr: Static Graph Backpropagate the gradients of targets to inputs. @@ -2042,8 +2041,9 @@ def gradients(targets, inputs, target_gradients=None, no_grad_set=None): will be None. Examples: + .. code-block:: python - + :name: code-example import paddle import paddle.nn.functional as F diff --git a/python/paddle/fluid/executor.py b/python/paddle/fluid/executor.py index 86b0d6560c927..56b743f4463ae 100644 --- a/python/paddle/fluid/executor.py +++ b/python/paddle/fluid/executor.py @@ -75,7 +75,6 @@ def _switch_scope(scope): @signature_safe_contextmanager def scope_guard(scope): """ - :api_attr: Static Graph This function switches scope through python `with` statement. Scope records the mapping between variable names and variables ( :ref:`api_guide_Variable` ), @@ -94,6 +93,7 @@ def scope_guard(scope): None Examples: + .. code-block:: python import paddle diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 314a502a3cbef..817e742fd1d8a 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -729,7 +729,7 @@ def is_compiled_with_rocm(): def cuda_places(device_ids=None): """ - **Note**: + Note: For multi-card tasks, please use `FLAGS_selected_gpus` environment variable to set the visible GPU device. The next version will fix the problem with `CUDA_VISIBLE_DEVICES` environment variable. @@ -754,6 +754,7 @@ def cuda_places(device_ids=None): list of paddle.CUDAPlace: Created GPU place list. Examples: + .. code-block:: python import paddle @@ -874,6 +875,7 @@ def cpu_places(device_count=None): list of paddle.CPUPlace: Created list of CPU places. Examples: + .. code-block:: python import paddle @@ -993,7 +995,6 @@ def name(self): @signature_safe_contextmanager def name_scope(prefix=None): """ - :api_attr: Static Graph Generate hierarchical name prefix for the operators in Static Graph. @@ -1006,6 +1007,7 @@ def name_scope(prefix=None): prefix(str, optional): prefix. Default is none. Examples: + .. code-block:: python import paddle @@ -6916,8 +6918,9 @@ def switch_device(device): @signature_safe_contextmanager def device_guard(device=None): """ - **Notes**: - **The API only supports static mode.** + + Note: + The API only supports static mode. A context manager that specifies the device on which the OP will be placed. @@ -6931,8 +6934,10 @@ def device_guard(device=None): assigned devices. Examples: + .. code-block:: python - + + # required: gpu import paddle paddle.enable_static() diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index a48cfd9150c65..7c7f101286e24 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -2154,7 +2154,6 @@ def set_var(var, ndarray): def load_program_state(model_path, var_list=None): """ - :api_attr: Static Graph Load program state from local file @@ -2169,6 +2168,7 @@ def load_program_state(model_path, var_list=None): state_dict(dict): the dict store Parameter and optimizer information Examples: + .. code-block:: python import paddle diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 1fdf59948345b..8b10a5f454e69 100755 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -11850,8 +11850,7 @@ def _elementwise_op(helper): def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): """ - Scale operator. - + Putting scale and bias to the input Tensor as following: ``bias_after_scale`` is True: @@ -11876,6 +11875,7 @@ def scale(x, scale=1.0, bias=0.0, bias_after_scale=True, act=None, name=None): Tensor: Output tensor of scale operator, with shape and data type same as input. Examples: + .. code-block:: python # scale as a float32 number diff --git a/python/paddle/fluid/param_attr.py b/python/paddle/fluid/param_attr.py index c3ee11ff5d906..a10ce1ce808f6 100644 --- a/python/paddle/fluid/param_attr.py +++ b/python/paddle/fluid/param_attr.py @@ -30,16 +30,17 @@ class ParamAttr(object): """ - Create a object to represent the attribute of parameter. The attributes are: - name, initializer, learning rate, regularizer, trainable, gradient clip, - and model average. - + Note: ``gradient_clip`` of ``ParamAttr`` HAS BEEN DEPRECATED since 2.0. Please use ``need_clip`` in ``ParamAttr`` to speficiy the clip scope. There are three clipping strategies: :ref:`api_paddle_nn_ClipGradByGlobalNorm` , :ref:`api_paddle_nn_ClipGradByNorm` , :ref:`api_paddle_nn_ClipGradByValue` . + Create a object to represent the attribute of parameter. The attributes are: + name, initializer, learning rate, regularizer, trainable, gradient clip, + and model average. + Parameters: name (str, optional): The parameter's name. Default None, meaning that the name would be created automatically. @@ -63,6 +64,7 @@ class ParamAttr(object): ParamAttr Object. Examples: + .. code-block:: python import paddle @@ -213,24 +215,22 @@ def _to_kwargs(self, with_initializer=False): class WeightNormParamAttr(ParamAttr): r""" - :api_attr: Static Graph Note: Please use 'paddle.nn.utils.weight_norm' in dygraph mode. - + + Note: + ``gradient_clip`` of ``ParamAttr`` HAS BEEN DEPRECATED since 2.0. + Please use ``need_clip`` in ``ParamAttr`` to speficiy the clip scope. + There are three clipping strategies: :ref:`api_paddle_nn_ClipGradByGlobalNorm` , + :ref:`api_paddle_nn_ClipGradByNorm` , :ref:`api_paddle_nn_ClipGradByValue` . + Parameter of weight Norm. Weight Norm is a reparameterization of the weight vectors in a neural network that decouples the magnitude of those weight vectors from their direction. Weight Norm has been implemented as discussed in this paper: `Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks `_. - - Note: - ``gradient_clip`` of ``ParamAttr`` HAS BEEN DEPRECATED since 2.0. - Please use ``need_clip`` in ``ParamAttr`` to speficiy the clip scope. - There are three clipping strategies: :ref:`api_paddle_nn_ClipGradByGlobalNorm` , - :ref:`api_paddle_nn_ClipGradByNorm` , :ref:`api_paddle_nn_ClipGradByValue` . - Args: dim(int, optional): Dimension over which to compute the norm. Dim is a non-negative @@ -258,6 +258,7 @@ class WeightNormParamAttr(ParamAttr): need_clip (bool, optional): Whether the parameter gradient need to be cliped in optimizer. Default is True. Examples: + .. code-block:: python import paddle From 30f65c2523daaad970d3147c2906985a35b986cc Mon Sep 17 00:00:00 2001 From: wenbin Date: Mon, 25 Apr 2022 15:57:23 +0800 Subject: [PATCH 65/66] int8 clone issue fix (#42218) --- paddle/fluid/framework/naive_executor.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/framework/naive_executor.cc b/paddle/fluid/framework/naive_executor.cc index f30d1ea1b83dd..dba3b3ff1e690 100644 --- a/paddle/fluid/framework/naive_executor.cc +++ b/paddle/fluid/framework/naive_executor.cc @@ -147,11 +147,16 @@ void NaiveExecutor::ResetTrtOps(int num) { int engine_predictor_id = trtop->Attr("predictor_id"); std::string engine_name = engine_key + std::to_string(engine_predictor_id); - operators::TensorRTEngine *trt_engine = - paddle::inference::Singleton< + operators::TensorRTEngine *trt_engine = nullptr; + // can't get trt engine if int8 calibration table data process. + if (paddle::inference::Singleton< inference::tensorrt::TRTEngineManager>::Global() - .Get(engine_name); - if (trt_engine->with_dynamic_shape()) { + .Has(engine_name)) { + trt_engine = paddle::inference::Singleton< + inference::tensorrt::TRTEngineManager>::Global() + .Get(engine_name); + } + if (trt_engine && trt_engine->with_dynamic_shape()) { LOG(INFO) << "rebuild trt engine, this may cost a lot of time!"; trt_engine->ResetContext(); trt_engine->ClearTensorMap(); From e52e6d0113ffe04b328f25c5ce4bb93a2dd5b138 Mon Sep 17 00:00:00 2001 From: Zhou Wei <1183042833@qq.com> Date: Mon, 25 Apr 2022 16:54:44 +0800 Subject: [PATCH 66/66] merge all phi kernel lib to several big static lib, reduce link command (#42185) * merge all phi lib to several big static lib * merge all phi lib to several big static lib --- CMakeLists.txt | 3 + cmake/generic.cmake | 128 +++++++++--------- .../distributed/collective/CMakeLists.txt | 14 +- .../fluid/eager/accumulation/CMakeLists.txt | 2 +- paddle/fluid/eager/api/utils/CMakeLists.txt | 2 +- paddle/fluid/eager/pylayer/CMakeLists.txt | 2 +- paddle/fluid/framework/CMakeLists.txt | 6 +- paddle/fluid/imperative/CMakeLists.txt | 4 +- paddle/fluid/inference/CMakeLists.txt | 8 +- .../fluid/inference/tensorrt/CMakeLists.txt | 4 +- paddle/phi/CMakeLists.txt | 2 +- paddle/phi/kernels/CMakeLists.txt | 3 +- 12 files changed, 90 insertions(+), 88 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e7d16ecfd7002..9002cb287e855 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,9 @@ else(APPLE AND WITH_ARM) cmake_minimum_required(VERSION 3.15) cmake_policy(VERSION 3.10) endif(APPLE AND WITH_ARM) +# use to get_property location of static lib +# https://cmake.org/cmake/help/v3.0/policy/CMP0026.html?highlight=cmp0026 +cmake_policy(SET CMP0026 OLD) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") set(PADDLE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(PADDLE_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index ba59eae392c66..35170b5198dc3 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -176,6 +176,36 @@ function(create_static_lib TARGET_NAME) endif() endfunction() +function(create_dummy_static_lib TARGET_NAME) + set(options "") + set(oneValueArgs "") + set(multiValueArgs LIBS DEPS LIMIT) + cmake_parse_arguments(merge "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + list(REMOVE_DUPLICATES merge_LIBS) + set(index 1) + set(offset 1) + # the dummy target would be consisted of limit size libraries + set(limit ${merge_LIMIT}) + list(LENGTH merge_LIBS libs_len) + foreach(lib ${merge_LIBS}) + list(APPEND merge_list ${lib}) + list(LENGTH merge_list listlen) + if ((${listlen} GREATER ${limit}) OR (${offset} EQUAL ${libs_len})) + message("Merge and generate static library: ${TARGET_NAME}_static_${index}") + merge_static_libs(${TARGET_NAME}_static_${index} ${merge_list}) + if(merge_DEPS) + target_link_libraries(${TARGET_NAME}_static_${index} ${merge_DEPS}) + endif() + set(merge_list) + list(APPEND ${TARGET_NAME}_list ${TARGET_NAME}_static_${index}) + MATH(EXPR index "${index}+1") + endif() + MATH(EXPR offset "${offset}+1") + endforeach() + cc_library(${TARGET_NAME} DEPS ${${TARGET_NAME}_list}) +endfunction() + function(merge_static_libs TARGET_NAME) set(libs ${ARGN}) list(REMOVE_DUPLICATES libs) @@ -193,92 +223,61 @@ function(merge_static_libs TARGET_NAME) # also help to track dependencies. set(target_SRCS ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}_dummy.c) - if(APPLE) # Use OSX's libtool to merge archives - # Make the generated dummy source file depended on all static input - # libs. If input lib changes,the source file is touched - # which causes the desired effect (relink). - add_custom_command(OUTPUT ${target_SRCS} - COMMAND ${CMAKE_COMMAND} -E touch ${target_SRCS} - DEPENDS ${libs}) - - # Generate dummy static lib - generate_dummy_static_lib(LIB_NAME ${TARGET_NAME} FILE_PATH ${target_SRCS} GENERATOR "generic.cmake:merge_static_libs") - - target_link_libraries(${TARGET_NAME} ${libs_deps}) + # Make the generated dummy source file depended on all static input + # libs. If input lib changes,the source file is touched + # which causes the desired effect (relink). + add_custom_command(OUTPUT ${target_SRCS} + COMMAND ${CMAKE_COMMAND} -E touch ${target_SRCS} + DEPENDS ${libs}) + + # Generate dummy staic lib + generate_dummy_static_lib(LIB_NAME ${TARGET_NAME} FILE_PATH ${target_SRCS} GENERATOR "generic.cmake:merge_static_libs") + target_link_libraries(${TARGET_NAME} ${libs_deps}) + # OSX: use 'libtool' to merge archives + if(APPLE) foreach(lib ${libs}) # Get the file names of the libraries to be merged set(libfiles ${libfiles} $) endforeach() add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMENT "Merge and generate static lib: lib${TARGET_NAME}.a" COMMAND rm "${CMAKE_CURRENT_BINARY_DIR}/lib${TARGET_NAME}.a" COMMAND /usr/bin/libtool -static -o "${CMAKE_CURRENT_BINARY_DIR}/lib${TARGET_NAME}.a" ${libfiles} ) - endif(APPLE) - if(LINUX) # general UNIX: use "ar" to extract objects and re-add to a common lib - set(target_DIR ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}.dir) - - foreach(lib ${libs}) - set(objlistfile ${target_DIR}/${lib}.objlist) # list of objects in the input library - set(objdir ${target_DIR}/${lib}.objdir) - - add_custom_command(OUTPUT ${objdir} - COMMAND ${CMAKE_COMMAND} -E make_directory ${objdir} - DEPENDS ${lib}) + endif() - add_custom_command(OUTPUT ${objlistfile} - COMMAND ${CMAKE_AR} -x "$" - COMMAND ${CMAKE_AR} -t "$" > ${objlistfile} - DEPENDS ${lib} ${objdir} - WORKING_DIRECTORY ${objdir}) + # LINUX: use "ar" to extract objects and re-add to a common lib + if(LINUX) + set(mri_file ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}.mri CACHE INTERNAL "phi_static.mri file") + get_property(ABS_MERGE_LIB_PATH TARGET ${TARGET_NAME} PROPERTY LOCATION) + file(WRITE ${mri_file} "create ${ABS_MERGE_LIB_PATH}\n") - list(APPEND target_OBJS "${objlistfile}") + foreach(lib ${libs}) + get_property(ABS_LIB_PATH TARGET ${lib} PROPERTY LOCATION) + file(APPEND ${mri_file} "addlib ${ABS_LIB_PATH}\n") endforeach() - - # Make the generated dummy source file depended on all static input - # libs. If input lib changes,the source file is touched - # which causes the desired effect (relink). - add_custom_command(OUTPUT ${target_SRCS} - COMMAND ${CMAKE_COMMAND} -E touch ${target_SRCS} - DEPENDS ${libs} ${target_OBJS}) - - # Generate dummy staic lib - generate_dummy_static_lib(LIB_NAME ${TARGET_NAME} FILE_PATH ${target_SRCS} GENERATOR "generic.cmake:merge_static_libs") - - target_link_libraries(${TARGET_NAME} ${libs_deps}) - - # Get the file name of the generated library - set(target_LIBNAME "$") + file(APPEND ${mri_file} "save\nend\n") add_custom_command(TARGET ${TARGET_NAME} POST_BUILD - COMMAND ${CMAKE_AR} crs ${target_LIBNAME} `find ${target_DIR} -name '*.o'` - COMMAND ${CMAKE_RANLIB} ${target_LIBNAME} - WORKING_DIRECTORY ${target_DIR}) - endif(LINUX) - if(WIN32) # windows do not support gcc/nvcc combined compiling. Use msvc lib.exe to merge libs. - # Make the generated dummy source file depended on all static input - # libs. If input lib changes,the source file is touched - # which causes the desired effect (relink). - add_custom_command(OUTPUT ${target_SRCS} - COMMAND ${CMAKE_COMMAND} -E touch ${target_SRCS} - DEPENDS ${libs}) - # Generate dummy staic lib - generate_dummy_static_lib(LIB_NAME ${TARGET_NAME} FILE_PATH ${target_SRCS} GENERATOR "generic.cmake:merge_static_libs") - - target_link_libraries(${TARGET_NAME} ${libs_deps}) + COMMENT "Merge and generate static lib: lib${TARGET_NAME}.a" + COMMAND ${CMAKE_AR} -M < ${mri_file} + COMMAND ${CMAKE_RANLIB} "$") + endif() + # Windows do not support gcc/nvcc combined compiling. Use msvc 'lib.exe' to merge libs. + if(WIN32) foreach(lib ${libs}) - # Get the file names of the libraries to be merged set(libfiles ${libfiles} $) endforeach() - # msvc will put libarary in directory of "/Release/xxxlib" by default - # COMMAND cmake -E remove "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/${TARGET_NAME}.lib" + # msvc compiler will put libarary in directory of "/Release/xxxlib" by default add_custom_command(TARGET ${TARGET_NAME} POST_BUILD + COMMENT "Merge and generate static lib: lib${TARGET_NAME}.lib" COMMAND cmake -E make_directory $ COMMAND lib /OUT:$ ${libfiles} ) - endif(WIN32) -endfunction(merge_static_libs) + endif() +endfunction() function(check_coverage_opt TARGET_NAME SRCS) if(WITH_COVERAGE AND WITH_INCREMENTAL_COVERAGE) @@ -1076,4 +1075,3 @@ function(math_library TARGET) cc_library(${TARGET} SRCS ${cc_srcs} DEPS ${math_library_DEPS} ${math_common_deps}) endif() endfunction() - diff --git a/paddle/fluid/distributed/collective/CMakeLists.txt b/paddle/fluid/distributed/collective/CMakeLists.txt index 6d736d5543ce4..f6b1bd47c1e46 100644 --- a/paddle/fluid/distributed/collective/CMakeLists.txt +++ b/paddle/fluid/distributed/collective/CMakeLists.txt @@ -1,20 +1,20 @@ -cc_library(processgroup SRCS ProcessGroup.cc DEPS phi phi_api eager_api) -cc_library(eager_reducer SRCS reducer.cc DEPS eager_api processgroup phi phi_api string_helper) +cc_library(processgroup SRCS ProcessGroup.cc DEPS phi_api eager_api) +cc_library(eager_reducer SRCS reducer.cc DEPS eager_api processgroup phi_api string_helper) if (WITH_DISTRIBUTE) - cc_library(processgroup_gloo SRCS ProcessGroupGloo.cc DEPS phi phi_api eager_api gloo_wrapper) + cc_library(processgroup_gloo SRCS ProcessGroupGloo.cc DEPS phi_api eager_api gloo_wrapper) endif() if(WITH_NCCL) - cc_library(processgroup_nccl SRCS ProcessGroupNCCL.cc NCCLTools.cc Common.cc DEPS place cuda_stream enforce collective_helper device_context phi phi_api eager_api) + cc_library(processgroup_nccl SRCS ProcessGroupNCCL.cc NCCLTools.cc Common.cc DEPS place cuda_stream enforce collective_helper device_context phi_api eager_api) if (WITH_DISTRIBUTE AND WITH_PSCORE) - cc_library(processgroup_heter SRCS ProcessGroupHeter.cc NCCLTools.cc Common.cc DEPS place cuda_stream enforce collective_helper device_context phi phi_api eager_api) + cc_library(processgroup_heter SRCS ProcessGroupHeter.cc NCCLTools.cc Common.cc DEPS place cuda_stream enforce collective_helper device_context phi_api eager_api) endif() endif() if(WITH_ASCEND_CL) - cc_library(processgroup_hccl SRCS ProcessGroupHCCL.cc HCCLTools.cc Common.cc DEPS place npu_stream enforce collective_helper device_context phi phi_api eager_api) + cc_library(processgroup_hccl SRCS ProcessGroupHCCL.cc HCCLTools.cc Common.cc DEPS place npu_stream enforce collective_helper device_context phi_api eager_api) if (WITH_DISTRIBUTE AND WITH_PSCORE) - cc_library(processgroup_heter SRCS ProcessGroupHeter.cc HCCLTools.cc Common.cc DEPS place npu_stream enforce collective_helper device_context phi phi_api eager_api) + cc_library(processgroup_heter SRCS ProcessGroupHeter.cc HCCLTools.cc Common.cc DEPS place npu_stream enforce collective_helper device_context phi_api eager_api) endif() endif() diff --git a/paddle/fluid/eager/accumulation/CMakeLists.txt b/paddle/fluid/eager/accumulation/CMakeLists.txt index 43ca707f4f6fb..0531aa5aab373 100644 --- a/paddle/fluid/eager/accumulation/CMakeLists.txt +++ b/paddle/fluid/eager/accumulation/CMakeLists.txt @@ -1 +1 @@ -cc_library(accumulation_node SRCS accumulation_node.cc DEPS gradient_accumulator phi phi_api grad_node_info) +cc_library(accumulation_node SRCS accumulation_node.cc DEPS gradient_accumulator phi_api grad_node_info) diff --git a/paddle/fluid/eager/api/utils/CMakeLists.txt b/paddle/fluid/eager/api/utils/CMakeLists.txt index c34df3972c23e..a2a380ebad6c5 100644 --- a/paddle/fluid/eager/api/utils/CMakeLists.txt +++ b/paddle/fluid/eager/api/utils/CMakeLists.txt @@ -1,3 +1,3 @@ -cc_library(tensor_utils SRCS tensor_utils.cc DEPS phi phi_api autograd_meta grad_node_info accumulation_node) +cc_library(tensor_utils SRCS tensor_utils.cc DEPS phi_api autograd_meta grad_node_info accumulation_node) cc_library(hook_utils SRCS hook_utils.cc DEPS phi tensor_utils autograd_meta grad_node_info utils accumulation_node) cc_library(global_utils SRCS global_utils.cc DEPS place tracer) diff --git a/paddle/fluid/eager/pylayer/CMakeLists.txt b/paddle/fluid/eager/pylayer/CMakeLists.txt index 8c660fa9694ed..59030342eccad 100644 --- a/paddle/fluid/eager/pylayer/CMakeLists.txt +++ b/paddle/fluid/eager/pylayer/CMakeLists.txt @@ -1 +1 @@ -cc_library(py_layer_node SRCS py_layer_node.cc DEPS pybind phi phi_api grad_node_info) +cc_library(py_layer_node SRCS py_layer_node.cc DEPS pybind phi_api grad_node_info) diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt index b6a7aea4f9cd7..bb7f3f26463d4 100755 --- a/paddle/fluid/framework/CMakeLists.txt +++ b/paddle/fluid/framework/CMakeLists.txt @@ -206,11 +206,11 @@ ENDIF() IF(WITH_XPU) cc_library(operator SRCS operator.cc DEPS xpu_op_list op_info device_context tensor scope glog trainer_desc_proto data_feed_proto shape_inference data_transform lod_tensor profiler transfer_scope_cache op_kernel_type op_call_stack unused_var_check nan_inf_utils - phi phi_utils kernel_factory infershape_utils op_utils) + phi_utils kernel_factory infershape_utils op_utils) ELSE() cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope glog trainer_desc_proto data_feed_proto shape_inference data_transform lod_tensor profiler transfer_scope_cache op_kernel_type op_call_stack unused_var_check nan_inf_utils - phi phi_utils kernel_factory infershape_utils op_utils) + phi_utils kernel_factory infershape_utils op_utils) ENDIF() cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry device_context) @@ -418,7 +418,7 @@ cc_library(save_load_util SRCS save_load_util.cc DEPS tensor scope layer) cc_test(save_load_util_test SRCS save_load_util_test.cc DEPS save_load_util tensor scope layer) cc_library(generator SRCS generator.cc DEPS enforce place) -cc_library(infershape_utils SRCS infershape_utils.cc DEPS lod_tensor selected_rows_utils attribute place phi var_type_traits phi phi_api_utils op_info shape_inference) +cc_library(infershape_utils SRCS infershape_utils.cc DEPS lod_tensor selected_rows_utils attribute place var_type_traits phi phi_api_utils op_info shape_inference) cc_test(infershape_utils_test SRCS infershape_utils_test.cc DEPS infershape_utils infermeta_utils meta_tensor) # Get the current working branch diff --git a/paddle/fluid/imperative/CMakeLists.txt b/paddle/fluid/imperative/CMakeLists.txt index 107bbdf09a021..92af1901b71ab 100644 --- a/paddle/fluid/imperative/CMakeLists.txt +++ b/paddle/fluid/imperative/CMakeLists.txt @@ -1,9 +1,9 @@ cc_library(imperative_flag SRCS flags.cc DEPS gflags flags) cc_library(var_helper SRCS var_helper.cc DEPS tensor phi_api) IF(WITH_XPU) -cc_library(prepared_operator SRCS prepared_operator.cc DEPS xpu_op_list proto_desc operator device_context lod_tensor selected_rows_utils var_type_traits op_kernel_type data_transform nan_inf_utils phi_api phi phi_utils var_helper) +cc_library(prepared_operator SRCS prepared_operator.cc DEPS xpu_op_list proto_desc operator device_context lod_tensor selected_rows_utils var_type_traits op_kernel_type data_transform nan_inf_utils phi_api phi_utils var_helper) ELSE() -cc_library(prepared_operator SRCS prepared_operator.cc DEPS proto_desc operator device_context lod_tensor selected_rows_utils var_type_traits op_kernel_type data_transform nan_inf_utils phi_api phi phi_utils var_helper) +cc_library(prepared_operator SRCS prepared_operator.cc DEPS proto_desc operator device_context lod_tensor selected_rows_utils var_type_traits op_kernel_type data_transform nan_inf_utils phi_api phi_utils var_helper) ENDIF() cc_library(layer SRCS layer.cc DEPS prepared_operator math_function imperative_flag variable_helper op_registry var_helper phi_api) add_subdirectory(jit) diff --git a/paddle/fluid/inference/CMakeLists.txt b/paddle/fluid/inference/CMakeLists.txt index bdf364aa9adcd..7a1f3e8326aeb 100644 --- a/paddle/fluid/inference/CMakeLists.txt +++ b/paddle/fluid/inference/CMakeLists.txt @@ -36,7 +36,7 @@ endif() # fluid_modules exclude API-interface of inference/api and inference/capi_exp get_property(fluid_modules GLOBAL PROPERTY FLUID_MODULES) get_property(phi_modules GLOBAL PROPERTY PHI_MODULES) -set(utils_modules stringpiece pretty_log string_helper) +set(utils_modules stringpiece pretty_log string_helper benchmark) add_subdirectory(api) @@ -50,9 +50,9 @@ if(WITH_ONNXRUNTIME) set(STATIC_INFERENCE_API ${STATIC_INFERENCE_API} onnxruntime_predictor) endif() -#TODO(wilber, T8T9): Do we still need to support windows gpu static library? +#windows GPU static library over the limit, so not create_static_lib, and cc_library is dummy if(WIN32 AND WITH_GPU) - cc_library(paddle_inference DEPS ${fluid_modules} ${phi_modules} ${STATIC_INFERENCE_API} ${utils_modules}) + cc_library(paddle_inference DEPS ${fluid_modules} phi ${STATIC_INFERENCE_API} ${utils_modules}) else() create_static_lib(paddle_inference ${fluid_modules} ${phi_modules} ${STATIC_INFERENCE_API} ${utils_modules}) endif() @@ -84,7 +84,7 @@ set(SHARED_INFERENCE_SRCS ${PADDLE_CUSTOM_OP_SRCS}) # shared inference library deps -set(SHARED_INFERENCE_DEPS ${fluid_modules} ${phi_modules} analysis_predictor) +set(SHARED_INFERENCE_DEPS ${fluid_modules} phi analysis_predictor ${utils_modules}) if (WITH_CRYPTO) set(SHARED_INFERENCE_DEPS ${SHARED_INFERENCE_DEPS} paddle_crypto) diff --git a/paddle/fluid/inference/tensorrt/CMakeLists.txt b/paddle/fluid/inference/tensorrt/CMakeLists.txt index d1d146b2ce5f6..c713e3a66ac71 100644 --- a/paddle/fluid/inference/tensorrt/CMakeLists.txt +++ b/paddle/fluid/inference/tensorrt/CMakeLists.txt @@ -1,8 +1,8 @@ # Compiling with WITH_PYTHON=ON and WITH_TENSORRT=ON failed on windows. Temporarily add paddle_inference_api dependency to solve the problem if(WIN32) -nv_library(tensorrt_engine SRCS engine.cc trt_int8_calibrator.cc DEPS ${GLOB_OPERATOR_DEPS} framework_proto device_context boost paddle_inference_api) + nv_library(tensorrt_engine SRCS engine.cc trt_int8_calibrator.cc DEPS ${GLOB_OPERATOR_DEPS} framework_proto device_context boost paddle_inference_api) else() -nv_library(tensorrt_engine SRCS engine.cc trt_int8_calibrator.cc DEPS ${GLOB_OPERATOR_DEPS} framework_proto device_context boost) + nv_library(tensorrt_engine SRCS engine.cc trt_int8_calibrator.cc DEPS ${GLOB_OPERATOR_DEPS} framework_proto device_context boost) endif() nv_library(tensorrt_op_teller SRCS op_teller.cc DEPS framework_proto device_context boost) nv_test(test_tensorrt SRCS test_tensorrt.cc DEPS dynload_cuda device_context dynamic_loader) diff --git a/paddle/phi/CMakeLists.txt b/paddle/phi/CMakeLists.txt index d43e327393f25..0595ea4d8bddf 100644 --- a/paddle/phi/CMakeLists.txt +++ b/paddle/phi/CMakeLists.txt @@ -27,7 +27,7 @@ set(PHI_DEPS convert_utils dense_tensor phi_context kernel_factory kernel_contex get_property(phi_kernels GLOBAL PROPERTY PHI_KERNELS) set(PHI_DEPS ${PHI_DEPS} ${phi_kernels}) -cc_library(phi DEPS ${PHI_DEPS}) +create_dummy_static_lib(phi LIBS ${PHI_DEPS} LIMIT 100) set(phi_extension_header_file ${CMAKE_CURRENT_SOURCE_DIR}/extension.h CACHE INTERNAL "phi/extension.h file") file(WRITE ${phi_extension_header_file} "// Header file generated by paddle/phi/CMakeLists.txt for external users,\n// DO NOT edit or include it within paddle.\n\n#pragma once\n\n") diff --git a/paddle/phi/kernels/CMakeLists.txt b/paddle/phi/kernels/CMakeLists.txt index a3a71ab692245..437c55c840f1a 100644 --- a/paddle/phi/kernels/CMakeLists.txt +++ b/paddle/phi/kernels/CMakeLists.txt @@ -36,7 +36,7 @@ set(MANUAL_BUILD_KERNELS ${AUTOTUNE_KERNELS} cross_entropy_kernel adam_kernel ad matrix_power_kernel matrix_power_grad_kernel maxout_kernel maxout_grad_kernel pool_kernel put_along_axis_kernel put_along_axis_grad_kernel segment_pool_kernel segment_pool_grad_kernel softmax_kernel softmax_grad_kernel take_along_axis_kernel take_along_axis_grad_kernel - triangular_solve_grad_kernel determinant_grad_kernel reduce_sum_kernel rnn_kernel rnn_grad_kernel warpctc_kernel warpctc_grad_kernel) + triangular_solve_grad_kernel determinant_grad_kernel reduce_sum_kernel reduce_mean_kernel rnn_kernel rnn_grad_kernel warpctc_kernel warpctc_grad_kernel) foreach(src ${AUTOTUNE_KERNELS}) kernel_library(${src} DEPS ${COMMON_KERNEL_DEPS} switch_autotune) endforeach() @@ -52,6 +52,7 @@ kernel_library(hierarchical_sigmoid_grad_kernel DEPS ${COMMON_KERNEL_DEPS} matri kernel_library(gumbel_softmax_kernel DEPS ${COMMON_KERNEL_DEPS} softmax) kernel_library(gumbel_softmax_grad_kernel DEPS ${COMMON_KERNEL_DEPS} softmax) kernel_library(reduce_sum_kernel DEPS ${COMMON_KERNEL_DEPS} cast_kernel) +kernel_library(reduce_mean_kernel DEPS ${COMMON_KERNEL_DEPS} cast_kernel) kernel_library(matrix_power_kernel DEPS ${COMMON_KERNEL_DEPS} matrix_inverse) kernel_library(matrix_power_grad_kernel DEPS ${COMMON_KERNEL_DEPS} matrix_inverse) kernel_library(maxout_kernel DEPS ${COMMON_KERNEL_DEPS} maxouting)