forked from PaddlePaddle/Paddle
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'develop' into heaviside_3
- Loading branch information
Showing
553 changed files
with
24,609 additions
and
6,273 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,20 +1,20 @@ | ||
| cc_library(processgroup SRCS ProcessGroup.cc DEPS phi phi_api eager_api) | ||
| cc_library(eager_reducer SRCS reducer.cc DEPS eager_api processgroup phi phi_api string_helper) | ||
| cc_library(processgroup SRCS ProcessGroup.cc DEPS phi_api eager_api) | ||
| cc_library(eager_reducer SRCS reducer.cc DEPS eager_api processgroup phi_api string_helper) | ||
|
|
||
| if (WITH_DISTRIBUTE) | ||
| cc_library(processgroup_gloo SRCS ProcessGroupGloo.cc DEPS phi phi_api eager_api gloo_wrapper) | ||
| cc_library(processgroup_gloo SRCS ProcessGroupGloo.cc DEPS phi_api eager_api gloo_wrapper) | ||
| endif() | ||
|
|
||
| if(WITH_NCCL) | ||
| cc_library(processgroup_nccl SRCS ProcessGroupNCCL.cc NCCLTools.cc Common.cc DEPS place cuda_stream enforce collective_helper device_context phi phi_api eager_api) | ||
| cc_library(processgroup_nccl SRCS ProcessGroupNCCL.cc NCCLTools.cc Common.cc DEPS place cuda_stream enforce collective_helper device_context phi_api eager_api) | ||
| if (WITH_DISTRIBUTE AND WITH_PSCORE) | ||
| cc_library(processgroup_heter SRCS ProcessGroupHeter.cc NCCLTools.cc Common.cc DEPS place cuda_stream enforce collective_helper device_context phi phi_api eager_api) | ||
| cc_library(processgroup_heter SRCS ProcessGroupHeter.cc NCCLTools.cc Common.cc DEPS place cuda_stream enforce collective_helper device_context phi_api eager_api) | ||
| endif() | ||
| endif() | ||
|
|
||
| if(WITH_ASCEND_CL) | ||
| cc_library(processgroup_hccl SRCS ProcessGroupHCCL.cc HCCLTools.cc Common.cc DEPS place npu_stream enforce collective_helper device_context phi phi_api eager_api) | ||
| cc_library(processgroup_hccl SRCS ProcessGroupHCCL.cc HCCLTools.cc Common.cc DEPS place npu_stream enforce collective_helper device_context phi_api eager_api) | ||
| if (WITH_DISTRIBUTE AND WITH_PSCORE) | ||
| cc_library(processgroup_heter SRCS ProcessGroupHeter.cc HCCLTools.cc Common.cc DEPS place npu_stream enforce collective_helper device_context phi phi_api eager_api) | ||
| cc_library(processgroup_heter SRCS ProcessGroupHeter.cc HCCLTools.cc Common.cc DEPS place npu_stream enforce collective_helper device_context phi_api eager_api) | ||
| endif() | ||
| endif() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| // Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| #pragma once | ||
| #include <queue> | ||
| #include <unordered_map> | ||
|
|
||
| namespace paddle { | ||
| namespace distributed { | ||
| class TopkCalculator { | ||
| public: | ||
| TopkCalculator(int shard_num, size_t k) | ||
| : _shard_num(shard_num), _total_max_size(k) { | ||
| _shard_max_size = _total_max_size / shard_num; | ||
| _shard_max_size = _shard_max_size > 1 ? _shard_max_size : 1; | ||
| for (int i = 0; i < shard_num; ++i) { | ||
| _mpq.emplace(i, std::priority_queue<double, std::vector<double>, | ||
| std::greater<double>>()); | ||
| } | ||
| } | ||
| ~TopkCalculator() {} | ||
| bool push(int shard_id, double value) { | ||
| if (_mpq.find(shard_id) == _mpq.end()) { | ||
| return false; | ||
| } | ||
| auto &pq = _mpq[shard_id]; | ||
| if (pq.size() < _shard_max_size) { | ||
| pq.push(value); | ||
| } else { | ||
| if (pq.top() < value) { | ||
| pq.pop(); | ||
| pq.push(value); | ||
| } | ||
| } | ||
| return true; | ||
| } | ||
| // TODO 再进行一次堆排序merge各个shard的结果 | ||
| int top() { | ||
| double total = 0; | ||
| for (const auto &item : _mpq) { | ||
| auto &pq = item.second; | ||
| if (!pq.empty()) { | ||
| total += pq.top(); | ||
| } | ||
| } | ||
| return total / _shard_num; | ||
| } | ||
|
|
||
| private: | ||
| std::unordered_map<int, std::priority_queue<double, std::vector<double>, | ||
| std::greater<double>>> | ||
| _mpq; | ||
| int _shard_num; | ||
| size_t _total_max_size; | ||
| size_t _shard_max_size; | ||
| }; | ||
|
|
||
| } // namespace distributed | ||
| } // namespace paddle |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.