1 /**
2 * Copyright 2022 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "src/extendrt/numa_adapter.h"
18 #include <dlfcn.h>
19 #include <fstream>
20 #include <string>
21 #include "src/common/log_adapter.h"
22 #include "src/common/common.h"
23 #include "src/common/utils.h"
24
25 namespace mindspore {
26 namespace numa {
27 namespace {
28 static constexpr auto kNodeBase = "/sys/devices/system/node/node";
29 constexpr int kBase = 10;
30 } // namespace
31
NUMAAdapter()32 NUMAAdapter::NUMAAdapter() {
33 available_ = false;
34 handle_ = dlopen("libnuma.so", RTLD_LAZY | RTLD_LOCAL);
35 if (handle_ == nullptr) {
36 MS_LOG(WARNING) << "Open libnuma.so failed!try libnuma.so.1 again.";
37 handle_ = dlopen("libnuma.so.1", RTLD_LAZY | RTLD_LOCAL);
38 if (handle_ == nullptr) {
39 MS_LOG(WARNING) << "Open numa so failed!";
40 return;
41 }
42 }
43
44 numa_interfaces_.numa_available = reinterpret_cast<int (*)(void)>(dlsym(handle_, "numa_available"));
45 if (MS_UNLIKELY(numa_interfaces_.numa_available == nullptr)) {
46 MS_LOG(ERROR) << "numa_available not found!";
47 }
48 if (numa_interfaces_.numa_available() < 0) {
49 MS_LOG(ERROR) << "numa is not available!";
50 (void)dlclose(handle_);
51 handle_ = nullptr;
52 return;
53 }
54 available_ = true;
55 numa_interfaces_.numa_num_configured_nodes =
56 reinterpret_cast<int (*)(void)>(dlsym(handle_, "numa_num_configured_nodes"));
57 if (MS_UNLIKELY(numa_interfaces_.numa_num_configured_nodes == nullptr)) {
58 MS_LOG(ERROR) << "numa_num_configured_nodes not found!";
59 available_ = false;
60 }
61 numa_interfaces_.numa_num_task_cpus = reinterpret_cast<int (*)(void)>(dlsym(handle_, "numa_num_task_cpus"));
62 if (MS_UNLIKELY(numa_interfaces_.numa_num_task_cpus == nullptr)) {
63 MS_LOG(ERROR) << "numa_num_task_cpus not found!";
64 available_ = false;
65 }
66 numa_interfaces_.numa_allocate_nodemask =
67 reinterpret_cast<struct bitmask *(*)(void)>(dlsym(handle_, "numa_allocate_nodemask"));
68 if (MS_UNLIKELY(numa_interfaces_.numa_allocate_nodemask == nullptr)) {
69 MS_LOG(ERROR) << "numa_allocate_nodemask not found!";
70 available_ = false;
71 }
72 numa_interfaces_.numa_bitmask_clearall =
73 reinterpret_cast<struct bitmask *(*)(struct bitmask *)>(dlsym(handle_, "numa_bitmask_clearall"));
74 if (MS_UNLIKELY(numa_interfaces_.numa_bitmask_clearall == nullptr)) {
75 MS_LOG(ERROR) << "numa_bitmask_clearall not found!";
76 available_ = false;
77 }
78 numa_interfaces_.numa_bitmask_setbit =
79 reinterpret_cast<struct bitmask *(*)(struct bitmask *, unsigned int)>(dlsym(handle_, "numa_bitmask_setbit"));
80 if (MS_UNLIKELY(numa_interfaces_.numa_bitmask_setbit == nullptr)) {
81 MS_LOG(ERROR) << "numa_bitmask_setbit not found!";
82 available_ = false;
83 }
84 numa_interfaces_.numa_bind = reinterpret_cast<void (*)(struct bitmask *)>(dlsym(handle_, "numa_bind"));
85 if (MS_UNLIKELY(numa_interfaces_.numa_bind == nullptr)) {
86 MS_LOG(ERROR) << "numa_bind not found!";
87 available_ = false;
88 }
89 numa_interfaces_.numa_bitmask_free =
90 reinterpret_cast<void (*)(struct bitmask *)>(dlsym(handle_, "numa_bitmask_free"));
91 if (MS_UNLIKELY(numa_interfaces_.numa_bitmask_free == nullptr)) {
92 MS_LOG(ERROR) << "numa_bitmask_free not found!";
93 available_ = false;
94 }
95 numa_interfaces_.numa_alloc_onnode =
96 reinterpret_cast<void *(*)(size_t size, int node)>(dlsym(handle_, "numa_alloc_onnode"));
97 if (MS_UNLIKELY(numa_interfaces_.numa_alloc_onnode == nullptr)) {
98 MS_LOG(ERROR) << "numa_bitmask_free not found!";
99 available_ = false;
100 }
101 numa_interfaces_.numa_node_size64 =
102 reinterpret_cast<int64_t (*)(int node, int64_t *freep)>(dlsym(handle_, "numa_node_size64"));
103 if (MS_UNLIKELY(numa_interfaces_.numa_node_size64 == nullptr)) {
104 MS_LOG(ERROR) << "numa_node_size64 not found!";
105 available_ = false;
106 }
107 numa_interfaces_.numa_free = reinterpret_cast<void (*)(void *start, size_t size)>(dlsym(handle_, "numa_free"));
108 if (MS_UNLIKELY(numa_interfaces_.numa_free == nullptr)) {
109 MS_LOG(ERROR) << "numa_free not found!";
110 available_ = false;
111 }
112 if (!available_) {
113 (void)dlclose(handle_);
114 handle_ = nullptr;
115 return;
116 }
117 }
118
Bind(int node_id) const119 void NUMAAdapter::Bind(int node_id) const {
120 if (!Available() || node_id < 0) {
121 return;
122 }
123 auto bitmask = numa_interfaces_.numa_allocate_nodemask();
124 if (MS_UNLIKELY(bitmask == nullptr)) {
125 MS_LOG(ERROR) << "bind numa_node " << node_id << " failed!";
126 return;
127 }
128 (void)numa_interfaces_.numa_bitmask_setbit(bitmask, node_id);
129 numa_interfaces_.numa_bind(bitmask);
130 numa_interfaces_.numa_bitmask_free(bitmask);
131 }
132
Malloc(int node_id,size_t size) const133 void *NUMAAdapter::Malloc(int node_id, size_t size) const {
134 if (!Available() || node_id < 0) {
135 return nullptr;
136 }
137 return numa_interfaces_.numa_alloc_onnode(size, node_id);
138 }
139
Free(void * data,size_t size) const140 void NUMAAdapter::Free(void *data, size_t size) const {
141 if (!Available() || data == nullptr) {
142 return;
143 }
144 numa_interfaces_.numa_free(data, size);
145 }
146
NodesNum() const147 int NUMAAdapter::NodesNum() const {
148 if (!Available()) {
149 return 0;
150 }
151 return numa_interfaces_.numa_num_configured_nodes();
152 }
153
CPUNum() const154 int NUMAAdapter::CPUNum() const {
155 if (!Available()) {
156 return 0;
157 }
158 return numa_interfaces_.numa_num_task_cpus();
159 }
160
GetCPUList(int node_id)161 std::vector<int> NUMAAdapter::GetCPUList(int node_id) {
162 std::vector<int> cpu_list;
163 if (!Available() || node_id < 0) {
164 return cpu_list;
165 }
166 auto iter = node_cpu_list_.find(node_id);
167 if (iter != node_cpu_list_.end()) {
168 return iter->second;
169 }
170 static constexpr auto kVectorDefaultSize = 32;
171 cpu_list.reserve(kVectorDefaultSize);
172 std::string cpu_list_file = kNodeBase + std::to_string(node_id) + "/cpulist";
173 std::ifstream cpu_list_ifs(cpu_list_file);
174 if (!cpu_list_ifs.good()) {
175 MS_LOG(ERROR) << "file: " << cpu_list_file << " is not exist";
176 return cpu_list;
177 }
178 if (!cpu_list_ifs.is_open()) {
179 MS_LOG(ERROR) << "file: " << cpu_list_file << " open failed";
180 return cpu_list;
181 }
182 std::string line;
183 if (!std::getline(cpu_list_ifs, line)) {
184 MS_LOG(ERROR) << cpu_list_file << " getline failed!";
185 return cpu_list;
186 }
187 lite::Trim(&line);
188 auto cpu_lists = lite::StrSplit(line, ",");
189 for (auto &&item : cpu_lists) {
190 auto cpu_range = lite::StrSplit(item, "-");
191 static constexpr size_t kMaxRangeNum = 2;
192 if (cpu_range.size() != kMaxRangeNum) {
193 continue;
194 }
195 int begin = static_cast<int>(strtol(cpu_range[0].c_str(), nullptr, kBase));
196 int end = static_cast<int>(strtol(cpu_range[1].c_str(), nullptr, kBase));
197 for (int j = begin; j <= end; ++j) {
198 cpu_list.emplace_back(j);
199 }
200 }
201 cpu_list_ifs.close();
202 node_cpu_list_[node_id] = cpu_list;
203 return cpu_list;
204 }
205
GetNodeSize(int node_id) const206 MemoryInfo NUMAAdapter::GetNodeSize(int node_id) const {
207 MemoryInfo mem_info;
208 if (!Available() || node_id < 0) {
209 return mem_info;
210 }
211 mem_info.total = numa_interfaces_.numa_node_size64(node_id, &mem_info.free);
212 return mem_info;
213 }
214
~NUMAAdapter()215 NUMAAdapter::~NUMAAdapter() {
216 MS_LOG(DEBUG) << "~NUMAAdapter() begin.";
217 if (handle_ == nullptr) {
218 return;
219 }
220 (void)dlclose(handle_);
221 handle_ = nullptr;
222 MS_LOG(DEBUG) << "~NUMAAdapter() end.";
223 }
224 } // namespace numa
225 } // namespace mindspore
226