1 /**
2 * Copyright 2023 Huawei Technologies Co., Ltd
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "plugin/device/ascend/hal/device/ascend_dma_handle.h"
18 #if defined(RT_MEMORY_P2PDMA)
19 #include <unistd.h>
20 #include <fcntl.h>
21 #include <sys/types.h>
22 #include <sys/stat.h>
23 #include <sys/mman.h>
24 #include <cstdlib>
25 #include <string>
26 #include "runtime/rt_error_codes.h"
27 #endif
28 #include "utils/log_adapter.h"
29 #include "include/common/utils/utils.h"
30 #include "runtime/device/kernel_runtime_manager.h"
31 #include "transform/symbol/acl_rt_symbol.h"
32 #include "transform/symbol/symbol_utils.h"
33
34 namespace mindspore {
35 namespace device {
36 namespace ascend {
GetInstance()37 AscendDmaHandle &AscendDmaHandle::GetInstance() {
38 static AscendDmaHandle instance{};
39 return instance;
40 }
41
GetBuf() const42 void *AscendDmaHandle::GetBuf() const { return buf_; }
GetDargs() const43 void *AscendDmaHandle::GetDargs() const { return dargs_; }
GetSize() const44 size_t AscendDmaHandle::GetSize() const { return hbm_alloc_size_; }
45
~AscendDmaHandle()46 AscendDmaHandle::~AscendDmaHandle() {
47 #if defined(RT_MEMORY_P2PDMA)
48 munmap(buf_, hbm_alloc_size_);
49 close(p2p_fd_);
50 CALL_ASCEND_API(aclrtFree, dargs_);
51 #endif
52 }
53
AscendDmaHandle()54 AscendDmaHandle::AscendDmaHandle() {
55 InitRuntimeInstance();
56 InitDmaMem();
57 }
58
InitRuntimeInstance()59 void AscendDmaHandle::InitRuntimeInstance() {
60 auto ms_context = MsContext::GetInstance();
61 MS_EXCEPTION_IF_NULL(ms_context);
62 device_id_ = ms_context->get_param<uint32_t>(MS_CTX_DEVICE_ID);
63 runtime_instance_ = device::KernelRuntimeManager::Instance().GetKernelRuntime(kAscendDevice, device_id_);
64 MS_EXCEPTION_IF_NULL(runtime_instance_);
65 }
66
InitDmaMem()67 void AscendDmaHandle::InitDmaMem() {
68 #if defined(RT_MEMORY_P2PDMA)
69 uint16_t app_module_id = static_cast<uint16_t>(APP);
70 auto ret = CALL_ASCEND_API(aclrtSetDevice, device_id_);
71 if (ret != ACL_ERROR_NONE) {
72 MS_LOG(EXCEPTION) << "aclrtSetDevice failed:" << ret;
73 }
74 ret = CALL_ASCEND_API(aclrtGetMemInfo, ACL_HBM_MEM, &device_hbm_free_size_, &device_hbm_total_size_);
75 MS_LOG(INFO) << "InitDmaMem device_hbm_free_size_:" << device_hbm_free_size_
76 << ", device_hbm_total_size_:" << device_hbm_total_size_;
77 if (ret != ACL_ERROR_NONE) {
78 MS_LOG(EXCEPTION) << "rtMemGetInfo failed:" << ret;
79 }
80 ret = rtMalloc(&dargs_, hbm_alloc_size_, RT_MEMORY_P2PDMA, app_module_id);
81 if (ret != ACL_ERROR_NONE) {
82 MS_LOG(EXCEPTION) << "rtMalloc failed:" << ret;
83 }
84 ret = CALL_ASCEND_API(aclrtMemset, dargs_, hbm_alloc_size_, 0x44, hbm_alloc_size_);
85 if (ret != ACL_ERROR_NONE) {
86 MS_LOG(EXCEPTION) << "aclrtMemset failed:" << ret;
87 }
88 std::string p2p_device_name = "/dev/p2pdma" + std::to_string(device_id_);
89 p2p_fd_ = open(p2p_device_name.c_str(), O_RDWR);
90 if (p2p_fd_ < 0) {
91 MS_LOG(EXCEPTION) << "Open device failed";
92 }
93 buf_ = mmap(nullptr, hbm_alloc_size_, PROT_READ | PROT_WRITE, MAP_SHARED, p2p_fd_, 0);
94 if (!buf_) {
95 MS_LOG(EXCEPTION) << "Fail to mmap";
96 }
97 MS_LOG(INFO) << "AscendDmaHandle mmap success device prt:" << dargs_ << " buffer ptr:" << buf_;
98 #endif
99 }
100 } // namespace ascend
101 } // namespace device
102 } // namespace mindspore
103