1 //===----------- api.cpp - Target independent OpenMP target RTL -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implementation of OpenMP API interface functions.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "device.h"
14 #include "private.h"
15 #include "rtl.h"
16
17 #include <climits>
18 #include <cstring>
19 #include <cstdlib>
20
omp_get_num_devices(void)21 EXTERN int omp_get_num_devices(void) {
22 PM->RTLsMtx.lock();
23 size_t DevicesSize = PM->Devices.size();
24 PM->RTLsMtx.unlock();
25
26 DP("Call to omp_get_num_devices returning %zd\n", DevicesSize);
27
28 return DevicesSize;
29 }
30
omp_get_initial_device(void)31 EXTERN int omp_get_initial_device(void) {
32 int hostDevice = omp_get_num_devices();
33 DP("Call to omp_get_initial_device returning %d\n", hostDevice);
34 return hostDevice;
35 }
36
omp_target_alloc(size_t size,int device_num)37 EXTERN void *omp_target_alloc(size_t size, int device_num) {
38 DP("Call to omp_target_alloc for device %d requesting %zu bytes\n",
39 device_num, size);
40
41 if (size <= 0) {
42 DP("Call to omp_target_alloc with non-positive length\n");
43 return NULL;
44 }
45
46 void *rc = NULL;
47
48 if (device_num == omp_get_initial_device()) {
49 rc = malloc(size);
50 DP("omp_target_alloc returns host ptr " DPxMOD "\n", DPxPTR(rc));
51 return rc;
52 }
53
54 if (!device_is_ready(device_num)) {
55 DP("omp_target_alloc returns NULL ptr\n");
56 return NULL;
57 }
58
59 rc = PM->Devices[device_num].allocData(size);
60 DP("omp_target_alloc returns device ptr " DPxMOD "\n", DPxPTR(rc));
61 return rc;
62 }
63
omp_target_free(void * device_ptr,int device_num)64 EXTERN void omp_target_free(void *device_ptr, int device_num) {
65 DP("Call to omp_target_free for device %d and address " DPxMOD "\n",
66 device_num, DPxPTR(device_ptr));
67
68 if (!device_ptr) {
69 DP("Call to omp_target_free with NULL ptr\n");
70 return;
71 }
72
73 if (device_num == omp_get_initial_device()) {
74 free(device_ptr);
75 DP("omp_target_free deallocated host ptr\n");
76 return;
77 }
78
79 if (!device_is_ready(device_num)) {
80 DP("omp_target_free returns, nothing to do\n");
81 return;
82 }
83
84 PM->Devices[device_num].deleteData(device_ptr);
85 DP("omp_target_free deallocated device ptr\n");
86 }
87
omp_target_is_present(void * ptr,int device_num)88 EXTERN int omp_target_is_present(void *ptr, int device_num) {
89 DP("Call to omp_target_is_present for device %d and address " DPxMOD "\n",
90 device_num, DPxPTR(ptr));
91
92 if (!ptr) {
93 DP("Call to omp_target_is_present with NULL ptr, returning false\n");
94 return false;
95 }
96
97 if (device_num == omp_get_initial_device()) {
98 DP("Call to omp_target_is_present on host, returning true\n");
99 return true;
100 }
101
102 PM->RTLsMtx.lock();
103 size_t DevicesSize = PM->Devices.size();
104 PM->RTLsMtx.unlock();
105 if (DevicesSize <= (size_t)device_num) {
106 DP("Call to omp_target_is_present with invalid device ID, returning "
107 "false\n");
108 return false;
109 }
110
111 DeviceTy &Device = PM->Devices[device_num];
112 bool IsLast; // not used
113 bool IsHostPtr;
114 void *TgtPtr = Device.getTgtPtrBegin(ptr, 0, IsLast, false, IsHostPtr);
115 int rc = (TgtPtr != NULL);
116 // Under unified memory the host pointer can be returned by the
117 // getTgtPtrBegin() function which means that there is no device
118 // corresponding point for ptr. This function should return false
119 // in that situation.
120 if (PM->RTLs.RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY)
121 rc = !IsHostPtr;
122 DP("Call to omp_target_is_present returns %d\n", rc);
123 return rc;
124 }
125
omp_target_memcpy(void * dst,void * src,size_t length,size_t dst_offset,size_t src_offset,int dst_device,int src_device)126 EXTERN int omp_target_memcpy(void *dst, void *src, size_t length,
127 size_t dst_offset, size_t src_offset, int dst_device, int src_device) {
128 DP("Call to omp_target_memcpy, dst device %d, src device %d, "
129 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offset %zu, "
130 "src offset %zu, length %zu\n", dst_device, src_device, DPxPTR(dst),
131 DPxPTR(src), dst_offset, src_offset, length);
132
133 if (!dst || !src || length <= 0) {
134 REPORT("Call to omp_target_memcpy with invalid arguments\n");
135 return OFFLOAD_FAIL;
136 }
137
138 if (src_device != omp_get_initial_device() && !device_is_ready(src_device)) {
139 REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
140 return OFFLOAD_FAIL;
141 }
142
143 if (dst_device != omp_get_initial_device() && !device_is_ready(dst_device)) {
144 REPORT("omp_target_memcpy returns OFFLOAD_FAIL\n");
145 return OFFLOAD_FAIL;
146 }
147
148 int rc = OFFLOAD_SUCCESS;
149 void *srcAddr = (char *)src + src_offset;
150 void *dstAddr = (char *)dst + dst_offset;
151
152 if (src_device == omp_get_initial_device() &&
153 dst_device == omp_get_initial_device()) {
154 DP("copy from host to host\n");
155 const void *p = memcpy(dstAddr, srcAddr, length);
156 if (p == NULL)
157 rc = OFFLOAD_FAIL;
158 } else if (src_device == omp_get_initial_device()) {
159 DP("copy from host to device\n");
160 DeviceTy &DstDev = PM->Devices[dst_device];
161 rc = DstDev.submitData(dstAddr, srcAddr, length, nullptr);
162 } else if (dst_device == omp_get_initial_device()) {
163 DP("copy from device to host\n");
164 DeviceTy &SrcDev = PM->Devices[src_device];
165 rc = SrcDev.retrieveData(dstAddr, srcAddr, length, nullptr);
166 } else {
167 DP("copy from device to device\n");
168 DeviceTy &SrcDev = PM->Devices[src_device];
169 DeviceTy &DstDev = PM->Devices[dst_device];
170 // First try to use D2D memcpy which is more efficient. If fails, fall back
171 // to unefficient way.
172 if (SrcDev.isDataExchangable(DstDev)) {
173 rc = SrcDev.dataExchange(srcAddr, DstDev, dstAddr, length, nullptr);
174 if (rc == OFFLOAD_SUCCESS)
175 return OFFLOAD_SUCCESS;
176 }
177
178 void *buffer = malloc(length);
179 rc = SrcDev.retrieveData(buffer, srcAddr, length, nullptr);
180 if (rc == OFFLOAD_SUCCESS)
181 rc = DstDev.submitData(dstAddr, buffer, length, nullptr);
182 free(buffer);
183 }
184
185 DP("omp_target_memcpy returns %d\n", rc);
186 return rc;
187 }
188
omp_target_memcpy_rect(void * dst,void * src,size_t element_size,int num_dims,const size_t * volume,const size_t * dst_offsets,const size_t * src_offsets,const size_t * dst_dimensions,const size_t * src_dimensions,int dst_device,int src_device)189 EXTERN int omp_target_memcpy_rect(void *dst, void *src, size_t element_size,
190 int num_dims, const size_t *volume, const size_t *dst_offsets,
191 const size_t *src_offsets, const size_t *dst_dimensions,
192 const size_t *src_dimensions, int dst_device, int src_device) {
193 DP("Call to omp_target_memcpy_rect, dst device %d, src device %d, "
194 "dst addr " DPxMOD ", src addr " DPxMOD ", dst offsets " DPxMOD ", "
195 "src offsets " DPxMOD ", dst dims " DPxMOD ", src dims " DPxMOD ", "
196 "volume " DPxMOD ", element size %zu, num_dims %d\n", dst_device,
197 src_device, DPxPTR(dst), DPxPTR(src), DPxPTR(dst_offsets),
198 DPxPTR(src_offsets), DPxPTR(dst_dimensions), DPxPTR(src_dimensions),
199 DPxPTR(volume), element_size, num_dims);
200
201 if (!(dst || src)) {
202 DP("Call to omp_target_memcpy_rect returns max supported dimensions %d\n",
203 INT_MAX);
204 return INT_MAX;
205 }
206
207 if (!dst || !src || element_size < 1 || num_dims < 1 || !volume ||
208 !dst_offsets || !src_offsets || !dst_dimensions || !src_dimensions) {
209 REPORT("Call to omp_target_memcpy_rect with invalid arguments\n");
210 return OFFLOAD_FAIL;
211 }
212
213 int rc;
214 if (num_dims == 1) {
215 rc = omp_target_memcpy(dst, src, element_size * volume[0],
216 element_size * dst_offsets[0], element_size * src_offsets[0],
217 dst_device, src_device);
218 } else {
219 size_t dst_slice_size = element_size;
220 size_t src_slice_size = element_size;
221 for (int i=1; i<num_dims; ++i) {
222 dst_slice_size *= dst_dimensions[i];
223 src_slice_size *= src_dimensions[i];
224 }
225
226 size_t dst_off = dst_offsets[0] * dst_slice_size;
227 size_t src_off = src_offsets[0] * src_slice_size;
228 for (size_t i=0; i<volume[0]; ++i) {
229 rc = omp_target_memcpy_rect((char *) dst + dst_off + dst_slice_size * i,
230 (char *) src + src_off + src_slice_size * i, element_size,
231 num_dims - 1, volume + 1, dst_offsets + 1, src_offsets + 1,
232 dst_dimensions + 1, src_dimensions + 1, dst_device, src_device);
233
234 if (rc) {
235 DP("Recursive call to omp_target_memcpy_rect returns unsuccessfully\n");
236 return rc;
237 }
238 }
239 }
240
241 DP("omp_target_memcpy_rect returns %d\n", rc);
242 return rc;
243 }
244
omp_target_associate_ptr(void * host_ptr,void * device_ptr,size_t size,size_t device_offset,int device_num)245 EXTERN int omp_target_associate_ptr(void *host_ptr, void *device_ptr,
246 size_t size, size_t device_offset, int device_num) {
247 DP("Call to omp_target_associate_ptr with host_ptr " DPxMOD ", "
248 "device_ptr " DPxMOD ", size %zu, device_offset %zu, device_num %d\n",
249 DPxPTR(host_ptr), DPxPTR(device_ptr), size, device_offset, device_num);
250
251 if (!host_ptr || !device_ptr || size <= 0) {
252 REPORT("Call to omp_target_associate_ptr with invalid arguments\n");
253 return OFFLOAD_FAIL;
254 }
255
256 if (device_num == omp_get_initial_device()) {
257 REPORT("omp_target_associate_ptr: no association possible on the host\n");
258 return OFFLOAD_FAIL;
259 }
260
261 if (!device_is_ready(device_num)) {
262 REPORT("omp_target_associate_ptr returns OFFLOAD_FAIL\n");
263 return OFFLOAD_FAIL;
264 }
265
266 DeviceTy &Device = PM->Devices[device_num];
267 void *device_addr = (void *)((uint64_t)device_ptr + (uint64_t)device_offset);
268 int rc = Device.associatePtr(host_ptr, device_addr, size);
269 DP("omp_target_associate_ptr returns %d\n", rc);
270 return rc;
271 }
272
omp_target_disassociate_ptr(void * host_ptr,int device_num)273 EXTERN int omp_target_disassociate_ptr(void *host_ptr, int device_num) {
274 DP("Call to omp_target_disassociate_ptr with host_ptr " DPxMOD ", "
275 "device_num %d\n", DPxPTR(host_ptr), device_num);
276
277 if (!host_ptr) {
278 REPORT("Call to omp_target_associate_ptr with invalid host_ptr\n");
279 return OFFLOAD_FAIL;
280 }
281
282 if (device_num == omp_get_initial_device()) {
283 REPORT(
284 "omp_target_disassociate_ptr: no association possible on the host\n");
285 return OFFLOAD_FAIL;
286 }
287
288 if (!device_is_ready(device_num)) {
289 REPORT("omp_target_disassociate_ptr returns OFFLOAD_FAIL\n");
290 return OFFLOAD_FAIL;
291 }
292
293 DeviceTy &Device = PM->Devices[device_num];
294 int rc = Device.disassociatePtr(host_ptr);
295 DP("omp_target_disassociate_ptr returns %d\n", rc);
296 return rc;
297 }
298