1 //===-------- interface.cpp - Target independent OpenMP target RTL --------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // Implementation of the interface to be used by Clang during the codegen of a
12 //===----------------------------------------------------------------------===//
23 ////////////////////////////////////////////////////////////////////////////////
24 /// manage the success or failure of a target construct
25 static void HandleDefaultTargetOffload() {
26 PM->TargetOffloadMtx.lock();
27 if (PM->TargetOffloadPolicy == tgt_default) {
28 if (omp_get_num_devices() > 0) {
29 DP("Default TARGET OFFLOAD policy is now mandatory "
30 "(devices were found)\n");
31 PM->TargetOffloadPolicy = tgt_mandatory;
33 DP("Default TARGET OFFLOAD policy is now disabled "
34 "(no devices were found)\n");
35 PM->TargetOffloadPolicy = tgt_disabled;
38 PM->TargetOffloadMtx.unlock();
41 static int IsOffloadDisabled() {
42 if (PM->TargetOffloadPolicy == tgt_default)
43 HandleDefaultTargetOffload();
44 return PM->TargetOffloadPolicy == tgt_disabled;
47 static void HandleTargetOutcome(bool success, ident_t *loc = nullptr) {
48 switch (PM->TargetOffloadPolicy) {
51 FATAL_MESSAGE0(1, "expected no offloading while offloading is disabled");
55 FATAL_MESSAGE0(1, "default offloading policy must be switched to "
56 "mandatory or disabled");
60 if (getInfoLevel() & OMP_INFOTYPE_DUMP_TABLE)
61 for (const auto &Device : PM->Devices)
62 dumpTargetPointerMappings(loc, Device);
64 FAILURE_MESSAGE("Run with LIBOMPTARGET_DEBUG=%d to dump host-target "
65 "pointer mappings.\n",
66 OMP_INFOTYPE_DUMP_TABLE);
69 if (info.isAvailible())
70 fprintf(stderr, "%s:%d:%d: ", info.getFilename(), info.getLine(),
73 FAILURE_MESSAGE("Source location information not present. Compile with "
74 "-g or -gline-tables-only.\n");
76 1, "failure of target construct while offloading is mandatory");
78 if (getInfoLevel() & OMP_INFOTYPE_DUMP_TABLE)
79 for (const auto &Device : PM->Devices)
80 dumpTargetPointerMappings(loc, Device);
86 ////////////////////////////////////////////////////////////////////////////////
87 /// adds requires flags
88 EXTERN void __tgt_register_requires(int64_t flags) {
90 PM->RTLs.RegisterRequires(flags);
93 ////////////////////////////////////////////////////////////////////////////////
94 /// adds a target shared library to the target execution image
95 EXTERN void __tgt_register_lib(__tgt_bin_desc *desc) {
97 PM->RTLs.RegisterLib(desc);
100 ////////////////////////////////////////////////////////////////////////////////
101 /// unloads a target shared library
102 EXTERN void __tgt_unregister_lib(__tgt_bin_desc *desc) {
104 PM->RTLs.UnregisterLib(desc);
107 /// creates host-to-target data mapping, stores it in the
108 /// libomptarget.so internal structure (an entry in a stack of data maps)
109 /// and passes the data to the device.
110 EXTERN void __tgt_target_data_begin(int64_t device_id, int32_t arg_num,
111 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
113 __tgt_target_data_begin_mapper(nullptr, device_id, arg_num, args_base, args,
114 arg_sizes, arg_types, nullptr, nullptr);
117 EXTERN void __tgt_target_data_begin_nowait(int64_t device_id, int32_t arg_num,
118 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
119 int32_t depNum, void *depList, int32_t noAliasDepNum,
120 void *noAliasDepList) {
122 if (depNum + noAliasDepNum > 0)
123 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
125 __tgt_target_data_begin_mapper(nullptr, device_id, arg_num, args_base, args,
126 arg_sizes, arg_types, nullptr, nullptr);
129 EXTERN void __tgt_target_data_begin_mapper(ident_t *loc, int64_t device_id,
130 int32_t arg_num, void **args_base,
131 void **args, int64_t *arg_sizes,
133 map_var_info_t *arg_names,
134 void **arg_mappers) {
136 if (IsOffloadDisabled()) return;
138 DP("Entering data begin region for device %" PRId64 " with %d mappings\n",
141 // No devices available?
142 if (device_id == OFFLOAD_DEVICE_DEFAULT) {
143 device_id = omp_get_default_device();
144 DP("Use default device id %" PRId64 "\n", device_id);
147 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
148 DP("Failed to get device %" PRId64 " ready\n", device_id);
149 HandleTargetOutcome(false, loc);
153 DeviceTy &Device = PM->Devices[device_id];
155 if (getInfoLevel() & OMP_INFOTYPE_KERNEL_ARGS)
156 printKernelArguments(loc, device_id, arg_num, arg_sizes, arg_types,
157 arg_names, "Entering OpenMP data region");
158 #ifdef OMPTARGET_DEBUG
159 for (int i = 0; i < arg_num; ++i) {
160 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
161 ", Type=0x%" PRIx64 ", Name=%s\n",
162 i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
163 (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "unknown");
167 int rc = targetDataBegin(Device, arg_num, args_base, args, arg_sizes,
168 arg_types, arg_names, arg_mappers, nullptr);
169 HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
172 EXTERN void __tgt_target_data_begin_nowait_mapper(
173 ident_t *loc, int64_t device_id, int32_t arg_num, void **args_base,
174 void **args, int64_t *arg_sizes, int64_t *arg_types,
175 map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
176 void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
178 if (depNum + noAliasDepNum > 0)
179 __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
181 __tgt_target_data_begin_mapper(loc, device_id, arg_num, args_base, args,
182 arg_sizes, arg_types, arg_names, arg_mappers);
185 /// passes data from the target, releases target memory and destroys
186 /// the host-target mapping (top entry from the stack of data maps)
187 /// created by the last __tgt_target_data_begin.
188 EXTERN void __tgt_target_data_end(int64_t device_id, int32_t arg_num,
189 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
191 __tgt_target_data_end_mapper(nullptr, device_id, arg_num, args_base, args,
192 arg_sizes, arg_types, nullptr, nullptr);
195 EXTERN void __tgt_target_data_end_nowait(int64_t device_id, int32_t arg_num,
196 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
197 int32_t depNum, void *depList, int32_t noAliasDepNum,
198 void *noAliasDepList) {
200 if (depNum + noAliasDepNum > 0)
201 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
203 __tgt_target_data_end_mapper(nullptr, device_id, arg_num, args_base, args,
204 arg_sizes, arg_types, nullptr, nullptr);
207 EXTERN void __tgt_target_data_end_mapper(ident_t *loc, int64_t device_id,
208 int32_t arg_num, void **args_base,
209 void **args, int64_t *arg_sizes,
211 map_var_info_t *arg_names,
212 void **arg_mappers) {
214 if (IsOffloadDisabled()) return;
215 DP("Entering data end region with %d mappings\n", arg_num);
217 // No devices available?
218 if (device_id == OFFLOAD_DEVICE_DEFAULT) {
219 device_id = omp_get_default_device();
223 size_t DevicesSize = PM->Devices.size();
224 PM->RTLsMtx.unlock();
225 if (DevicesSize <= (size_t)device_id) {
226 DP("Device ID %" PRId64 " does not have a matching RTL.\n", device_id);
227 HandleTargetOutcome(false, loc);
231 DeviceTy &Device = PM->Devices[device_id];
232 if (!Device.IsInit) {
233 DP("Uninit device: ignore");
234 HandleTargetOutcome(false, loc);
238 if (getInfoLevel() & OMP_INFOTYPE_KERNEL_ARGS)
239 printKernelArguments(loc, device_id, arg_num, arg_sizes, arg_types,
240 arg_names, "Exiting OpenMP data region");
241 #ifdef OMPTARGET_DEBUG
242 for (int i=0; i<arg_num; ++i) {
243 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
244 ", Type=0x%" PRIx64 ", Name=%s\n",
245 i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
246 (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "unknown");
250 int rc = targetDataEnd(Device, arg_num, args_base, args, arg_sizes, arg_types,
251 arg_names, arg_mappers, nullptr);
252 HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
255 EXTERN void __tgt_target_data_end_nowait_mapper(
256 ident_t *loc, int64_t device_id, int32_t arg_num, void **args_base,
257 void **args, int64_t *arg_sizes, int64_t *arg_types,
258 map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
259 void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
261 if (depNum + noAliasDepNum > 0)
262 __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
264 __tgt_target_data_end_mapper(loc, device_id, arg_num, args_base, args,
265 arg_sizes, arg_types, arg_names, arg_mappers);
268 EXTERN void __tgt_target_data_update(int64_t device_id, int32_t arg_num,
269 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
271 __tgt_target_data_update_mapper(nullptr, device_id, arg_num, args_base, args,
272 arg_sizes, arg_types, nullptr, nullptr);
275 EXTERN void __tgt_target_data_update_nowait(int64_t device_id, int32_t arg_num,
276 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
277 int32_t depNum, void *depList, int32_t noAliasDepNum,
278 void *noAliasDepList) {
280 if (depNum + noAliasDepNum > 0)
281 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
283 __tgt_target_data_update_mapper(nullptr, device_id, arg_num, args_base, args,
284 arg_sizes, arg_types, nullptr, nullptr);
287 EXTERN void __tgt_target_data_update_mapper(ident_t *loc, int64_t device_id,
288 int32_t arg_num, void **args_base,
289 void **args, int64_t *arg_sizes,
291 map_var_info_t *arg_names,
292 void **arg_mappers) {
294 if (IsOffloadDisabled()) return;
295 DP("Entering data update with %d mappings\n", arg_num);
297 // No devices available?
298 if (device_id == OFFLOAD_DEVICE_DEFAULT) {
299 device_id = omp_get_default_device();
302 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
303 DP("Failed to get device %" PRId64 " ready\n", device_id);
304 HandleTargetOutcome(false, loc);
308 if (getInfoLevel() & OMP_INFOTYPE_KERNEL_ARGS)
309 printKernelArguments(loc, device_id, arg_num, arg_sizes, arg_types,
310 arg_names, "Updating OpenMP data");
312 DeviceTy &Device = PM->Devices[device_id];
313 int rc = targetDataUpdate(Device, arg_num, args_base, args, arg_sizes,
314 arg_types, arg_names, arg_mappers);
315 HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
318 EXTERN void __tgt_target_data_update_nowait_mapper(
319 ident_t *loc, int64_t device_id, int32_t arg_num, void **args_base,
320 void **args, int64_t *arg_sizes, int64_t *arg_types,
321 map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
322 void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
324 if (depNum + noAliasDepNum > 0)
325 __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
327 __tgt_target_data_update_mapper(loc, device_id, arg_num, args_base, args,
328 arg_sizes, arg_types, arg_names, arg_mappers);
331 EXTERN int __tgt_target(int64_t device_id, void *host_ptr, int32_t arg_num,
332 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types) {
334 return __tgt_target_mapper(nullptr, device_id, host_ptr, arg_num, args_base,
335 args, arg_sizes, arg_types, nullptr, nullptr);
338 EXTERN int __tgt_target_nowait(int64_t device_id, void *host_ptr,
339 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
340 int64_t *arg_types, int32_t depNum, void *depList, int32_t noAliasDepNum,
341 void *noAliasDepList) {
343 if (depNum + noAliasDepNum > 0)
344 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
346 return __tgt_target_mapper(nullptr, device_id, host_ptr, arg_num, args_base,
347 args, arg_sizes, arg_types, nullptr, nullptr);
350 EXTERN int __tgt_target_mapper(ident_t *loc, int64_t device_id, void *host_ptr,
351 int32_t arg_num, void **args_base, void **args,
352 int64_t *arg_sizes, int64_t *arg_types,
353 map_var_info_t *arg_names, void **arg_mappers) {
355 if (IsOffloadDisabled()) return OFFLOAD_FAIL;
356 DP("Entering target region with entry point " DPxMOD " and device Id %"
357 PRId64 "\n", DPxPTR(host_ptr), device_id);
359 if (device_id == OFFLOAD_DEVICE_DEFAULT) {
360 device_id = omp_get_default_device();
363 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
364 REPORT("Failed to get device %" PRId64 " ready\n", device_id);
365 HandleTargetOutcome(false, loc);
369 if (getInfoLevel() & OMP_INFOTYPE_KERNEL_ARGS)
370 printKernelArguments(loc, device_id, arg_num, arg_sizes, arg_types,
371 arg_names, "Entering OpenMP kernel");
372 #ifdef OMPTARGET_DEBUG
373 for (int i=0; i<arg_num; ++i) {
374 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
375 ", Type=0x%" PRIx64 ", Name=%s\n",
376 i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
377 (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "unknown");
381 int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
382 arg_types, arg_names, arg_mappers, 0, 0, false /*team*/);
383 HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
387 EXTERN int __tgt_target_nowait_mapper(
388 ident_t *loc, int64_t device_id, void *host_ptr, int32_t arg_num,
389 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
390 map_var_info_t *arg_names, void **arg_mappers, int32_t depNum,
391 void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
393 if (depNum + noAliasDepNum > 0)
394 __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
396 return __tgt_target_mapper(loc, device_id, host_ptr, arg_num, args_base, args,
397 arg_sizes, arg_types, arg_names, arg_mappers);
400 EXTERN int __tgt_target_teams(int64_t device_id, void *host_ptr,
401 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
402 int64_t *arg_types, int32_t team_num, int32_t thread_limit) {
404 return __tgt_target_teams_mapper(nullptr, device_id, host_ptr, arg_num,
405 args_base, args, arg_sizes, arg_types,
406 nullptr, nullptr, team_num, thread_limit);
409 EXTERN int __tgt_target_teams_nowait(int64_t device_id, void *host_ptr,
410 int32_t arg_num, void **args_base, void **args, int64_t *arg_sizes,
411 int64_t *arg_types, int32_t team_num, int32_t thread_limit, int32_t depNum,
412 void *depList, int32_t noAliasDepNum, void *noAliasDepList) {
414 if (depNum + noAliasDepNum > 0)
415 __kmpc_omp_taskwait(NULL, __kmpc_global_thread_num(NULL));
417 return __tgt_target_teams_mapper(nullptr, device_id, host_ptr, arg_num,
418 args_base, args, arg_sizes, arg_types,
419 nullptr, nullptr, team_num, thread_limit);
422 EXTERN int __tgt_target_teams_mapper(ident_t *loc, int64_t device_id,
423 void *host_ptr, int32_t arg_num,
424 void **args_base, void **args,
425 int64_t *arg_sizes, int64_t *arg_types,
426 map_var_info_t *arg_names,
427 void **arg_mappers, int32_t team_num,
428 int32_t thread_limit) {
430 if (IsOffloadDisabled()) return OFFLOAD_FAIL;
431 DP("Entering target region with entry point " DPxMOD " and device Id %"
432 PRId64 "\n", DPxPTR(host_ptr), device_id);
434 if (device_id == OFFLOAD_DEVICE_DEFAULT) {
435 device_id = omp_get_default_device();
438 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
439 REPORT("Failed to get device %" PRId64 " ready\n", device_id);
440 HandleTargetOutcome(false, loc);
444 if (getInfoLevel() & OMP_INFOTYPE_KERNEL_ARGS)
445 printKernelArguments(loc, device_id, arg_num, arg_sizes, arg_types,
446 arg_names, "Entering OpenMP kernel");
447 #ifdef OMPTARGET_DEBUG
448 for (int i=0; i<arg_num; ++i) {
449 DP("Entry %2d: Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
450 ", Type=0x%" PRIx64 ", Name=%s\n",
451 i, DPxPTR(args_base[i]), DPxPTR(args[i]), arg_sizes[i], arg_types[i],
452 (arg_names) ? getNameFromMapping(arg_names[i]).c_str() : "unknown");
456 int rc = target(device_id, host_ptr, arg_num, args_base, args, arg_sizes,
457 arg_types, arg_names, arg_mappers, team_num, thread_limit,
459 HandleTargetOutcome(rc == OFFLOAD_SUCCESS, loc);
463 EXTERN int __tgt_target_teams_nowait_mapper(
464 ident_t *loc, int64_t device_id, void *host_ptr, int32_t arg_num,
465 void **args_base, void **args, int64_t *arg_sizes, int64_t *arg_types,
466 map_var_info_t *arg_names, void **arg_mappers, int32_t team_num,
467 int32_t thread_limit, int32_t depNum, void *depList, int32_t noAliasDepNum,
468 void *noAliasDepList) {
470 if (depNum + noAliasDepNum > 0)
471 __kmpc_omp_taskwait(loc, __kmpc_global_thread_num(loc));
473 return __tgt_target_teams_mapper(loc, device_id, host_ptr, arg_num, args_base,
474 args, arg_sizes, arg_types, arg_names,
475 arg_mappers, team_num, thread_limit);
478 // Get the current number of components for a user-defined mapper.
479 EXTERN int64_t __tgt_mapper_num_components(void *rt_mapper_handle) {
481 auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
482 int64_t size = MapperComponentsPtr->Components.size();
483 DP("__tgt_mapper_num_components(Handle=" DPxMOD ") returns %" PRId64 "\n",
484 DPxPTR(rt_mapper_handle), size);
488 // Push back one component for a user-defined mapper.
489 EXTERN void __tgt_push_mapper_component(void *rt_mapper_handle, void *base,
490 void *begin, int64_t size, int64_t type,
493 DP("__tgt_push_mapper_component(Handle=" DPxMOD
494 ") adds an entry (Base=" DPxMOD ", Begin=" DPxMOD ", Size=%" PRId64
495 ", Type=0x%" PRIx64 ", Name=%s).\n",
496 DPxPTR(rt_mapper_handle), DPxPTR(base), DPxPTR(begin), size, type,
497 (name) ? getNameFromMapping(name).c_str() : "unknown");
498 auto *MapperComponentsPtr = (struct MapperComponentsTy *)rt_mapper_handle;
499 MapperComponentsPtr->Components.push_back(
500 MapComponentInfoTy(base, begin, size, type, name));
503 EXTERN void __kmpc_push_target_tripcount(ident_t *loc, int64_t device_id,
504 uint64_t loop_tripcount) {
506 if (IsOffloadDisabled())
509 if (device_id == OFFLOAD_DEVICE_DEFAULT) {
510 device_id = omp_get_default_device();
513 if (CheckDeviceAndCtors(device_id) != OFFLOAD_SUCCESS) {
514 DP("Failed to get device %" PRId64 " ready\n", device_id);
515 HandleTargetOutcome(false, loc);
519 DP("__kmpc_push_target_tripcount(%" PRId64 ", %" PRIu64 ")\n", device_id,
521 PM->TblMapMtx.lock();
522 PM->Devices[device_id].LoopTripCnt.emplace(__kmpc_global_thread_num(NULL),
524 PM->TblMapMtx.unlock();