1 //===------- interface.h - OpenMP interface definitions ---------- CUDA -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains all the definitions that are relevant to 10 // the interface. The first section contains the interface as 11 // declared by OpenMP. The second section includes the compiler 12 // specific interfaces. 13 // 14 //===----------------------------------------------------------------------===// 15 16 #ifndef _INTERFACES_H_ 17 #define _INTERFACES_H_ 18 19 #include <stddef.h> 20 #include <stdint.h> 21 22 #ifdef __AMDGCN__ 23 #include "amdgcn/src/amdgcn_interface.h" 24 #endif 25 #ifdef __CUDACC__ 26 #include "nvptx/src/nvptx_interface.h" 27 #endif 28 29 //////////////////////////////////////////////////////////////////////////////// 30 // OpenMP interface 31 //////////////////////////////////////////////////////////////////////////////// 32 33 typedef uint64_t omp_nest_lock_t; /* arbitrary type of the right length */ 34 35 typedef enum omp_sched_t { 36 omp_sched_static = 1, /* chunkSize >0 */ 37 omp_sched_dynamic = 2, /* chunkSize >0 */ 38 omp_sched_guided = 3, /* chunkSize >0 */ 39 omp_sched_auto = 4, /* no chunkSize */ 40 } omp_sched_t; 41 42 typedef enum omp_proc_bind_t { 43 omp_proc_bind_false = 0, 44 omp_proc_bind_true = 1, 45 omp_proc_bind_master = 2, 46 omp_proc_bind_close = 3, 47 omp_proc_bind_spread = 4 48 } omp_proc_bind_t; 49 50 EXTERN double omp_get_wtick(void); 51 EXTERN double omp_get_wtime(void); 52 53 EXTERN void omp_set_num_threads(int num); 54 EXTERN int omp_get_num_threads(void); 55 EXTERN int omp_get_max_threads(void); 56 EXTERN int omp_get_thread_limit(void); 57 EXTERN int omp_get_thread_num(void); 58 EXTERN int omp_get_num_procs(void); 59 EXTERN int omp_in_parallel(void); 60 EXTERN int omp_in_final(void); 61 EXTERN void omp_set_dynamic(int flag); 62 EXTERN int omp_get_dynamic(void); 63 EXTERN void omp_set_nested(int flag); 64 EXTERN int omp_get_nested(void); 65 EXTERN void omp_set_max_active_levels(int level); 66 EXTERN int omp_get_max_active_levels(void); 67 EXTERN int omp_get_level(void); 68 EXTERN int omp_get_active_level(void); 69 EXTERN int omp_get_ancestor_thread_num(int level); 70 EXTERN int omp_get_team_size(int level); 71 72 EXTERN void omp_init_lock(omp_lock_t *lock); 73 EXTERN void omp_init_nest_lock(omp_nest_lock_t *lock); 74 EXTERN void omp_destroy_lock(omp_lock_t *lock); 75 EXTERN void omp_destroy_nest_lock(omp_nest_lock_t *lock); 76 EXTERN void omp_set_lock(omp_lock_t *lock); 77 EXTERN void omp_set_nest_lock(omp_nest_lock_t *lock); 78 EXTERN void omp_unset_lock(omp_lock_t *lock); 79 EXTERN void omp_unset_nest_lock(omp_nest_lock_t *lock); 80 EXTERN int omp_test_lock(omp_lock_t *lock); 81 EXTERN int omp_test_nest_lock(omp_nest_lock_t *lock); 82 83 EXTERN void omp_get_schedule(omp_sched_t *kind, int *modifier); 84 EXTERN void omp_set_schedule(omp_sched_t kind, int modifier); 85 EXTERN omp_proc_bind_t omp_get_proc_bind(void); 86 EXTERN int omp_get_cancellation(void); 87 EXTERN void omp_set_default_device(int deviceId); 88 EXTERN int omp_get_default_device(void); 89 EXTERN int omp_get_num_devices(void); 90 EXTERN int omp_get_num_teams(void); 91 EXTERN int omp_get_team_num(void); 92 EXTERN int omp_is_initial_device(void); 93 EXTERN int omp_get_initial_device(void); 94 EXTERN int omp_get_max_task_priority(void); 95 96 //////////////////////////////////////////////////////////////////////////////// 97 // file below is swiped from kmpc host interface 98 //////////////////////////////////////////////////////////////////////////////// 99 100 //////////////////////////////////////////////////////////////////////////////// 101 // kmp specific types 102 //////////////////////////////////////////////////////////////////////////////// 103 104 typedef enum kmp_sched_t { 105 kmp_sched_static_chunk = 33, 106 kmp_sched_static_nochunk = 34, 107 kmp_sched_dynamic = 35, 108 kmp_sched_guided = 36, 109 kmp_sched_runtime = 37, 110 kmp_sched_auto = 38, 111 112 kmp_sched_static_balanced_chunk = 45, 113 114 kmp_sched_static_ordered = 65, 115 kmp_sched_static_nochunk_ordered = 66, 116 kmp_sched_dynamic_ordered = 67, 117 kmp_sched_guided_ordered = 68, 118 kmp_sched_runtime_ordered = 69, 119 kmp_sched_auto_ordered = 70, 120 121 kmp_sched_distr_static_chunk = 91, 122 kmp_sched_distr_static_nochunk = 92, 123 kmp_sched_distr_static_chunk_sched_static_chunkone = 93, 124 125 kmp_sched_default = kmp_sched_static_nochunk, 126 kmp_sched_unordered_first = kmp_sched_static_chunk, 127 kmp_sched_unordered_last = kmp_sched_auto, 128 kmp_sched_ordered_first = kmp_sched_static_ordered, 129 kmp_sched_ordered_last = kmp_sched_auto_ordered, 130 kmp_sched_distribute_first = kmp_sched_distr_static_chunk, 131 kmp_sched_distribute_last = 132 kmp_sched_distr_static_chunk_sched_static_chunkone, 133 134 /* Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers. 135 * Since we need to distinguish the three possible cases (no modifier, 136 * monotonic modifier, nonmonotonic modifier), we need separate bits for 137 * each modifier. The absence of monotonic does not imply nonmonotonic, 138 * especially since 4.5 says that the behaviour of the "no modifier" case 139 * is implementation defined in 4.5, but will become "nonmonotonic" in 5.0. 140 * 141 * Since we're passing a full 32 bit value, we can use a couple of high 142 * bits for these flags; out of paranoia we avoid the sign bit. 143 * 144 * These modifiers can be or-ed into non-static schedules by the compiler 145 * to pass the additional information. They will be stripped early in the 146 * processing in __kmp_dispatch_init when setting up schedules, so 147 * most of the code won't ever see schedules with these bits set. 148 */ 149 kmp_sched_modifier_monotonic = (1 << 29), 150 /**< Set if the monotonic schedule modifier was present */ 151 kmp_sched_modifier_nonmonotonic = (1 << 30), 152 /**< Set if the nonmonotonic schedule modifier was present */ 153 154 #define SCHEDULE_WITHOUT_MODIFIERS(s) \ 155 (enum kmp_sched_t)( \ 156 (s) & ~(kmp_sched_modifier_nonmonotonic | kmp_sched_modifier_monotonic)) 157 #define SCHEDULE_HAS_MONOTONIC(s) (((s)&kmp_sched_modifier_monotonic) != 0) 158 #define SCHEDULE_HAS_NONMONOTONIC(s) \ 159 (((s)&kmp_sched_modifier_nonmonotonic) != 0) 160 #define SCHEDULE_HAS_NO_MODIFIERS(s) \ 161 (((s) & (kmp_sched_modifier_nonmonotonic | kmp_sched_modifier_monotonic)) == \ 162 0) 163 164 } kmp_sched_t; 165 166 /*! 167 * Enum for accesseing the reserved_2 field of the ident_t struct below. 168 */ 169 enum { 170 /*! Bit set to 1 when in SPMD mode. */ 171 KMP_IDENT_SPMD_MODE = 0x01, 172 /*! Bit set to 1 when a simplified runtime is used. */ 173 KMP_IDENT_SIMPLE_RT_MODE = 0x02, 174 }; 175 176 /*! 177 * The ident structure that describes a source location. 178 * The struct is identical to the one in the kmp.h file. 179 * We maintain the same data structure for compatibility. 180 */ 181 typedef int kmp_int32; 182 typedef struct ident { 183 kmp_int32 reserved_1; /**< might be used in Fortran; see above */ 184 kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags; KMP_IDENT_KMPC 185 identifies this union member */ 186 kmp_int32 reserved_2; /**< not really used in Fortran any more; see above */ 187 kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for C++ */ 188 char const *psource; /**< String describing the source location. 189 The string is composed of semi-colon separated fields 190 which describe the source file, the function and a pair 191 of line numbers that delimit the construct. */ 192 } ident_t; 193 194 // parallel defs 195 typedef ident_t kmp_Ident; 196 typedef void (*kmp_InterWarpCopyFctPtr)(void *src, int32_t warp_num); 197 typedef void (*kmp_ShuffleReductFctPtr)(void *rhsData, int16_t lane_id, 198 int16_t lane_offset, 199 int16_t shortCircuit); 200 typedef void (*kmp_ListGlobalFctPtr)(void *buffer, int idx, void *reduce_data); 201 202 // task defs 203 typedef struct kmp_TaskDescr kmp_TaskDescr; 204 typedef int32_t (*kmp_TaskFctPtr)(int32_t global_tid, kmp_TaskDescr *taskDescr); 205 typedef struct kmp_TaskDescr { 206 void *sharedPointerTable; // ptr to a table of shared var ptrs 207 kmp_TaskFctPtr sub; // task subroutine 208 int32_t partId; // unused 209 kmp_TaskFctPtr destructors; // destructor of c++ first private 210 } kmp_TaskDescr; 211 212 // sync defs 213 typedef int32_t kmp_CriticalName[8]; 214 215 //////////////////////////////////////////////////////////////////////////////// 216 // external interface 217 //////////////////////////////////////////////////////////////////////////////// 218 219 // parallel 220 EXTERN int32_t __kmpc_global_thread_num(kmp_Ident *loc); 221 EXTERN void __kmpc_push_num_threads(kmp_Ident *loc, int32_t global_tid, 222 int32_t num_threads); 223 EXTERN void __kmpc_serialized_parallel(kmp_Ident *loc, uint32_t global_tid); 224 EXTERN void __kmpc_end_serialized_parallel(kmp_Ident *loc, 225 uint32_t global_tid); 226 EXTERN uint16_t __kmpc_parallel_level(kmp_Ident *loc, uint32_t global_tid); 227 228 // proc bind 229 EXTERN void __kmpc_push_proc_bind(kmp_Ident *loc, uint32_t global_tid, 230 int proc_bind); 231 EXTERN int omp_get_num_places(void); 232 EXTERN int omp_get_place_num_procs(int place_num); 233 EXTERN void omp_get_place_proc_ids(int place_num, int *ids); 234 EXTERN int omp_get_place_num(void); 235 EXTERN int omp_get_partition_num_places(void); 236 EXTERN void omp_get_partition_place_nums(int *place_nums); 237 238 // for static (no chunk or chunk) 239 EXTERN void __kmpc_for_static_init_4(kmp_Ident *loc, int32_t global_tid, 240 int32_t sched, int32_t *plastiter, 241 int32_t *plower, int32_t *pupper, 242 int32_t *pstride, int32_t incr, 243 int32_t chunk); 244 EXTERN void __kmpc_for_static_init_4u(kmp_Ident *loc, int32_t global_tid, 245 int32_t sched, int32_t *plastiter, 246 uint32_t *plower, uint32_t *pupper, 247 int32_t *pstride, int32_t incr, 248 int32_t chunk); 249 EXTERN void __kmpc_for_static_init_8(kmp_Ident *loc, int32_t global_tid, 250 int32_t sched, int32_t *plastiter, 251 int64_t *plower, int64_t *pupper, 252 int64_t *pstride, int64_t incr, 253 int64_t chunk); 254 EXTERN void __kmpc_for_static_init_8u(kmp_Ident *loc, int32_t global_tid, 255 int32_t sched, int32_t *plastiter1, 256 uint64_t *plower, uint64_t *pupper, 257 int64_t *pstride, int64_t incr, 258 int64_t chunk); 259 EXTERN 260 void __kmpc_for_static_init_4_simple_spmd(kmp_Ident *loc, int32_t global_tid, 261 int32_t sched, int32_t *plastiter, 262 int32_t *plower, int32_t *pupper, 263 int32_t *pstride, int32_t incr, 264 int32_t chunk); 265 EXTERN 266 void __kmpc_for_static_init_4u_simple_spmd(kmp_Ident *loc, int32_t global_tid, 267 int32_t sched, int32_t *plastiter, 268 uint32_t *plower, uint32_t *pupper, 269 int32_t *pstride, int32_t incr, 270 int32_t chunk); 271 EXTERN 272 void __kmpc_for_static_init_8_simple_spmd(kmp_Ident *loc, int32_t global_tid, 273 int32_t sched, int32_t *plastiter, 274 int64_t *plower, int64_t *pupper, 275 int64_t *pstride, int64_t incr, 276 int64_t chunk); 277 EXTERN 278 void __kmpc_for_static_init_8u_simple_spmd(kmp_Ident *loc, int32_t global_tid, 279 int32_t sched, int32_t *plastiter1, 280 uint64_t *plower, uint64_t *pupper, 281 int64_t *pstride, int64_t incr, 282 int64_t chunk); 283 EXTERN 284 void __kmpc_for_static_init_4_simple_generic(kmp_Ident *loc, 285 int32_t global_tid, int32_t sched, 286 int32_t *plastiter, 287 int32_t *plower, int32_t *pupper, 288 int32_t *pstride, int32_t incr, 289 int32_t chunk); 290 EXTERN 291 void __kmpc_for_static_init_4u_simple_generic( 292 kmp_Ident *loc, int32_t global_tid, int32_t sched, int32_t *plastiter, 293 uint32_t *plower, uint32_t *pupper, int32_t *pstride, int32_t incr, 294 int32_t chunk); 295 EXTERN 296 void __kmpc_for_static_init_8_simple_generic(kmp_Ident *loc, 297 int32_t global_tid, int32_t sched, 298 int32_t *plastiter, 299 int64_t *plower, int64_t *pupper, 300 int64_t *pstride, int64_t incr, 301 int64_t chunk); 302 EXTERN 303 void __kmpc_for_static_init_8u_simple_generic( 304 kmp_Ident *loc, int32_t global_tid, int32_t sched, int32_t *plastiter1, 305 uint64_t *plower, uint64_t *pupper, int64_t *pstride, int64_t incr, 306 int64_t chunk); 307 308 EXTERN void __kmpc_for_static_fini(kmp_Ident *loc, int32_t global_tid); 309 310 // for dynamic 311 EXTERN void __kmpc_dispatch_init_4(kmp_Ident *loc, int32_t global_tid, 312 int32_t sched, int32_t lower, int32_t upper, 313 int32_t incr, int32_t chunk); 314 EXTERN void __kmpc_dispatch_init_4u(kmp_Ident *loc, int32_t global_tid, 315 int32_t sched, uint32_t lower, 316 uint32_t upper, int32_t incr, 317 int32_t chunk); 318 EXTERN void __kmpc_dispatch_init_8(kmp_Ident *loc, int32_t global_tid, 319 int32_t sched, int64_t lower, int64_t upper, 320 int64_t incr, int64_t chunk); 321 EXTERN void __kmpc_dispatch_init_8u(kmp_Ident *loc, int32_t global_tid, 322 int32_t sched, uint64_t lower, 323 uint64_t upper, int64_t incr, 324 int64_t chunk); 325 326 EXTERN int __kmpc_dispatch_next_4(kmp_Ident *loc, int32_t global_tid, 327 int32_t *plastiter, int32_t *plower, 328 int32_t *pupper, int32_t *pstride); 329 EXTERN int __kmpc_dispatch_next_4u(kmp_Ident *loc, int32_t global_tid, 330 int32_t *plastiter, uint32_t *plower, 331 uint32_t *pupper, int32_t *pstride); 332 EXTERN int __kmpc_dispatch_next_8(kmp_Ident *loc, int32_t global_tid, 333 int32_t *plastiter, int64_t *plower, 334 int64_t *pupper, int64_t *pstride); 335 EXTERN int __kmpc_dispatch_next_8u(kmp_Ident *loc, int32_t global_tid, 336 int32_t *plastiter, uint64_t *plower, 337 uint64_t *pupper, int64_t *pstride); 338 339 EXTERN void __kmpc_dispatch_fini_4(kmp_Ident *loc, int32_t global_tid); 340 EXTERN void __kmpc_dispatch_fini_4u(kmp_Ident *loc, int32_t global_tid); 341 EXTERN void __kmpc_dispatch_fini_8(kmp_Ident *loc, int32_t global_tid); 342 EXTERN void __kmpc_dispatch_fini_8u(kmp_Ident *loc, int32_t global_tid); 343 344 // reduction 345 EXTERN void __kmpc_nvptx_end_reduce(int32_t global_tid); 346 EXTERN void __kmpc_nvptx_end_reduce_nowait(int32_t global_tid); 347 EXTERN int32_t __kmpc_nvptx_parallel_reduce_nowait_v2( 348 kmp_Ident *loc, int32_t global_tid, int32_t num_vars, size_t reduce_size, 349 void *reduce_data, kmp_ShuffleReductFctPtr shflFct, 350 kmp_InterWarpCopyFctPtr cpyFct); 351 EXTERN int32_t __kmpc_nvptx_teams_reduce_nowait_v2( 352 kmp_Ident *loc, int32_t global_tid, void *global_buffer, 353 int32_t num_of_records, void *reduce_data, kmp_ShuffleReductFctPtr shflFct, 354 kmp_InterWarpCopyFctPtr cpyFct, kmp_ListGlobalFctPtr lgcpyFct, 355 kmp_ListGlobalFctPtr lgredFct, kmp_ListGlobalFctPtr glcpyFct, 356 kmp_ListGlobalFctPtr glredFct); 357 EXTERN int32_t __kmpc_shuffle_int32(int32_t val, int16_t delta, int16_t size); 358 EXTERN int64_t __kmpc_shuffle_int64(int64_t val, int16_t delta, int16_t size); 359 360 // sync barrier 361 EXTERN void __kmpc_barrier(kmp_Ident *loc_ref, int32_t tid); 362 EXTERN void __kmpc_barrier_simple_spmd(kmp_Ident *loc_ref, int32_t tid); 363 EXTERN int32_t __kmpc_cancel_barrier(kmp_Ident *loc, int32_t global_tid); 364 365 // single 366 EXTERN int32_t __kmpc_single(kmp_Ident *loc, int32_t global_tid); 367 EXTERN void __kmpc_end_single(kmp_Ident *loc, int32_t global_tid); 368 369 // sync 370 EXTERN int32_t __kmpc_master(kmp_Ident *loc, int32_t global_tid); 371 EXTERN void __kmpc_end_master(kmp_Ident *loc, int32_t global_tid); 372 EXTERN void __kmpc_ordered(kmp_Ident *loc, int32_t global_tid); 373 EXTERN void __kmpc_end_ordered(kmp_Ident *loc, int32_t global_tid); 374 EXTERN void __kmpc_critical(kmp_Ident *loc, int32_t global_tid, 375 kmp_CriticalName *crit); 376 EXTERN void __kmpc_end_critical(kmp_Ident *loc, int32_t global_tid, 377 kmp_CriticalName *crit); 378 EXTERN void __kmpc_flush(kmp_Ident *loc); 379 380 // vote 381 EXTERN __kmpc_impl_lanemask_t __kmpc_warp_active_thread_mask(); 382 // syncwarp 383 EXTERN void __kmpc_syncwarp(__kmpc_impl_lanemask_t); 384 385 // tasks 386 EXTERN kmp_TaskDescr *__kmpc_omp_task_alloc(kmp_Ident *loc, 387 uint32_t global_tid, int32_t flag, 388 size_t sizeOfTaskInclPrivate, 389 size_t sizeOfSharedTable, 390 kmp_TaskFctPtr sub); 391 EXTERN int32_t __kmpc_omp_task(kmp_Ident *loc, uint32_t global_tid, 392 kmp_TaskDescr *newLegacyTaskDescr); 393 EXTERN int32_t __kmpc_omp_task_with_deps(kmp_Ident *loc, uint32_t global_tid, 394 kmp_TaskDescr *newLegacyTaskDescr, 395 int32_t depNum, void *depList, 396 int32_t noAliasDepNum, 397 void *noAliasDepList); 398 EXTERN void __kmpc_omp_task_begin_if0(kmp_Ident *loc, uint32_t global_tid, 399 kmp_TaskDescr *newLegacyTaskDescr); 400 EXTERN void __kmpc_omp_task_complete_if0(kmp_Ident *loc, uint32_t global_tid, 401 kmp_TaskDescr *newLegacyTaskDescr); 402 EXTERN void __kmpc_omp_wait_deps(kmp_Ident *loc, uint32_t global_tid, 403 int32_t depNum, void *depList, 404 int32_t noAliasDepNum, void *noAliasDepList); 405 EXTERN void __kmpc_taskgroup(kmp_Ident *loc, uint32_t global_tid); 406 EXTERN void __kmpc_end_taskgroup(kmp_Ident *loc, uint32_t global_tid); 407 EXTERN int32_t __kmpc_omp_taskyield(kmp_Ident *loc, uint32_t global_tid, 408 int end_part); 409 EXTERN int32_t __kmpc_omp_taskwait(kmp_Ident *loc, uint32_t global_tid); 410 EXTERN void __kmpc_taskloop(kmp_Ident *loc, uint32_t global_tid, 411 kmp_TaskDescr *newKmpTaskDescr, int if_val, 412 uint64_t *lb, uint64_t *ub, int64_t st, int nogroup, 413 int32_t sched, uint64_t grainsize, void *task_dup); 414 415 // cancel 416 EXTERN int32_t __kmpc_cancellationpoint(kmp_Ident *loc, int32_t global_tid, 417 int32_t cancelVal); 418 EXTERN int32_t __kmpc_cancel(kmp_Ident *loc, int32_t global_tid, 419 int32_t cancelVal); 420 421 // non standard 422 EXTERN void __kmpc_kernel_init(int ThreadLimit, int16_t RequiresOMPRuntime); 423 EXTERN void __kmpc_kernel_deinit(int16_t IsOMPRuntimeInitialized); 424 EXTERN void __kmpc_spmd_kernel_init(int ThreadLimit, 425 int16_t RequiresOMPRuntime); 426 EXTERN void __kmpc_spmd_kernel_deinit_v2(int16_t RequiresOMPRuntime); 427 EXTERN void __kmpc_kernel_prepare_parallel(void *WorkFn); 428 EXTERN bool __kmpc_kernel_parallel(void **WorkFn); 429 EXTERN void __kmpc_kernel_end_parallel(); 430 431 EXTERN void __kmpc_data_sharing_init_stack(); 432 EXTERN void __kmpc_data_sharing_init_stack_spmd(); 433 EXTERN void *__kmpc_data_sharing_coalesced_push_stack(size_t size, 434 int16_t UseSharedMemory); 435 EXTERN void *__kmpc_data_sharing_push_stack(size_t size, int16_t UseSharedMemory); 436 EXTERN void __kmpc_data_sharing_pop_stack(void *a); 437 EXTERN void __kmpc_begin_sharing_variables(void ***GlobalArgs, size_t nArgs); 438 EXTERN void __kmpc_end_sharing_variables(); 439 EXTERN void __kmpc_get_shared_variables(void ***GlobalArgs); 440 441 // The slot used for data sharing by the master and worker threads. We use a 442 // complete (default size version and an incomplete one so that we allow sizes 443 // greater than the default). 444 struct __kmpc_data_sharing_slot { 445 __kmpc_data_sharing_slot *Next; 446 __kmpc_data_sharing_slot *Prev; 447 void *PrevSlotStackPtr; 448 void *DataEnd; 449 char Data[]; 450 }; 451 452 // SPMD execution mode interrogation function. 453 EXTERN int8_t __kmpc_is_spmd_exec_mode(); 454 455 EXTERN void __kmpc_get_team_static_memory(int16_t isSPMDExecutionMode, 456 const void *buf, size_t size, 457 int16_t is_shared, const void **res); 458 459 EXTERN void __kmpc_restore_team_static_memory(int16_t isSPMDExecutionMode, 460 int16_t is_shared); 461 462 #endif 463