diff options
Diffstat (limited to 'contrib/llvm-project/openmp/runtime/src/z_Linux_util.cpp')
| -rw-r--r-- | contrib/llvm-project/openmp/runtime/src/z_Linux_util.cpp | 256 |
1 files changed, 213 insertions, 43 deletions
diff --git a/contrib/llvm-project/openmp/runtime/src/z_Linux_util.cpp b/contrib/llvm-project/openmp/runtime/src/z_Linux_util.cpp index 3b5910fc95e8..95feafbec8e5 100644 --- a/contrib/llvm-project/openmp/runtime/src/z_Linux_util.cpp +++ b/contrib/llvm-project/openmp/runtime/src/z_Linux_util.cpp @@ -25,13 +25,14 @@ #include <alloca.h> #endif #include <math.h> // HUGE_VAL. +#include <semaphore.h> #include <sys/resource.h> #include <sys/syscall.h> #include <sys/time.h> #include <sys/times.h> #include <unistd.h> -#if KMP_OS_LINUX && !KMP_OS_CNK +#if KMP_OS_LINUX #include <sys/sysinfo.h> #if KMP_USE_FUTEX // We should really include <futex.h>, but that causes compatibility problems on @@ -70,7 +71,8 @@ struct kmp_sys_timer { }; // Convert timespec to nanoseconds. -#define TS2NS(timespec) (((timespec).tv_sec * 1e9) + (timespec).tv_nsec) +#define TS2NS(timespec) \ + (((timespec).tv_sec * (long int)1e9) + (timespec).tv_nsec) static struct kmp_sys_timer __kmp_sys_timer_data; @@ -133,13 +135,13 @@ void __kmp_affinity_determine_capable(const char *env_var) { // If Linux* OS: // If the syscall fails or returns a suggestion for the size, // then we don't have to search for an appropriate size. - int gCode; - int sCode; + long gCode; + long sCode; unsigned char *buf; buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT); gCode = syscall(__NR_sched_getaffinity, 0, KMP_CPU_SET_SIZE_LIMIT, buf); KA_TRACE(30, ("__kmp_affinity_determine_capable: " - "initial getaffinity call returned %d errno = %d\n", + "initial getaffinity call returned %ld errno = %d\n", gCode, errno)); // if ((gCode < 0) && (errno == ENOSYS)) @@ -168,7 +170,7 @@ void __kmp_affinity_determine_capable(const char *env_var) { // buffer with the same size fails with errno set to EFAULT. sCode = syscall(__NR_sched_setaffinity, 0, gCode, NULL); KA_TRACE(30, ("__kmp_affinity_determine_capable: " - "setaffinity for mask size %d returned %d errno = %d\n", + "setaffinity for mask size %ld returned %ld errno = %d\n", gCode, sCode, errno)); if (sCode < 0) { if (errno == ENOSYS) { @@ -207,7 +209,7 @@ void __kmp_affinity_determine_capable(const char *env_var) { for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) { gCode = syscall(__NR_sched_getaffinity, 0, size, buf); KA_TRACE(30, ("__kmp_affinity_determine_capable: " - "getaffinity for mask size %d returned %d errno = %d\n", + "getaffinity for mask size %ld returned %ld errno = %d\n", size, gCode, errno)); if (gCode < 0) { @@ -239,7 +241,7 @@ void __kmp_affinity_determine_capable(const char *env_var) { sCode = syscall(__NR_sched_setaffinity, 0, gCode, NULL); KA_TRACE(30, ("__kmp_affinity_determine_capable: " - "setaffinity for mask size %d returned %d errno = %d\n", + "setaffinity for mask size %ld returned %ld errno = %d\n", gCode, sCode, errno)); if (sCode < 0) { if (errno == ENOSYS) { // Linux* OS only @@ -276,7 +278,7 @@ void __kmp_affinity_determine_capable(const char *env_var) { } } #elif KMP_OS_FREEBSD - int gCode; + long gCode; unsigned char *buf; buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT); gCode = pthread_getaffinity_np(pthread_self(), KMP_CPU_SET_SIZE_LIMIT, reinterpret_cast<cpuset_t *>(buf)); @@ -316,7 +318,7 @@ void __kmp_affinity_determine_capable(const char *env_var) { int __kmp_futex_determine_capable() { int loc = 0; - int rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0); + long rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0); int retval = (rc == 0) || (errno != ENOSYS); KA_TRACE(10, @@ -1149,6 +1151,7 @@ static void __kmp_team_handler(int signo) { if (__kmp_debug_buf) { __kmp_dump_debug_buffer(); } + __kmp_unregister_library(); // cleanup shared memory KMP_MB(); // Flush all pending memory write invalidates. TCW_4(__kmp_global.g.g_abort, signo); KMP_MB(); // Flush all pending memory write invalidates. @@ -1271,8 +1274,8 @@ static void __kmp_atfork_prepare(void) { } static void __kmp_atfork_parent(void) { - __kmp_release_bootstrap_lock(&__kmp_initz_lock); __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); + __kmp_release_bootstrap_lock(&__kmp_initz_lock); } /* Reset the library so execution in the child starts "all over again" with @@ -1280,6 +1283,7 @@ static void __kmp_atfork_parent(void) { allocated by parent, just abandon it to be safe. */ static void __kmp_atfork_child(void) { __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock); + __kmp_release_bootstrap_lock(&__kmp_initz_lock); /* TODO make sure this is done right for nested/sibling */ // ATT: Memory leaks are here? TODO: Check it and fix. /* KMP_ASSERT( 0 ); */ @@ -1301,7 +1305,6 @@ static void __kmp_atfork_child(void) { } #endif // KMP_AFFINITY_SUPPORTED - __kmp_init_runtime = FALSE; #if KMP_USE_MONITOR __kmp_init_monitor = 0; #endif @@ -1354,6 +1357,8 @@ static void __kmp_atfork_child(void) { __kmp_itt_reset(); // reset ITT's global state #endif /* USE_ITT_BUILD */ + __kmp_serial_initialize(); + /* This is necessary to make sure no stale data is left around */ /* AC: customers complain that we use unsafe routines in the atfork handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen @@ -1459,8 +1464,7 @@ static inline void __kmp_suspend_template(int th_gtid, C *flag) { __kmp_suspend_initialize_thread(th); - status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex); - KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); + __kmp_lock_suspend_mx(th); KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n", th_gtid, flag->get())); @@ -1471,8 +1475,7 @@ static inline void __kmp_suspend_template(int th_gtid, C *flag) { if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME && __kmp_pause_status != kmp_soft_paused) { flag->unset_sleeping(); - status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex); - KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); + __kmp_unlock_suspend_mx(th); return; } KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x," @@ -1535,7 +1538,7 @@ static inline void __kmp_suspend_template(int th_gtid, C *flag) { th_gtid)); status = pthread_cond_wait(&th->th.th_suspend_cv.c_cond, &th->th.th_suspend_mx.m_mutex); -#endif +#endif // USE_SUSPEND_TIMEOUT if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) { KMP_SYSFAIL("pthread_cond_wait", status); @@ -1575,21 +1578,26 @@ static inline void __kmp_suspend_template(int th_gtid, C *flag) { } #endif - status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex); - KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); + __kmp_unlock_suspend_mx(th); KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid)); } -void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) { +template <bool C, bool S> +void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) { __kmp_suspend_template(th_gtid, flag); } -void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) { +template <bool C, bool S> +void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) { __kmp_suspend_template(th_gtid, flag); } void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) { __kmp_suspend_template(th_gtid, flag); } +template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *); +template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *); +template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *); + /* This routine signals the thread specified by target_gtid to wake up after setting the sleep bit indicated by the flag argument to FALSE. The target thread must already have called __kmp_suspend_template() */ @@ -1609,8 +1617,7 @@ static inline void __kmp_resume_template(int target_gtid, C *flag) { __kmp_suspend_initialize_thread(th); - status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex); - KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); + __kmp_lock_suspend_mx(th); if (!flag) { // coming from __kmp_null_resume_wrapper flag = (C *)CCAST(void *, th->th.th_sleep_loc); @@ -1619,13 +1626,11 @@ static inline void __kmp_resume_template(int target_gtid, C *flag) { // First, check if the flag is null or its type has changed. If so, someone // else woke it up. if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type - // simply shows what - // flag was cast to + // simply shows what flag was cast to KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already " "awake: flag(%p)\n", gtid, target_gtid, NULL)); - status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex); - KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); + __kmp_unlock_suspend_mx(th); return; } else { // if multiple threads are sleeping, flag should be internally // referring to a specific thread here @@ -1635,8 +1640,7 @@ static inline void __kmp_resume_template(int target_gtid, C *flag) { "awake: flag(%p): " "%u => %u\n", gtid, target_gtid, flag->get(), old_spin, flag->load())); - status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex); - KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); + __kmp_unlock_suspend_mx(th); return; } KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset " @@ -1656,23 +1660,27 @@ static inline void __kmp_resume_template(int target_gtid, C *flag) { #endif status = pthread_cond_signal(&th->th.th_suspend_cv.c_cond); KMP_CHECK_SYSFAIL("pthread_cond_signal", status); - status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex); - KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); + __kmp_unlock_suspend_mx(th); KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up" " for T#%d\n", gtid, target_gtid)); } -void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) { +template <bool C, bool S> +void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) { __kmp_resume_template(target_gtid, flag); } -void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) { +template <bool C, bool S> +void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) { __kmp_resume_template(target_gtid, flag); } void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) { __kmp_resume_template(target_gtid, flag); } +template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *); +template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *); + #if KMP_USE_MONITOR void __kmp_resume_monitor() { KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume); @@ -1740,7 +1748,8 @@ double __kmp_read_cpu_time(void) { /*t =*/times(&buffer); - return (buffer.tms_utime + buffer.tms_cutime) / (double)CLOCKS_PER_SEC; + return (double)(buffer.tms_utime + buffer.tms_cutime) / + (double)CLOCKS_PER_SEC; } int __kmp_read_system_info(struct kmp_sys_info *info) { @@ -1781,7 +1790,7 @@ void __kmp_read_system_time(double *delta) { status = gettimeofday(&tval, NULL); KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status); TIMEVAL_TO_TIMESPEC(&tval, &stop); - t_ns = TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start); + t_ns = (double)(TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start)); *delta = (t_ns * 1e-9); } @@ -1800,7 +1809,7 @@ static int __kmp_get_xproc(void) { #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \ KMP_OS_OPENBSD || KMP_OS_HURD - r = sysconf(_SC_NPROCESSORS_ONLN); + __kmp_type_convert(sysconf(_SC_NPROCESSORS_ONLN), &(r)); #elif KMP_OS_DARWIN @@ -1875,7 +1884,7 @@ void __kmp_runtime_initialize(void) { if (sysconf(_SC_THREADS)) { /* Query the maximum number of threads */ - __kmp_sys_max_nth = sysconf(_SC_THREAD_THREADS_MAX); + __kmp_type_convert(sysconf(_SC_THREAD_THREADS_MAX), &(__kmp_sys_max_nth)); if (__kmp_sys_max_nth == -1) { /* Unlimited threads for NPTL */ __kmp_sys_max_nth = INT_MAX; @@ -1989,7 +1998,7 @@ void __kmp_initialize_system_tick() { nsec2 = __kmp_now_nsec(); diff = nsec2 - nsec; if (diff > 0) { - kmp_uint64 tpms = (kmp_uint64)(1e6 * (delay + (now - goal)) / diff); + kmp_uint64 tpms = ((kmp_uint64)1e6 * (delay + (now - goal)) / diff); if (tpms > 0) __kmp_ticks_per_msec = tpms; } @@ -2191,13 +2200,13 @@ int __kmp_get_load_balance(int max) { // getloadavg() may return the number of samples less than requested that is // less than 3. if (__kmp_load_balance_interval < 180 && (res >= 1)) { - ret_avg = averages[0]; // 1 min + ret_avg = (int)averages[0]; // 1 min } else if ((__kmp_load_balance_interval >= 180 && __kmp_load_balance_interval < 600) && (res >= 2)) { - ret_avg = averages[1]; // 5 min + ret_avg = (int)averages[1]; // 5 min } else if ((__kmp_load_balance_interval >= 600) && (res == 3)) { - ret_avg = averages[2]; // 15 min + ret_avg = (int)averages[2]; // 15 min } else { // Error occurred return -1; } @@ -2364,7 +2373,7 @@ int __kmp_get_load_balance(int max) { -- ln */ char buffer[65]; - int len; + ssize_t len; len = read(stat_file, buffer, sizeof(buffer) - 1); if (len >= 0) { buffer[len] = 0; @@ -2439,7 +2448,7 @@ int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc, , void **exit_frame_ptr #endif - ) { +) { #if OMPT_SUPPORT *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0); #endif @@ -2518,4 +2527,165 @@ int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc, #endif +// Functions for hidden helper task +namespace { +// Condition variable for initializing hidden helper team +pthread_cond_t hidden_helper_threads_initz_cond_var; +pthread_mutex_t hidden_helper_threads_initz_lock; +volatile int hidden_helper_initz_signaled = FALSE; + +// Condition variable for deinitializing hidden helper team +pthread_cond_t hidden_helper_threads_deinitz_cond_var; +pthread_mutex_t hidden_helper_threads_deinitz_lock; +volatile int hidden_helper_deinitz_signaled = FALSE; + +// Condition variable for the wrapper function of main thread +pthread_cond_t hidden_helper_main_thread_cond_var; +pthread_mutex_t hidden_helper_main_thread_lock; +volatile int hidden_helper_main_thread_signaled = FALSE; + +// Semaphore for worker threads. We don't use condition variable here in case +// that when multiple signals are sent at the same time, only one thread might +// be waken. +sem_t hidden_helper_task_sem; +} // namespace + +void __kmp_hidden_helper_worker_thread_wait() { + int status = sem_wait(&hidden_helper_task_sem); + KMP_CHECK_SYSFAIL("sem_wait", status); +} + +void __kmp_do_initialize_hidden_helper_threads() { + // Initialize condition variable + int status = + pthread_cond_init(&hidden_helper_threads_initz_cond_var, nullptr); + KMP_CHECK_SYSFAIL("pthread_cond_init", status); + + status = pthread_cond_init(&hidden_helper_threads_deinitz_cond_var, nullptr); + KMP_CHECK_SYSFAIL("pthread_cond_init", status); + + status = pthread_cond_init(&hidden_helper_main_thread_cond_var, nullptr); + KMP_CHECK_SYSFAIL("pthread_cond_init", status); + + status = pthread_mutex_init(&hidden_helper_threads_initz_lock, nullptr); + KMP_CHECK_SYSFAIL("pthread_mutex_init", status); + + status = pthread_mutex_init(&hidden_helper_threads_deinitz_lock, nullptr); + KMP_CHECK_SYSFAIL("pthread_mutex_init", status); + + status = pthread_mutex_init(&hidden_helper_main_thread_lock, nullptr); + KMP_CHECK_SYSFAIL("pthread_mutex_init", status); + + // Initialize the semaphore + status = sem_init(&hidden_helper_task_sem, 0, 0); + KMP_CHECK_SYSFAIL("sem_init", status); + + // Create a new thread to finish initialization + pthread_t handle; + status = pthread_create( + &handle, nullptr, + [](void *) -> void * { + __kmp_hidden_helper_threads_initz_routine(); + return nullptr; + }, + nullptr); + KMP_CHECK_SYSFAIL("pthread_create", status); +} + +void __kmp_hidden_helper_threads_initz_wait() { + // Initial thread waits here for the completion of the initialization. The + // condition variable will be notified by main thread of hidden helper teams. + int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock); + KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); + + if (!TCR_4(hidden_helper_initz_signaled)) { + status = pthread_cond_wait(&hidden_helper_threads_initz_cond_var, + &hidden_helper_threads_initz_lock); + KMP_CHECK_SYSFAIL("pthread_cond_wait", status); + } + + status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock); + KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); +} + +void __kmp_hidden_helper_initz_release() { + // After all initialization, reset __kmp_init_hidden_helper_threads to false. + int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock); + KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); + + status = pthread_cond_signal(&hidden_helper_threads_initz_cond_var); + KMP_CHECK_SYSFAIL("pthread_cond_wait", status); + + TCW_SYNC_4(hidden_helper_initz_signaled, TRUE); + + status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock); + KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); +} + +void __kmp_hidden_helper_main_thread_wait() { + // The main thread of hidden helper team will be blocked here. The + // condition variable can only be signal in the destructor of RTL. + int status = pthread_mutex_lock(&hidden_helper_main_thread_lock); + KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); + + if (!TCR_4(hidden_helper_main_thread_signaled)) { + status = pthread_cond_wait(&hidden_helper_main_thread_cond_var, + &hidden_helper_main_thread_lock); + KMP_CHECK_SYSFAIL("pthread_cond_wait", status); + } + + status = pthread_mutex_unlock(&hidden_helper_main_thread_lock); + KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); +} + +void __kmp_hidden_helper_main_thread_release() { + // The initial thread of OpenMP RTL should call this function to wake up the + // main thread of hidden helper team. + int status = pthread_mutex_lock(&hidden_helper_main_thread_lock); + KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); + + status = pthread_cond_signal(&hidden_helper_main_thread_cond_var); + KMP_CHECK_SYSFAIL("pthread_cond_signal", status); + + // The hidden helper team is done here + TCW_SYNC_4(hidden_helper_main_thread_signaled, TRUE); + + status = pthread_mutex_unlock(&hidden_helper_main_thread_lock); + KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); +} + +void __kmp_hidden_helper_worker_thread_signal() { + int status = sem_post(&hidden_helper_task_sem); + KMP_CHECK_SYSFAIL("sem_post", status); +} + +void __kmp_hidden_helper_threads_deinitz_wait() { + // Initial thread waits here for the completion of the deinitialization. The + // condition variable will be notified by main thread of hidden helper teams. + int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock); + KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); + + if (!TCR_4(hidden_helper_deinitz_signaled)) { + status = pthread_cond_wait(&hidden_helper_threads_deinitz_cond_var, + &hidden_helper_threads_deinitz_lock); + KMP_CHECK_SYSFAIL("pthread_cond_wait", status); + } + + status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock); + KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); +} + +void __kmp_hidden_helper_threads_deinitz_release() { + int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock); + KMP_CHECK_SYSFAIL("pthread_mutex_lock", status); + + status = pthread_cond_signal(&hidden_helper_threads_deinitz_cond_var); + KMP_CHECK_SYSFAIL("pthread_cond_wait", status); + + TCW_SYNC_4(hidden_helper_deinitz_signaled, TRUE); + + status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock); + KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status); +} + // end of file // |