12#ifdef THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION
14#include "internal/gc.h"
15#include "internal/sanitizers.h"
17#ifdef HAVE_SYS_RESOURCE_H
18#include <sys/resource.h>
20#ifdef HAVE_THR_STKSEGMENT
23#if defined(HAVE_FCNTL_H)
25#elif defined(HAVE_SYS_FCNTL_H)
28#ifdef HAVE_SYS_PRCTL_H
31#if defined(HAVE_SYS_TIME_H)
38#include <sys/syscall.h>
44# include <AvailabilityMacros.h>
47#if defined(HAVE_SYS_EVENTFD_H) && defined(HAVE_EVENTFD)
48# define USE_EVENTFD (1)
49# include <sys/eventfd.h>
51# define USE_EVENTFD (0)
54#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \
55 defined(CLOCK_REALTIME) && defined(CLOCK_MONOTONIC) && \
56 defined(HAVE_CLOCK_GETTIME)
57static pthread_condattr_t condattr_mono;
58static pthread_condattr_t *condattr_monotonic = &condattr_mono;
60static const void *
const condattr_monotonic = NULL;
65#define COROUTINE_TARGET_MAY_BE_FREED 1
69#ifndef HAVE_SYS_EVENT_H
70#define HAVE_SYS_EVENT_H 0
73#ifndef HAVE_SYS_EPOLL_H
74#define HAVE_SYS_EPOLL_H 0
82 #if defined(__EMSCRIPTEN__) || defined(COROUTINE_PTHREAD_CONTEXT)
85 #define USE_MN_THREADS 0
86 #elif HAVE_SYS_EPOLL_H
87 #include <sys/epoll.h>
88 #define USE_MN_THREADS 1
89 #elif HAVE_SYS_EVENT_H
90 #include <sys/event.h>
91 #define USE_MN_THREADS 1
93 #define USE_MN_THREADS 0
97#ifdef HAVE_SCHED_YIELD
98#define native_thread_yield() (void)sched_yield()
100#define native_thread_yield() ((void)0)
105#define NATIVE_MUTEX_LOCK_DEBUG 0
106#define NATIVE_MUTEX_LOCK_DEBUG_YIELD 0
109mutex_debug(
const char *msg,
void *lock)
111 if (NATIVE_MUTEX_LOCK_DEBUG) {
113 static pthread_mutex_t dbglock = PTHREAD_MUTEX_INITIALIZER;
115 if ((r = pthread_mutex_lock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
116 fprintf(stdout,
"%s: %p\n", msg, lock);
117 if ((r = pthread_mutex_unlock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
125#if NATIVE_MUTEX_LOCK_DEBUG_YIELD
126 native_thread_yield();
128 mutex_debug(
"lock", lock);
129 if ((r = pthread_mutex_lock(lock)) != 0) {
138 mutex_debug(
"unlock", lock);
139 if ((r = pthread_mutex_unlock(lock)) != 0) {
148 mutex_debug(
"trylock", lock);
149 if ((r = pthread_mutex_trylock(lock)) != 0) {
163 int r = pthread_mutex_init(lock, 0);
164 mutex_debug(
"init", lock);
173 int r = pthread_mutex_destroy(lock);
174 mutex_debug(
"destroy", lock);
183 int r = pthread_cond_init(cond, condattr_monotonic);
192 int r = pthread_cond_destroy(cond);
213 r = pthread_cond_signal(cond);
214 }
while (r == EAGAIN);
225 r = pthread_cond_broadcast(cond);
226 }
while (r == EAGAIN);
235 int r = pthread_cond_wait(cond, mutex);
242native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex,
const rb_hrtime_t *abs)
254 rb_hrtime2timespec(&ts, abs);
255 r = pthread_cond_timedwait(cond, mutex, &ts);
256 }
while (r == EINTR);
258 if (r != 0 && r != ETIMEDOUT) {
266native_cond_timeout(rb_nativethread_cond_t *cond,
const rb_hrtime_t rel)
268 if (condattr_monotonic) {
269 return rb_hrtime_add(rb_hrtime_now(), rel);
275 return rb_hrtime_add(rb_timespec2hrtime(&ts), rel);
282 rb_hrtime_t hrmsec = native_cond_timeout(cond, RB_HRTIME_PER_MSEC * msec);
283 native_cond_timedwait(cond, mutex, &hrmsec);
288static rb_internal_thread_event_hook_t *rb_internal_thread_event_hooks = NULL;
289static void rb_thread_execute_hooks(
rb_event_flag_t event, rb_thread_t *th);
310#define RB_INTERNAL_THREAD_HOOK(event, th) \
311 if (UNLIKELY(rb_internal_thread_event_hooks)) { \
312 fprintf(stderr, "[thread=%"PRIxVALUE"] %s in %s (%s:%d)\n", th->self, event_name(event), __func__, __FILE__, __LINE__); \
313 rb_thread_execute_hooks(event, th); \
316#define RB_INTERNAL_THREAD_HOOK(event, th) if (UNLIKELY(rb_internal_thread_event_hooks)) { rb_thread_execute_hooks(event, th); }
319static rb_serial_t current_fork_gen = 1;
321#if defined(SIGVTALRM) && !defined(__EMSCRIPTEN__)
322# define USE_UBF_LIST 1
325static void threadptr_trap_interrupt(rb_thread_t *);
327static void native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr,
struct rb_native_thread *nt);
328static void native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr,
struct rb_native_thread *nt);
329static void native_thread_assign(
struct rb_native_thread *nt, rb_thread_t *th);
331static void ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r);
332static void timer_thread_wakeup(
void);
333static void timer_thread_wakeup_locked(rb_vm_t *vm);
334static void timer_thread_wakeup_force(
void);
335static void thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th);
339#define thread_sched_dump(s) thread_sched_dump_(__FILE__, __LINE__, s)
342th_has_dedicated_nt(
const rb_thread_t *th)
345 return th->nt->dedicated > 0;
350thread_sched_dump_(const
char *file,
int line, struct
rb_thread_sched *sched)
352 fprintf(stderr,
"@%s:%d running:%d\n", file, line, sched->running ? (
int)sched->running->serial : -1);
355 ccan_list_for_each(&sched->readyq, th, sched.node.readyq) {
356 i++;
if (i>10) rb_bug(
"too many");
357 fprintf(stderr,
" ready:%d (%sNT:%d)\n", th->serial,
358 th->nt ? (th->nt->dedicated ?
"D" :
"S") :
"x",
359 th->nt ? (
int)th->nt->serial : -1);
363#define ractor_sched_dump(s) ractor_sched_dump_(__FILE__, __LINE__, s)
367ractor_sched_dump_(const
char *file,
int line, rb_vm_t *vm)
371 fprintf(stderr,
"ractor_sched_dump %s:%d\n", file, line);
374 ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) {
376 if (i>10) rb_bug(
"!!");
377 fprintf(stderr,
" %d ready:%d\n", i, rb_ractor_id(r));
381#define thread_sched_lock(a, b) thread_sched_lock_(a, b, __FILE__, __LINE__)
382#define thread_sched_unlock(a, b) thread_sched_unlock_(a, b, __FILE__, __LINE__)
388 VM_ASSERT(sched->lock_owner == NULL);
390 sched->lock_owner = th;
395thread_sched_set_unlocked(
struct rb_thread_sched *sched, rb_thread_t *th)
398 VM_ASSERT(sched->lock_owner == th);
400 sched->lock_owner = NULL;
405thread_sched_lock_(
struct rb_thread_sched *sched, rb_thread_t *th,
const char *file,
int line)
410 RUBY_DEBUG_LOG2(file, line,
"r:%d th:%u", th ? (
int)rb_ractor_id(th->ractor) : -1, rb_th_serial(th));
412 RUBY_DEBUG_LOG2(file, line,
"th:%u", rb_th_serial(th));
415 thread_sched_set_locked(sched, th);
419thread_sched_unlock_(
struct rb_thread_sched *sched, rb_thread_t *th,
const char *file,
int line)
421 RUBY_DEBUG_LOG2(file, line,
"th:%u", rb_th_serial(th));
423 thread_sched_set_unlocked(sched, th);
429ASSERT_thread_sched_locked(
struct rb_thread_sched *sched, rb_thread_t *th)
435 VM_ASSERT(sched->lock_owner == th);
438 VM_ASSERT(sched->lock_owner != NULL);
443#define ractor_sched_lock(a, b) ractor_sched_lock_(a, b, __FILE__, __LINE__)
444#define ractor_sched_unlock(a, b) ractor_sched_unlock_(a, b, __FILE__, __LINE__)
448rb_ractor_serial(const rb_ractor_t *r)
451 return rb_ractor_id(r);
459ractor_sched_set_locked(rb_vm_t *vm, rb_ractor_t *cr)
462 VM_ASSERT(vm->ractor.sched.lock_owner == NULL);
463 VM_ASSERT(vm->ractor.sched.locked ==
false);
465 vm->ractor.sched.lock_owner = cr;
466 vm->ractor.sched.locked =
true;
471ractor_sched_set_unlocked(rb_vm_t *vm, rb_ractor_t *cr)
474 VM_ASSERT(vm->ractor.sched.locked);
475 VM_ASSERT(vm->ractor.sched.lock_owner == cr);
477 vm->ractor.sched.locked =
false;
478 vm->ractor.sched.lock_owner = NULL;
483ractor_sched_lock_(rb_vm_t *vm, rb_ractor_t *cr,
const char *file,
int line)
488 RUBY_DEBUG_LOG2(file, line,
"cr:%u prev_owner:%u", rb_ractor_serial(cr), rb_ractor_serial(vm->ractor.sched.lock_owner));
490 RUBY_DEBUG_LOG2(file, line,
"cr:%u", rb_ractor_serial(cr));
493 ractor_sched_set_locked(vm, cr);
497ractor_sched_unlock_(rb_vm_t *vm, rb_ractor_t *cr,
const char *file,
int line)
499 RUBY_DEBUG_LOG2(file, line,
"cr:%u", rb_ractor_serial(cr));
501 ractor_sched_set_unlocked(vm, cr);
506ASSERT_ractor_sched_locked(rb_vm_t *vm, rb_ractor_t *cr)
509 VM_ASSERT(vm->ractor.sched.locked);
510 VM_ASSERT(cr == NULL || vm->ractor.sched.lock_owner == cr);
515ractor_sched_running_threads_contain_p(rb_vm_t *vm, rb_thread_t *th)
518 ccan_list_for_each(&vm->ractor.sched.running_threads, rth, sched.node.running_threads) {
519 if (rth == th)
return true;
526ractor_sched_running_threads_size(rb_vm_t *vm)
530 ccan_list_for_each(&vm->ractor.sched.running_threads, th, sched.node.running_threads) {
538ractor_sched_timeslice_threads_size(rb_vm_t *vm)
542 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, th, sched.node.timeslice_threads) {
550ractor_sched_timeslice_threads_contain_p(rb_vm_t *vm, rb_thread_t *th)
553 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, rth, sched.node.timeslice_threads) {
554 if (rth == th)
return true;
559static void ractor_sched_barrier_join_signal_locked(rb_vm_t *vm);
563thread_sched_setup_running_threads(
struct rb_thread_sched *sched, rb_ractor_t *cr, rb_vm_t *vm,
564 rb_thread_t *add_th, rb_thread_t *del_th, rb_thread_t *add_timeslice_th)
566#if USE_RUBY_DEBUG_LOG
567 unsigned int prev_running_cnt = vm->ractor.sched.running_cnt;
570 rb_thread_t *del_timeslice_th;
572 if (del_th && sched->is_running_timeslice) {
573 del_timeslice_th = del_th;
574 sched->is_running_timeslice =
false;
577 del_timeslice_th = NULL;
580 RUBY_DEBUG_LOG(
"+:%u -:%u +ts:%u -ts:%u",
581 rb_th_serial(add_th), rb_th_serial(del_th),
582 rb_th_serial(add_timeslice_th), rb_th_serial(del_timeslice_th));
584 ractor_sched_lock(vm, cr);
588 VM_ASSERT(ractor_sched_running_threads_contain_p(vm, del_th));
589 VM_ASSERT(del_timeslice_th != NULL ||
590 !ractor_sched_timeslice_threads_contain_p(vm, del_th));
592 ccan_list_del_init(&del_th->sched.node.running_threads);
593 vm->ractor.sched.running_cnt--;
595 if (UNLIKELY(vm->ractor.sched.barrier_waiting)) {
596 ractor_sched_barrier_join_signal_locked(vm);
598 sched->is_running =
false;
602 if (vm->ractor.sched.barrier_waiting) {
604 RUBY_DEBUG_LOG(
"barrier_waiting");
605 RUBY_VM_SET_VM_BARRIER_INTERRUPT(add_th->ec);
608 VM_ASSERT(!ractor_sched_running_threads_contain_p(vm, add_th));
609 VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(vm, add_th));
611 ccan_list_add(&vm->ractor.sched.running_threads, &add_th->sched.node.running_threads);
612 vm->ractor.sched.running_cnt++;
613 sched->is_running =
true;
616 if (add_timeslice_th) {
618 int was_empty = ccan_list_empty(&vm->ractor.sched.timeslice_threads);
619 VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(vm, add_timeslice_th));
620 ccan_list_add(&vm->ractor.sched.timeslice_threads, &add_timeslice_th->sched.node.timeslice_threads);
621 sched->is_running_timeslice =
true;
623 timer_thread_wakeup_locked(vm);
627 if (del_timeslice_th) {
628 VM_ASSERT(ractor_sched_timeslice_threads_contain_p(vm, del_timeslice_th));
629 ccan_list_del_init(&del_timeslice_th->sched.node.timeslice_threads);
632 VM_ASSERT(ractor_sched_running_threads_size(vm) == vm->ractor.sched.running_cnt);
633 VM_ASSERT(ractor_sched_timeslice_threads_size(vm) <= vm->ractor.sched.running_cnt);
635 ractor_sched_unlock(vm, cr);
640 RUBY_DEBUG_LOG(
"run:%u->%u", prev_running_cnt, vm->ractor.sched.running_cnt);
644thread_sched_add_running_thread(
struct rb_thread_sched *sched, rb_thread_t *th)
646 ASSERT_thread_sched_locked(sched, th);
647 VM_ASSERT(sched->running == th);
649 rb_vm_t *vm = th->vm;
650 thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL, ccan_list_empty(&sched->readyq) ? NULL : th);
654thread_sched_del_running_thread(
struct rb_thread_sched *sched, rb_thread_t *th)
656 ASSERT_thread_sched_locked(sched, th);
658 rb_vm_t *vm = th->vm;
659 thread_sched_setup_running_threads(sched, th->ractor, vm, NULL, th, NULL);
663rb_add_running_thread(rb_thread_t *th)
667 thread_sched_lock(sched, th);
669 thread_sched_add_running_thread(sched, th);
671 thread_sched_unlock(sched, th);
675rb_del_running_thread(rb_thread_t *th)
679 thread_sched_lock(sched, th);
681 thread_sched_del_running_thread(sched, th);
683 thread_sched_unlock(sched, th);
691thread_sched_set_running(
struct rb_thread_sched *sched, rb_thread_t *th)
693 RUBY_DEBUG_LOG(
"th:%u->th:%u", rb_th_serial(sched->running), rb_th_serial(th));
694 VM_ASSERT(sched->running != th);
701thread_sched_readyq_contain_p(struct
rb_thread_sched *sched, rb_thread_t *th)
704 ccan_list_for_each(&sched->readyq, rth, sched.node.readyq) {
706 VM_ASSERT(th->sched.node.is_ready);
710 VM_ASSERT(!th->sched.node.is_ready);
721 ASSERT_thread_sched_locked(sched, NULL);
722 rb_thread_t *next_th;
724 VM_ASSERT(sched->running != NULL);
726 if (ccan_list_empty(&sched->readyq)) {
730 next_th = ccan_list_pop(&sched->readyq, rb_thread_t, sched.node.readyq);
731 VM_ASSERT(next_th->sched.node.is_ready);
732 next_th->sched.node.is_ready =
false;
734 VM_ASSERT(sched->readyq_cnt > 0);
736 ccan_list_node_init(&next_th->sched.node.readyq);
739 RUBY_DEBUG_LOG(
"next_th:%u readyq_cnt:%d", rb_th_serial(next_th), sched->readyq_cnt);
748 ASSERT_thread_sched_locked(sched, NULL);
749 RUBY_DEBUG_LOG(
"ready_th:%u readyq_cnt:%d", rb_th_serial(ready_th), sched->readyq_cnt);
751 VM_ASSERT(sched->running != NULL);
752 VM_ASSERT(!thread_sched_readyq_contain_p(sched, ready_th));
754 if (sched->is_running) {
755 if (ccan_list_empty(&sched->readyq)) {
757 thread_sched_setup_running_threads(sched, ready_th->ractor, ready_th->vm, NULL, NULL, sched->running);
765 ccan_list_add_tail(&sched->readyq, &ready_th->sched.node.readyq);
766 ready_th->sched.node.is_ready =
true;
773thread_sched_wakeup_running_thread(
struct rb_thread_sched *sched, rb_thread_t *next_th,
bool will_switch)
775 ASSERT_thread_sched_locked(sched, NULL);
776 VM_ASSERT(sched->running == next_th);
780 if (th_has_dedicated_nt(next_th)) {
781 RUBY_DEBUG_LOG(
"pinning th:%u", next_th->serial);
786 RUBY_DEBUG_LOG(
"th:%u is already running.", next_th->serial);
791 RUBY_DEBUG_LOG(
"th:%u (do nothing)", rb_th_serial(next_th));
794 RUBY_DEBUG_LOG(
"th:%u (enq)", rb_th_serial(next_th));
795 ractor_sched_enq(next_th->vm, next_th->ractor);
800 RUBY_DEBUG_LOG(
"no waiting threads%s",
"");
806thread_sched_to_ready_common(
struct rb_thread_sched *sched, rb_thread_t *th,
bool wakeup,
bool will_switch)
808 RUBY_DEBUG_LOG(
"th:%u running:%u redyq_cnt:%d", rb_th_serial(th), rb_th_serial(sched->running), sched->readyq_cnt);
810 VM_ASSERT(sched->running != th);
811 VM_ASSERT(!thread_sched_readyq_contain_p(sched, th));
814 if (sched->running == NULL) {
815 thread_sched_set_running(sched, th);
816 if (wakeup) thread_sched_wakeup_running_thread(sched, th, will_switch);
819 thread_sched_enq(sched, th);
831 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
833 thread_sched_lock(sched, th);
835 thread_sched_to_ready_common(sched, th,
true,
false);
837 thread_sched_unlock(sched, th);
842thread_sched_wait_running_turn(
struct rb_thread_sched *sched, rb_thread_t *th,
bool can_direct_transfer)
844 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
846 ASSERT_thread_sched_locked(sched, th);
847 VM_ASSERT(th == rb_ec_thread_ptr(rb_current_ec_noinline()));
849 if (th != sched->running) {
851 if (th->has_dedicated_nt && th == sched->runnable_hot_th && (sched->running == NULL || sched->running->has_dedicated_nt)) {
852 RUBY_DEBUG_LOG(
"(nt) stealing: hot-th:%u. running:%u", rb_th_serial(th), rb_th_serial(sched->running));
855 if (sched->running != NULL) {
856 rb_thread_t *running = sched->running;
857 VM_ASSERT(!thread_sched_readyq_contain_p(sched, running));
858 running->sched.node.is_ready =
true;
859 ccan_list_add(&sched->readyq, &running->sched.node.readyq);
864 if (th->sched.node.is_ready) {
865 VM_ASSERT(thread_sched_readyq_contain_p(sched, th));
866 ccan_list_del_init(&th->sched.node.readyq);
867 th->sched.node.is_ready =
false;
870 thread_sched_set_running(sched, th);
871 rb_ractor_thread_switch(th->ractor, th,
false);
878 rb_thread_t *next_th;
879 while((next_th = sched->running) != th) {
880 if (th_has_dedicated_nt(th)) {
881 RUBY_DEBUG_LOG(
"(nt) sleep th:%u running:%u", rb_th_serial(th), rb_th_serial(sched->running));
883 thread_sched_set_unlocked(sched, th);
885 RUBY_DEBUG_LOG(
"nt:%d cond:%p", th->nt->serial, &th->nt->cond.readyq);
888 thread_sched_set_locked(sched, th);
890 if (sched->runnable_hot_th != NULL && sched->runnable_hot_th_waiting) {
891 VM_ASSERT(sched->runnable_hot_th != th);
895 thread_sched_unlock(sched, th);
896 thread_sched_lock(sched, th);
899 RUBY_DEBUG_LOG(
"(nt) wakeup %s", sched->running == th ?
"success" :
"failed");
900 if (th == sched->running) {
901 rb_ractor_thread_switch(th->ractor, th,
false);
906 if (can_direct_transfer &&
907 (next_th = sched->running) != NULL &&
911 RUBY_DEBUG_LOG(
"th:%u->%u (direct)", rb_th_serial(th), rb_th_serial(next_th));
913 thread_sched_set_unlocked(sched, th);
915 rb_ractor_set_current_ec(th->ractor, NULL);
916 thread_sched_switch(th, next_th);
918 thread_sched_set_locked(sched, th);
923 native_thread_assign(NULL, th);
925 RUBY_DEBUG_LOG(
"th:%u->%u (ractor scheduling)", rb_th_serial(th), rb_th_serial(next_th));
927 thread_sched_set_unlocked(sched, th);
929 rb_ractor_set_current_ec(th->ractor, NULL);
930 coroutine_transfer0(th->sched.context, nt->nt_context,
false);
932 thread_sched_set_locked(sched, th);
935 VM_ASSERT(rb_current_ec_noinline() == th->ec);
939 VM_ASSERT(th->nt != NULL);
940 VM_ASSERT(rb_current_ec_noinline() == th->ec);
941 VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none);
944 thread_sched_add_running_thread(sched, th);
949 sched->runnable_hot_th = NULL;
950 sched->runnable_hot_th_waiting = 0;
958thread_sched_to_running_common(
struct rb_thread_sched *sched, rb_thread_t *th)
960 RUBY_DEBUG_LOG(
"th:%u dedicated:%d", rb_th_serial(th), th_has_dedicated_nt(th));
962 VM_ASSERT(sched->running != th);
963 VM_ASSERT(th_has_dedicated_nt(th));
964 VM_ASSERT(GET_THREAD() == th);
966 native_thread_dedicated_dec(th->vm, th->ractor, th->nt);
969 thread_sched_to_ready_common(sched, th,
false,
false);
971 if (sched->running == th) {
972 thread_sched_add_running_thread(sched, th);
976 thread_sched_wait_running_turn(sched, th,
false);
992 if (sched->runnable_hot_th == th) {
993 sched->runnable_hot_th_waiting = 1;
995 thread_sched_lock(sched, th);
997 thread_sched_to_running_common(sched, th);
999 thread_sched_unlock(sched, th);
1010thread_sched_wakeup_next_thread(
struct rb_thread_sched *sched, rb_thread_t *th,
bool will_switch)
1012 ASSERT_thread_sched_locked(sched, th);
1014 VM_ASSERT(sched->running == th);
1015 VM_ASSERT(sched->running->nt != NULL);
1017 rb_thread_t *next_th = thread_sched_deq(sched);
1019 RUBY_DEBUG_LOG(
"next_th:%u", rb_th_serial(next_th));
1020 VM_ASSERT(th != next_th);
1022 thread_sched_set_running(sched, next_th);
1023 VM_ASSERT(next_th == sched->running);
1024 thread_sched_wakeup_running_thread(sched, next_th, will_switch);
1026 if (th != next_th) {
1027 thread_sched_del_running_thread(sched, th);
1033thread_sched_to_dead_common(
struct rb_thread_sched *sched, rb_thread_t *th)
1035 RUBY_DEBUG_LOG(
"th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated);
1039 thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th));
1048 thread_sched_lock(sched, th);
1050 thread_sched_to_dead_common(sched, th);
1052 thread_sched_unlock(sched, th);
1059thread_sched_to_waiting_common(
struct rb_thread_sched *sched, rb_thread_t *th,
bool yield_immediately)
1061 RUBY_DEBUG_LOG(
"th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated);
1065 native_thread_dedicated_inc(th->vm, th->ractor, th->nt);
1066 if (!yield_immediately) {
1067 sched->runnable_hot_th = th;
1068 sched->runnable_hot_th_waiting = 0;
1070 thread_sched_wakeup_next_thread(sched, th,
false);
1077thread_sched_to_waiting(
struct rb_thread_sched *sched, rb_thread_t *th,
bool yield_immediately)
1079 thread_sched_lock(sched, th);
1081 thread_sched_to_waiting_common(sched, th, yield_immediately);
1083 thread_sched_unlock(sched, th);
1091 VM_ASSERT(func != NULL);
1094 if (RUBY_VM_INTERRUPTED(th->ec)) {
1095 RUBY_DEBUG_LOG(
"interrupted:0x%x", th->ec->interrupt_flag);
1101 if (!th->ec->raised_flag && RUBY_VM_INTERRUPTED(th->ec)) {
1106 VM_ASSERT(th->unblock.func == NULL);
1107 th->unblock.func = func;
1108 th->unblock.arg = arg;
1116ubf_clear(rb_thread_t *th)
1118 if (th->unblock.func) {
1121 th->unblock.func = NULL;
1122 th->unblock.arg = NULL;
1129ubf_waiting(
void *
ptr)
1131 rb_thread_t *th = (rb_thread_t *)
ptr;
1135 th->unblock.func = NULL;
1136 th->unblock.arg = NULL;
1138 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
1140 thread_sched_lock(sched, th);
1142 if (sched->running == th) {
1146 thread_sched_to_ready_common(sched, th,
true,
false);
1149 thread_sched_unlock(sched, th);
1156thread_sched_to_waiting_until_wakeup(
struct rb_thread_sched *sched, rb_thread_t *th)
1158 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
1160 RB_VM_SAVE_MACHINE_CONTEXT(th);
1165 thread_sched_lock(sched, th);
1168 if (ubf_set(th, ubf_waiting, (
void *)th)) {
1169 RUBY_DEBUG_LOG(
"th:%u interrupted", rb_th_serial(th));
1172 bool can_direct_transfer = !th_has_dedicated_nt(th);
1174 thread_sched_wakeup_next_thread(sched, th, can_direct_transfer);
1175 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1178 thread_sched_unlock(sched, th);
1188 RUBY_DEBUG_LOG(
"th:%d sched->readyq_cnt:%d", (
int)th->serial, sched->readyq_cnt);
1190 thread_sched_lock(sched, th);
1192 if (!ccan_list_empty(&sched->readyq)) {
1194 thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th));
1195 bool can_direct_transfer = !th_has_dedicated_nt(th);
1196 thread_sched_to_ready_common(sched, th,
false, can_direct_transfer);
1197 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1198 th->status = THREAD_RUNNABLE;
1201 VM_ASSERT(sched->readyq_cnt == 0);
1204 thread_sched_unlock(sched, th);
1213 sched->lock_owner = NULL;
1216 ccan_list_head_init(&sched->readyq);
1217 sched->readyq_cnt = 0;
1220 if (!atfork) sched->enable_mn_threads =
true;
1227#ifdef RUBY_ASAN_ENABLED
1228 void **fake_stack = to_dead ? NULL : &transfer_from->fake_stack;
1229 __sanitizer_start_switch_fiber(fake_stack, transfer_to->stack_base, transfer_to->stack_size);
1233 struct
coroutine_context *returning_from = coroutine_transfer(transfer_from, transfer_to);
1237 VM_ASSERT(!to_dead);
1238#ifdef RUBY_ASAN_ENABLED
1239 __sanitizer_finish_switch_fiber(transfer_from->fake_stack,
1240 (
const void**)&returning_from->stack_base, &returning_from->stack_size);
1248 VM_ASSERT(!nt->dedicated);
1249 VM_ASSERT(next_th->nt == NULL);
1251 RUBY_DEBUG_LOG(
"next_th:%u", rb_th_serial(next_th));
1253 ruby_thread_set_native(next_th);
1254 native_thread_assign(nt, next_th);
1256 coroutine_transfer0(current_cont, next_th->sched.context, to_dead);
1260thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th)
1263 native_thread_assign(NULL, cth);
1264 RUBY_DEBUG_LOG(
"th:%u->%u on nt:%d", rb_th_serial(cth), rb_th_serial(next_th), nt->serial);
1265 thread_sched_switch0(cth->sched.context, next_th, nt, cth->status == THREAD_KILLED);
1268#if VM_CHECK_MODE > 0
1271grq_size(rb_vm_t *vm, rb_ractor_t *cr)
1273 ASSERT_ractor_sched_locked(vm, cr);
1275 rb_ractor_t *r, *prev_r = NULL;
1278 ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) {
1281 VM_ASSERT(r != prev_r);
1289ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r)
1292 rb_ractor_t *cr = NULL;
1294 VM_ASSERT(sched->running != NULL);
1295 VM_ASSERT(sched->running->nt == NULL);
1297 ractor_sched_lock(vm, cr);
1299#if VM_CHECK_MODE > 0
1302 ccan_list_for_each(&vm->ractor.sched.grq,
tr, threads.sched.grq_node) {
1307 ccan_list_add_tail(&vm->ractor.sched.grq, &sched->grq_node);
1308 vm->ractor.sched.grq_cnt++;
1309 VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt);
1311 RUBY_DEBUG_LOG(
"r:%u th:%u grq_cnt:%u", rb_ractor_id(r), rb_th_serial(sched->running), vm->ractor.sched.grq_cnt);
1317 ractor_sched_unlock(vm, cr);
1321#ifndef SNT_KEEP_SECONDS
1322#define SNT_KEEP_SECONDS 0
1327#define MINIMUM_SNT 0
1331ractor_sched_deq(rb_vm_t *vm, rb_ractor_t *cr)
1335 ractor_sched_lock(vm, cr);
1337 RUBY_DEBUG_LOG(
"empty? %d", ccan_list_empty(&vm->ractor.sched.grq));
1340 VM_ASSERT(rb_current_execution_context(
false) == NULL);
1341 VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt);
1343 while ((r = ccan_list_pop(&vm->ractor.sched.grq, rb_ractor_t, threads.sched.grq_node)) == NULL) {
1344 RUBY_DEBUG_LOG(
"wait grq_cnt:%d", (
int)vm->ractor.sched.grq_cnt);
1346#if SNT_KEEP_SECONDS > 0
1347 rb_hrtime_t abs = rb_hrtime_add(rb_hrtime_now(), RB_HRTIME_PER_SEC * SNT_KEEP_SECONDS);
1348 if (native_cond_timedwait(&vm->ractor.sched.cond, &vm->ractor.sched.lock, &abs) == ETIMEDOUT) {
1349 RUBY_DEBUG_LOG(
"timeout, grq_cnt:%d", (
int)vm->ractor.sched.grq_cnt);
1350 VM_ASSERT(r == NULL);
1351 vm->ractor.sched.snt_cnt--;
1352 vm->ractor.sched.running_cnt--;
1356 RUBY_DEBUG_LOG(
"wakeup grq_cnt:%d", (
int)vm->ractor.sched.grq_cnt);
1359 ractor_sched_set_unlocked(vm, cr);
1361 ractor_sched_set_locked(vm, cr);
1363 RUBY_DEBUG_LOG(
"wakeup grq_cnt:%d", (
int)vm->ractor.sched.grq_cnt);
1367 VM_ASSERT(rb_current_execution_context(
false) == NULL);
1370 VM_ASSERT(vm->ractor.sched.grq_cnt > 0);
1371 vm->ractor.sched.grq_cnt--;
1372 RUBY_DEBUG_LOG(
"r:%d grq_cnt:%u", (
int)rb_ractor_id(r), vm->ractor.sched.grq_cnt);
1375 VM_ASSERT(SNT_KEEP_SECONDS > 0);
1379 ractor_sched_unlock(vm, cr);
1384void rb_ractor_lock_self(rb_ractor_t *r);
1385void rb_ractor_unlock_self(rb_ractor_t *r);
1390rb_ractor_sched_wait(rb_execution_context_t *ec, rb_ractor_t *cr,
rb_unblock_function_t *ubf,
void *ubf_arg)
1394 RUBY_DEBUG_LOG(
"start%s",
"");
1396 rb_thread_t *
volatile th = rb_ec_thread_ptr(ec);
1399 if (ubf_set(th, ubf, ubf_arg)) {
1404 thread_sched_lock(sched, th);
1405 rb_ractor_unlock_self(cr);
1408 bool can_direct_transfer = !th_has_dedicated_nt(th);
1409 RB_VM_SAVE_MACHINE_CONTEXT(th);
1410 th->status = THREAD_STOPPED_FOREVER;
1412 thread_sched_wakeup_next_thread(sched, th, can_direct_transfer);
1414 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1415 th->status = THREAD_RUNNABLE;
1417 thread_sched_unlock(sched, th);
1418 rb_ractor_lock_self(cr);
1422 RUBY_DEBUG_LOG(
"end%s",
"");
1426rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *r_th)
1431 RUBY_DEBUG_LOG(
"r:%u th:%d", (
unsigned int)rb_ractor_id(r), r_th->serial);
1433 thread_sched_lock(sched, r_th);
1435 if (r_th->status == THREAD_STOPPED_FOREVER) {
1436 thread_sched_to_ready_common(sched, r_th,
true,
false);
1439 thread_sched_unlock(sched, r_th);
1443ractor_sched_barrier_completed_p(rb_vm_t *vm)
1445 RUBY_DEBUG_LOG(
"run:%u wait:%u", vm->ractor.sched.running_cnt, vm->ractor.sched.barrier_waiting_cnt);
1446 VM_ASSERT(vm->ractor.sched.running_cnt - 1 >= vm->ractor.sched.barrier_waiting_cnt);
1448 return (vm->ractor.sched.running_cnt - vm->ractor.sched.barrier_waiting_cnt) == 1;
1452rb_ractor_sched_barrier_start(rb_vm_t *vm, rb_ractor_t *cr)
1454 VM_ASSERT(cr == GET_RACTOR());
1455 VM_ASSERT(vm->ractor.sync.lock_owner == cr);
1456 VM_ASSERT(!vm->ractor.sched.barrier_waiting);
1457 VM_ASSERT(vm->ractor.sched.barrier_waiting_cnt == 0);
1458 VM_ASSERT(vm->ractor.sched.barrier_ractor == NULL);
1459 VM_ASSERT(vm->ractor.sched.barrier_lock_rec == 0);
1461 RUBY_DEBUG_LOG(
"start serial:%u", vm->ractor.sched.barrier_serial);
1463 unsigned int lock_rec;
1465 ractor_sched_lock(vm, cr);
1467 vm->ractor.sched.barrier_waiting =
true;
1468 vm->ractor.sched.barrier_ractor = cr;
1469 vm->ractor.sched.barrier_lock_rec = vm->ractor.sync.lock_rec;
1472 lock_rec = vm->ractor.sync.lock_rec;
1473 vm->ractor.sync.lock_rec = 0;
1474 vm->ractor.sync.lock_owner = NULL;
1479 ccan_list_for_each(&vm->ractor.sched.running_threads, ith, sched.node.running_threads) {
1480 if (ith->ractor != cr) {
1481 RUBY_DEBUG_LOG(
"barrier request to th:%u", rb_th_serial(ith));
1482 RUBY_VM_SET_VM_BARRIER_INTERRUPT(ith->ec);
1487 while (!ractor_sched_barrier_completed_p(vm)) {
1488 ractor_sched_set_unlocked(vm, cr);
1490 ractor_sched_set_locked(vm, cr);
1493 RUBY_DEBUG_LOG(
"completed seirial:%u", vm->ractor.sched.barrier_serial);
1496 vm->ractor.sched.barrier_serial++;
1497 vm->ractor.sched.barrier_waiting_cnt = 0;
1502 vm->ractor.sync.lock_rec = lock_rec;
1503 vm->ractor.sync.lock_owner = cr;
1512rb_ractor_sched_barrier_end(rb_vm_t *vm, rb_ractor_t *cr)
1514 RUBY_DEBUG_LOG(
"serial:%u", (
unsigned int)vm->ractor.sched.barrier_serial - 1);
1515 VM_ASSERT(vm->ractor.sched.barrier_waiting);
1516 VM_ASSERT(vm->ractor.sched.barrier_ractor);
1517 VM_ASSERT(vm->ractor.sched.barrier_lock_rec > 0);
1519 vm->ractor.sched.barrier_waiting =
false;
1520 vm->ractor.sched.barrier_ractor = NULL;
1521 vm->ractor.sched.barrier_lock_rec = 0;
1522 ractor_sched_unlock(vm, cr);
1526ractor_sched_barrier_join_signal_locked(rb_vm_t *vm)
1528 if (ractor_sched_barrier_completed_p(vm)) {
1534ractor_sched_barrier_join_wait_locked(rb_vm_t *vm, rb_thread_t *th)
1536 VM_ASSERT(vm->ractor.sched.barrier_waiting);
1538 unsigned int barrier_serial = vm->ractor.sched.barrier_serial;
1540 while (vm->ractor.sched.barrier_serial == barrier_serial) {
1541 RUBY_DEBUG_LOG(
"sleep serial:%u", barrier_serial);
1542 RB_VM_SAVE_MACHINE_CONTEXT(th);
1544 rb_ractor_t *cr = th->ractor;
1545 ractor_sched_set_unlocked(vm, cr);
1547 ractor_sched_set_locked(vm, cr);
1549 RUBY_DEBUG_LOG(
"wakeup serial:%u", barrier_serial);
1554rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr)
1556 VM_ASSERT(cr->threads.sched.running != NULL);
1557 VM_ASSERT(cr == GET_RACTOR());
1558 VM_ASSERT(vm->ractor.sync.lock_owner == NULL);
1559 VM_ASSERT(vm->ractor.sched.barrier_waiting);
1561#if USE_RUBY_DEBUG_LOG || VM_CHECK_MODE > 0
1562 unsigned int barrier_serial = vm->ractor.sched.barrier_serial;
1565 RUBY_DEBUG_LOG(
"join");
1569 VM_ASSERT(vm->ractor.sched.barrier_waiting);
1570 VM_ASSERT(vm->ractor.sched.barrier_serial == barrier_serial);
1572 ractor_sched_lock(vm, cr);
1575 vm->ractor.sched.barrier_waiting_cnt++;
1576 RUBY_DEBUG_LOG(
"waiting_cnt:%u serial:%u", vm->ractor.sched.barrier_waiting_cnt, barrier_serial);
1578 ractor_sched_barrier_join_signal_locked(vm);
1579 ractor_sched_barrier_join_wait_locked(vm, cr->threads.sched.running);
1581 ractor_sched_unlock(vm, cr);
1591static void clear_thread_cache_altstack(
void);
1604 clear_thread_cache_altstack();
1608#ifdef RB_THREAD_T_HAS_NATIVE_ID
1610get_native_thread_id(
void)
1613 return (
int)syscall(SYS_gettid);
1614#elif defined(__FreeBSD__)
1615 return pthread_getthreadid_np();
1620#if defined(HAVE_WORKING_FORK)
1621void rb_internal_thread_event_hooks_rw_lock_atfork(
void);
1627 rb_thread_sched_init(sched,
true);
1628 rb_thread_t *th = GET_THREAD();
1629 rb_vm_t *vm = GET_VM();
1631 if (th_has_dedicated_nt(th)) {
1632 vm->ractor.sched.snt_cnt = 0;
1635 vm->ractor.sched.snt_cnt = 1;
1637 vm->ractor.sched.running_cnt = 0;
1640#if VM_CHECK_MODE > 0
1641 vm->ractor.sched.lock_owner = NULL;
1642 vm->ractor.sched.locked =
false;
1650 ccan_list_head_init(&vm->ractor.sched.grq);
1651 ccan_list_head_init(&vm->ractor.sched.timeslice_threads);
1652 ccan_list_head_init(&vm->ractor.sched.running_threads);
1654 rb_internal_thread_event_hooks_rw_lock_atfork();
1656 VM_ASSERT(sched->is_running);
1657 sched->is_running_timeslice =
false;
1659 if (sched->running != th) {
1660 thread_sched_to_running(sched, th);
1663 thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL, NULL);
1666#ifdef RB_THREAD_T_HAS_NATIVE_ID
1668 th->nt->tid = get_native_thread_id();
1675#ifdef RB_THREAD_LOCAL_SPECIFIER
1676static RB_THREAD_LOCAL_SPECIFIER rb_thread_t *ruby_native_thread;
1678static pthread_key_t ruby_native_thread_key;
1690ruby_thread_from_native(
void)
1692#ifdef RB_THREAD_LOCAL_SPECIFIER
1693 return ruby_native_thread;
1695 return pthread_getspecific(ruby_native_thread_key);
1700ruby_thread_set_native(rb_thread_t *th)
1704 ccan_list_node_init(&th->sched.node.ubf);
1711 rb_ractor_set_current_ec(th->ractor, th->ec);
1713#ifdef RB_THREAD_LOCAL_SPECIFIER
1714 ruby_native_thread = th;
1717 return pthread_setspecific(ruby_native_thread_key, th) == 0;
1725Init_native_thread(rb_thread_t *main_th)
1727#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK)
1728 if (condattr_monotonic) {
1729 int r = pthread_condattr_init(condattr_monotonic);
1731 r = pthread_condattr_setclock(condattr_monotonic, CLOCK_MONOTONIC);
1733 if (r) condattr_monotonic = NULL;
1737#ifndef RB_THREAD_LOCAL_SPECIFIER
1738 if (pthread_key_create(&ruby_native_thread_key, 0) == EAGAIN) {
1739 rb_bug(
"pthread_key_create failed (ruby_native_thread_key)");
1741 if (pthread_key_create(&ruby_current_ec_key, 0) == EAGAIN) {
1742 rb_bug(
"pthread_key_create failed (ruby_current_ec_key)");
1745 ruby_posix_signal(SIGVTALRM, null_func);
1748 rb_vm_t *vm = main_th->vm;
1754 ccan_list_head_init(&vm->ractor.sched.grq);
1755 ccan_list_head_init(&vm->ractor.sched.timeslice_threads);
1756 ccan_list_head_init(&vm->ractor.sched.running_threads);
1759 main_th->nt->thread_id = pthread_self();
1760 main_th->nt->serial = 1;
1761#ifdef RUBY_NT_SERIAL
1764 ruby_thread_set_native(main_th);
1765 native_thread_setup(main_th->nt);
1766 native_thread_setup_on_thread(main_th->nt);
1768 TH_SCHED(main_th)->running = main_th;
1769 main_th->has_dedicated_nt = 1;
1771 thread_sched_setup_running_threads(TH_SCHED(main_th), main_th->ractor, vm, main_th, NULL, NULL);
1774 main_th->nt->dedicated = 1;
1775 main_th->nt->vm = vm;
1778 vm->ractor.sched.dnt_cnt = 1;
1781extern int ruby_mn_threads_enabled;
1784ruby_mn_threads_params(
void)
1786 rb_vm_t *vm = GET_VM();
1787 rb_ractor_t *main_ractor = GET_RACTOR();
1789 const char *mn_threads_cstr = getenv(
"RUBY_MN_THREADS");
1790 bool enable_mn_threads =
false;
1792 if (USE_MN_THREADS && mn_threads_cstr && (enable_mn_threads = atoi(mn_threads_cstr) > 0)) {
1794 ruby_mn_threads_enabled = 1;
1796 main_ractor->threads.sched.enable_mn_threads = enable_mn_threads;
1798 const char *max_cpu_cstr = getenv(
"RUBY_MAX_CPU");
1799 const int default_max_cpu = 8;
1800 int max_cpu = default_max_cpu;
1802 if (USE_MN_THREADS && max_cpu_cstr) {
1803 int given_max_cpu = atoi(max_cpu_cstr);
1804 if (given_max_cpu > 0) {
1805 max_cpu = given_max_cpu;
1809 vm->ractor.sched.max_cpu = max_cpu;
1813native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr,
struct rb_native_thread *nt)
1815 RUBY_DEBUG_LOG(
"nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated + 1);
1817 if (nt->dedicated == 0) {
1818 ractor_sched_lock(vm, cr);
1820 vm->ractor.sched.snt_cnt--;
1821 vm->ractor.sched.dnt_cnt++;
1823 ractor_sched_unlock(vm, cr);
1830native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr,
struct rb_native_thread *nt)
1832 RUBY_DEBUG_LOG(
"nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated - 1);
1833 VM_ASSERT(nt->dedicated > 0);
1836 if (nt->dedicated == 0) {
1837 ractor_sched_lock(vm, cr);
1839 nt->vm->ractor.sched.snt_cnt++;
1840 nt->vm->ractor.sched.dnt_cnt--;
1842 ractor_sched_unlock(vm, cr);
1849#if USE_RUBY_DEBUG_LOG
1852 RUBY_DEBUG_LOG(
"th:%d nt:%d->%d", (
int)th->serial, (
int)th->nt->serial, (
int)nt->serial);
1855 RUBY_DEBUG_LOG(
"th:%d nt:NULL->%d", (
int)th->serial, (
int)nt->serial);
1860 RUBY_DEBUG_LOG(
"th:%d nt:%d->NULL", (
int)th->serial, (
int)th->nt->serial);
1863 RUBY_DEBUG_LOG(
"th:%d nt:NULL->NULL", (
int)th->serial);
1886 RB_ALTSTACK_FREE(nt->altstack);
1898 if (&nt->cond.readyq != &nt->cond.intr) {
1902 native_thread_destroy_atfork(nt);
1906#if defined HAVE_PTHREAD_GETATTR_NP || defined HAVE_PTHREAD_ATTR_GET_NP
1907#define STACKADDR_AVAILABLE 1
1908#elif defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP
1909#define STACKADDR_AVAILABLE 1
1910#undef MAINSTACKADDR_AVAILABLE
1911#define MAINSTACKADDR_AVAILABLE 1
1912void *pthread_get_stackaddr_np(pthread_t);
1913size_t pthread_get_stacksize_np(pthread_t);
1914#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
1915#define STACKADDR_AVAILABLE 1
1916#elif defined HAVE_PTHREAD_GETTHRDS_NP
1917#define STACKADDR_AVAILABLE 1
1918#elif defined __HAIKU__
1919#define STACKADDR_AVAILABLE 1
1922#ifndef MAINSTACKADDR_AVAILABLE
1923# ifdef STACKADDR_AVAILABLE
1924# define MAINSTACKADDR_AVAILABLE 1
1926# define MAINSTACKADDR_AVAILABLE 0
1929#if MAINSTACKADDR_AVAILABLE && !defined(get_main_stack)
1930# define get_main_stack(addr, size) get_stack(addr, size)
1933#ifdef STACKADDR_AVAILABLE
1938get_stack(
void **addr,
size_t *size)
1940#define CHECK_ERR(expr) \
1941 {int err = (expr); if (err) return err;}
1942#ifdef HAVE_PTHREAD_GETATTR_NP
1943 pthread_attr_t attr;
1945 STACK_GROW_DIR_DETECTION;
1946 CHECK_ERR(pthread_getattr_np(pthread_self(), &attr));
1947# ifdef HAVE_PTHREAD_ATTR_GETSTACK
1948 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
1949 STACK_DIR_UPPER((
void)0, (
void)(*addr = (
char *)*addr + *size));
1951 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
1952 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
1954# ifdef HAVE_PTHREAD_ATTR_GETGUARDSIZE
1955 CHECK_ERR(pthread_attr_getguardsize(&attr, &guard));
1957 guard = getpagesize();
1960 pthread_attr_destroy(&attr);
1961#elif defined HAVE_PTHREAD_ATTR_GET_NP
1962 pthread_attr_t attr;
1963 CHECK_ERR(pthread_attr_init(&attr));
1964 CHECK_ERR(pthread_attr_get_np(pthread_self(), &attr));
1965# ifdef HAVE_PTHREAD_ATTR_GETSTACK
1966 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
1968 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
1969 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
1971 STACK_DIR_UPPER((
void)0, (
void)(*addr = (
char *)*addr + *size));
1972 pthread_attr_destroy(&attr);
1973#elif (defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP)
1974 pthread_t th = pthread_self();
1975 *addr = pthread_get_stackaddr_np(th);
1976 *size = pthread_get_stacksize_np(th);
1977#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
1979# if defined HAVE_THR_STKSEGMENT
1980 CHECK_ERR(thr_stksegment(&stk));
1982 CHECK_ERR(pthread_stackseg_np(pthread_self(), &stk));
1985 *size = stk.ss_size;
1986#elif defined HAVE_PTHREAD_GETTHRDS_NP
1987 pthread_t th = pthread_self();
1988 struct __pthrdsinfo thinfo;
1990 int regsiz=
sizeof(reg);
1991 CHECK_ERR(pthread_getthrds_np(&th, PTHRDSINFO_QUERY_ALL,
1992 &thinfo,
sizeof(thinfo),
1994 *addr = thinfo.__pi_stackaddr;
1998 *size = thinfo.__pi_stackend - thinfo.__pi_stackaddr;
1999 STACK_DIR_UPPER((
void)0, (
void)(*addr = (
char *)*addr + *size));
2000#elif defined __HAIKU__
2002 STACK_GROW_DIR_DETECTION;
2003 CHECK_ERR(get_thread_info(find_thread(NULL), &info));
2004 *addr = info.stack_base;
2005 *size = (uintptr_t)info.stack_end - (uintptr_t)info.stack_base;
2006 STACK_DIR_UPPER((
void)0, (
void)(*addr = (
char *)*addr + *size));
2008#error STACKADDR_AVAILABLE is defined but not implemented.
2016 rb_nativethread_id_t id;
2017 size_t stack_maxsize;
2019} native_main_thread;
2021#ifdef STACK_END_ADDRESS
2022extern void *STACK_END_ADDRESS;
2026 RUBY_STACK_SPACE_LIMIT = 1024 * 1024,
2027 RUBY_STACK_SPACE_RATIO = 5
2031space_size(
size_t stack_size)
2033 size_t space_size = stack_size / RUBY_STACK_SPACE_RATIO;
2034 if (space_size > RUBY_STACK_SPACE_LIMIT) {
2035 return RUBY_STACK_SPACE_LIMIT;
2043native_thread_init_main_thread_stack(
void *addr)
2045 native_main_thread.id = pthread_self();
2046#ifdef RUBY_ASAN_ENABLED
2047 addr = asan_get_real_stack_addr((
void *)addr);
2050#if MAINSTACKADDR_AVAILABLE
2051 if (native_main_thread.stack_maxsize)
return;
2055 if (get_main_stack(&stackaddr, &size) == 0) {
2056 native_main_thread.stack_maxsize = size;
2057 native_main_thread.stack_start = stackaddr;
2062#ifdef STACK_END_ADDRESS
2063 native_main_thread.stack_start = STACK_END_ADDRESS;
2065 if (!native_main_thread.stack_start ||
2066 STACK_UPPER((
VALUE *)(
void *)&addr,
2067 native_main_thread.stack_start > (
VALUE *)addr,
2068 native_main_thread.stack_start < (
VALUE *)addr)) {
2069 native_main_thread.stack_start = (
VALUE *)addr;
2073#if defined(HAVE_GETRLIMIT)
2074#if defined(PTHREAD_STACK_DEFAULT)
2075# if PTHREAD_STACK_DEFAULT < RUBY_STACK_SPACE*5
2076# error "PTHREAD_STACK_DEFAULT is too small"
2078 size_t size = PTHREAD_STACK_DEFAULT;
2080 size_t size = RUBY_VM_THREAD_VM_STACK_SIZE;
2083 int pagesize = getpagesize();
2085 STACK_GROW_DIR_DETECTION;
2086 if (getrlimit(RLIMIT_STACK, &rlim) == 0) {
2087 size = (size_t)rlim.rlim_cur;
2089 addr = native_main_thread.stack_start;
2090 if (IS_STACK_DIR_UPPER()) {
2091 space = ((size_t)((
char *)addr + size) / pagesize) * pagesize - (size_t)addr;
2094 space = (size_t)addr - ((
size_t)((
char *)addr - size) / pagesize + 1) * pagesize;
2096 native_main_thread.stack_maxsize = space;
2100#if MAINSTACKADDR_AVAILABLE
2107 STACK_GROW_DIR_DETECTION;
2109 if (IS_STACK_DIR_UPPER()) {
2110 start = native_main_thread.stack_start;
2111 end = (
char *)native_main_thread.stack_start + native_main_thread.stack_maxsize;
2114 start = (
char *)native_main_thread.stack_start - native_main_thread.stack_maxsize;
2115 end = native_main_thread.stack_start;
2118 if ((
void *)addr < start || (
void *)addr > end) {
2120 native_main_thread.stack_start = (
VALUE *)addr;
2121 native_main_thread.stack_maxsize = 0;
2126#define CHECK_ERR(expr) \
2127 {int err = (expr); if (err) {rb_bug_errno(#expr, err);}}
2130native_thread_init_stack(rb_thread_t *th,
void *local_in_parent_frame)
2132 rb_nativethread_id_t curr = pthread_self();
2133#ifdef RUBY_ASAN_ENABLED
2134 local_in_parent_frame = asan_get_real_stack_addr(local_in_parent_frame);
2135 th->ec->machine.asan_fake_stack_handle = asan_get_thread_fake_stack_handle();
2138 if (!native_main_thread.id) {
2141 native_thread_init_main_thread_stack(local_in_parent_frame);
2144 if (pthread_equal(curr, native_main_thread.id)) {
2145 th->ec->machine.stack_start = native_main_thread.stack_start;
2146 th->ec->machine.stack_maxsize = native_main_thread.stack_maxsize;
2149#ifdef STACKADDR_AVAILABLE
2150 if (th_has_dedicated_nt(th)) {
2154 if (get_stack(&start, &size) == 0) {
2155 uintptr_t diff = (uintptr_t)start - (uintptr_t)local_in_parent_frame;
2156 th->ec->machine.stack_start = local_in_parent_frame;
2157 th->ec->machine.stack_maxsize = size - diff;
2161 rb_raise(
rb_eNotImpError,
"ruby engine can initialize only in the main thread");
2170 struct rb_native_thread *nt;
2180 pthread_attr_t attr;
2182 const size_t stack_size = nt->vm->default_params.thread_machine_stack_size;
2183 const size_t space = space_size(stack_size);
2185 nt->machine_stack_maxsize = stack_size - space;
2187#ifdef USE_SIGALTSTACK
2188 nt->altstack = rb_allocate_sigaltstack();
2191 CHECK_ERR(pthread_attr_init(&attr));
2193# ifdef PTHREAD_STACK_MIN
2194 RUBY_DEBUG_LOG(
"stack size: %lu", (
unsigned long)stack_size);
2195 CHECK_ERR(pthread_attr_setstacksize(&attr, stack_size));
2198# ifdef HAVE_PTHREAD_ATTR_SETINHERITSCHED
2199 CHECK_ERR(pthread_attr_setinheritsched(&attr, PTHREAD_INHERIT_SCHED));
2201 CHECK_ERR(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED));
2203 err = pthread_create(&nt->thread_id, &attr, nt_start, nt);
2205 RUBY_DEBUG_LOG(
"nt:%d err:%d", (
int)nt->serial, err);
2207 CHECK_ERR(pthread_attr_destroy(&attr));
2218 if (&nt->cond.readyq != &nt->cond.intr) {
2227#ifdef RB_THREAD_T_HAS_NATIVE_ID
2228 nt->tid = get_native_thread_id();
2232 RB_ALTSTACK_INIT(nt->altstack, nt->altstack);
2236native_thread_alloc(
void)
2239 native_thread_setup(nt);
2245#if USE_RUBY_DEBUG_LOG
2253native_thread_create_dedicated(rb_thread_t *th)
2255 th->nt = native_thread_alloc();
2256 th->nt->vm = th->vm;
2257 th->nt->running_thread = th;
2258 th->nt->dedicated = 1;
2261 size_t vm_stack_word_size = th->vm->default_params.thread_vm_stack_size /
sizeof(
VALUE);
2262 void *vm_stack = ruby_xmalloc(vm_stack_word_size *
sizeof(
VALUE));
2263 th->sched.malloc_stack =
true;
2264 rb_ec_initialize_vm_stack(th->ec, vm_stack, vm_stack_word_size);
2265 th->sched.context_stack = vm_stack;
2268 int err = native_thread_create0(th->nt);
2271 thread_sched_to_ready(TH_SCHED(th), th);
2277call_thread_start_func_2(rb_thread_t *th)
2285 VALUE stack_start = 0;
2286 VALUE *stack_start_addr = asan_get_real_stack_addr(&stack_start);
2288 native_thread_init_stack(th, stack_start_addr);
2289 thread_start_func_2(th, th->ec->machine.stack_start);
2296 rb_vm_t *vm = nt->vm;
2298 native_thread_setup_on_thread(nt);
2301#ifdef RB_THREAD_T_HAS_NATIVE_ID
2302 nt->tid = get_native_thread_id();
2305#if USE_RUBY_DEBUG_LOG && defined(RUBY_NT_SERIAL)
2306 ruby_nt_serial = nt->serial;
2309 RUBY_DEBUG_LOG(
"nt:%u", nt->serial);
2311 if (!nt->dedicated) {
2312 coroutine_initialize_main(nt->nt_context);
2316 if (nt->dedicated) {
2318 rb_thread_t *th = nt->running_thread;
2321 RUBY_DEBUG_LOG(
"on dedicated th:%u", rb_th_serial(th));
2322 ruby_thread_set_native(th);
2324 thread_sched_lock(sched, th);
2326 if (sched->running == th) {
2327 thread_sched_add_running_thread(sched, th);
2329 thread_sched_wait_running_turn(sched, th,
false);
2331 thread_sched_unlock(sched, th);
2334 call_thread_start_func_2(th);
2338 RUBY_DEBUG_LOG(
"check next");
2339 rb_ractor_t *r = ractor_sched_deq(vm, NULL);
2344 thread_sched_lock(sched, NULL);
2346 rb_thread_t *next_th = sched->running;
2348 if (next_th && next_th->nt == NULL) {
2349 RUBY_DEBUG_LOG(
"nt:%d next_th:%d", (
int)nt->serial, (
int)next_th->serial);
2350 thread_sched_switch0(nt->nt_context, next_th, nt,
false);
2353 RUBY_DEBUG_LOG(
"no schedulable threads -- next_th:%p", next_th);
2356 thread_sched_unlock(sched, NULL);
2363 if (nt->dedicated) {
2373static int native_thread_create_shared(rb_thread_t *th);
2376static void nt_free_stack(
void *mstack);
2380rb_threadptr_remove(rb_thread_t *th)
2383 if (th->sched.malloc_stack) {
2388 rb_vm_t *vm = th->vm;
2389 th->sched.finished =
false;
2392 ccan_list_add(&vm->ractor.sched.zombie_threads, &th->sched.node.zombie_threads);
2399rb_threadptr_sched_free(rb_thread_t *th)
2402 if (th->sched.malloc_stack) {
2405 native_thread_destroy(th->nt);
2408 nt_free_stack(th->sched.context_stack);
2413 th->sched.context = NULL;
2417 native_thread_destroy(th->nt);
2424rb_thread_sched_mark_zombies(rb_vm_t *vm)
2426 if (!ccan_list_empty(&vm->ractor.sched.zombie_threads)) {
2427 rb_thread_t *zombie_th, *next_zombie_th;
2428 ccan_list_for_each_safe(&vm->ractor.sched.zombie_threads, zombie_th, next_zombie_th, sched.node.zombie_threads) {
2429 if (zombie_th->sched.finished) {
2430 ccan_list_del_init(&zombie_th->sched.node.zombie_threads);
2433 rb_gc_mark(zombie_th->self);
2440native_thread_create(rb_thread_t *th)
2442 VM_ASSERT(th->nt == 0);
2443 RUBY_DEBUG_LOG(
"th:%d has_dnt:%d", th->serial, th->has_dedicated_nt);
2446 if (!th->ractor->threads.sched.enable_mn_threads) {
2447 th->has_dedicated_nt = 1;
2450 if (th->has_dedicated_nt) {
2451 return native_thread_create_dedicated(th);
2454 return native_thread_create_shared(th);
2458#if USE_NATIVE_THREAD_PRIORITY
2461native_thread_apply_priority(rb_thread_t *th)
2463#if defined(_POSIX_PRIORITY_SCHEDULING) && (_POSIX_PRIORITY_SCHEDULING > 0)
2464 struct sched_param sp;
2466 int priority = 0 - th->priority;
2468 pthread_getschedparam(th->nt->thread_id, &policy, &sp);
2469 max = sched_get_priority_max(policy);
2470 min = sched_get_priority_min(policy);
2472 if (min > priority) {
2475 else if (max < priority) {
2479 sp.sched_priority = priority;
2480 pthread_setschedparam(th->nt->thread_id, policy, &sp);
2491 return rb_fd_select(n, readfds, writefds, exceptfds, timeout);
2495ubf_pthread_cond_signal(
void *
ptr)
2497 rb_thread_t *th = (rb_thread_t *)
ptr;
2498 RUBY_DEBUG_LOG(
"th:%u on nt:%d", rb_th_serial(th), (
int)th->nt->serial);
2503native_cond_sleep(rb_thread_t *th, rb_hrtime_t *rel)
2505 rb_nativethread_lock_t *lock = &th->interrupt_lock;
2506 rb_nativethread_cond_t *cond = &th->nt->cond.intr;
2516 const rb_hrtime_t max = (rb_hrtime_t)100000000 * RB_HRTIME_PER_SEC;
2518 THREAD_BLOCKING_BEGIN(th);
2521 th->unblock.func = ubf_pthread_cond_signal;
2522 th->unblock.arg = th;
2524 if (RUBY_VM_INTERRUPTED(th->ec)) {
2526 RUBY_DEBUG_LOG(
"interrupted before sleep th:%u", rb_th_serial(th));
2539 end = native_cond_timeout(cond, *rel);
2540 native_cond_timedwait(cond, lock, &end);
2543 th->unblock.func = 0;
2547 THREAD_BLOCKING_END(th);
2549 RUBY_DEBUG_LOG(
"done th:%u", rb_th_serial(th));
2553static CCAN_LIST_HEAD(ubf_list_head);
2554static rb_nativethread_lock_t ubf_list_lock = RB_NATIVETHREAD_LOCK_INIT;
2557ubf_list_atfork(
void)
2559 ccan_list_head_init(&ubf_list_head);
2565ubf_list_contain_p(rb_thread_t *th)
2567 rb_thread_t *list_th;
2568 ccan_list_for_each(&ubf_list_head, list_th, sched.node.ubf) {
2569 if (list_th == th)
return true;
2576register_ubf_list(rb_thread_t *th)
2578 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
2579 struct ccan_list_node *node = &th->sched.node.ubf;
2581 VM_ASSERT(th->unblock.func != NULL);
2586 if (ccan_list_empty((
struct ccan_list_head*)node)) {
2587 VM_ASSERT(!ubf_list_contain_p(th));
2588 ccan_list_add(&ubf_list_head, node);
2593 timer_thread_wakeup();
2598unregister_ubf_list(rb_thread_t *th)
2600 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
2601 struct ccan_list_node *node = &th->sched.node.ubf;
2604 VM_ASSERT(th->unblock.func == NULL);
2606 if (!ccan_list_empty((
struct ccan_list_head*)node)) {
2609 VM_ASSERT(ubf_list_contain_p(th));
2610 ccan_list_del_init(node);
2621ubf_wakeup_thread(rb_thread_t *th)
2623 RUBY_DEBUG_LOG(
"th:%u thread_id:%p", rb_th_serial(th), (
void *)th->nt->thread_id);
2625 pthread_kill(th->nt->thread_id, SIGVTALRM);
2629ubf_select(
void *
ptr)
2631 rb_thread_t *th = (rb_thread_t *)
ptr;
2632 RUBY_DEBUG_LOG(
"wakeup th:%u", rb_th_serial(th));
2633 ubf_wakeup_thread(th);
2634 register_ubf_list(th);
2638ubf_threads_empty(
void)
2640 return ccan_list_empty(&ubf_list_head) != 0;
2644ubf_wakeup_all_threads(
void)
2649 ccan_list_for_each(&ubf_list_head, th, sched.node.ubf) {
2650 ubf_wakeup_thread(th);
2657#define register_ubf_list(th) (void)(th)
2658#define unregister_ubf_list(th) (void)(th)
2660static void ubf_wakeup_all_threads(
void) {
return; }
2661static bool ubf_threads_empty(
void) {
return true; }
2662#define ubf_list_atfork() do {} while (0)
2666#define WRITE_CONST(fd, str) (void)(write((fd),(str),sizeof(str)-1)<0)
2669rb_thread_wakeup_timer_thread(
int sig)
2675 timer_thread_wakeup_force();
2679 rb_vm_t *vm = GET_VM();
2680 rb_thread_t *main_th = vm->ractor.main_thread;
2683 volatile rb_execution_context_t *main_th_ec = ACCESS_ONCE(rb_execution_context_t *, main_th->ec);
2686 RUBY_VM_SET_TRAP_INTERRUPT(main_th_ec);
2688 if (vm->ubf_async_safe && main_th->unblock.func) {
2689 (main_th->unblock.func)(main_th->unblock.arg);
2696#define CLOSE_INVALIDATE_PAIR(expr) \
2697 close_invalidate_pair(expr,"close_invalidate: "#expr)
2699close_invalidate(
int *fdp,
const char *msg)
2704 if (close(fd) < 0) {
2705 async_bug_fd(msg,
errno, fd);
2710close_invalidate_pair(
int fds[2],
const char *msg)
2712 if (USE_EVENTFD && fds[0] == fds[1]) {
2714 close_invalidate(&fds[0], msg);
2717 close_invalidate(&fds[1], msg);
2718 close_invalidate(&fds[0], msg);
2728 oflags = fcntl(fd, F_GETFL);
2731 oflags |= O_NONBLOCK;
2732 err = fcntl(fd, F_SETFL, oflags);
2739setup_communication_pipe_internal(
int pipes[2])
2743 if (pipes[0] > 0 || pipes[1] > 0) {
2744 VM_ASSERT(pipes[0] > 0);
2745 VM_ASSERT(pipes[1] > 0);
2753#if USE_EVENTFD && defined(EFD_NONBLOCK) && defined(EFD_CLOEXEC)
2754 pipes[0] = pipes[1] = eventfd(0, EFD_NONBLOCK|EFD_CLOEXEC);
2756 if (pipes[0] >= 0) {
2764 rb_bug(
"can not create communication pipe");
2768 set_nonblock(pipes[0]);
2769 set_nonblock(pipes[1]);
2772#if !defined(SET_CURRENT_THREAD_NAME) && defined(__linux__) && defined(PR_SET_NAME)
2773# define SET_CURRENT_THREAD_NAME(name) prctl(PR_SET_NAME, name)
2778#if defined(__linux__)
2780#elif defined(__APPLE__)
2788static VALUE threadptr_invoke_proc_location(rb_thread_t *th);
2791native_set_thread_name(rb_thread_t *th)
2793#ifdef SET_CURRENT_THREAD_NAME
2795 if (!
NIL_P(loc = th->name)) {
2796 SET_CURRENT_THREAD_NAME(RSTRING_PTR(loc));
2798 else if ((loc = threadptr_invoke_proc_location(th)) !=
Qnil) {
2800 char buf[THREAD_NAME_MAX];
2805 p = strrchr(name,
'/');
2813 if (
len >=
sizeof(buf)) {
2814 buf[
sizeof(buf)-2] =
'*';
2815 buf[
sizeof(buf)-1] =
'\0';
2817 SET_CURRENT_THREAD_NAME(buf);
2823native_set_another_thread_name(rb_nativethread_id_t thread_id,
VALUE name)
2825#if defined SET_ANOTHER_THREAD_NAME || defined SET_CURRENT_THREAD_NAME
2826 char buf[THREAD_NAME_MAX];
2828# if !defined SET_ANOTHER_THREAD_NAME
2829 if (!pthread_equal(pthread_self(), thread_id))
return;
2834 if (n >= (
int)
sizeof(buf)) {
2835 memcpy(buf, s,
sizeof(buf)-1);
2836 buf[
sizeof(buf)-1] =
'\0';
2840# if defined SET_ANOTHER_THREAD_NAME
2841 SET_ANOTHER_THREAD_NAME(thread_id, s);
2842# elif defined SET_CURRENT_THREAD_NAME
2843 SET_CURRENT_THREAD_NAME(s);
2848#if defined(RB_THREAD_T_HAS_NATIVE_ID) || defined(__APPLE__)
2850native_thread_native_thread_id(rb_thread_t *target_th)
2852 if (!target_th->nt)
return Qnil;
2854#ifdef RB_THREAD_T_HAS_NATIVE_ID
2855 int tid = target_th->nt->tid;
2856 if (tid == 0)
return Qnil;
2858#elif defined(__APPLE__)
2864# if (!defined(MAC_OS_X_VERSION_10_6) || \
2865 (MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_X_VERSION_10_6) || \
2866 defined(__POWERPC__) )
2867 const bool no_pthread_threadid_np =
true;
2868# define NO_PTHREAD_MACH_THREAD_NP 1
2869# elif MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_6
2870 const bool no_pthread_threadid_np =
false;
2872# if !(defined(__has_attribute) && __has_attribute(availability))
2874 __attribute__((weak))
int pthread_threadid_np(pthread_t, uint64_t*);
2877 const bool no_pthread_threadid_np = !&pthread_threadid_np;
2879 if (no_pthread_threadid_np) {
2880 return ULL2NUM(pthread_mach_thread_np(pthread_self()));
2882# ifndef NO_PTHREAD_MACH_THREAD_NP
2883 int e = pthread_threadid_np(target_th->nt->thread_id, &tid);
2885 return ULL2NUM((
unsigned long long)tid);
2889# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 1
2891# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 0
2895 rb_serial_t created_fork_gen;
2896 pthread_t pthread_id;
2900#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
2903#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
2904#define EPOLL_EVENTS_MAX 0x10
2905 struct epoll_event finished_events[EPOLL_EVENTS_MAX];
2906#elif HAVE_SYS_EVENT_H && USE_MN_THREADS
2907#define KQUEUE_EVENTS_MAX 0x10
2908 struct kevent finished_events[KQUEUE_EVENTS_MAX];
2912 struct ccan_list_head waiting;
2913 pthread_mutex_t waiting_lock;
2915 .created_fork_gen = 0,
2918#define TIMER_THREAD_CREATED_P() (timer_th.created_fork_gen == current_fork_gen)
2920static void timer_thread_check_timeslice(rb_vm_t *vm);
2921static int timer_thread_set_timeout(rb_vm_t *vm);
2922static void timer_thread_wakeup_thread(rb_thread_t *th, uint32_t event_serial);
2924#include "thread_pthread_mn.c"
2930 return (rb_thread_t *)((size_t)w - offsetof(rb_thread_t, sched.waiting_reason));
2938timer_thread_set_timeout(rb_vm_t *vm)
2945 ractor_sched_lock(vm, NULL);
2947 if ( !ccan_list_empty(&vm->ractor.sched.timeslice_threads)
2948 || !ubf_threads_empty()
2949 || vm->ractor.sched.grq_cnt > 0
2952 RUBY_DEBUG_LOG(
"timeslice:%d ubf:%d grq:%d",
2953 !ccan_list_empty(&vm->ractor.sched.timeslice_threads),
2954 !ubf_threads_empty(),
2955 (vm->ractor.sched.grq_cnt > 0));
2958 vm->ractor.sched.timeslice_wait_inf =
false;
2961 vm->ractor.sched.timeslice_wait_inf =
true;
2964 ractor_sched_unlock(vm, NULL);
2971 rb_thread_t *th = thread_sched_waiting_thread(w);
2973 if (th && (th->sched.waiting_reason.flags & thread_sched_waiting_timeout)) {
2974 rb_hrtime_t now = rb_hrtime_now();
2975 rb_hrtime_t hrrel = rb_hrtime_sub(th->sched.waiting_reason.data.timeout, now);
2977 RUBY_DEBUG_LOG(
"th:%u now:%lu rel:%lu", rb_th_serial(th), (
unsigned long)now, (
unsigned long)hrrel);
2980 int thread_timeout = (int)((hrrel + RB_HRTIME_PER_MSEC - 1) / RB_HRTIME_PER_MSEC);
2983 if (timeout < 0 || thread_timeout < timeout) {
2984 timeout = thread_timeout;
2990 RUBY_DEBUG_LOG(
"timeout:%d inf:%d", timeout, (
int)vm->ractor.sched.timeslice_wait_inf);
2998timer_thread_check_signal(rb_vm_t *vm)
3002 int signum = rb_signal_buff_size();
3003 if (UNLIKELY(signum > 0) && vm->ractor.main_thread) {
3004 RUBY_DEBUG_LOG(
"signum:%d", signum);
3005 threadptr_trap_interrupt(vm->ractor.main_thread);
3010timer_thread_check_exceed(rb_hrtime_t abs, rb_hrtime_t now)
3015 else if (abs - now < RB_HRTIME_PER_MSEC) {
3024timer_thread_deq_wakeup(rb_vm_t *vm, rb_hrtime_t now, uint32_t *event_serial)
3029 (w->flags & thread_sched_waiting_timeout) &&
3030 timer_thread_check_exceed(w->data.timeout, now)) {
3032 RUBY_DEBUG_LOG(
"wakeup th:%u", rb_th_serial(thread_sched_waiting_thread(w)));
3035 ccan_list_del_init(&w->node);
3038 w->flags = thread_sched_waiting_none;
3041 rb_thread_t *th = thread_sched_waiting_thread(w);
3042 *event_serial = w->data.event_serial;
3050timer_thread_wakeup_thread_locked(
struct rb_thread_sched *sched, rb_thread_t *th, uint32_t event_serial)
3052 if (sched->running != th && th->sched.event_serial == event_serial) {
3053 thread_sched_to_ready_common(sched, th,
true,
false);
3058timer_thread_wakeup_thread(rb_thread_t *th, uint32_t event_serial)
3060 RUBY_DEBUG_LOG(
"th:%u", rb_th_serial(th));
3063 thread_sched_lock(sched, th);
3065 timer_thread_wakeup_thread_locked(sched, th, event_serial);
3067 thread_sched_unlock(sched, th);
3071timer_thread_check_timeout(rb_vm_t *vm)
3073 rb_hrtime_t now = rb_hrtime_now();
3075 uint32_t event_serial;
3079 while ((th = timer_thread_deq_wakeup(vm, now, &event_serial)) != NULL) {
3081 timer_thread_wakeup_thread(th, event_serial);
3089timer_thread_check_timeslice(rb_vm_t *vm)
3093 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, th, sched.node.timeslice_threads) {
3094 RUBY_DEBUG_LOG(
"timeslice th:%u", rb_th_serial(th));
3095 RUBY_VM_SET_TIMER_INTERRUPT(th->ec);
3103 pthread_sigmask(0, NULL, &oldmask);
3104 if (sigismember(&oldmask, SIGVTALRM)) {
3108 RUBY_DEBUG_LOG(
"ok");
3113timer_thread_func(
void *
ptr)
3115 rb_vm_t *vm = (rb_vm_t *)
ptr;
3116#if defined(RUBY_NT_SERIAL)
3120 RUBY_DEBUG_LOG(
"started%s",
"");
3123 timer_thread_check_signal(vm);
3124 timer_thread_check_timeout(vm);
3125 ubf_wakeup_all_threads();
3128 timer_thread_polling(vm);
3131 RUBY_DEBUG_LOG(
"terminated");
3137signal_communication_pipe(
int fd)
3140 const uint64_t buff = 1;
3142 const char buff =
'!';
3149 if ((result = write(fd, &buff,
sizeof(buff))) <= 0) {
3152 case EINTR:
goto retry;
3154#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN
3159 async_bug_fd(
"rb_thread_wakeup_timer_thread: write", e, fd);
3162 if (TT_DEBUG) WRITE_CONST(2,
"rb_thread_wakeup_timer_thread: write\n");
3170timer_thread_wakeup_force(
void)
3173 signal_communication_pipe(timer_th.comm_fds[1]);
3177timer_thread_wakeup_locked(rb_vm_t *vm)
3180 ASSERT_ractor_sched_locked(vm, NULL);
3182 if (timer_th.created_fork_gen == current_fork_gen) {
3183 if (vm->ractor.sched.timeslice_wait_inf) {
3184 RUBY_DEBUG_LOG(
"wakeup with fd:%d", timer_th.comm_fds[1]);
3185 timer_thread_wakeup_force();
3188 RUBY_DEBUG_LOG(
"will be wakeup...");
3194timer_thread_wakeup(
void)
3196 rb_vm_t *vm = GET_VM();
3198 ractor_sched_lock(vm, NULL);
3200 timer_thread_wakeup_locked(vm);
3202 ractor_sched_unlock(vm, NULL);
3206rb_thread_create_timer_thread(
void)
3208 rb_serial_t created_fork_gen = timer_th.created_fork_gen;
3210 RUBY_DEBUG_LOG(
"fork_gen create:%d current:%d", (
int)created_fork_gen, (
int)current_fork_gen);
3212 timer_th.created_fork_gen = current_fork_gen;
3214 if (created_fork_gen != current_fork_gen) {
3215 if (created_fork_gen != 0) {
3216 RUBY_DEBUG_LOG(
"forked child process");
3218 CLOSE_INVALIDATE_PAIR(timer_th.comm_fds);
3219#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
3220 close_invalidate(&timer_th.event_fd,
"close event_fd");
3225 ccan_list_head_init(&timer_th.waiting);
3229 setup_communication_pipe_internal(timer_th.comm_fds);
3232 timer_thread_setup_mn();
3235 pthread_create(&timer_th.pthread_id, NULL, timer_thread_func, GET_VM());
3239native_stop_timer_thread(
void)
3243 RUBY_DEBUG_LOG(
"wakeup send %d", timer_th.comm_fds[1]);
3244 timer_thread_wakeup_force();
3245 RUBY_DEBUG_LOG(
"wakeup sent");
3246 pthread_join(timer_th.pthread_id, NULL);
3248 if (TT_DEBUG) fprintf(stderr,
"stop timer thread\n");
3254native_reset_timer_thread(
void)
3259#ifdef HAVE_SIGALTSTACK
3261ruby_stack_overflowed_p(
const rb_thread_t *th,
const void *addr)
3265 const size_t water_mark = 1024 * 1024;
3266 STACK_GROW_DIR_DETECTION;
3269 size = th->ec->machine.stack_maxsize;
3270 base = (
char *)th->ec->machine.stack_start - STACK_DIR_UPPER(0, size);
3272#ifdef STACKADDR_AVAILABLE
3273 else if (get_stack(&base, &size) == 0) {
3275 if (pthread_equal(th->nt->thread_id, native_main_thread.id)) {
3277 if (getrlimit(RLIMIT_STACK, &rlim) == 0 && rlim.rlim_cur > size) {
3278 size = (size_t)rlim.rlim_cur;
3282 base = (
char *)base + STACK_DIR_UPPER(+size, -size);
3289 size /= RUBY_STACK_SPACE_RATIO;
3290 if (size > water_mark) size = water_mark;
3291 if (IS_STACK_DIR_UPPER()) {
3292 if (size > ~(
size_t)base+1) size = ~(size_t)base+1;
3293 if (addr > base && addr <= (
void *)((
char *)base + size))
return 1;
3296 if (size > (
size_t)base) size = (size_t)base;
3297 if (addr > (
void *)((
char *)base - size) && addr <= base)
return 1;
3307 if (fd < 0)
return 0;
3309 if (fd == timer_th.comm_fds[0] ||
3310 fd == timer_th.comm_fds[1]
3311#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
3312 || fd == timer_th.event_fd
3315 goto check_fork_gen;
3320 if (timer_th.created_fork_gen == current_fork_gen) {
3332 return pthread_self();
3335#if defined(USE_POLL) && !defined(HAVE_PPOLL)
3338ruby_ppoll(
struct pollfd *fds, nfds_t nfds,
3339 const struct timespec *ts,
const sigset_t *sigmask)
3346 if (ts->tv_sec > INT_MAX/1000)
3347 timeout_ms = INT_MAX;
3349 tmp = (int)(ts->tv_sec * 1000);
3351 tmp2 = (int)((ts->tv_nsec + 999999L) / (1000L * 1000L));
3352 if (INT_MAX - tmp < tmp2)
3353 timeout_ms = INT_MAX;
3355 timeout_ms = (int)(tmp + tmp2);
3361 return poll(fds, nfds, timeout_ms);
3363# define ppoll(fds,nfds,ts,sigmask) ruby_ppoll((fds),(nfds),(ts),(sigmask))
3376#define THREAD_BLOCKING_YIELD(th) do { \
3377 const rb_thread_t *next_th; \
3378 struct rb_thread_sched *sched = TH_SCHED(th); \
3379 RB_VM_SAVE_MACHINE_CONTEXT(th); \
3380 thread_sched_to_waiting(sched, (th)); \
3381 next_th = sched->running; \
3382 rb_native_mutex_unlock(&sched->lock_); \
3383 native_thread_yield(); \
3384 if (!next_th && rb_ractor_living_thread_num(th->ractor) > 1) { \
3385 native_thread_yield(); \
3389native_sleep(rb_thread_t *th, rb_hrtime_t *rel)
3393 RUBY_DEBUG_LOG(
"rel:%d", rel ? (
int)*rel : 0);
3395 if (th_has_dedicated_nt(th)) {
3396 native_cond_sleep(th, rel);
3399 thread_sched_wait_events(sched, th, -1, thread_sched_waiting_timeout, rel);
3403 thread_sched_to_waiting_until_wakeup(sched, th);
3406 RUBY_DEBUG_LOG(
"wakeup");
3410static pthread_rwlock_t rb_thread_fork_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
3413static void *last_exiting_thread;
3414STATIC_ASSERT(pthread_t_fits_in_pointer,
sizeof(pthread_t) <=
sizeof(
void *));
3417rb_thread_release_fork_lock(
void)
3420 if ((r = pthread_rwlock_unlock(&rb_thread_fork_rw_lock))) {
3426rb_thread_reset_fork_lock(
void)
3431 rb_thread_fork_rw_lock = (pthread_rwlock_t)PTHREAD_RWLOCK_INITIALIZER;
3435rb_thread_prevent_fork(
void *(*func)(
void *),
void *data)
3438 if ((r = pthread_rwlock_rdlock(&rb_thread_fork_rw_lock))) {
3441 void *result = func(data);
3442 rb_thread_release_fork_lock();
3447thread_register_exiting(
void *thread)
3450 if (previous_thread) {
3452 if ((r = pthread_join((pthread_t)(uintptr_t)previous_thread, NULL))) {
3461rb_thread_register_exiting(
void)
3463 thread_register_exiting((
void *)(uintptr_t)pthread_self());
3467rb_thread_acquire_fork_lock(
void)
3470 if ((r = pthread_rwlock_wrlock(&rb_thread_fork_rw_lock))) {
3477 thread_register_exiting(NULL);
3482struct rb_internal_thread_event_hook {
3483 rb_internal_thread_event_callback callback;
3487 struct rb_internal_thread_event_hook *next;
3490static pthread_rwlock_t rb_internal_thread_event_hooks_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
3492#if defined(HAVE_WORKING_FORK)
3494rb_internal_thread_event_hooks_rw_lock_atfork(
void)
3503 rb_internal_thread_event_hooks_rw_lock =
3504 (pthread_rwlock_t)PTHREAD_RWLOCK_INITIALIZER;
3508rb_internal_thread_event_hook_t *
3511 rb_internal_thread_event_hook_t *hook =
ALLOC_N(rb_internal_thread_event_hook_t, 1);
3512 hook->callback = callback;
3513 hook->user_data = user_data;
3514 hook->event = internal_event;
3517 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
3521 hook->next = rb_internal_thread_event_hooks;
3522 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook);
3524 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3534 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
3538 bool success = FALSE;
3540 if (rb_internal_thread_event_hooks == hook) {
3541 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook->next);
3545 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
3548 if (h->next == hook) {
3549 h->next = hook->next;
3553 }
while ((h = h->next));
3556 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3570 if ((r = pthread_rwlock_rdlock(&rb_internal_thread_event_hooks_rw_lock))) {
3574 if (rb_internal_thread_event_hooks) {
3575 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
3577 if (h->event & event) {
3578 rb_internal_thread_event_data_t event_data = {
3581 (*h->callback)(event, &event_data, h->user_data);
3583 }
while((h = h->next));
3585 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3595 rb_thread_t *th = GET_THREAD();
3596 bool is_snt = th->nt->dedicated == 0;
3597 native_thread_dedicated_inc(th->vm, th->ractor, th->nt);
3603rb_thread_malloc_stack_set(rb_thread_t *th,
void *stack)
3605 th->sched.malloc_stack =
true;
3606 th->sched.context_stack = stack;
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
#define RUBY_ATOMIC_PTR_EXCHANGE(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except it expects its arguments are void*.
#define RUBY_ATOMIC_FETCH_ADD(var, val)
Atomically replaces the value pointed by var with the result of addition of val to the old value of v...
#define RUBY_ATOMIC_LOAD(var)
Atomic load.
#define RUBY_ATOMIC_SET(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except for the return type.
uint32_t rb_event_flag_t
Represents event(s).
#define INT2FIX
Old name of RB_INT2FIX.
#define ZALLOC
Old name of RB_ZALLOC.
#define ALLOC_N
Old name of RB_ALLOC_N.
#define ULL2NUM
Old name of RB_ULL2NUM.
#define NUM2INT
Old name of RB_NUM2INT.
#define Qnil
Old name of RUBY_Qnil.
#define NIL_P
Old name of RB_NIL_P.
VALUE rb_eNotImpError
NotImplementedError exception.
void rb_syserr_fail(int e, const char *mesg)
Raises appropriate exception that represents a C errno.
void rb_bug_errno(const char *mesg, int errno_arg)
This is a wrapper of rb_bug() which automatically constructs appropriate message from the passed errn...
int rb_cloexec_pipe(int fildes[2])
Opens a pipe with closing on exec.
void rb_update_max_fd(int fd)
Informs the interpreter that the passed fd can be the max.
int rb_reserved_fd_p(int fd)
Queries if the given FD is reserved or not.
void rb_unblock_function_t(void *)
This is the type of UBFs.
void rb_timespec_now(struct timespec *ts)
Fills the current time into the given struct.
char * ptr
Pointer to the underlying memory region, of at least capa bytes.
int len
Length of the buffer.
#define RUBY_INTERNAL_THREAD_EVENT_RESUMED
Triggered when a thread successfully acquired the GVL.
rb_internal_thread_event_hook_t * rb_internal_thread_add_event_hook(rb_internal_thread_event_callback func, rb_event_flag_t events, void *data)
Registers a thread event hook function.
#define RUBY_INTERNAL_THREAD_EVENT_EXITED
Triggered when a thread exits.
#define RUBY_INTERNAL_THREAD_EVENT_SUSPENDED
Triggered when a thread released the GVL.
bool rb_thread_lock_native_thread(void)
Declare the current Ruby thread should acquire a dedicated native thread on M:N thread scheduler.
#define RUBY_INTERNAL_THREAD_EVENT_STARTED
Triggered when a new thread is started.
bool rb_internal_thread_remove_event_hook(rb_internal_thread_event_hook_t *hook)
Unregister the passed hook.
#define RUBY_INTERNAL_THREAD_EVENT_READY
Triggered when a thread attempt to acquire the GVL.
#define RBIMPL_ATTR_MAYBE_UNUSED()
Wraps (or simulates) [[maybe_unused]].
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
#define rb_fd_select
Waits for multiple file descriptors at once.
#define RARRAY_AREF(a, i)
#define RSTRING_GETMEM(str, ptrvar, lenvar)
Convenient macro to obtain the contents and length at once.
#define errno
Ractor-aware version of errno.
The data structure which wraps the fd_set bitmap used by select(2).
rb_nativethread_id_t rb_nativethread_self(void)
Queries the ID of the native thread that is calling this function.
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_cond_initialize(rb_nativethread_cond_t *cond)
Fills the passed condition variable with an initial value.
int rb_native_mutex_trylock(rb_nativethread_lock_t *lock)
Identical to rb_native_mutex_lock(), except it doesn't block in case rb_native_mutex_lock() would.
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
void rb_native_cond_destroy(rb_nativethread_cond_t *cond)
Destroys the passed condition variable.
void rb_native_cond_signal(rb_nativethread_cond_t *cond)
Signals a condition variable.
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.
void rb_native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, unsigned long msec)
Identical to rb_native_cond_wait(), except it additionally takes timeout in msec resolution.
uintptr_t VALUE
Type that represents a Ruby object.
void ruby_xfree(void *ptr)
Deallocates a storage instance.