Index: kern_timeout.c =================================================================== RCS file: /cvs/src/sys/kern/kern_timeout.c,v diff -u -p -r1.101 kern_timeout.c --- kern_timeout.c 13 Jan 2025 03:21:10 -0000 1.101 +++ kern_timeout.c 30 Apr 2025 01:43:50 -0000 @@ -48,6 +48,11 @@ #include #endif +struct timeout_ctx { + struct circq *tctx_todo; + struct timeout *tctx_running; +}; + /* * Locks used to protect global variables in this file: * @@ -74,9 +79,21 @@ struct circq timeout_wheel[BUCKETS]; /* struct circq timeout_wheel_kc[BUCKETS]; /* [T] Clock-based timeouts */ struct circq timeout_new; /* [T] New, unscheduled timeouts */ struct circq timeout_todo; /* [T] Due or needs rescheduling */ +static struct timeout_ctx timeout_ctx_si = { + .tctx_todo = &timeout_todo, /* [I] */ + .tctx_running = NULL, /* [T] */ +}; struct circq timeout_proc; /* [T] Due + needs process context */ +static struct timeout_ctx timeout_ctx_proc = { + .tctx_todo = &timeout_proc, /* [I] */ + .tctx_running = NULL, /* [T] */ +}; #ifdef MULTIPROCESSOR struct circq timeout_proc_mp; /* [T] Process ctx + no kernel lock */ +static struct timeout_ctx timeout_ctx_proc_mp = { + .tctx_todo = &timeout_proc_mp, /* [I] */ + .tctx_running = NULL, /* [T] */ +}; #endif time_t timeout_level_width[WHEELCOUNT]; /* [I] Wheel level width (seconds) */ @@ -113,6 +130,14 @@ struct kclock { (elem)->prev = (elem); \ } while (0) +#define CIRCQ_INSERT_HEAD(list, elem) do { \ + (elem)->next = (list)->next; \ + (list)->next->prev = (elem); \ + (list)->next = (elem); \ + (elem)->prev = (list); \ + tostat.tos_pending++; \ +} while (0) + #define CIRCQ_INSERT_TAIL(list, elem) do { \ (elem)->prev = (list)->prev; \ (elem)->next = (list); \ @@ -180,7 +205,7 @@ void softclock_thread_mp(void *); void timeout_barrier_timeout(void *); uint32_t timeout_bucket(const struct timeout *); uint32_t timeout_maskwheel(uint32_t, const struct timespec *); -void timeout_run(struct timeout *); +void timeout_run(struct timeout_ctx *, struct timeout *); /* * The first thing in a struct timeout is its struct circq, so we @@ -465,6 +490,7 @@ timeout_del_barrier(struct timeout *to) void timeout_barrier(struct timeout *to) { + struct timeout_ctx *tctx; struct timeout barrier; struct cond c; int flags; @@ -478,30 +504,31 @@ timeout_barrier(struct timeout *to) cond_init(&c); mtx_enter(&timeout_mutex); - - barrier.to_time = ticks; - SET(barrier.to_flags, TIMEOUT_ONQUEUE); if (ISSET(flags, TIMEOUT_PROC)) { #ifdef MULTIPROCESSOR if (ISSET(flags, TIMEOUT_MPSAFE)) - CIRCQ_INSERT_TAIL(&timeout_proc_mp, &barrier.to_list); + tctx = &timeout_ctx_proc_mp; else #endif - CIRCQ_INSERT_TAIL(&timeout_proc, &barrier.to_list); + tctx = &timeout_ctx_proc; } else - CIRCQ_INSERT_TAIL(&timeout_todo, &barrier.to_list); + tctx = &timeout_ctx_si; + + if (tctx->tctx_running != to) { + mtx_leave(&timeout_mutex); + return; + } + barrier.to_time = ticks; + SET(barrier.to_flags, TIMEOUT_ONQUEUE); + CIRCQ_INSERT_HEAD(tctx->tctx_todo, &barrier.to_list); mtx_leave(&timeout_mutex); - if (ISSET(flags, TIMEOUT_PROC)) { -#ifdef MULTIPROCESSOR - if (ISSET(flags, TIMEOUT_MPSAFE)) - wakeup_one(&timeout_proc_mp); - else -#endif - wakeup_one(&timeout_proc); - } else - softintr_schedule(softclock_si); + /* + * We know the relevant timeout context was running something + * and now also has the barrier to run, so we just have to + * wait for it to pick up the barrier task now. + */ cond_wait(&c, "tmobar"); } @@ -634,7 +661,7 @@ timeout_hardclock_update(void) } void -timeout_run(struct timeout *to) +timeout_run(struct timeout_ctx *tctx, struct timeout *to) { void (*fn)(void *); void *arg; @@ -652,6 +679,7 @@ timeout_run(struct timeout *to) struct process *kcov_process = to->to_process; #endif + tctx->tctx_running = to; mtx_leave(&timeout_mutex); timeout_sync_enter(needsproc); #if NKCOV > 0 @@ -663,6 +691,7 @@ timeout_run(struct timeout *to) #endif timeout_sync_leave(needsproc); mtx_enter(&timeout_mutex); + tctx->tctx_running = NULL; } void @@ -689,7 +718,7 @@ softclock_process_kclock_timeout(struct CIRCQ_INSERT_TAIL(&timeout_proc, &to->to_list); return; } - timeout_run(to); + timeout_run(&timeout_ctx_si, to); tostat.tos_run_softclock++; } @@ -716,7 +745,7 @@ softclock_process_tick_timeout(struct ti CIRCQ_INSERT_TAIL(&timeout_proc, &to->to_list); return; } - timeout_run(to); + timeout_run(&timeout_ctx_si, to); tostat.tos_run_softclock++; } @@ -782,12 +811,37 @@ softclock_create_thread(void *arg) #endif } +static void +softclock_thread_run(struct timeout_ctx *tctx) +{ + struct circq *todo = tctx->tctx_todo; + struct timeout *to; + + for (;;) { + /* + * Avoid holding both timeout_mutex and SCHED_LOCK + * at the same time. + */ + sleep_setup(todo, PSWP, "tmoslp"); + sleep_finish(0, CIRCQ_EMPTY(tctx->tctx_todo)); + + mtx_enter(&timeout_mutex); + tostat.tos_thread_wakeups++; + while (!CIRCQ_EMPTY(todo)) { + to = timeout_from_circq(CIRCQ_FIRST(todo)); + CIRCQ_REMOVE(&to->to_list); + timeout_run(tctx, to); + tostat.tos_run_thread++; + } + mtx_leave(&timeout_mutex); + } +} + void softclock_thread(void *arg) { CPU_INFO_ITERATOR cii; struct cpu_info *ci; - struct timeout *to; int s; KERNEL_ASSERT_LOCKED(); @@ -801,18 +855,7 @@ softclock_thread(void *arg) sched_peg_curproc(ci); s = splsoftclock(); - mtx_enter(&timeout_mutex); - for (;;) { - while (!CIRCQ_EMPTY(&timeout_proc)) { - to = timeout_from_circq(CIRCQ_FIRST(&timeout_proc)); - CIRCQ_REMOVE(&to->to_list); - timeout_run(to); - tostat.tos_run_thread++; - } - tostat.tos_thread_wakeups++; - msleep_nsec(&timeout_proc, &timeout_mutex, PSWP, "tmoslp", - INFSLP); - } + softclock_thread_run(&timeout_ctx_proc); splx(s); } @@ -820,23 +863,10 @@ softclock_thread(void *arg) void softclock_thread_mp(void *arg) { - struct timeout *to; - KERNEL_ASSERT_LOCKED(); KERNEL_UNLOCK(); - mtx_enter(&timeout_mutex); - for (;;) { - while (!CIRCQ_EMPTY(&timeout_proc_mp)) { - to = timeout_from_circq(CIRCQ_FIRST(&timeout_proc_mp)); - CIRCQ_REMOVE(&to->to_list); - timeout_run(to); - tostat.tos_run_thread++; - } - tostat.tos_thread_wakeups++; - msleep_nsec(&timeout_proc_mp, &timeout_mutex, PSWP, "tmoslp", - INFSLP); - } + softclock_thread_run(&timeout_ctx_proc_mp); } #endif /* MULTIPROCESSOR */