Index: kern_rwlock.c =================================================================== RCS file: /cvs/src/sys/kern/kern_rwlock.c,v diff -u -p -r1.57 kern_rwlock.c --- kern_rwlock.c 3 Jun 2025 00:20:31 -0000 1.57 +++ kern_rwlock.c 24 Jun 2025 20:37:27 -0000 @@ -25,6 +25,7 @@ #include #include #include +#include #ifdef RWDIAG #define RW_SLEEP_TMO 10000000000ULL /* 10 seconds */ @@ -42,8 +43,8 @@ #ifdef MULTIPROCESSOR #define rw_cas(p, e, n) atomic_cas_ulong(p, e, n) -#define rw_inc(p) atomic_inc_int(p) -#define rw_dec(p) atomic_dec_int(p) +#define rw_inc_nv(p) atomic_inc_int_nv(p) +#define rw_dec_nv(p) atomic_dec_int_nv(p) #else static inline unsigned long rw_cas(volatile unsigned long *p, unsigned long e, unsigned long n) @@ -56,24 +57,45 @@ rw_cas(volatile unsigned long *p, unsign return (o); } -static inline void +static inline unsigned int +rw_inc_nv(volatile unsigned int *p) +{ + return ++(*p); +} + +static inline unsigned int +rw_dec_nv(volatile unsigned int *p) +{ + return --(*p); +} +#endif + +static unsigned int rw_inc(volatile unsigned int *p) { - ++(*p); + unsigned int nv; + + nv = rw_inc_nv(p); + KASSERT(nv < INT_MAX); + return (nv); } -static inline void +static unsigned int rw_dec(volatile unsigned int *p) { - (*p)--; + unsigned int nv; + + nv = rw_dec_nv(p); + KASSERT(nv < INT_MAX); + return (nv); } -#endif -static int rw_do_enter_read(struct rwlock *, int); -static void rw_do_exit_read(struct rwlock *, unsigned long); -static int rw_do_enter_write(struct rwlock *, int); -static int rw_downgrade(struct rwlock *, int); -static int rw_upgrade(struct rwlock *, int); +static int rw_do_enter_read(struct rwlock *, int, unsigned long); +static void rw_do_exit_read(struct rwlock *, unsigned long, unsigned long); +static int rw_do_enter_write(struct rwlock *, int, unsigned long); +static void rw_do_exit_write(struct rwlock *, unsigned long); +static int rw_downgrade(struct rwlock *, int, unsigned long); +static int rw_upgrade(struct rwlock *, int, unsigned long); static void rw_exited(struct rwlock *); @@ -91,29 +113,33 @@ rw_self(void) void rw_enter_read(struct rwlock *rwl) { - rw_do_enter_read(rwl, 0); + unsigned long pc = (unsigned long)__builtin_return_address(0); + rw_do_enter_read(rwl, 0, pc); } void rw_enter_write(struct rwlock *rwl) { - rw_do_enter_write(rwl, 0); + unsigned long pc = (unsigned long)__builtin_return_address(0); + rw_do_enter_write(rwl, 0, pc); } void rw_exit_read(struct rwlock *rwl) { + unsigned long pc = (unsigned long)__builtin_return_address(0); /* maybe we're the last one? */ - rw_do_exit_read(rwl, RWLOCK_READ_INCR); + rw_do_exit_read(rwl, RWLOCK_READ_INCR, pc); } static void -rw_do_exit_read(struct rwlock *rwl, unsigned long owner) +rw_do_exit_read(struct rwlock *rwl, unsigned long owner, unsigned long pc) { unsigned long decr; unsigned long nowner; WITNESS_UNLOCK(&rwl->rwl_lock_obj, 0); + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, LLTRACE_LK_R_SHARED, pc); for (;;) { decr = owner - RWLOCK_READ_INCR; @@ -144,10 +170,18 @@ rw_do_exit_read(struct rwlock *rwl, unsi void rw_exit_write(struct rwlock *rwl) { + unsigned long pc = (unsigned long)__builtin_return_address(0); + rw_do_exit_write(rwl, pc); +} + +static void +rw_do_exit_write(struct rwlock *rwl, unsigned long pc) +{ unsigned long self = rw_self(); unsigned long owner; WITNESS_UNLOCK(&rwl->rwl_lock_obj, LOP_EXCLUSIVE); + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, LLTRACE_LK_R_EXCL, pc); membar_exit_before_atomic(); owner = rw_cas(&rwl->rwl_owner, self, 0); @@ -168,6 +202,10 @@ _rw_init_flags_witness(struct rwlock *rw rwl->rwl_waiters = 0; rwl->rwl_readers = 0; rwl->rwl_name = name; +#if 0 + rwl->rwl_next = NULL; + rwl->rwl_tail = &rwl->rwl_next; +#endif #ifdef WITNESS rwl->rwl_lock_obj.lo_flags = lo_flags; @@ -190,21 +228,22 @@ _rw_init_flags(struct rwlock *rwl, const int rw_enter(struct rwlock *rwl, int flags) { + unsigned long pc = (unsigned long)__builtin_return_address(0); int op = flags & RW_OPMASK; int error; switch (op) { case RW_WRITE: - error = rw_do_enter_write(rwl, flags); + error = rw_do_enter_write(rwl, flags, pc); break; case RW_READ: - error = rw_do_enter_read(rwl, flags); + error = rw_do_enter_read(rwl, flags, pc); break; case RW_DOWNGRADE: - error = rw_downgrade(rwl, flags); + error = rw_downgrade(rwl, flags, pc); break; case RW_UPGRADE: - error = rw_upgrade(rwl, flags); + error = rw_upgrade(rwl, flags, pc); break; default: panic("%s rwlock %p: %s unexpected op 0x%x", @@ -216,7 +255,7 @@ rw_enter(struct rwlock *rwl, int flags) } static int -rw_do_enter_write(struct rwlock *rwl, int flags) +rw_do_enter_write(struct rwlock *rwl, int flags, unsigned long pc) { unsigned long self = rw_self(); unsigned long owner; @@ -235,6 +274,8 @@ rw_do_enter_write(struct rwlock *rwl, in owner = rw_cas(&rwl->rwl_owner, 0, self); if (owner == 0) { /* wow, we won. so easy */ + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, LLTRACE_LK_I_EXCL, + pc); goto locked; } if (__predict_false(owner == self)) { @@ -242,6 +283,7 @@ rw_do_enter_write(struct rwlock *rwl, in rwl->rwl_name, rwl); } + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, LLTRACE_LK_A_START, pc); #ifdef MULTIPROCESSOR /* * If process holds the kernel lock, then we want to give up on CPU @@ -268,6 +310,8 @@ rw_do_enter_write(struct rwlock *rwl, in if (owner == 0) { spc->spc_spinning--; /* ok, we won now. */ + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, + LLTRACE_LK_A_EXCL, pc); goto locked; } } @@ -275,8 +319,11 @@ rw_do_enter_write(struct rwlock *rwl, in } #endif - if (ISSET(flags, RW_NOSLEEP)) + if (ISSET(flags, RW_NOSLEEP)) { + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, LLTRACE_LK_A_ABORT, + pc); return (EBUSY); + } prio = PLOCK - 4; if (ISSET(flags, RW_INTR)) @@ -299,16 +346,19 @@ rw_do_enter_write(struct rwlock *rwl, in #endif if (ISSET(flags, RW_INTR) && (error != 0)) { rw_dec(&rwl->rwl_waiters); + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, + LLTRACE_LK_A_ABORT, pc); return (error); } owner = rw_cas(&rwl->rwl_owner, 0, self); } while (owner != 0); rw_dec(&rwl->rwl_waiters); + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, LLTRACE_LK_A_EXCL, pc); locked: membar_enter_after_atomic(); - WITNESS_LOCK(&rwl->rwl_lock_obj, lop_flags); + WITNESS_LOCK(&rwl->rwl_lock_obj, LOP_EXCLUSIVE); return (0); } @@ -332,7 +382,7 @@ rw_read_incr(struct rwlock *rwl, unsigne } static int -rw_do_enter_read(struct rwlock *rwl, int flags) +rw_do_enter_read(struct rwlock *rwl, int flags, unsigned long pc) { unsigned long owner; int error; @@ -349,6 +399,8 @@ rw_do_enter_read(struct rwlock *rwl, int owner = rw_cas(&rwl->rwl_owner, 0, RWLOCK_READ_INCR); if (owner == 0) { /* ermagerd, we won! */ + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, LLTRACE_LK_I_SHARED, + pc); goto locked; } @@ -360,12 +412,19 @@ rw_do_enter_read(struct rwlock *rwl, int } else if (atomic_load_int(&rwl->rwl_waiters) == 0) { if (rw_read_incr(rwl, owner)) { /* nailed it */ + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, + LLTRACE_LK_I_SHARED, pc); goto locked; } } - if (ISSET(flags, RW_NOSLEEP)) + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, LLTRACE_LK_A_START, pc); + + if (ISSET(flags, RW_NOSLEEP)) { + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, LLTRACE_LK_A_ABORT, + pc); return (EBUSY); + } prio = PLOCK; if (ISSET(flags, RW_INTR)) @@ -392,16 +451,17 @@ rw_do_enter_read(struct rwlock *rwl, int } } while (!rw_read_incr(rwl, 0)); rw_dec(&rwl->rwl_readers); + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, LLTRACE_LK_A_SHARED, pc); locked: membar_enter_after_atomic(); - WITNESS_LOCK(&rwl->rwl_lock_obj, lop_flags); + WITNESS_LOCK(&rwl->rwl_lock_obj, 0); return (0); } static int -rw_downgrade(struct rwlock *rwl, int flags) +rw_downgrade(struct rwlock *rwl, int flags, unsigned long pc) { unsigned long self = rw_self(); unsigned long owner; @@ -422,6 +482,7 @@ rw_downgrade(struct rwlock *rwl, int fla WITNESS_DOWNGRADE(&rwl->rwl_lock_obj, lop_flags); } #endif + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, LLTRACE_LK_DOWNGRADE, pc); membar_consumer(); if (atomic_load_int(&rwl->rwl_waiters) == 0 && @@ -432,7 +493,7 @@ rw_downgrade(struct rwlock *rwl, int fla } static int -rw_upgrade(struct rwlock *rwl, int flags) +rw_upgrade(struct rwlock *rwl, int flags, unsigned long pc) { unsigned long self = rw_self(); unsigned long owner; @@ -451,6 +512,8 @@ rw_upgrade(struct rwlock *rwl, int flags owner, self); } + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, LLTRACE_LK_A_ABORT, + pc); return (EBUSY); } @@ -462,6 +525,7 @@ rw_upgrade(struct rwlock *rwl, int flags WITNESS_UPGRADE(&rwl->rwl_lock_obj, lop_flags); } #endif + LLTRACE(lltrace_lock, rwl, LLTRACE_LK_RW, LLTRACE_LK_UPGRADE, pc); return (0); } @@ -469,6 +533,7 @@ rw_upgrade(struct rwlock *rwl, int flags void rw_exit(struct rwlock *rwl) { + unsigned long pc = (unsigned long)__builtin_return_address(0); unsigned long owner; owner = atomic_load_long(&rwl->rwl_owner); @@ -478,9 +543,9 @@ rw_exit(struct rwlock *rwl) } if (ISSET(owner, RWLOCK_WRLOCK)) - rw_exit_write(rwl); + rw_do_exit_write(rwl, pc); else - rw_do_exit_read(rwl, owner); + rw_do_exit_read(rwl, owner, pc); } static void