diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 30e4c07..b8a3631 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -450,7 +450,8 @@ _tsleep_wakeup(struct thread *td) if (td->td_gd != gd) { lwkt_hold(td); - lwkt_send_ipiq(td->td_gd, (ipifunc1_t)tsleep_wakeup_remote, td); + lwkt_send_ipiq_adaptive(td->td_gd, + (ipifunc1_t)tsleep_wakeup_remote, td); return; } #endif diff --git a/sys/kern/lwkt_ipiq.c b/sys/kern/lwkt_ipiq.c index 2235f59..3767cd1 100644 --- a/sys/kern/lwkt_ipiq.c +++ b/sys/kern/lwkt_ipiq.c @@ -73,6 +73,8 @@ static __int64_t ipiq_fifofull; /* number of fifo full conditions detected */ static __int64_t ipiq_avoided; /* interlock with target avoids cpu ipi */ static __int64_t ipiq_passive; /* passive IPI messages */ static __int64_t ipiq_cscount; /* number of cpu synchronizations */ +int ipiq_adaptive_threshold; /* passive IPI when target cpu lwkt runq length + * approx. greater than this */ static int ipiq_debug; /* set to 1 for debug */ #ifdef PANIC_DEBUG static int panic_ipiq_cpu = -1; @@ -91,6 +93,9 @@ SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_passive, CTLFLAG_RW, &ipiq_passive, 0, "Number of passive IPI messages sent"); SYSCTL_QUAD(_lwkt, OID_AUTO, ipiq_cscount, CTLFLAG_RW, &ipiq_cscount, 0, "Number of cpu synchronizations"); +SYSCTL_INT(_lwkt, OID_AUTO, ipiq_adapative_threshold, CTLFLAG_RW, + &ipiq_adaptive_threshold, 0, + "Threshold where adaptive IPI's become passive"); SYSCTL_INT(_lwkt, OID_AUTO, ipiq_debug, CTLFLAG_RW, &ipiq_debug, 0, ""); #ifdef PANIC_DEBUG diff --git a/sys/kern/lwkt_thread.c b/sys/kern/lwkt_thread.c index c55ebdd..b888494 100644 --- a/sys/kern/lwkt_thread.c +++ b/sys/kern/lwkt_thread.c @@ -36,7 +36,7 @@ * Each cpu in a system has its own self-contained light weight kernel * thread scheduler, which means that generally speaking we only need * to use a critical section to avoid problems. Foreign thread - * scheduling is queued via (async) IPIs. + * scheduling is done via IPIs. */ #include @@ -243,6 +243,7 @@ _lwkt_dequeue(thread_t td) TAILQ_REMOVE(&gd->gd_tdrunq, td, td_threadq); if (TAILQ_FIRST(&gd->gd_tdrunq) == NULL) atomic_clear_int(&gd->gd_reqflags, RQF_RUNNING); + --gd->gd_lwkt_runq_len; } } @@ -280,6 +281,8 @@ _lwkt_enqueue(thread_t td) */ if (TAILQ_FIRST(&gd->gd_tdrunq) == td) need_lwkt_resched(); + + ++gd->gd_lwkt_runq_len; } } @@ -491,7 +494,7 @@ lwkt_init_thread(thread_t td, void *stack, int stksize, int flags, TAILQ_INSERT_TAIL(&gd->gd_tdallq, td, td_allq); crit_exit_gd(mygd); } else { - lwkt_send_ipiq(gd, lwkt_init_thread_remote, td); + lwkt_send_ipiq_adaptive(gd, lwkt_init_thread_remote, td); } #else crit_enter_gd(mygd); @@ -974,7 +977,7 @@ lwkt_switch_return(thread_t otd) (TDF_MIGRATING | TDF_RUNNING)); otd->td_migrate_gd = NULL; otd->td_flags &= ~TDF_RUNNING; - lwkt_send_ipiq(rgd, lwkt_setcpu_remote, otd); + lwkt_send_ipiq_adaptive(rgd, lwkt_setcpu_remote, otd); } else { otd->td_flags &= ~TDF_RUNNING; } @@ -1314,7 +1317,7 @@ _lwkt_schedule(thread_t td) _lwkt_enqueue(td); _lwkt_schedule_post(mygd, td, 1); } else { - lwkt_send_ipiq3(td->td_gd, lwkt_schedule_remote, td, 0); + lwkt_send_ipiq3_adaptive(td->td_gd, lwkt_schedule_remote, td, 0); } #else _lwkt_enqueue(td); @@ -1447,7 +1450,7 @@ lwkt_deschedule(thread_t td) if (td->td_gd == mycpu) { _lwkt_dequeue(td); } else { - lwkt_send_ipiq(td->td_gd, (ipifunc1_t)lwkt_deschedule, td); + lwkt_send_ipiq_adaptive(td->td_gd, (ipifunc1_t)lwkt_deschedule, td); } } #else diff --git a/sys/sys/globaldata.h b/sys/sys/globaldata.h index 51424a3..276e658 100644 --- a/sys/sys/globaldata.h +++ b/sys/sys/globaldata.h @@ -166,7 +166,8 @@ struct globaldata { struct systimer *gd_systimer_inprog; /* in-progress systimer */ int gd_timer_running; u_int gd_idle_repeat; /* repeated switches to idle */ - int gd_ireserved[7]; + int gd_lwkt_runq_len; /* size of lwkt run queue */ + int gd_ireserved[6]; const char *gd_infomsg; /* debugging */ struct lwkt_tokref gd_handoff; /* hand-off tokref */ void *gd_preserved[8]; /* future fields */ diff --git a/sys/sys/thread2.h b/sys/sys/thread2.h index 250c4a5..2d3484f 100644 --- a/sys/sys/thread2.h +++ b/sys/sys/thread2.h @@ -29,6 +29,8 @@ #endif #include +extern int ipiq_adaptive_threshold; + /* * Is a token held by the specified thread? */ @@ -311,6 +313,16 @@ lwkt_send_ipiq_passive(globaldata_t target, ipifunc1_t func, void *arg) } static __inline int +lwkt_send_ipiq_adaptive(globaldata_t target, ipifunc1_t func, void *arg) +{ + if (ipiq_adaptive_threshold > 0 && + target->gd_lwkt_runq_len >= ipiq_adaptive_threshold) + return(lwkt_send_ipiq3_passive(target, (ipifunc3_t)func, arg, 0)); + else + return(lwkt_send_ipiq3(target, (ipifunc3_t)func, arg, 0)); +} + +static __inline int lwkt_send_ipiq2_passive(globaldata_t target, ipifunc2_t func, void *arg1, int arg2) { @@ -329,6 +341,17 @@ lwkt_send_ipiq2_bycpu(int dcpu, ipifunc2_t func, void *arg1, int arg2) return(lwkt_send_ipiq3_bycpu(dcpu, (ipifunc3_t)func, arg1, arg2)); } +static __inline int +lwkt_send_ipiq3_adaptive(globaldata_t target, ipifunc3_t func, void *arg, + int arg2) +{ + if (ipiq_adaptive_threshold > 0 && + target->gd_lwkt_runq_len >= ipiq_adaptive_threshold) + return(lwkt_send_ipiq3_passive(target, func, arg, arg2)); + else + return(lwkt_send_ipiq3(target, func, arg, arg2)); +} + #endif /* SMP */ #endif /* _KERNEL */ #endif /* _SYS_THREAD2_H_ */