Loading...
--- xnu/xnu-12377.121.6/bsd/pthread/pthread_workqueue.c
+++ xnu/xnu-8020.121.3/bsd/pthread/pthread_workqueue.c
@@ -41,7 +41,6 @@
#include <kern/thread.h>
#include <kern/thread_group.h>
#include <kern/zalloc.h>
-#include <kern/work_interval.h>
#include <mach/kern_return.h>
#include <mach/mach_param.h>
#include <mach/mach_port.h>
@@ -58,7 +57,6 @@
#include <machine/machine_routines.h>
#include <machine/smp.h>
#include <vm/vm_map.h>
-#include <vm/vm_fault_xnu.h>
#include <vm/vm_protos.h>
#include <sys/eventvar.h>
@@ -84,13 +82,6 @@
#include <os/log.h>
static void workq_unpark_continue(void *uth, wait_result_t wr) __dead2;
-
-static void workq_bound_thread_unpark_continue(void *uth, wait_result_t wr) __dead2;
-
-static void workq_bound_thread_initialize_and_unpark_continue(void *uth, wait_result_t wr) __dead2;
-
-static void workq_bound_thread_setup_and_run(struct uthread *uth, int setup_flags) __dead2;
-
static void workq_schedule_creator(proc_t p, struct workqueue *wq,
workq_kern_threadreq_flags_t flags);
@@ -98,8 +89,7 @@
workq_threadreq_t req);
static uint32_t workq_constrained_allowance(struct workqueue *wq,
- thread_qos_t at_qos, struct uthread *uth,
- bool may_start_timer, bool record_failed_allowance);
+ thread_qos_t at_qos, struct uthread *uth, bool may_start_timer);
static bool _wq_cooperative_queue_refresh_best_req_qos(struct workqueue *wq);
@@ -456,8 +446,6 @@
return count;
}
-/* The input qos here should be the requested QoS of the thread, not accounting
- * for any overrides */
static inline void
_wq_cooperative_queue_scheduled_count_dec(struct workqueue *wq, thread_qos_t qos)
{
@@ -465,8 +453,6 @@
assert(old_scheduled_count > 0);
}
-/* The input qos here should be the requested QoS of the thread, not accounting
- * for any overrides */
static inline void
_wq_cooperative_queue_scheduled_count_inc(struct workqueue *wq, thread_qos_t qos)
{
@@ -610,18 +596,11 @@
return false;
}
-/* Input thread must be self. Called during self override, resetting overrides
- * or while processing kevents
- *
- * Called with workq lock held. Sometimes also the thread mutex
- */
static void
workq_thread_update_bucket(proc_t p, struct workqueue *wq, struct uthread *uth,
struct uu_workq_policy old_pri, struct uu_workq_policy new_pri,
bool force_run)
{
- assert(uth == current_uthread());
-
thread_qos_t old_bucket = old_pri.qos_bucket;
thread_qos_t new_bucket = workq_pri_bucket(new_pri);
@@ -632,12 +611,11 @@
new_pri.qos_bucket = new_bucket;
uth->uu_workq_pri = new_pri;
- if (old_pri.qos_override != new_pri.qos_override) {
- thread_set_workq_override(get_machthread(uth), new_pri.qos_override);
- }
-
- if (wq->wq_reqcount &&
- (old_bucket > new_bucket || force_run)) {
+ if (workq_pri_override(old_pri) != new_bucket) {
+ thread_set_workq_override(get_machthread(uth), new_bucket);
+ }
+
+ if (wq->wq_reqcount && (old_bucket > new_bucket || force_run)) {
int flags = WORKQ_THREADREQ_CAN_CREATE_THREADS;
if (old_bucket > new_bucket) {
/*
@@ -686,11 +664,7 @@
}
}
-/*
- * This function is always called with the workq lock, except for the
- * permanently bound workqueue thread, which instead requires the kqlock.
- * See locking model for bound thread's uu_workq_flags.
- */
+/* Called with the workq lock held */
static void
workq_thread_reset_pri(struct workqueue *wq, struct uthread *uth,
workq_threadreq_t req, bool unpark)
@@ -718,7 +692,7 @@
uint32_t mgr_pri = wq->wq_event_manager_priority;
assert(trp.trp_value == 0); // manager qos and thread policy don't mix
- if (_pthread_priority_has_sched_pri(mgr_pri)) {
+ if (mgr_pri & _PTHREAD_PRIORITY_SCHED_PRI_FLAG) {
mgr_pri &= _PTHREAD_PRIORITY_SCHED_PRI_MASK;
thread_set_workq_pri(th, THREAD_QOS_UNSPECIFIED, mgr_pri,
POLICY_TIMESHARE);
@@ -741,12 +715,8 @@
#if CONFIG_PREADOPT_TG
if (req && (req->tr_flags & WORKQ_TR_FLAG_WORKLOOP)) {
/*
- * For kqwl permanently configured with a thread group, we can safely borrow
- * +1 ref from kqwl_preadopt_tg. A thread then takes additional +1 ref
- * for itself via thread_set_preadopt_thread_group.
- *
- * In all other cases, we cannot safely read and borrow the reference from the kqwl
- * since it can disappear from under us at any time due to the max-ing logic in
+ * We cannot safely read and borrow the reference from the kqwl since it
+ * can disappear from under us at any time due to the max-ing logic in
* kqueue_set_preadopted_thread_group.
*
* As such, we do the following dance:
@@ -768,8 +738,7 @@
int ret = 0;
again:
ret = os_atomic_rmw_loop(tg_loc, old_tg, new_tg, relaxed, {
- if ((!KQWL_HAS_VALID_PREADOPTED_TG(old_tg)) ||
- KQWL_HAS_PERMANENT_PREADOPTED_TG(old_tg)) {
+ if (!KQWL_HAS_VALID_PREADOPTED_TG(old_tg)) {
os_atomic_rmw_loop_give_up(break);
}
@@ -805,12 +774,8 @@
new_tg = thread_group_to_set;
});
} else {
- if (KQWL_HAS_PERMANENT_PREADOPTED_TG(old_tg)) {
- thread_set_preadopt_thread_group(th, KQWL_GET_PREADOPTED_TG(old_tg));
- } else {
- /* Nothing valid on the kqwl, just clear what's on the thread */
- thread_set_preadopt_thread_group(th, NULL);
- }
+ /* Nothing valid on the kqwl, just clear what's on the thread */
+ thread_set_preadopt_thread_group(th, NULL);
}
} else {
/* Not even a kqwl, clear what's on the thread */
@@ -959,13 +924,11 @@
struct workqueue *wq = proc_get_wqptr_fast(p);
workq_lock_spin(wq);
- if (!workq_thread_is_permanently_bound(uth)) {
- TAILQ_REMOVE(&wq->wq_thrunlist, uth, uu_workq_entry);
- if (uth->uu_workq_flags & UT_WORKQ_DYING) {
- WQ_TRACE_WQ(TRACE_wq_thread_terminate | DBG_FUNC_END,
- wq, wq->wq_thidlecount, 0, 0);
- workq_death_policy_evaluate(wq, 1);
- }
+ TAILQ_REMOVE(&wq->wq_thrunlist, uth, uu_workq_entry);
+ if (uth->uu_workq_flags & UT_WORKQ_DYING) {
+ WQ_TRACE_WQ(TRACE_wq_thread_terminate | DBG_FUNC_END,
+ wq, wq->wq_thidlecount, 0, 0);
+ workq_death_policy_evaluate(wq, 1);
}
if (wq->wq_nthreads-- == wq_max_threads) {
/*
@@ -1061,13 +1024,8 @@
* - dropped and retaken around thread creation
* - return with workq lock held
*/
-static kern_return_t
-workq_add_new_idle_thread(
- proc_t p,
- struct workqueue *wq,
- thread_continue_t continuation,
- bool is_permanently_bound,
- thread_t *new_thread)
+static bool
+workq_add_new_idle_thread(proc_t p, struct workqueue *wq)
{
mach_vm_offset_t th_stackaddr;
kern_return_t kret;
@@ -1077,7 +1035,7 @@
workq_unlock(wq);
- vm_map_t vmap = get_task_map(proc_task(p));
+ vm_map_t vmap = get_task_map(p->task);
kret = pthread_functions->workq_create_threadstack(p, vmap, &th_stackaddr);
if (kret != KERN_SUCCESS) {
@@ -1086,10 +1044,7 @@
goto out;
}
- kret = thread_create_workq_waiting(proc_task(p),
- continuation,
- &th,
- is_permanently_bound);
+ kret = thread_create_workq_waiting(p->task, workq_unpark_continue, &th);
if (kret != KERN_SUCCESS) {
WQ_TRACE_WQ(TRACE_wq_thread_create_failed | DBG_FUNC_NONE, wq,
kret, 0, 0);
@@ -1101,20 +1056,14 @@
// on success, because it calls workq_thread_init_and_wq_lock() above
struct uthread *uth = get_bsdthread_info(th);
+
+ wq->wq_creations++;
+ wq->wq_thidlecount++;
uth->uu_workq_stackaddr = (user_addr_t)th_stackaddr;
-
- wq->wq_creations++;
- if (!is_permanently_bound) {
- wq->wq_thidlecount++;
- TAILQ_INSERT_TAIL(&wq->wq_thnewlist, uth, uu_workq_entry);
- }
-
- if (new_thread) {
- *new_thread = th;
- }
+ TAILQ_INSERT_TAIL(&wq->wq_thnewlist, uth, uu_workq_entry);
WQ_TRACE_WQ(TRACE_wq_thread_create | DBG_FUNC_NONE, wq, 0, 0, 0);
- return kret;
+ return true;
out:
workq_lock_spin(wq);
@@ -1124,7 +1073,7 @@
* to do so when it fails.
*/
wq->wq_nthreads--;
- return kret;
+ return false;
}
static inline bool
@@ -1136,20 +1085,13 @@
static inline bool
workq_thread_is_nonovercommit(struct uthread *uth)
{
- return (uth->uu_workq_flags & (UT_WORKQ_OVERCOMMIT |
- UT_WORKQ_COOPERATIVE)) == 0;
+ return (uth->uu_workq_flags & (UT_WORKQ_OVERCOMMIT | UT_WORKQ_COOPERATIVE)) == 0;
}
static inline bool
workq_thread_is_cooperative(struct uthread *uth)
{
return (uth->uu_workq_flags & UT_WORKQ_COOPERATIVE) != 0;
-}
-
-bool
-workq_thread_is_permanently_bound(struct uthread *uth)
-{
- return (uth->uu_workq_flags & UT_WORKQ_PERMANENT_BIND) != 0;
}
static inline void
@@ -1197,7 +1139,7 @@
uint32_t flags = WQ_FLAG_THREAD_NEWSPI | qos | WQ_FLAG_THREAD_PRIO_QOS;
thread_t th = get_machthread(uth);
- vm_map_t vmap = get_task_map(proc_task(p));
+ vm_map_t vmap = get_task_map(p->task);
if (!first_use) {
flags |= WQ_FLAG_THREAD_REUSE;
@@ -1252,7 +1194,7 @@
if (workq_thread_is_cooperative(uth)) {
assert(!is_creator);
- thread_qos_t thread_qos = uth->uu_workq_pri.qos_req;
+ thread_qos_t thread_qos = uth->uu_workq_pri.qos_bucket;
_wq_cooperative_queue_scheduled_count_dec(wq, thread_qos);
/* Before we get here, we always go through
@@ -1362,9 +1304,7 @@
static inline bool
workq_tr_is_nonovercommit(workq_tr_flags_t tr_flags)
{
- return (tr_flags & (WORKQ_TR_FLAG_OVERCOMMIT |
- WORKQ_TR_FLAG_COOPERATIVE |
- WORKQ_TR_FLAG_PERMANENT_BIND)) == 0;
+ return (tr_flags & (WORKQ_TR_FLAG_OVERCOMMIT | WORKQ_TR_FLAG_COOPERATIVE)) == 0;
}
static inline bool
@@ -1404,10 +1344,13 @@
}
}
+
/* Calculates the number of threads scheduled >= the input QoS */
static uint64_t
-workq_num_cooperative_threads_scheduled_to_qos_internal(struct workqueue *wq, thread_qos_t qos)
-{
+workq_num_cooperative_threads_scheduled_to_qos(struct workqueue *wq, thread_qos_t qos)
+{
+ workq_lock_held(wq);
+
uint64_t num_cooperative_threads = 0;
for (thread_qos_t cur_qos = WORKQ_THREAD_QOS_MAX; cur_qos >= qos; cur_qos--) {
@@ -1418,20 +1361,13 @@
return num_cooperative_threads;
}
-/* Calculates the number of threads scheduled >= the input QoS */
-static uint64_t
-workq_num_cooperative_threads_scheduled_to_qos_locked(struct workqueue *wq, thread_qos_t qos)
-{
- workq_lock_held(wq);
- return workq_num_cooperative_threads_scheduled_to_qos_internal(wq, qos);
-}
-
static uint64_t
workq_num_cooperative_threads_scheduled_total(struct workqueue *wq)
{
- return workq_num_cooperative_threads_scheduled_to_qos_locked(wq, WORKQ_THREAD_QOS_MIN);
-}
-
+ return workq_num_cooperative_threads_scheduled_to_qos(wq, WORKQ_THREAD_QOS_MIN);
+}
+
+#if DEBUG || DEVELOPMENT
static bool
workq_has_cooperative_thread_requests(struct workqueue *wq)
{
@@ -1444,6 +1380,7 @@
return false;
}
+#endif
/*
* Determines the next QoS bucket we should service next in the cooperative
@@ -1528,7 +1465,7 @@
wq->wq_cooperative_queue_best_req_qos = highest_qos_req;
}
-#if MACH_ASSERT
+#if DEBUG || DEVELOPMENT
/* Assert that if we are showing up the next best req as UN, then there
* actually is no thread request in the cooperative pool buckets */
if (wq->wq_cooperative_queue_best_req_qos == THREAD_QOS_UNSPECIFIED) {
@@ -1570,7 +1507,7 @@
if (uth && workq_thread_is_cooperative(uth)) {
exclude_thread_as_scheduled = true;
- _wq_cooperative_queue_scheduled_count_dec(wq, uth->uu_workq_pri.qos_req);
+ _wq_cooperative_queue_scheduled_count_dec(wq, uth->uu_workq_pri.qos_bucket);
}
/*
@@ -1604,7 +1541,7 @@
* If number of threads at the QoS bucket >= input QoS exceeds the max we want
* for the pool, deny this thread
*/
- uint64_t aggregate_down_to_qos = workq_num_cooperative_threads_scheduled_to_qos_locked(wq, qos);
+ uint64_t aggregate_down_to_qos = workq_num_cooperative_threads_scheduled_to_qos(wq, qos);
passed_admissions = (aggregate_down_to_qos < wq_cooperative_queue_max_size(wq));
WQ_TRACE(TRACE_wq_cooperative_admission | DBG_FUNC_NONE, aggregate_down_to_qos,
qos, passed_admissions, WQ_COOPERATIVE_POOL_SATURATED_UP_TO_QOS);
@@ -1615,7 +1552,7 @@
out:
if (exclude_thread_as_scheduled) {
- _wq_cooperative_queue_scheduled_count_inc(wq, uth->uu_workq_pri.qos_req);
+ _wq_cooperative_queue_scheduled_count_inc(wq, uth->uu_workq_pri.qos_bucket);
}
return passed_admissions;
}
@@ -2323,8 +2260,7 @@
bsdthread_part_of_cooperative_workqueue(struct uthread *uth)
{
return (workq_thread_is_cooperative(uth) || workq_thread_is_nonovercommit(uth)) &&
- (uth->uu_workq_pri.qos_bucket != WORKQ_THREAD_QOS_MANAGER) &&
- (!workq_thread_is_permanently_bound(uth));
+ (uth->uu_workq_pri.qos_bucket != WORKQ_THREAD_QOS_MANAGER);
}
static bool
@@ -2351,7 +2287,6 @@
int unbind_rv = 0, qos_rv = 0, voucher_rv = 0, fixedpri_rv = 0;
bool is_wq_thread = (thread_get_tag(th) & THREAD_TAG_WORKQUEUE);
- assert(th == current_thread());
if (flags & WORKQ_SET_SELF_WQ_KEVENT_UNBIND) {
if (!is_wq_thread) {
unbind_rv = EINVAL;
@@ -2378,38 +2313,12 @@
}
qos:
- if (flags & (WORKQ_SET_SELF_QOS_FLAG | WORKQ_SET_SELF_QOS_OVERRIDE_FLAG)) {
- assert(flags & WORKQ_SET_SELF_QOS_FLAG);
-
+ if (flags & WORKQ_SET_SELF_QOS_FLAG) {
thread_qos_policy_data_t new_policy;
- thread_qos_t qos_override = THREAD_QOS_UNSPECIFIED;
if (!_pthread_priority_to_policy(priority, &new_policy)) {
qos_rv = EINVAL;
goto voucher;
- }
-
- if (flags & WORKQ_SET_SELF_QOS_OVERRIDE_FLAG) {
- /*
- * If the WORKQ_SET_SELF_QOS_OVERRIDE_FLAG is set, we definitely
- * should have an override QoS in the pthread_priority_t and we should
- * only come into this path for cooperative thread requests
- */
- if (!_pthread_priority_has_override_qos(priority) ||
- !_pthread_priority_is_cooperative(priority)) {
- qos_rv = EINVAL;
- goto voucher;
- }
- qos_override = _pthread_priority_thread_override_qos(priority);
- } else {
- /*
- * If the WORKQ_SET_SELF_QOS_OVERRIDE_FLAG is not set, we definitely
- * should not have an override QoS in the pthread_priority_t
- */
- if (_pthread_priority_has_override_qos(priority)) {
- qos_rv = EINVAL;
- goto voucher;
- }
}
if (!is_wq_thread) {
@@ -2456,37 +2365,10 @@
struct uu_workq_policy old_pri, new_pri;
bool force_run = false;
- if (qos_override) {
- /*
- * We're in the case of a thread clarifying that it is for eg. not IN
- * req QoS but rather, UT req QoS with IN override. However, this can
- * race with a concurrent override happening to the thread via
- * workq_thread_add_dispatch_override so this needs to be
- * synchronized with the thread mutex.
- */
- thread_mtx_lock(th);
- }
-
workq_lock_spin(wq);
old_pri = new_pri = uth->uu_workq_pri;
new_pri.qos_req = (thread_qos_t)new_policy.qos_tier;
-
- if (old_pri.qos_override < qos_override) {
- /*
- * Since this can race with a concurrent override via
- * workq_thread_add_dispatch_override, only adjust override value if we
- * are higher - this is a saturating function.
- *
- * We should not be changing the final override values, we should simply
- * be redistributing the current value with a different breakdown of req
- * vs override QoS - assert to that effect. Therefore, buckets should
- * not change.
- */
- new_pri.qos_override = qos_override;
- assert(workq_pri_override(new_pri) == workq_pri_override(old_pri));
- assert(workq_pri_bucket(new_pri) == workq_pri_bucket(old_pri));
- }
/* Adjust schedule counts for various types of transitions */
@@ -2502,24 +2384,17 @@
/* cooperative -> cooperative */
} else if (workq_thread_is_cooperative(uth)) {
- _wq_cooperative_queue_scheduled_count_dec(wq, old_pri.qos_req);
- _wq_cooperative_queue_scheduled_count_inc(wq, new_pri.qos_req);
+ _wq_cooperative_queue_scheduled_count_dec(wq, old_pri.qos_bucket);
+ _wq_cooperative_queue_scheduled_count_inc(wq, workq_pri_bucket(new_pri));
/* We're changing schedule counts within cooperative pool, we
* need to refresh best cooperative QoS logic again */
force_run = _wq_cooperative_queue_refresh_best_req_qos(wq);
}
- /*
- * This will set up an override on the thread if any and will also call
- * schedule_creator if needed
- */
+ /* This will also call schedule_creator if needed */
workq_thread_update_bucket(p, wq, uth, old_pri, new_pri, force_run);
workq_unlock(wq);
-
- if (qos_override) {
- thread_mtx_unlock(th);
- }
if (workq_thread_is_overcommit(uth)) {
thread_disarm_workqueue_quantum(th);
@@ -2623,7 +2498,7 @@
return ESRCH;
}
- int rv = proc_thread_qos_add_override(proc_task(p), th, 0, qos, TRUE,
+ int rv = proc_thread_qos_add_override(p->task, th, 0, qos, TRUE,
resource, THREAD_QOS_OVERRIDE_TYPE_PTHREAD_EXPLICIT_OVERRIDE);
thread_deallocate(th);
@@ -2640,7 +2515,7 @@
return ESRCH;
}
- int rv = proc_thread_qos_remove_override(proc_task(p), th, 0, resource,
+ int rv = proc_thread_qos_remove_override(p->task, th, 0, resource,
THREAD_QOS_OVERRIDE_TYPE_PTHREAD_EXPLICIT_OVERRIDE);
thread_deallocate(th);
@@ -2679,9 +2554,13 @@
if (ulock_addr) {
uint32_t val;
int rc;
- vm_fault_disable();
+ /*
+ * Workaround lack of explicit support for 'no-fault copyin'
+ * <rdar://problem/24999882>, as disabling preemption prevents paging in
+ */
+ disable_preemption();
rc = copyin_atomic32(ulock_addr, &val);
- vm_fault_enable();
+ enable_preemption();
if (rc == 0 && ulock_owner_value_to_port_name(val) != kport) {
goto out;
}
@@ -2724,20 +2603,11 @@
WQ_TRACE_WQ(TRACE_wq_override_reset | DBG_FUNC_NONE, wq, 0, 0, 0);
- /*
- * workq_thread_add_dispatch_override takes the thread mutex before doing the
- * copyin to validate the drainer and apply the override. We need to do the
- * same here. See rdar://84472518
- */
- thread_mtx_lock(thread);
-
workq_lock_spin(wq);
old_pri = new_pri = uth->uu_workq_pri;
new_pri.qos_override = THREAD_QOS_UNSPECIFIED;
workq_thread_update_bucket(p, wq, uth, old_pri, new_pri, false);
workq_unlock(wq);
-
- thread_mtx_unlock(thread);
return 0;
}
@@ -2759,20 +2629,6 @@
}
static int
-workq_allow_sigmask(proc_t p, sigset_t mask)
-{
- if (mask & workq_threadmask) {
- return EINVAL;
- }
-
- proc_lock(p);
- p->p_workq_allow_sigmask |= mask;
- proc_unlock(p);
-
- return 0;
-}
-
-static int
bsdthread_get_max_parallelism(thread_qos_t qos, unsigned long flags,
int *retval)
{
@@ -2787,12 +2643,10 @@
return EINVAL;
}
-#if !HAS_ARM_FEAT_SME
/* No units are present */
if (flags & QOS_PARALLELISM_CLUSTER_SHARED_RESOURCE) {
return ENOTSUP;
}
-#endif /* !HAS_ARM_FEAT_SME */
if (flags & QOS_PARALLELISM_REALTIME) {
if (qos) {
@@ -2868,8 +2722,6 @@
return bsdthread_dispatch_apply_attr(p, current_thread(),
(unsigned long)uap->arg1, (uint64_t)uap->arg2,
(uint64_t)uap->arg3);
- case BSDTHREAD_CTL_WORKQ_ALLOW_SIGMASK:
- return workq_allow_sigmask(p, (int)uap->arg1);
case BSDTHREAD_CTL_SET_QOS:
case BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_ADD:
case BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET:
@@ -2983,7 +2835,7 @@
assert(!workq_threadreq_is_cooperative(req));
if (workq_threadreq_is_nonovercommit(req)) {
- unpaced = workq_constrained_allowance(wq, qos, NULL, false, true);
+ unpaced = workq_constrained_allowance(wq, qos, NULL, false);
if (unpaced >= reqcount - 1) {
unpaced = reqcount - 1;
}
@@ -3024,8 +2876,7 @@
reqcount--;
}
} while (unpaced && wq->wq_nthreads < wq_max_threads &&
- (workq_add_new_idle_thread(p, wq, workq_unpark_continue,
- false, NULL) == KERN_SUCCESS));
+ workq_add_new_idle_thread(p, wq));
if (_wq_exiting(wq)) {
goto unlock_and_exit;
@@ -3056,18 +2907,7 @@
struct uthread *uth = NULL;
assert(req->tr_flags & (WORKQ_TR_FLAG_WORKLOOP | WORKQ_TR_FLAG_KEVENT));
- /*
- * Thread bound kqworkloops overlay the priority queue entry with other
- * data (the thread_t and work interval), so should never have their
- * threadreq passed here.
- */
- assert(!(req->tr_flags & WORKQ_TR_FLAG_PERMANENT_BIND));
-
- /*
- * For any new initialization changes done to workqueue thread request below,
- * please also consider if they are relevant to permanently bound thread
- * request. See workq_kern_threadreq_permanent_bind.
- */
+
if (req->tr_flags & WORKQ_TR_FLAG_WL_OUTSIDE_QOS) {
workq_threadreq_param_t trp = kqueue_threadreq_workloop_param(req);
qos = thread_workq_qos_for_pri(trp.trp_pri);
@@ -3254,34 +3094,6 @@
}
void
-workq_kern_bound_thread_reset_pri(workq_threadreq_t req, struct uthread *uth)
-{
- assert(workq_thread_is_permanently_bound(uth));
-
- if (req && (req->tr_flags & WORKQ_TR_FLAG_WL_OUTSIDE_QOS)) {
- /*
- * For requests outside-of-QoS, we set the scheduling policy and
- * absolute priority for the bound thread right at the initialization
- * time. See workq_kern_threadreq_permanent_bind.
- */
- return;
- }
-
- struct workqueue *wq = proc_get_wqptr_fast(current_proc());
- if (req) {
- assert(req->tr_qos != WORKQ_THREAD_QOS_MANAGER);
- workq_thread_reset_pri(wq, uth, req, /*unpark*/ true);
- } else {
- thread_qos_t qos = workq_pri_override(uth->uu_workq_pri);
- if (qos > WORKQ_THREAD_QOS_CLEANUP) {
- workq_thread_reset_pri(wq, uth, NULL, /*unpark*/ true);
- } else {
- uth->uu_save.uus_workq_park_data.qos = qos;
- }
- }
-}
-
-void
workq_kern_threadreq_lock(struct proc *p)
{
workq_lock_spin(proc_get_wqptr_fast(p));
@@ -3307,7 +3119,7 @@
if (req->tr_state == WORKQ_TR_STATE_BINDING) {
kqueue_threadreq_bind(p, req, req->tr_thread,
- KQUEUE_THREADREQ_BIND_NO_INHERITOR_UPDATE);
+ KQUEUE_THREADERQ_BIND_NO_INHERITOR_UPDATE);
return;
}
@@ -3330,223 +3142,6 @@
workq_perform_turnstile_operation_locked(wq, ^{
turnstile_update_inheritor(wl_ts, inheritor, flags);
});
-}
-
-/*
- * An entry point for kevent to request a newly created workqueue thread
- * and bind it permanently to the given workqueue thread request.
- *
- * It currently only supports fixed scheduler priority thread requests.
- *
- * The newly created thread counts towards wq_nthreads. This function returns
- * an error if we are above that limit. There is no concept of delayed thread
- * creation for such specially configured kqworkloops.
- *
- * If successful, the newly created thread will be parked in
- * workq_bound_thread_initialize_and_unpark_continue waiting for
- * new incoming events.
- */
-kern_return_t
-workq_kern_threadreq_permanent_bind(struct proc *p, struct workq_threadreq_s *kqr)
-{
- kern_return_t ret = 0;
- thread_t new_thread = NULL;
- struct workqueue *wq = proc_get_wqptr_fast(p);
-
- workq_lock_spin(wq);
-
- if (wq->wq_nthreads >= wq_max_threads) {
- ret = EDOM;
- } else {
- if (kqr->tr_flags & WORKQ_TR_FLAG_WL_OUTSIDE_QOS) {
- workq_threadreq_param_t trp = kqueue_threadreq_workloop_param(kqr);
- /*
- * For requests outside-of-QoS, we fully initialize the thread
- * request here followed by preadopting the scheduling properties
- * on the newly created bound thread.
- */
- thread_qos_t qos = thread_workq_qos_for_pri(trp.trp_pri);
- if (qos == THREAD_QOS_UNSPECIFIED) {
- qos = WORKQ_THREAD_QOS_ABOVEUI;
- }
- kqr->tr_qos = qos;
- }
- kqr->tr_count = 1;
-
- /* workq_lock dropped and retaken around thread creation below. */
- ret = workq_add_new_idle_thread(p, wq,
- workq_bound_thread_initialize_and_unpark_continue,
- true, &new_thread);
- if (ret == KERN_SUCCESS) {
- struct uthread *uth = get_bsdthread_info(new_thread);
- if (kqr->tr_flags & WORKQ_TR_FLAG_WL_OUTSIDE_QOS) {
- workq_thread_reset_pri(wq, uth, kqr, /*unpark*/ true);
- }
- /*
- * The newly created thread goes through a full bind to the kqwl
- * right upon creation.
- * It then falls back to soft bind/unbind upon wakeup/park.
- */
- kqueue_threadreq_bind_prepost(p, kqr, uth);
- uth->uu_workq_flags |= UT_WORKQ_PERMANENT_BIND;
- }
- }
-
- workq_unlock(wq);
-
- if (ret == KERN_SUCCESS) {
- kqueue_threadreq_bind_commit(p, new_thread);
- }
- return ret;
-}
-
-/*
- * Called with kqlock held. It does not need to take the process wide
- * global workq lock -> making it faster.
- */
-void
-workq_kern_bound_thread_wakeup(struct workq_threadreq_s *kqr)
-{
- struct uthread *uth = get_bsdthread_info(kqr->tr_thread);
- workq_threadreq_param_t trp = kqueue_threadreq_workloop_param(kqr);
-
- /*
- * See "Locking model for accessing uu_workq_flags" for more information
- * on how access to uu_workq_flags for the bound thread is synchronized.
- */
- assert((uth->uu_workq_flags & (UT_WORKQ_RUNNING | UT_WORKQ_DYING)) == 0);
-
- if (trp.trp_flags & TRP_RELEASED) {
- uth->uu_workq_flags |= UT_WORKQ_DYING;
- } else {
- uth->uu_workq_flags |= UT_WORKQ_RUNNING;
- }
-
- workq_thread_wakeup(uth);
-}
-
-/*
- * Called with kqlock held. Dropped before parking.
- * It does not need to take process wide global workqueue
- * lock -> making it faster.
- */
-__attribute__((noreturn, noinline))
-void
-workq_kern_bound_thread_park(struct workq_threadreq_s *kqr)
-{
- struct uthread *uth = get_bsdthread_info(kqr->tr_thread);
- assert(uth == current_uthread());
-
- /*
- * See "Locking model for accessing uu_workq_flags" for more information
- * on how access to uu_workq_flags for the bound thread is synchronized.
- */
- uth->uu_workq_flags &= ~(UT_WORKQ_RUNNING);
-
- thread_disarm_workqueue_quantum(get_machthread(uth));
-
- /*
- * TODO (pavhad) We could do the reusable userspace stack performance
- * optimization here.
- */
-
- kqworkloop_bound_thread_park_prepost(kqr);
- /* KQ_SLEEP bit is set and kqlock is dropped. */
-
- __assert_only kern_return_t kr;
- kr = thread_set_voucher_name(MACH_PORT_NULL);
- assert(kr == KERN_SUCCESS);
-
- /*
- * Bound threads park (and unpark) with the scheduler callback cleared and
- * not counting towards thactive. If we unpark to do work (as opposed to
- * waking up to thread exit), the scheduler callback is added via
- * workq_setup_and_run, and thactive is incremented in
- * workq_bound_thread_setup_and_run.
- */
- thread_sched_call(get_machthread(uth), NULL);
- proc_t p = current_proc();
- struct workqueue *wq = proc_get_wqptr_fast(p);
- _wq_thactive_dec(wq, uth->uu_workq_pri.qos_bucket);
-
- kqworkloop_bound_thread_park_commit(kqr,
- workq_parked_wait_event(uth), workq_bound_thread_unpark_continue);
-
- __builtin_unreachable();
-}
-
-/*
- * To terminate the permenantly bound workqueue thread. It unbinds itself
- * with the kqwl during uthread_cleanup -> kqueue_threadreq_unbind.
- * It is also when it will release its reference on the kqwl.
- */
-__attribute__((noreturn, noinline))
-void
-workq_kern_bound_thread_terminate(struct workq_threadreq_s *kqr)
-{
- proc_t p = current_proc();
- struct uthread *uth = get_bsdthread_info(kqr->tr_thread);
- uint16_t uu_workq_flags_orig;
-
- assert(uth == current_uthread());
-
- /*
- * See "Locking model for accessing uu_workq_flags" for more information
- * on how access to uu_workq_flags for the bound thread is synchronized.
- */
- kqworkloop_bound_thread_terminate(kqr, &uu_workq_flags_orig);
-
- if (uu_workq_flags_orig & UT_WORKQ_WORK_INTERVAL_JOINED) {
- __assert_only kern_return_t kr;
- kr = kern_work_interval_join(get_machthread(uth), MACH_PORT_NULL);
- /* The bound thread un-joins the work interval and drops its +1 ref. */
- assert(kr == KERN_SUCCESS);
- }
-
- /*
- * Drop the voucher now that we are on our way to termination.
- */
- __assert_only kern_return_t kr;
- kr = thread_set_voucher_name(MACH_PORT_NULL);
- assert(kr == KERN_SUCCESS);
-
- uint32_t upcall_flags = WQ_FLAG_THREAD_NEWSPI;
- upcall_flags |= uth->uu_save.uus_workq_park_data.qos |
- WQ_FLAG_THREAD_PRIO_QOS;
-
- thread_t th = get_machthread(uth);
- vm_map_t vmap = get_task_map(proc_task(p));
-
- uint32_t setup_flags = WQ_SETUP_EXIT_THREAD;
-
- if ((uu_workq_flags_orig & UT_WORKQ_NEW) == 0) {
- upcall_flags |= WQ_FLAG_THREAD_REUSE;
- } else {
- /*
- * The bound thread is exiting before ever having gone to userspace. We
- * need pthread to perform initial setup before calling _pthread_exit.
- * TODO(aaron): We should investigate an optimization both here and in
- * workq_unpark_for_death_and_unlock to never send these threads to
- * userspace, instead processing ASTs and calling destroying the stack.
- */
- setup_flags |= WQ_SETUP_FIRST_USE;
-
- /*
- * Typically we'd set the workq_thport in workq_setup_and_run, but we
- * haven't been through that path yet.
- */
- assert(uth->uu_workq_thport == MACH_PORT_NULL);
- /* convert_thread_to_port_immovable() consumes a reference */
- thread_reference(th);
- /* Convert to immovable thread port, then pin the entry */
- uth->uu_workq_thport = ipc_port_copyout_send_pinned(
- convert_thread_to_port_immovable(th),
- get_task_ipcspace(proc_task(p)));
- }
-
- pthread_functions->workq_setup_thread(p, th, vmap, uth->uu_workq_stackaddr,
- uth->uu_workq_thport, 0, setup_flags, upcall_flags);
- __builtin_unreachable();
}
void
@@ -3612,7 +3207,7 @@
flags |= PTHREAD_WQ_QUANTUM_EXPIRY_SHUFFLE;
}
} else if (workq_thread_is_nonovercommit(uth)) {
- if (!workq_constrained_allowance(wq, qos, uth, false, false)) {
+ if (!workq_constrained_allowance(wq, qos, uth, false)) {
flags |= PTHREAD_WQ_QUANTUM_EXPIRY_NARROW;
}
}
@@ -3657,14 +3252,10 @@
return EINVAL;
}
- /*
- * Reset signal mask on the workqueue thread to default state,
- * but do not touch any signals that are marked for preservation.
- */
- sigset_t resettable = uth->uu_sigmask & ~p->p_workq_allow_sigmask;
- if (resettable != (sigset_t)~workq_threadmask) {
+ /* reset signal mask on the workqueue thread to default state */
+ if (uth->uu_sigmask != (sigset_t)(~workq_threadmask)) {
proc_lock(p);
- uth->uu_sigmask |= ~workq_threadmask & ~p->p_workq_allow_sigmask;
+ uth->uu_sigmask = ~workq_threadmask;
proc_unlock(p);
}
@@ -3674,10 +3265,6 @@
* the kqr from this thread.
*/
trp = kqueue_threadreq_workloop_param(kqr);
- }
-
- if (kqr && kqr->tr_flags & WORKQ_TR_FLAG_PERMANENT_BIND) {
- goto handle_stack_events;
}
/*
@@ -3688,8 +3275,6 @@
* - or we proceed to workq_select_threadreq_or_park_and_unlock() who will.
*/
thread_freeze_base_pri(th);
-
-handle_stack_events:
if (kqr) {
uint32_t upcall_flags = WQ_FLAG_THREAD_NEWSPI | WQ_FLAG_THREAD_REUSE;
@@ -3712,7 +3297,7 @@
}
}
error = pthread_functions->workq_handle_stack_events(p, th,
- get_task_map(proc_task(p)), uth->uu_workq_stackaddr,
+ get_task_map(p->task), uth->uu_workq_stackaddr,
uth->uu_workq_thport, eventlist, nevents, upcall_flags);
if (error) {
assert(uth->uu_kqr_bound == kqr);
@@ -3808,7 +3393,7 @@
/*
* Normalize the incoming priority so that it is ordered numerically.
*/
- if (_pthread_priority_has_sched_pri(pri)) {
+ if (pri & _PTHREAD_PRIORITY_SCHED_PRI_FLAG) {
pri &= (_PTHREAD_PRIORITY_SCHED_PRI_MASK |
_PTHREAD_PRIORITY_SCHED_PRI_FLAG);
} else {
@@ -3859,7 +3444,7 @@
break;
}
workq_lock_spin(wq);
- bool should_narrow = !workq_constrained_allowance(wq, qos, uth, false, false);
+ bool should_narrow = !workq_constrained_allowance(wq, qos, uth, false);
workq_unlock(wq);
*retval = should_narrow;
@@ -3950,7 +3535,7 @@
*/
if (!uth->uu_save.uus_workq_park_data.has_stack) {
pthread_functions->workq_markfree_threadstack(p,
- get_machthread(uth), get_task_map(proc_task(p)),
+ get_machthread(uth), get_task_map(p->task),
uth->uu_workq_stackaddr);
}
@@ -4015,13 +3600,11 @@
(uth && uth->uu_workq_pri.qos_bucket == WORKQ_THREAD_QOS_MANAGER);
}
-/* Called with workq lock held. */
static uint32_t
workq_constrained_allowance(struct workqueue *wq, thread_qos_t at_qos,
- struct uthread *uth, bool may_start_timer, bool record_failed_allowance)
+ struct uthread *uth, bool may_start_timer)
{
assert(at_qos != WORKQ_THREAD_QOS_MANAGER);
- uint32_t allowance_passed = 0;
uint32_t count = 0;
uint32_t max_count = wq->wq_constrained_threads_scheduled;
@@ -4040,8 +3623,7 @@
* we need 1 or more constrained threads to return to the kernel before
* we can dispatch additional work
*/
- allowance_passed = 0;
- goto out;
+ return 0;
}
max_count -= wq_max_constrained_threads;
@@ -4078,12 +3660,10 @@
count -= thactive_count + busycount;
WQ_TRACE_WQ(TRACE_wq_constrained_admission | DBG_FUNC_NONE, wq, 2,
thactive_count, busycount);
- allowance_passed = MIN(count, max_count);
- goto out;
+ return MIN(count, max_count);
} else {
WQ_TRACE_WQ(TRACE_wq_constrained_admission | DBG_FUNC_NONE, wq, 3,
thactive_count, busycount);
- allowance_passed = 0;
}
if (may_start_timer) {
@@ -4094,11 +3674,7 @@
workq_schedule_delayed_thread_creation(wq, 0);
}
-out:
- if (record_failed_allowance) {
- wq->wq_exceeded_active_constrained_thread_limit = !allowance_passed;
- }
- return allowance_passed;
+ return 0;
}
static bool
@@ -4112,7 +3688,7 @@
return workq_cooperative_allowance(wq, req->tr_qos, uth, true);
}
if (workq_threadreq_is_nonovercommit(req)) {
- return workq_constrained_allowance(wq, req->tr_qos, uth, true, true);
+ return workq_constrained_allowance(wq, req->tr_qos, uth, true);
}
return true;
@@ -4134,11 +3710,11 @@
* requires us to reeevaluate the next best request for it.
*/
if (uth && workq_thread_is_cooperative(uth)) {
- _wq_cooperative_queue_scheduled_count_dec(wq, uth->uu_workq_pri.qos_req);
+ _wq_cooperative_queue_scheduled_count_dec(wq, uth->uu_workq_pri.qos_bucket);
(void) _wq_cooperative_queue_refresh_best_req_qos(wq);
- _wq_cooperative_queue_scheduled_count_inc(wq, uth->uu_workq_pri.qos_req);
+ _wq_cooperative_queue_scheduled_count_inc(wq, uth->uu_workq_pri.qos_bucket);
} else {
/*
* The old value that was already precomputed should be safe to use -
@@ -4242,7 +3818,7 @@
return req_pri;
}
- if (workq_constrained_allowance(wq, req_tmp->tr_qos, NULL, true, true)) {
+ if (workq_constrained_allowance(wq, req_tmp->tr_qos, NULL, true)) {
/*
* If the constrained thread request is the best one and passes
* the admission check, pick it.
@@ -4305,7 +3881,7 @@
*
* Note that the creator thread is an overcommit thread.
*/
- thread_qos_t new_thread_qos = uth->uu_workq_pri.qos_req;
+ thread_qos_t new_thread_qos = uth->uu_workq_pri.qos_bucket;
/*
* Anytime a cooperative bucket's schedule count changes, we need to
@@ -4451,7 +4027,7 @@
return req_pri;
}
- if (workq_constrained_allowance(wq, req_tmp->tr_qos, uth, true, true)) {
+ if (workq_constrained_allowance(wq, req_tmp->tr_qos, uth, true)) {
/*
* If the constrained thread request is the best one and passes
* the admission check, pick it.
@@ -4574,8 +4150,7 @@
} else if (!(flags & WORKQ_THREADREQ_CAN_CREATE_THREADS)) {
/* This can drop the workqueue lock, and take it again */
workq_schedule_immediate_thread_creation(wq);
- } else if ((workq_add_new_idle_thread(p, wq,
- workq_unpark_continue, false, NULL) == KERN_SUCCESS)) {
+ } else if (workq_add_new_idle_thread(p, wq)) {
goto again;
} else {
workq_schedule_delayed_thread_creation(wq, 0);
@@ -4629,7 +4204,7 @@
goto park;
}
- struct uu_workq_policy old_pri = uth->uu_workq_pri;
+ thread_qos_t old_thread_bucket = uth->uu_workq_pri.qos_bucket;
uint8_t tr_flags = req->tr_flags;
struct turnstile *req_ts = kqueue_threadreq_get_turnstile(req);
@@ -4664,8 +4239,8 @@
wq->wq_creator = NULL;
_wq_thactive_inc(wq, req->tr_qos);
wq->wq_thscheduled_count[_wq_bucket(req->tr_qos)]++;
- } else if (old_pri.qos_bucket != req->tr_qos) {
- _wq_thactive_move(wq, old_pri.qos_bucket, req->tr_qos);
+ } else if (old_thread_bucket != req->tr_qos) {
+ _wq_thactive_move(wq, old_thread_bucket, req->tr_qos);
}
workq_thread_reset_pri(wq, uth, req, /*unpark*/ true);
@@ -4677,7 +4252,7 @@
*/
bool cooperative_sched_count_changed =
workq_adjust_cooperative_constrained_schedule_counts(wq, uth,
- old_pri.qos_req, tr_flags);
+ old_thread_bucket, tr_flags);
if (workq_tr_is_overcommit(tr_flags)) {
workq_thread_set_type(uth, UT_WORKQ_OVERCOMMIT);
@@ -4697,21 +4272,6 @@
});
}
WQ_TRACE_WQ(TRACE_wq_select_threadreq | DBG_FUNC_NONE, wq, 3, 0, 0);
-
- /*
- * If a cooperative thread was the one which picked up the manager
- * thread request, we need to reevaluate the cooperative pool before
- * it goes and parks.
- *
- * For every other of thread request that it picks up, the logic in
- * workq_threadreq_select should have done this refresh.
- * See workq_push_idle_thread.
- */
- if (cooperative_sched_count_changed) {
- if (req->tr_qos == WORKQ_THREAD_QOS_MANAGER) {
- _wq_cooperative_queue_refresh_best_req_qos(wq);
- }
- }
goto park_thawed;
}
@@ -4881,7 +4441,7 @@
}
/**
- * parked idle thread wakes up
+ * parked thread wakes up
*/
__attribute__((noreturn, noinline))
static void
@@ -4941,7 +4501,7 @@
workq_setup_and_run(proc_t p, struct uthread *uth, int setup_flags)
{
thread_t th = get_machthread(uth);
- vm_map_t vmap = get_task_map(proc_task(p));
+ vm_map_t vmap = get_task_map(p->task);
if (setup_flags & WQ_SETUP_CLEAR_VOUCHER) {
/*
@@ -4985,12 +4545,12 @@
}
if (uth->uu_workq_thport == MACH_PORT_NULL) {
- /* convert_thread_to_port_immovable() consumes a reference */
+ /* convert_thread_to_port_pinned() consumes a reference */
thread_reference(th);
- /* Convert to immovable thread port, then pin the entry */
- uth->uu_workq_thport = ipc_port_copyout_send_pinned(
- convert_thread_to_port_immovable(th),
- get_task_ipcspace(proc_task(p)));
+ /* Convert to immovable/pinned thread port, but port is not pinned yet */
+ ipc_port_t port = convert_thread_to_port_pinned(th);
+ /* Atomically, pin and copy out the port */
+ uth->uu_workq_thport = ipc_port_copyout_send_pinned(port, get_task_ipcspace(p->task));
}
/* Thread has been set up to run, arm its next workqueue quantum or disarm
@@ -5017,166 +4577,6 @@
pthread_functions->workq_setup_thread(p, th, vmap, uth->uu_workq_stackaddr,
uth->uu_workq_thport, 0, setup_flags, upcall_flags);
- __builtin_unreachable();
-}
-
-/**
- * A wrapper around workq_setup_and_run for permanently bound thread.
- */
-__attribute__((noreturn, noinline))
-static void
-workq_bound_thread_setup_and_run(struct uthread *uth, int setup_flags)
-{
- struct workq_threadreq_s * kqr = uth->uu_kqr_bound;
-
- uint32_t upcall_flags = (WQ_FLAG_THREAD_NEWSPI |
- WQ_FLAG_THREAD_WORKLOOP | WQ_FLAG_THREAD_KEVENT);
- if (workq_tr_is_overcommit(kqr->tr_flags)) {
- workq_thread_set_type(uth, UT_WORKQ_OVERCOMMIT);
- upcall_flags |= WQ_FLAG_THREAD_OVERCOMMIT;
- }
- uth->uu_save.uus_workq_park_data.upcall_flags = upcall_flags;
-
- /*
- * Increment thactive since we've decided this thread should go to
- * userspace. The scheduler callback is set in workq_setup_and_run.
- */
- proc_t p = current_proc();
- struct workqueue *wq = proc_get_wqptr_fast(p);
- _wq_thactive_inc(wq, uth->uu_workq_pri.qos_bucket);
-
- workq_setup_and_run(p, uth, setup_flags);
- __builtin_unreachable();
-}
-
-/**
- * A parked bound thread wakes up for the first time.
- */
-__attribute__((noreturn, noinline))
-static void
-workq_bound_thread_initialize_and_unpark_continue(void *parameter __unused,
- wait_result_t wr)
-{
- /*
- * Locking model for accessing uu_workq_flags :
- *
- * The concurrent access to uu_workq_flags is synchronized with workq lock
- * until a thread gets permanently bound to a kqwl. Post that, kqlock
- * is used for subsequent synchronizations. This gives us a significant
- * benefit by avoiding having to take a process wide workq lock on every
- * wakeup of the bound thread.
- * This flip in locking model is tracked with UT_WORKQ_PERMANENT_BIND flag.
- *
- * There is one more optimization we can perform for when the thread is
- * awakened for running (i.e THREAD_AWAKENED) until it parks.
- * During this window, we know KQ_SLEEP bit is reset so there should not
- * be any concurrent attempts to modify uu_workq_flags by
- * kqworkloop_bound_thread_wakeup because the thread is already "awake".
- * So we can safely access uu_workq_flags within this window without having
- * to take kqlock. This KQ_SLEEP is later set by the bound thread under
- * kqlock on its way to parking.
- */
- struct uthread *uth = get_bsdthread_info(current_thread());
-
- if (__probable(wr == THREAD_AWAKENED)) {
- /* At most one flag. */
- assert((uth->uu_workq_flags & (UT_WORKQ_RUNNING | UT_WORKQ_DYING))
- != (UT_WORKQ_RUNNING | UT_WORKQ_DYING));
-
- assert(workq_thread_is_permanently_bound(uth));
-
- if (uth->uu_workq_flags & UT_WORKQ_RUNNING) {
- assert(uth->uu_workq_flags & UT_WORKQ_NEW);
- uth->uu_workq_flags &= ~UT_WORKQ_NEW;
-
- struct workq_threadreq_s * kqr = uth->uu_kqr_bound;
- if (kqr->tr_work_interval) {
- kern_return_t kr;
- kr = kern_work_interval_explicit_join(get_machthread(uth),
- kqr->tr_work_interval);
- /*
- * The work interval functions requires to be called on the
- * current thread. If we fail here, we record the fact and
- * continue.
- * In the future, we can preflight checking that this join will
- * always be successful when the paird kqwl is configured; but,
- * for now, this should be a rare case (e.g. if you have passed
- * invalid arguments to the join).
- */
- if (kr == KERN_SUCCESS) {
- uth->uu_workq_flags |= UT_WORKQ_WORK_INTERVAL_JOINED;
- /* Thread and kqwl both have +1 ref on the work interval. */
- } else {
- uth->uu_workq_flags |= UT_WORKQ_WORK_INTERVAL_FAILED;
- }
- }
- workq_thread_reset_cpupercent(kqr, uth);
- workq_bound_thread_setup_and_run(uth, WQ_SETUP_FIRST_USE);
- __builtin_unreachable();
- } else {
- /*
- * The permanently bound kqworkloop is getting destroyed so we
- * are woken up to cleanly unbind ourselves from it and terminate.
- * See KQ_WORKLOOP_DESTROY -> workq_kern_bound_thread_wakeup.
- *
- * The actual full unbind happens from
- * uthread_cleanup -> kqueue_threadreq_unbind.
- */
- assert(uth->uu_workq_flags & UT_WORKQ_DYING);
- }
- } else {
- /*
- * The process is getting terminated so we are woken up to die.
- * E.g. SIGKILL'd.
- */
- assert(wr == THREAD_INTERRUPTED);
- /*
- * It is possible we started running as the process is aborted
- * due to termination; but, workq_kern_threadreq_permanent_bind
- * has not had a chance to bind us to the kqwl yet.
- *
- * We synchronize with it using workq lock.
- */
- proc_t p = current_proc();
- struct workqueue *wq = proc_get_wqptr_fast(p);
- workq_lock_spin(wq);
- assert(workq_thread_is_permanently_bound(uth));
- workq_unlock(wq);
-
- /*
- * We do the bind commit ourselves if workq_kern_threadreq_permanent_bind
- * has not done it for us yet so our state is aligned with what the
- * termination path below expects.
- */
- kqueue_threadreq_bind_commit(p, get_machthread(uth));
- }
- workq_kern_bound_thread_terminate(uth->uu_kqr_bound);
- __builtin_unreachable();
-}
-
-/**
- * A parked bound thread wakes up. Not the first time.
- */
-__attribute__((noreturn, noinline))
-static void
-workq_bound_thread_unpark_continue(void *parameter __unused, wait_result_t wr)
-{
- struct uthread *uth = get_bsdthread_info(current_thread());
- assert(workq_thread_is_permanently_bound(uth));
-
- if (__probable(wr == THREAD_AWAKENED)) {
- /* At most one flag. */
- assert((uth->uu_workq_flags & (UT_WORKQ_RUNNING | UT_WORKQ_DYING))
- != (UT_WORKQ_RUNNING | UT_WORKQ_DYING));
- if (uth->uu_workq_flags & UT_WORKQ_RUNNING) {
- workq_bound_thread_setup_and_run(uth, WQ_SETUP_NONE);
- } else {
- assert(uth->uu_workq_flags & UT_WORKQ_DYING);
- }
- } else {
- assert(wr == THREAD_INTERRUPTED);
- }
- workq_kern_bound_thread_terminate(uth->uu_kqr_bound);
__builtin_unreachable();
}
@@ -5224,17 +4624,6 @@
pwqinfo->pwq_state |= WQ_EXCEEDED_TOTAL_THREAD_LIMIT;
}
- uint64_t total_cooperative_threads;
- total_cooperative_threads = workq_num_cooperative_threads_scheduled_total(wq);
- if ((total_cooperative_threads == wq_cooperative_queue_max_size(wq)) &&
- workq_has_cooperative_thread_requests(wq)) {
- pwqinfo->pwq_state |= WQ_EXCEEDED_COOPERATIVE_THREAD_LIMIT;
- }
-
- if (wq->wq_exceeded_active_constrained_thread_limit) {
- pwqinfo->pwq_state |= WQ_EXCEEDED_ACTIVE_CONSTRAINED_THREAD_LIMIT;
- }
-
workq_unlock(wq);
return error;
}
@@ -5265,22 +4654,16 @@
return TRUE;
}
-uint64_t
-workqueue_get_task_ss_flags_from_pwq_state_kdp(void * v)
+uint32_t
+workqueue_get_pwq_state_kdp(void * v)
{
static_assert((WQ_EXCEEDED_CONSTRAINED_THREAD_LIMIT << 17) ==
kTaskWqExceededConstrainedThreadLimit);
static_assert((WQ_EXCEEDED_TOTAL_THREAD_LIMIT << 17) ==
kTaskWqExceededTotalThreadLimit);
static_assert((WQ_FLAGS_AVAILABLE << 17) == kTaskWqFlagsAvailable);
- static_assert(((uint64_t)WQ_EXCEEDED_COOPERATIVE_THREAD_LIMIT << 34) ==
- (uint64_t)kTaskWqExceededCooperativeThreadLimit);
- static_assert(((uint64_t)WQ_EXCEEDED_ACTIVE_CONSTRAINED_THREAD_LIMIT << 34) ==
- (uint64_t)kTaskWqExceededActiveConstrainedThreadLimit);
static_assert((WQ_FLAGS_AVAILABLE | WQ_EXCEEDED_TOTAL_THREAD_LIMIT |
- WQ_EXCEEDED_CONSTRAINED_THREAD_LIMIT |
- WQ_EXCEEDED_COOPERATIVE_THREAD_LIMIT |
- WQ_EXCEEDED_ACTIVE_CONSTRAINED_THREAD_LIMIT) == 0x1F);
+ WQ_EXCEEDED_CONSTRAINED_THREAD_LIMIT) == 0x7);
if (v == NULL) {
return 0;
@@ -5293,29 +4676,17 @@
return 0;
}
- uint64_t ss_flags = kTaskWqFlagsAvailable;
+ uint32_t pwq_state = WQ_FLAGS_AVAILABLE;
if (wq->wq_constrained_threads_scheduled >= wq_max_constrained_threads) {
- ss_flags |= kTaskWqExceededConstrainedThreadLimit;
+ pwq_state |= WQ_EXCEEDED_CONSTRAINED_THREAD_LIMIT;
}
if (wq->wq_nthreads >= wq_max_threads) {
- ss_flags |= kTaskWqExceededTotalThreadLimit;
- }
-
- uint64_t total_cooperative_threads;
- total_cooperative_threads = workq_num_cooperative_threads_scheduled_to_qos_internal(wq,
- WORKQ_THREAD_QOS_MIN);
- if ((total_cooperative_threads == wq_cooperative_queue_max_size(wq)) &&
- workq_has_cooperative_thread_requests(wq)) {
- ss_flags |= kTaskWqExceededCooperativeThreadLimit;
- }
-
- if (wq->wq_exceeded_active_constrained_thread_limit) {
- ss_flags |= kTaskWqExceededActiveConstrainedThreadLimit;
- }
-
- return ss_flags;
+ pwq_state |= WQ_EXCEEDED_TOTAL_THREAD_LIMIT;
+ }
+
+ return pwq_state;
}
void