Loading...
--- xnu/xnu-12377.121.6/osfmk/kern/stack.c
+++ xnu/xnu-792/osfmk/kern/stack.c
@@ -1,29 +1,23 @@
/*
- * Copyright (c) 2003-2019 Apple Inc. All rights reserved.
- *
- * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
- *
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the License
- * may not be used to create, or enable the creation or redistribution of,
- * unlawful or unlicensed copies of an Apple operating system, or to
- * circumvent, violate, or enable the circumvention or violation of, any
- * terms of an Apple operating system software license agreement.
- *
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this file.
- *
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_LICENSE_HEADER_START@
+ *
+ * The contents of this file constitute Original Code as defined in and
+ * are subject to the Apple Public Source License Version 1.1 (the
+ * "License"). You may not use this file except in compliance with the
+ * License. Please obtain a copy of the License at
+ * http://www.apple.com/publicsource and read it before using this file.
+ *
+ * This Original Code and all software distributed under the License are
+ * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- *
- * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
+ * License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * @APPLE_LICENSE_HEADER_END@
*/
/*
* Kernel stack management routines.
@@ -34,24 +28,16 @@
#include <mach/processor_set.h>
#include <kern/kern_types.h>
-#include <kern/lock_group.h>
#include <kern/mach_param.h>
-#include <kern/misc_protos.h>
-#include <kern/percpu.h>
#include <kern/processor.h>
#include <kern/thread.h>
#include <kern/zalloc.h>
#include <kern/kalloc.h>
-#include <kern/ledger.h>
-
-#include <vm/vm_map_xnu.h>
-#include <vm/vm_kern_xnu.h>
-
-#include <san/kasan.h>
-
-#if __has_feature(ptrauth_calls)
-#include <ptrauth.h>
-#endif
+
+#include <vm/vm_map.h>
+#include <vm/vm_kern.h>
+
+#include <mach_debug.h>
/*
* We allocate stacks from generic kernel VM.
@@ -60,123 +46,57 @@
* because stack_alloc_try/thread_invoke operate at splsched.
*/
-static SIMPLE_LOCK_DECLARE(stack_lock_data, 0);
-#define stack_lock() simple_lock(&stack_lock_data, LCK_GRP_NULL)
-#define stack_unlock() simple_unlock(&stack_lock_data)
-
-#define STACK_CACHE_SIZE 2
-
-static vm_offset_t stack_free_list;
-
-static unsigned int stack_free_count, stack_free_hiwat; /* free list count */
-static unsigned int stack_hiwat;
-unsigned int stack_total; /* current total count */
-unsigned long long stack_allocs; /* total count of allocations */
-
-static unsigned int stack_free_target;
-static int stack_free_delta;
-
-static unsigned int stack_new_count; /* total new stack allocations */
-
-static SECURITY_READ_ONLY_LATE(vm_offset_t) stack_addr_mask;
-SECURITY_READ_ONLY_LATE(vm_offset_t) kernel_stack_size;
-SECURITY_READ_ONLY_LATE(vm_offset_t) kernel_stack_mask;
-vm_offset_t kernel_stack_depth_max;
-
-struct stack_cache {
- vm_offset_t free;
- unsigned int count;
-};
-static struct stack_cache PERCPU_DATA(stack_cache);
-
-static inline vm_offset_t *
-stack_get_next_ptr(vm_offset_t stack)
-{
- /*
- * The next field is at the base of the stack,
- * so the low end is left unsullied.
- */
- return (vm_offset_t *)((stack) + kernel_stack_size) - 1;
-}
-
-#if __has_feature(ptrauth_calls)
-static inline uint64_t
-stack_next_ptr_discriminator(vm_offset_t containing_stack)
-{
- /* diversify by both type and location */
- return ptrauth_blend_discriminator(
- (void *)containing_stack,
- ptrauth_string_discriminator("stack.next"));
-}
-#endif /* __has_feature(ptrauth_calls) */
-
-static inline void
-stack_set_next(vm_offset_t stack, vm_offset_t stack_next)
-{
- vm_offset_t *next_ptr = stack_get_next_ptr(stack);
-
-#if __has_feature(ptrauth_calls)
- *next_ptr = (vm_offset_t)ptrauth_sign_unauthenticated(
- (void *)stack_next,
- ptrauth_key_process_independent_data,
- stack_next_ptr_discriminator(stack));
-#else
- *next_ptr = stack_next;
-#endif
-}
-
-static inline vm_offset_t
-stack_get_next(vm_offset_t stack)
-{
- vm_offset_t *next_ptr = stack_get_next_ptr(stack);
-
-#if __has_feature(ptrauth_calls)
- return (vm_offset_t)ptrauth_auth_data(
- (void *)*next_ptr,
- ptrauth_key_process_independent_data,
- stack_next_ptr_discriminator(stack));
-#else
- return *next_ptr;
-#endif
-}
-
-static inline vm_offset_t
-roundup_pow2(vm_offset_t size)
-{
- if ((size & (size - 1)) == 0) {
- /* if size is a power of 2 we're good */
- return size;
- }
-
- return 1ul << flsll(size);
-}
-
-static vm_offset_t stack_alloc_internal(void);
-static void stack_free_stack(vm_offset_t);
-
-static void
+decl_simple_lock_data(static,stack_lock_data)
+#define stack_lock() simple_lock(&stack_lock_data)
+#define stack_unlock() simple_unlock(&stack_lock_data)
+
+#define STACK_CACHE_SIZE 2
+
+static vm_map_t stack_map;
+static vm_offset_t stack_free_list;
+
+static unsigned int stack_free_count, stack_free_hiwat; /* free list count */
+static unsigned int stack_total, stack_hiwat; /* current total count */
+
+static unsigned int stack_free_target;
+static int stack_free_delta;
+
+static unsigned int stack_new_count; /* total new stack allocations */
+
+static vm_offset_t stack_addr_mask;
+
+/*
+ * The next field is at the base of the stack,
+ * so the low end is left unsullied.
+ */
+#define stack_next(stack) \
+ (*((vm_offset_t *)((stack) + KERNEL_STACK_SIZE) - 1))
+
+void
stack_init(void)
{
- uint32_t kernel_stack_pages = atop(KERNEL_STACK_SIZE);
-
- kernel_stack_size = KERNEL_STACK_SIZE;
- kernel_stack_mask = -KERNEL_STACK_SIZE;
-
- if (PE_parse_boot_argn("kernel_stack_pages",
- &kernel_stack_pages,
- sizeof(kernel_stack_pages))) {
- kernel_stack_size = kernel_stack_pages * PAGE_SIZE;
- }
-
- if (kernel_stack_size < round_page(kernel_stack_size)) {
- panic("stack_init: stack size %p not a multiple of page size %d",
- (void *) kernel_stack_size, PAGE_SIZE);
- }
-
- stack_addr_mask = roundup_pow2(kernel_stack_size) - 1;
- kernel_stack_mask = ~stack_addr_mask;
-}
-STARTUP(TUNABLES, STARTUP_RANK_MIDDLE, stack_init);
+ vm_offset_t stacks, boundary;
+ vm_map_offset_t map_addr;
+
+ simple_lock_init(&stack_lock_data, 0);
+
+ if (KERNEL_STACK_SIZE < round_page(KERNEL_STACK_SIZE))
+ panic("stack_init: stack size %d not a multiple of page size %d\n", KERNEL_STACK_SIZE, PAGE_SIZE);
+
+ for (boundary = PAGE_SIZE; boundary <= KERNEL_STACK_SIZE; )
+ boundary <<= 1;
+
+ stack_addr_mask = boundary - 1;
+
+ if (kmem_suballoc(kernel_map, &stacks, (boundary * (2 * THREAD_MAX + 64)),
+ FALSE, VM_FLAGS_ANYWHERE, &stack_map) != KERN_SUCCESS)
+ panic("stack_init: kmem_suballoc");
+
+ map_addr = vm_map_min(stack_map);
+ if (vm_map_enter(stack_map, &map_addr, vm_map_round_page(PAGE_SIZE), 0, VM_FLAGS_FIXED,
+ VM_OBJECT_NULL, 0, FALSE, VM_PROT_NONE, VM_PROT_NONE, VM_INHERIT_DEFAULT) != KERN_SUCCESS)
+ panic("stack_init: vm_map_enter");
+}
/*
* stack_alloc:
@@ -184,66 +104,37 @@
* Allocate a stack for a thread, may
* block.
*/
-
-static vm_offset_t
-stack_alloc_internal(void)
-{
- vm_offset_t stack = 0;
- spl_t s;
- kma_flags_t flags = KMA_NOFAIL | KMA_GUARD_FIRST | KMA_GUARD_LAST |
- KMA_GUARD_STACK | KMA_KSTACK | KMA_KOBJECT | KMA_ZERO;
+void
+stack_alloc(
+ thread_t thread)
+{
+ vm_offset_t stack;
+ spl_t s;
+
+ assert(thread->kernel_stack == 0);
s = splsched();
stack_lock();
- stack_allocs++;
stack = stack_free_list;
if (stack != 0) {
- stack_free_list = stack_get_next(stack);
+ stack_free_list = stack_next(stack);
stack_free_count--;
- } else {
- if (++stack_total > stack_hiwat) {
+ }
+ else {
+ if (++stack_total > stack_hiwat)
stack_hiwat = stack_total;
- }
stack_new_count++;
}
stack_free_delta--;
stack_unlock();
splx(s);
-
+
if (stack == 0) {
- /*
- * Request two guard pages below the stack and one above. Ask
- * kernel_memory_allocate() for three extra pages to account
- * for these.
- */
-
- kernel_memory_allocate(kernel_map, &stack,
- kernel_stack_size + ptoa(3), stack_addr_mask,
- flags, VM_KERN_MEMORY_STACK);
-
- /*
- * The stack address that comes back is the address of the lower
- * guard pages. Skip past them to get the actual stack base address.
- */
-
- stack += ptoa(2);
- }
- return stack;
-}
-
-void
-stack_alloc(
- thread_t thread)
-{
- assert(thread->kernel_stack == 0);
- machine_stack_attach(thread, stack_alloc_internal());
-}
-
-void
-stack_handoff(thread_t from, thread_t to)
-{
- assert(from == current_thread());
- machine_stack_handoff(from, to);
+ if (kernel_memory_allocate(stack_map, &stack, KERNEL_STACK_SIZE, stack_addr_mask, KMA_KOBJECT) != KERN_SUCCESS)
+ panic("stack_alloc: kernel_memory_allocate");
+ }
+
+ machine_stack_attach(thread, stack);
}
/*
@@ -253,53 +144,55 @@
*/
void
stack_free(
- thread_t thread)
-{
- vm_offset_t stack = machine_stack_detach(thread);
-
-#if CONFIG_SPTM
- assert(thread->machine.kredzonestack == 0);
-#endif
+ thread_t thread)
+{
+ vm_offset_t stack = machine_stack_detach(thread);
+
assert(stack);
if (stack != thread->reserved_stack) {
- stack_free_stack(stack);
- }
-}
-
-void
-stack_free_reserved(
- thread_t thread)
-{
- if (thread->reserved_stack != thread->kernel_stack) {
- stack_free_stack(thread->reserved_stack);
- }
-}
-
-static void
+ struct stack_cache *cache;
+ spl_t s;
+
+ s = splsched();
+ cache = &PROCESSOR_DATA(current_processor(), stack_cache);
+ if (cache->count < STACK_CACHE_SIZE) {
+ stack_next(stack) = cache->free;
+ cache->free = stack;
+ cache->count++;
+ }
+ else {
+ stack_lock();
+ stack_next(stack) = stack_free_list;
+ stack_free_list = stack;
+ if (++stack_free_count > stack_free_hiwat)
+ stack_free_hiwat = stack_free_count;
+ stack_free_delta++;
+ stack_unlock();
+ }
+ splx(s);
+ }
+}
+
+void
stack_free_stack(
- vm_offset_t stack)
-{
- struct stack_cache *cache;
- spl_t s;
-
-#if KASAN_DEBUG
- /* Sanity check - stack should be unpoisoned by now */
- assert(kasan_check_shadow(stack, kernel_stack_size, 0));
-#endif
+ vm_offset_t stack)
+{
+ struct stack_cache *cache;
+ spl_t s;
s = splsched();
- cache = PERCPU_GET(stack_cache);
+ cache = &PROCESSOR_DATA(current_processor(), stack_cache);
if (cache->count < STACK_CACHE_SIZE) {
- stack_set_next(stack, cache->free);
+ stack_next(stack) = cache->free;
cache->free = stack;
cache->count++;
- } else {
+ }
+ else {
stack_lock();
- stack_set_next(stack, stack_free_list);
+ stack_next(stack) = stack_free_list;
stack_free_list = stack;
- if (++stack_free_count > stack_free_hiwat) {
+ if (++stack_free_count > stack_free_hiwat)
stack_free_hiwat = stack_free_count;
- }
stack_free_delta++;
stack_unlock();
}
@@ -318,22 +211,23 @@
*/
boolean_t
stack_alloc_try(
- thread_t thread)
-{
- struct stack_cache *cache;
- vm_offset_t stack;
-
- cache = PERCPU_GET(stack_cache);
+ thread_t thread)
+{
+ struct stack_cache *cache;
+ vm_offset_t stack;
+
+ cache = &PROCESSOR_DATA(current_processor(), stack_cache);
stack = cache->free;
if (stack != 0) {
- cache->free = stack_get_next(stack);
+ cache->free = stack_next(stack);
cache->count--;
- } else {
+ }
+ else {
if (stack_free_list != 0) {
stack_lock();
stack = stack_free_list;
if (stack != 0) {
- stack_free_list = stack_get_next(stack);
+ stack_free_list = stack_next(stack);
stack_free_count--;
stack_free_delta--;
}
@@ -343,13 +237,13 @@
if (stack != 0 || (stack = thread->reserved_stack) != 0) {
machine_stack_attach(thread, stack);
- return TRUE;
- }
-
- return FALSE;
-}
-
-static unsigned int stack_collect_tick, last_stack_tick;
+ return (TRUE);
+ }
+
+ return (FALSE);
+}
+
+static unsigned int stack_collect_tick, last_stack_tick;
/*
* stack_collect:
@@ -361,9 +255,9 @@
stack_collect(void)
{
if (stack_collect_tick != last_stack_tick) {
- unsigned int target;
- vm_offset_t stack;
- spl_t s;
+ unsigned int target;
+ vm_offset_t stack;
+ spl_t s;
s = splsched();
stack_lock();
@@ -373,26 +267,14 @@
while (stack_free_count > target) {
stack = stack_free_list;
- stack_free_list = stack_get_next(stack);
+ stack_free_list = stack_next(stack);
stack_free_count--; stack_total--;
stack_unlock();
splx(s);
- /*
- * Get the stack base address, then decrement by two pages
- * to account for the lower guard pages. Add three extra pages
- * to the size to account for the guard pages (two below, one above)
- * that were originally requested when the stack was allocated
- * back in stack_alloc().
- */
-
- stack = (vm_offset_t)vm_map_trunc_page(
- stack,
- VM_MAP_PAGE_MASK(kernel_map));
- stack -= ptoa(2);
- kmem_free(kernel_map, stack, kernel_stack_size + ptoa(3),
- KMF_GUARD_FIRST | KMF_GUARD_LAST | KMF_GUARD_STACK);
- stack = 0;
+ if (vm_map_remove(stack_map, vm_map_trunc_page(stack),
+ vm_map_round_page(stack + KERNEL_STACK_SIZE), VM_MAP_REMOVE_KUNWIRE) != KERN_SUCCESS)
+ panic("stack_collect: vm_map_remove");
s = splsched();
stack_lock();
@@ -419,18 +301,18 @@
*/
void
compute_stack_target(
- __unused void *arg)
-{
- spl_t s;
+__unused void *arg)
+{
+ spl_t s;
s = splsched();
stack_lock();
- if (stack_free_target > 5) {
+ if (stack_free_target > 5)
stack_free_target = (4 * stack_free_target) / 5;
- } else if (stack_free_target > 0) {
+ else
+ if (stack_free_target > 0)
stack_free_target--;
- }
stack_free_target += (stack_free_delta >= 0)? stack_free_delta: -stack_free_delta;
@@ -441,25 +323,148 @@
splx(s);
}
+void
+stack_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size,
+ vm_size_t *alloc_size, int *collectable, int *exhaustable)
+{
+ unsigned int total, hiwat, free;
+ spl_t s;
+
+ s = splsched();
+ stack_lock();
+ total = stack_total;
+ hiwat = stack_hiwat;
+ free = stack_free_count;
+ stack_unlock();
+ splx(s);
+
+ *count = total - free;
+ *cur_size = KERNEL_STACK_SIZE * total;
+ *max_size = KERNEL_STACK_SIZE * hiwat;
+ *elem_size = KERNEL_STACK_SIZE;
+ *alloc_size = KERNEL_STACK_SIZE;
+ *collectable = 1;
+ *exhaustable = 0;
+}
+
/* OBSOLETE */
-void stack_privilege(
- thread_t thread);
+void stack_privilege(
+ thread_t thread);
void
stack_privilege(
- __unused thread_t thread)
+ __unused thread_t thread)
{
/* OBSOLETE */
}
-__mockable vm_offset_t
-min_valid_stack_address(void)
-{
- return (vm_offset_t)vm_map_min(kernel_map);
-}
-
-__mockable vm_offset_t
-max_valid_stack_address(void)
-{
- return (vm_offset_t)vm_map_max(kernel_map);
-}
+/*
+ * Return info on stack usage for threads in a specific processor set
+ */
+kern_return_t
+processor_set_stack_usage(
+ processor_set_t pset,
+ unsigned int *totalp,
+ vm_size_t *spacep,
+ vm_size_t *residentp,
+ vm_size_t *maxusagep,
+ vm_offset_t *maxstackp)
+{
+#if !MACH_DEBUG
+ return KERN_NOT_SUPPORTED;
+#else
+ unsigned int total;
+ vm_size_t maxusage;
+ vm_offset_t maxstack;
+
+ register thread_t *threads;
+ register thread_t thread;
+
+ unsigned int actual; /* this many things */
+ unsigned int i;
+
+ vm_size_t size, size_needed;
+ void *addr;
+
+ if (pset == PROCESSOR_SET_NULL)
+ return KERN_INVALID_ARGUMENT;
+
+ size = 0; addr = 0;
+
+ for (;;) {
+ pset_lock(pset);
+ if (!pset->active) {
+ pset_unlock(pset);
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ actual = pset->thread_count;
+
+ /* do we have the memory we need? */
+
+ size_needed = actual * sizeof(thread_t);
+ if (size_needed <= size)
+ break;
+
+ /* unlock the pset and allocate more memory */
+ pset_unlock(pset);
+
+ if (size != 0)
+ kfree(addr, size);
+
+ assert(size_needed > 0);
+ size = size_needed;
+
+ addr = kalloc(size);
+ if (addr == 0)
+ return KERN_RESOURCE_SHORTAGE;
+ }
+
+ /* OK, have memory and the processor_set is locked & active */
+ threads = (thread_t *) addr;
+ for (i = 0, thread = (thread_t) queue_first(&pset->threads);
+ !queue_end(&pset->threads, (queue_entry_t) thread);
+ thread = (thread_t) queue_next(&thread->pset_threads)) {
+ thread_reference_internal(thread);
+ threads[i++] = thread;
+ }
+ assert(i <= actual);
+
+ /* can unlock processor set now that we have the thread refs */
+ pset_unlock(pset);
+
+ /* calculate maxusage and free thread references */
+
+ total = 0;
+ maxusage = 0;
+ maxstack = 0;
+ while (i > 0) {
+ thread_t threadref = threads[--i];
+
+ if (threadref->kernel_stack != 0)
+ total++;
+
+ thread_deallocate(threadref);
+ }
+
+ if (size != 0)
+ kfree(addr, size);
+
+ *totalp = total;
+ *residentp = *spacep = total * round_page(KERNEL_STACK_SIZE);
+ *maxusagep = maxusage;
+ *maxstackp = maxstack;
+ return KERN_SUCCESS;
+
+#endif /* MACH_DEBUG */
+}
+
+vm_offset_t min_valid_stack_address(void)
+{
+ return vm_map_min(stack_map);
+}
+
+vm_offset_t max_valid_stack_address(void)
+{
+ return vm_map_max(stack_map);
+}