Loading...
--- libmalloc/libmalloc-374.60.3/src/pgm_malloc.c
+++ libmalloc/libmalloc-715.120.13/src/pgm_malloc.c
@@ -24,11 +24,7 @@
#include "pgm_malloc.h"
#include <TargetConditionals.h>
-#if !TARGET_OS_DRIVERKIT
-# include <dlfcn.h> // dladdr()
-#endif
#include <mach/mach_time.h> // mach_absolute_time()
-#include <sys/codesign.h> // csops()
#include "internal.h"
@@ -88,8 +84,7 @@
uint32_t max_allocations;
uint32_t max_metadata;
uint32_t sample_counter_range;
- uint32_t min_alignment;
- bool signal_handler;
+ uint32_t left_align_pct;
bool debug;
uint64_t debug_log_throttle_ms;
@@ -117,8 +112,8 @@
uint64_t last_log_time;
} pgm_zone_t;
-MALLOC_STATIC_ASSERT(__offsetof(pgm_zone_t, malloc_zone) == 0,
- "pgm_zone_t instances must be usable as regular zones");
+ASSERT_WRAPPER_ZONE(pgm_zone_t);
+
MALLOC_STATIC_ASSERT(__offsetof(pgm_zone_t, padding) < PAGE_MAX_SIZE,
"First page is mapped read-only");
MALLOC_STATIC_ASSERT(__offsetof(pgm_zone_t, lock) >= PAGE_MAX_SIZE,
@@ -128,6 +123,27 @@
#pragma mark -
+#pragma mark Thread Local Sample Counter
+
+MALLOC_STATIC_ASSERT(sizeof(void *) >= sizeof(uint32_t), "Pointer is used as 32bit counter");
+
+#define TSD_GET_COUNTER() ((uint32_t)_pthread_getspecific_direct(__TSD_MALLOC_PROB_GUARD_SAMPLE_COUNTER))
+#define TSD_SET_COUNTER(val) _pthread_setspecific_direct(__TSD_MALLOC_PROB_GUARD_SAMPLE_COUNTER, (void *)(uintptr_t)val)
+
+static const uint32_t k_no_sample = UINT32_MAX;
+
+void
+pgm_thread_set_disabled(bool disabled)
+{
+ if (disabled) {
+ TSD_SET_COUNTER(k_no_sample);
+ } else {
+ TSD_SET_COUNTER(0);
+ }
+}
+
+
+#pragma mark -
#pragma mark Decider Functions
// The "decider" functions are performance critical. They should be inlinable and must not lock.
@@ -146,15 +162,17 @@
static inline boolean_t
should_sample_counter(uint32_t counter_range)
{
- MALLOC_STATIC_ASSERT(sizeof(void *) >= sizeof(uint32_t), "Pointer is used as 32bit counter");
- uint32_t counter = (uint32_t)_pthread_getspecific_direct(__TSD_MALLOC_PROB_GUARD_SAMPLE_COUNTER);
+ uint32_t counter = TSD_GET_COUNTER();
+ if (counter == k_no_sample) {
+ return false;
+ }
// 0 -> regenerate counter; 1 -> sample allocation
if (counter == 0) {
counter = rand_uniform(counter_range);
} else {
counter--;
}
- _pthread_setspecific_direct(__TSD_MALLOC_PROB_GUARD_SAMPLE_COUNTER, (void *)(uintptr_t)counter);
+ TSD_SET_COUNTER(counter);
return counter == 0;
}
#endif
@@ -295,19 +313,21 @@
#pragma mark -
#pragma mark Allocator Helpers
+// Darwin ABI requires 16 byte alignment.
+static const size_t k_min_alignment = 16;
+
static bool
is_power_of_2(size_t n) {
return __builtin_popcountl(n) == 1;
}
static size_t
-block_size(size_t size, size_t min_alignment)
-{
- MALLOC_ASSERT(is_power_of_2(min_alignment));
+block_size(size_t size)
+{
if (size == 0) {
- return min_alignment;
- }
- const size_t mask = (min_alignment - 1);
+ return k_min_alignment;
+ }
+ const size_t mask = (k_min_alignment - 1);
return (size + mask) & ~mask;
}
@@ -347,12 +367,17 @@
}
static uint16_t
-choose_offset_on_page(size_t size, size_t alignment, uint16_t page_size) {
+choose_offset_on_page(size_t size,
+ size_t alignment,
+ uint32_t left_align_pct,
+ uint16_t page_size)
+{
MALLOC_ASSERT(size <= page_size);
MALLOC_ASSERT(alignment <= page_size && is_power_of_2(alignment));
MALLOC_ASSERT(is_power_of_2(page_size));
- boolean_t left_align = rand_uniform(2);
- if (left_align) {
+ MALLOC_ASSERT(left_align_pct <= 100);
+
+ if (rand_uniform(100) < left_align_pct) {
return 0;
}
size_t mask = ~(alignment - 1);
@@ -406,17 +431,18 @@
allocate(pgm_zone_t *zone, size_t size, size_t alignment)
{
MALLOC_ASSERT(size <= PAGE_SIZE);
- MALLOC_ASSERT(zone->min_alignment <= alignment && alignment <= PAGE_SIZE);
+ MALLOC_ASSERT(k_min_alignment <= alignment && alignment <= PAGE_SIZE);
MALLOC_ASSERT(is_power_of_2(alignment));
if (is_full(zone)) {
return (vm_address_t)NULL;
}
- size = block_size(size, zone->min_alignment);
+ size = block_size(size);
uint32_t slot = choose_available_slot(zone);
uint32_t metadata = choose_metadata(zone, slot);
- uint16_t offset = choose_offset_on_page(size, alignment, PAGE_SIZE);
+ uint16_t offset = choose_offset_on_page(
+ size, alignment, zone->left_align_pct, PAGE_SIZE);
// Mark page writable before updating metadata. Ensures metadata is correct
// whenever a fault could be triggered.
@@ -492,7 +518,7 @@
vm_address_t new_addr;
if (sample && !is_full(zone)) {
- new_addr = allocate(zone, new_size, zone->min_alignment);
+ new_addr = allocate(zone, new_size, k_min_alignment);
MALLOC_ASSERT(new_addr);
} else {
new_addr = (vm_address_t)DELEGATE(malloc, new_size);
@@ -558,7 +584,14 @@
static void *
pgm_malloc(pgm_zone_t *zone, size_t size)
{
- SAMPLED_ALLOCATE(size, zone->min_alignment, malloc, size);
+ SAMPLED_ALLOCATE(size, k_min_alignment, malloc, size);
+ return ptr;
+}
+
+static void *
+pgm_malloc_type_malloc(pgm_zone_t *zone, size_t size, malloc_type_id_t type_id)
+{
+ SAMPLED_ALLOCATE(size, k_min_alignment, malloc_type_malloc, size, type_id);
return ptr;
}
@@ -569,8 +602,22 @@
if (os_unlikely(os_mul_overflow(num_items, size, &total_size))) {
return DELEGATE(calloc, num_items, size);
}
- SAMPLED_ALLOCATE(total_size, zone->min_alignment, calloc, num_items, size);
- memset(ptr, 0, total_size);
+ SAMPLED_ALLOCATE(total_size, k_min_alignment, calloc, num_items, size);
+ bzero(ptr, total_size);
+ return ptr;
+}
+
+static void *
+pgm_malloc_type_calloc(pgm_zone_t *zone, size_t num_items, size_t size,
+ malloc_type_id_t type_id)
+{
+ size_t total_size;
+ if (os_unlikely(os_mul_overflow(num_items, size, &total_size))) {
+ return DELEGATE(malloc_type_calloc, num_items, size, type_id);
+ }
+ SAMPLED_ALLOCATE(total_size, k_min_alignment, malloc_type_calloc, num_items,
+ size, type_id);
+ bzero(ptr, total_size);
return ptr;
}
@@ -603,6 +650,23 @@
return new_ptr;
}
+static void *
+pgm_malloc_type_realloc(pgm_zone_t *zone, void *ptr, size_t new_size,
+ malloc_type_id_t type_id)
+{
+ if (os_unlikely(!ptr)) {
+ return pgm_malloc_type_malloc(zone, new_size, type_id);
+ }
+ boolean_t sample = should_sample(zone, new_size);
+ if (os_likely(!sample)) {
+ DELEGATE_UNGUARDED(ptr, malloc_type_realloc, ptr, new_size, type_id);
+ }
+ lock(zone);
+ void *new_ptr = (void *)reallocate(zone, (vm_address_t)ptr, new_size, sample);
+ unlock(zone);
+ return new_ptr;
+}
+
static void my_vm_deallocate(vm_address_t addr, size_t size);
static void
pgm_destroy(pgm_zone_t *zone)
@@ -612,69 +676,30 @@
my_vm_deallocate((vm_address_t)zone, sizeof(pgm_zone_t));
}
-static unsigned
-pgm_batch_malloc(pgm_zone_t *zone, size_t size, void **results, unsigned count)
-{
- if (os_unlikely(count == 0)) {
- return 0;
- }
- DELEGATE_UNSAMPLED(size, batch_malloc, size, results, count);
-
- uint32_t sample_count = 1; // Sample at least one allocation.
- for (uint32_t i = 1; i < count; i++) {
- if (should_sample_counter(zone->sample_counter_range)) {
- sample_count++;
- }
- }
- // TODO(yln): Express the above with only one call to rand_uniform(). "n choose k"?
-
- for (uint32_t i = 0; i < sample_count; i++) {
- lock(zone);
- void *ptr = (void *)allocate(zone, size, zone->min_alignment);
- unlock(zone);
- if (!ptr) {
- sample_count = i;
- break; // Zone full.
- }
- results[i] = ptr;
- }
-
- void **remaining_results = results + sample_count;
- uint32_t remaining_count = count - sample_count;
- remaining_count = DELEGATE(batch_malloc, size, remaining_results, remaining_count) ;
-
- // TODO(yln): sampled allocations will always be in the beginning of the results
- // array. We could shuffle it: https://en.wikipedia.org/wiki/Fisher–Yates_shuffle
- return sample_count + remaining_count;
-}
-
-static void
-pgm_batch_free(pgm_zone_t *zone, void **to_be_freed, unsigned count)
-{
- for (uint32_t i = 0; i < count; i++) {
- vm_address_t addr = (vm_address_t)to_be_freed[i];
- if (os_unlikely(is_guarded(zone, addr))) {
- lock(zone);
- deallocate(zone, addr);
- unlock(zone);
- to_be_freed[i] = NULL;
- }
- }
- return DELEGATE(batch_free, to_be_freed, count);
-}
-
static void *
pgm_memalign(pgm_zone_t *zone, size_t alignment, size_t size)
{
- // Delegate for (alignment > page size) and invalid alignment sizes.
- if (alignment > PAGE_SIZE || !is_power_of_2(alignment) || alignment < sizeof(void *)) {
+ if (os_unlikely(alignment > PAGE_SIZE)) {
return DELEGATE(memalign, alignment, size);
}
- size_t adj_alignment = MAX(alignment, zone->min_alignment);
+ size_t adj_alignment = MAX(alignment, k_min_alignment);
SAMPLED_ALLOCATE(size, adj_alignment, memalign, alignment, size);
return ptr;
}
+static void *
+pgm_malloc_type_memalign(pgm_zone_t *zone, size_t alignment, size_t size,
+ malloc_type_id_t type_id)
+{
+ if (os_unlikely(alignment > PAGE_SIZE)) {
+ return DELEGATE(malloc_type_memalign, alignment, size, type_id);
+ }
+ size_t adj_alignment = MAX(alignment, k_min_alignment);
+ SAMPLED_ALLOCATE(size, adj_alignment, malloc_type_memalign, alignment, size,
+ type_id);
+ return ptr;
+}
+
static void
pgm_free_definite_size(pgm_zone_t *zone, void *ptr, size_t size)
{
@@ -684,9 +709,41 @@
static boolean_t
pgm_claimed_address(pgm_zone_t *zone, void *ptr)
{
- return is_guarded(zone, (vm_address_t)ptr);
-}
-
+ DELEGATE_UNGUARDED(ptr, claimed_address, ptr);
+ return true;
+}
+
+static void *
+pgm_malloc_with_options(pgm_zone_t *zone, size_t align, size_t size,
+ uint64_t options)
+{
+ if (os_unlikely(align > PAGE_SIZE)) {
+ return DELEGATE(malloc_with_options, align, size, options);
+ }
+ size_t adj_alignment = MAX(align, k_min_alignment);
+ SAMPLED_ALLOCATE(size, adj_alignment, malloc_with_options, align, size, options);
+ if (options & MALLOC_NP_OPTION_CLEAR) {
+ bzero(ptr, size);
+ }
+ return ptr;
+}
+
+static void *
+pgm_malloc_type_malloc_with_options(pgm_zone_t *zone, size_t align, size_t size,
+ uint64_t options, malloc_type_id_t type_id)
+{
+ if (os_unlikely(align > PAGE_SIZE)) {
+ return DELEGATE(malloc_type_malloc_with_options, align, size, options,
+ type_id);
+ }
+ size_t adj_alignment = MAX(align, k_min_alignment);
+ SAMPLED_ALLOCATE(size, adj_alignment, malloc_type_malloc_with_options,
+ align, size, options, type_id);
+ if (options & MALLOC_NP_OPTION_CLEAR) {
+ bzero(ptr, size);
+ }
+ return ptr;
+}
#pragma mark -
#pragma mark Integrity Checks
@@ -702,7 +759,7 @@
(zone->max_metadata <= zone->num_slots) &&
(zone->num_slots <= k_max_slots) &&
(zone->sample_counter_range > 0) &&
- (zone->min_alignment == 1 || zone->min_alignment == 16); // strict alignment || Darwin ABI alignment
+ (0 <= zone->left_align_pct && zone->left_align_pct <= 100);
}
static bool
@@ -711,6 +768,8 @@
return check_configuration(zone) &&
// Quarantine
(zone->size == quarantine_size(zone->num_slots)) &&
+ (zone->begin % PAGE_SIZE == 0) &&
+ (zone->size % PAGE_SIZE == 0) &&
(zone->begin + zone->size == zone->end) &&
(zone->begin < zone->end) &&
(zone->region_size == 2 * k_zone_spacer + zone->size) &&
@@ -736,9 +795,9 @@
return (slot->state <= ss_freed) &&
(slot->metadata < zone->num_metadata) &&
(slot->size <= PAGE_SIZE) &&
- (slot->size == block_size(slot->size, zone->min_alignment)) &&
+ (slot->size == block_size(slot->size)) &&
(slot->offset <= PAGE_SIZE) &&
- (slot->offset % zone->min_alignment == 0) &&
+ (slot->offset % k_min_alignment == 0) &&
((size_t)slot->offset + slot->size <= PAGE_SIZE);
}
@@ -788,30 +847,32 @@
#pragma mark -
#pragma mark Introspection Functions
-typedef enum { rt_zone_only, rt_slots, rt_slots_and_metadata } read_type_t;
-
-#define READ(remote_address, size, local_memory, checker, zone) \
+typedef enum {
+ rt_zone_only = 1 << 0,
+ rt_slots = 1 << 1,
+ rt_metadata = 1 << 2,
+} read_type_t;
+
+#define READ(remote_address, size, local_memory, checker, check_data) \
{ \
kern_return_t kr = reader(task, (vm_address_t)remote_address, size, (void **)local_memory); \
if (kr != KERN_SUCCESS) return kr; \
- if (!checker(zone)) return KERN_FAILURE; \
+ if (!checker(check_data)) return KERN_FAILURE; \
}
static kern_return_t
read_zone(task_t task, vm_address_t zone_address, memory_reader_t reader, pgm_zone_t *zone, read_type_t read_type)
{
- if (!reader && task == mach_task_self()) {
- reader = _malloc_default_reader;
- }
+ reader = reader_or_in_memory_fallback(reader, task);
pgm_zone_t *zone_ptr;
READ(zone_address, sizeof(pgm_zone_t), &zone_ptr, check_zone, zone_ptr);
*zone = *zone_ptr; // Copy to writable memory
- if (read_type >= rt_slots) {
+ if (read_type & rt_slots) {
READ(zone->slots, zone->num_slots * sizeof(slot_t), &zone->slots, check_slots, zone);
}
- if (read_type >= rt_slots_and_metadata) {
+ if (read_type & rt_metadata) {
READ(zone->metadata, zone->max_metadata * sizeof(metadata_t), &zone->metadata, check_metadata, zone);
}
return KERN_SUCCESS;
@@ -823,7 +884,7 @@
kern_return_t kr = read_zone(task, zone_address, reader, &zone_copy, read_type); \
if (kr != KERN_SUCCESS) return kr; \
} \
- pgm_zone_t *zone = &zone_copy;
+ const pgm_zone_t *zone = &zone_copy;
#define RECORD(remote_address, size_, type) \
{ \
@@ -866,7 +927,7 @@
}
static void
-pgm_statistics(pgm_zone_t *zone, malloc_statistics_t *stats)
+pgm_statistics(const pgm_zone_t *zone, malloc_statistics_t *stats)
{
*stats = (malloc_statistics_t){
.blocks_in_use = zone->num_allocations,
@@ -977,6 +1038,9 @@
#pragma mark -
#pragma mark Zone Templates
+#define PGM_ZONE_VERSION 16
+#define MIN_WRAPPED_ZONE_VERSION 16
+
// Suppress warning: incompatible function pointer types
#define FN_PTR(fn) (void *)(&fn)
@@ -1012,6 +1076,9 @@
#else
.enumerate_unavailable_without_blocks = NULL,
#endif
+
+ // Zone type
+ .zone_type = MALLOC_ZONE_TYPE_PGM,
};
static const malloc_zone_t malloc_zone_template = {
@@ -1029,19 +1096,29 @@
.destroy = FN_PTR(pgm_destroy),
// Batch operations
- .batch_malloc = FN_PTR(pgm_batch_malloc),
- .batch_free = FN_PTR(pgm_batch_free),
+ .batch_malloc = malloc_zone_batch_malloc_fallback,
+ .batch_free = malloc_zone_batch_free_fallback,
// Introspection
.zone_name = "ProbGuardMallocZone",
- .version = 12,
+ .version = PGM_ZONE_VERSION,
.introspect = (malloc_introspection_t *)&introspection_template, // Effectively const.
// Specialized operations
.memalign = FN_PTR(pgm_memalign),
.free_definite_size = FN_PTR(pgm_free_definite_size),
- .pressure_relief = NULL,
- .claimed_address = FN_PTR(pgm_claimed_address)
+ .pressure_relief = malloc_zone_pressure_relief_fallback,
+ .claimed_address = FN_PTR(pgm_claimed_address),
+ .try_free_default = NULL,
+ .malloc_with_options = FN_PTR(pgm_malloc_with_options),
+
+ // Typed entrypoints
+ .malloc_type_malloc = FN_PTR(pgm_malloc_type_malloc),
+ .malloc_type_calloc = FN_PTR(pgm_malloc_type_calloc),
+ .malloc_type_realloc = FN_PTR(pgm_malloc_type_realloc),
+ .malloc_type_memalign = FN_PTR(pgm_malloc_type_memalign),
+ .malloc_type_malloc_with_options =
+ FN_PTR(pgm_malloc_type_malloc_with_options),
};
@@ -1051,8 +1128,7 @@
static const char *
env_var(const char *name)
{
- const char **env = (const char **)*_NSGetEnviron();
- return _simple_getenv(env, name);
+ return getenv(name);
}
static uint32_t
@@ -1079,15 +1155,41 @@
#pragma mark -
#pragma mark Zone Configuration
+static struct {
+ bool internal_build;
+ bool MallocProbGuard_is_set;
+ bool MallocProbGuard;
+ bool targeted_coverage;
+} g_env;
+
+void
+pgm_init_config(bool internal_build)
+{
+ // Avoid dirty memory; do not write in the common case
+ if (internal_build) {
+ g_env.internal_build = internal_build;
+ }
+ if (env_var("MallocProbGuard")) {
+ g_env.MallocProbGuard_is_set = true;
+ g_env.MallocProbGuard = env_bool("MallocProbGuard");
+ }
+
+ const char *all_factors = env_var("__TRIFactors");
+ const char *pgm_factor =
+ "PROBABILISTIC_GUARD_MALLOC_TARGETED_COVERAGE:enable=1";
+ if (all_factors && strstr(all_factors, pgm_factor)) {
+ g_env.targeted_coverage = true;
+ }
+}
+
static bool
is_platform_binary(void)
{
- uint32_t flags = 0;
- int err = csops(getpid(), CS_OPS_STATUS, &flags, sizeof(flags));
- if (err) {
- return false;
- }
- return (flags & CS_PLATFORM_BINARY);
+#if CONFIG_CHECK_PLATFORM_BINARY
+ return malloc_is_platform_binary;
+#else
+ return _malloc_is_platform_binary();
+#endif
}
extern bool main_image_has_section(const char* segname, const char *sectname);
@@ -1097,54 +1199,59 @@
if (!internal_build && !is_platform_binary()) {
return false;
}
-#if TARGET_OS_OSX
uint32_t activation_rate = (internal_build ? 250 : 1000);
if (rand_uniform(activation_rate) != 0) {
return false;
}
-#else
- if (!env_bool("MallocProbGuardViaLaunchd")) {
- return false;
- }
-#endif
if (main_image_has_section("__DATA", "__pgm_opt_out")) {
return false;
}
return true;
}
-#if TARGET_OS_WATCH
+#if TARGET_OS_WATCH || TARGET_OS_TV
static bool
-is_high_memory_device(void)
-{
- uint64_t high_memory = 1.2 * 1024 * 1024 * 1024; // 1.2 GB
- return platform_hw_memsize() > high_memory;
+is_low_memory_device(void)
+{
+ uint64_t low_memory = 1.2 * 1024 * 1024 * 1024; // 1.2 GB
+ return platform_hw_memsize() <= low_memory;
}
#endif
bool
-pgm_should_enable(bool internal_build)
-{
- if (env_var("MallocProbGuard")) {
- return env_bool("MallocProbGuard");
- }
- if (FEATURE_FLAG(ProbGuard, true) && should_activate(internal_build)) {
-#if TARGET_OS_OSX || TARGET_OS_IOS
- return true;
-#elif TARGET_OS_TV
- if (internal_build) {
- return true;
- }
-#elif TARGET_OS_WATCH
- if (internal_build && is_high_memory_device()) {
- return true;
- }
-#endif
+pgm_should_enable(void)
+{
+ // Env var override
+ if (g_env.MallocProbGuard_is_set) {
+ return g_env.MallocProbGuard;
+ }
+
+ // Feature flags
+ if (!FEATURE_FLAG(ProbGuard, true)) {
+ return false;
}
if (FEATURE_FLAG(ProbGuardAllProcesses, false)) {
return true;
}
- return false;
+
+ // Excluded configurations
+#if TARGET_OS_WATCH || TARGET_OS_TV
+ if (is_low_memory_device()) {
+ return false;
+ }
+#elif TARGET_OS_BRIDGE || TARGET_OS_DRIVERKIT
+ if (!g_env.internal_build) {
+ return false;
+ }
+#endif
+
+ // Targeted coverage via Trial
+ if (g_env.targeted_coverage) {
+ return true;
+ }
+
+ // Random activation
+ return should_activate(g_env.internal_build);
}
static uint32_t
@@ -1153,15 +1260,10 @@
return (TARGET_OS_OSX ? 8 : 2) * 1024;
}
-// TODO(yln): uniform sampling is likely not optimal here, since we will tend to
-// sample around the average of our range, which is probably more frequent than
-// what we want. We probably want the average to be less frequent, but still be
-// able to reach the "very frequent" end of our range occassionally. Consider
-// using a geometric (or other weighted distribution) here.
static uint32_t
choose_sample_rate(void)
{
- uint32_t min = 500, max = 10000;
+ uint32_t min = 500, max = 5000;
return rand_uniform(max - min) + min;
}
@@ -1197,16 +1299,14 @@
uint32_t sample_rate = env_uint("MallocProbGuardSampleRate", choose_sample_rate());
// Approximate a (1 / sample_rate) chance for sampling; 1 means "always sample".
zone->sample_counter_range = (sample_rate != 1) ? (2 * sample_rate) : 1;
- bool strict_alignment = env_var("MallocProbGuardStrictAlignment") ? env_bool("MallocProbGuardStrictAlignment") : FEATURE_FLAG(ProbGuardStrictAlignment, false);
- zone->min_alignment = (strict_alignment && MALLOC_TARGET_64BIT) ? 1 : 16; // Darwin ABI requires 16 byte alignment.
- zone->signal_handler = env_bool("MallocProbGuardSignalHandler");
+ zone->left_align_pct = env_uint("MallocProbGuardLeftAlignPercentage", 10);
zone->debug = env_bool("MallocProbGuardDebug");
zone->debug_log_throttle_ms = env_uint("MallocProbGuardDebugLogThrottleInMillis", 1000);
if (zone->debug) {
malloc_report(ASL_LEVEL_INFO,
- "ProbGuard configuration: %u kB budget, 1/%u sample rate, %u/%u/%u allocations/metadata/slots, strict alignment: %d\n",
- memory_budget_in_kb, sample_rate, zone->max_allocations, zone->max_metadata, zone->num_slots, strict_alignment);
+ "ProbGuard configuration: %u kB budget, 1/%u sample rate, %u/%u/%u allocations/metadata/slots\n",
+ memory_budget_in_kb, sample_rate, zone->max_allocations, zone->max_metadata, zone->num_slots);
}
if (!check_configuration(zone)) {
MALLOC_REPORT_FATAL_ERROR(0, "ProbGuard: bad configuration");
@@ -1225,11 +1325,30 @@
static void my_vm_protect(vm_address_t addr, size_t size, vm_prot_t protection);
static void
+disable_unsupported_apis(malloc_zone_t *pgm_zone, const malloc_zone_t *wrapped_zone)
+{
+ #define DISABLE_UNSUPPORTED(api) if (!wrapped_zone->api) pgm_zone->api = NULL;
+
+ // In practice, there are no zones we support wrapping right now that don't
+ // have these entrypoints
+ DISABLE_UNSUPPORTED(memalign)
+ DISABLE_UNSUPPORTED(malloc_type_memalign)
+ DISABLE_UNSUPPORTED(free_definite_size)
+ DISABLE_UNSUPPORTED(claimed_address)
+
+ // These ones are actually load-bearing: the nano and scalable zones do not
+ // implement these entrypoints
+ DISABLE_UNSUPPORTED(malloc_with_options)
+ DISABLE_UNSUPPORTED(malloc_type_malloc_with_options)
+}
+
+static void
setup_zone(pgm_zone_t *zone, malloc_zone_t *wrapped_zone)
{
// Malloc zone
zone->malloc_zone = malloc_zone_template;
zone->wrapped_zone = wrapped_zone;
+ disable_unsupported_apis(&zone->malloc_zone, wrapped_zone);
// Configuration
configure_zone(zone);
@@ -1251,23 +1370,14 @@
init_lock(zone);
}
-static void install_signal_handler(void *unused);
malloc_zone_t *
pgm_create_zone(malloc_zone_t *wrapped_zone)
{
- // rdar://74948496 ([PGM] Drop all requirements for wrapped_zone)
- MALLOC_ASSERT(wrapped_zone->version >= 6);
- MALLOC_ASSERT(wrapped_zone->batch_malloc && wrapped_zone->batch_free &&
- wrapped_zone->memalign && wrapped_zone->free_definite_size);
+ MALLOC_ASSERT(wrapped_zone->version >= MIN_WRAPPED_ZONE_VERSION);
pgm_zone_t *zone = (pgm_zone_t *)my_vm_map(sizeof(pgm_zone_t), VM_PROT_READ_WRITE, VM_MEMORY_MALLOC);
setup_zone(zone, wrapped_zone);
my_vm_protect((vm_address_t)zone, PAGE_MAX_SIZE, VM_PROT_READ);
-
- if (zone->signal_handler) {
- static os_once_t once_pred;
- os_once(&once_pred, NULL, &install_signal_handler);
- }
return (malloc_zone_t *)zone;
}
@@ -1304,8 +1414,10 @@
return;
}
if (should_log(zone)) {
- malloc_report(ASL_LEVEL_INFO, "ProbGuard: %9s 0x%lx, fill state: %3u/%u\n",
- label, addr, zone->num_allocations, zone->max_allocations);
+ malloc_report(ASL_LEVEL_INFO,
+ "ProbGuard: %9s 0x%llx, fill state: %3u/%u\n", label,
+ (unsigned long long)addr, zone->num_allocations,
+ zone->max_allocations);
}
if (!pgm_check(zone)) {
MALLOC_REPORT_FATAL_ERROR(addr, "ProbGuard: zone integrity check failed");
@@ -1347,16 +1459,29 @@
}
}
-static void
+#define KR_NO_PGM (KERN_RETURN_MAX + 1)
+static kern_return_t
diagnose_page_fault(const pgm_zone_t *zone, vm_address_t fault_address, pgm_report_t *report)
{
+ if (!is_guarded(zone, fault_address)) {
+ return KR_NO_PGM;
+ }
+
slot_lookup_t res = lookup_slot(zone, fault_address);
+ // Guaranteed by lookup_slot()
MALLOC_ASSERT(res.slot < zone->num_slots);
+ // Checked by read_zone()
MALLOC_ASSERT(zone->slots[res.slot].metadata < zone->max_metadata);
slot_state_t ss = zone->slots[res.slot].state;
- // We got here because of a page fault.
- MALLOC_ASSERT(ss != ss_allocated || res.bounds == b_oob_guard_page);
+ // We should have gotten here because of a page fault.
+ if (ss == ss_allocated && res.bounds != b_oob_guard_page) {
+ malloc_report(ASL_LEVEL_ERR | MALLOC_REPORT_LOG_ONLY,
+ "Failed to generate PGM report for fault address %p: "
+ "slot is unexpectedly allocated with bounds %d\n",
+ (void *)fault_address, (int)res.bounds);
+ return KERN_FAILURE;
+ }
// Note that all of the following error conditions may also be caused by:
// *) Randomly corrupted pointer
@@ -1391,82 +1516,26 @@
}
break;
default:
+ // Checked by read_zone()
__builtin_unreachable();
}
report->fault_address = fault_address;
fill_in_report(zone, res.slot, report);
-}
-
-
-#pragma mark -
-#pragma mark Error Printing
-
-static const uint32_t k_buf_len = 1024;
-static void
-get_symbol_and_module_name(vm_address_t addr, char buf[k_buf_len])
-{
- int success = 0;
-#if !TARGET_OS_DRIVERKIT
- Dl_info info;
- success = dladdr((void *)addr, &info);
- if (success) {
- snprintf(buf, k_buf_len, "%s (%s)", info.dli_sname, info.dli_fname);
- }
-#endif
- if (!success) {
- snprintf(buf, k_buf_len, "%p", (void *)addr);
- }
-}
-
-static void
-print_trace(stack_trace_t *trace, const char *label)
-{
- malloc_report(ASL_LEVEL_ERR, "%s trace (thread %llu, time: %llu):\n", label, trace->thread_id, trace->time);
- for (uint32_t i = 0; i < trace->num_frames; i++) {
- char sym_name[k_buf_len];
- get_symbol_and_module_name(trace->frames[i], sym_name);
- malloc_report(ASL_LEVEL_ERR, " #%u %s\n", i, sym_name);
- }
- malloc_report(ASL_LEVEL_ERR, "\n", label);
-}
-
-static void
-print_report(pgm_report_t *report)
-{
- malloc_report(ASL_LEVEL_ERR, "ProbGuard: invalid access at 0x%lx\n",
- report->fault_address);
- malloc_report(ASL_LEVEL_ERR, "Error type: %s (%s confidence)\n",
- report->error_type, report->confidence);
- malloc_report(ASL_LEVEL_ERR, "Nearest allocation: 0x%lx, size: %lu, state: %s\n",
- report->nearest_allocation, report->allocation_size, report->allocation_state);
-
- if (report->num_traces >= 1) {
- print_trace(&report->alloc_trace, "Allocation");
- if (report->num_traces >= 2) {
- print_trace(&report->dealloc_trace, "Deallocation");
- }
- } else {
- malloc_report(ASL_LEVEL_ERR, "Allocation stack traces not available. "
- "Try increasing `MallocProbGuardMetadata` and rerun.\n");
- }
-}
-
-
-#pragma mark -
-#pragma mark Crash Reporter API
-
-static kern_return_t
-diagnose_fault_from_external_process(vm_address_t fault_address, pgm_report_t *report,
- task_t task, vm_address_t zone_address, memory_reader_t reader)
-{
- READ_ZONE(zone, rt_slots_and_metadata);
- diagnose_page_fault(zone, fault_address, report);
return KERN_SUCCESS;
}
+
+#pragma mark -
+#pragma mark Crash Reporter SPI
+
+// KERN_FAILURE - memory read error
+// KERN_SUCCESS - memory read ok, PGM report filled
+// KR_NO_PGM - memory read ok, but no PGM result, try next zone
+MALLOC_STATIC_ASSERT(KR_NO_PGM > KERN_RETURN_MAX, "KR_NO_PGM");
+
static crash_reporter_memory_reader_t g_crm_reader;
-static const uint32_t k_max_read_memory = 3;
+static const uint32_t k_max_read_memory = 3; // See read_zone() and read_type_t
static void *read_memory[k_max_read_memory];
static uint32_t num_read_memory;
static kern_return_t
@@ -1475,13 +1544,15 @@
MALLOC_ASSERT(num_read_memory < k_max_read_memory);
void *ptr = g_crm_reader(task, address, size);
*local_memory = ptr;
+ if (!ptr) return KERN_FAILURE;
read_memory[num_read_memory++] = ptr;
- return ptr ? KERN_SUCCESS : KERN_FAILURE;
+ return KERN_SUCCESS;
}
static memory_reader_t *
setup_memory_reader(crash_reporter_memory_reader_t crm_reader)
{
+ MALLOC_ASSERT(crm_reader);
g_crm_reader = crm_reader;
num_read_memory = 0;
return memory_reader_adapter;
@@ -1491,82 +1562,61 @@
free_read_memory()
{
for (uint32_t i = 0; i < num_read_memory; i++) {
- free(read_memory[i]);
- }
+ _free(read_memory[i]);
+ }
+ num_read_memory = 0;
+}
+
+static kern_return_t
+is_pgm_zone(vm_address_t zone_address, task_t task, memory_reader_t reader)
+{
+ unsigned zone_type;
+ kern_return_t kr = get_zone_type(task, reader, zone_address, &zone_type);
+ if (kr != KERN_SUCCESS) {
+ return kr;
+ }
+
+ return (zone_type == MALLOC_ZONE_TYPE_PGM) ? KERN_SUCCESS : KR_NO_PGM;
+}
+
+static kern_return_t
+diagnose_fault_from_external_process(vm_address_t fault_address, pgm_report_t *report,
+ task_t task, vm_address_t zone_address, memory_reader_t reader)
+{
+ READ_ZONE(zone, rt_slots | rt_metadata);
+ return diagnose_page_fault(zone, fault_address, report);
+}
+
+static kern_return_t
+extract_report_select_zone(vm_address_t fault_address, pgm_report_t *report,
+ task_t task, vm_address_t *zone_addresses, uint32_t zone_count, memory_reader_t reader)
+{
+ for (uint32_t i = 0; i < zone_count; i++) {
+ kern_return_t kr = is_pgm_zone(zone_addresses[i], task, reader);
+ free_read_memory();
+ if (kr == KR_NO_PGM) continue;
+ if (kr != KERN_SUCCESS) return kr;
+
+ kr = diagnose_fault_from_external_process(fault_address, report, task, zone_addresses[i], reader);
+ free_read_memory();
+ if (kr == KR_NO_PGM) continue;
+ return kr;
+ }
+ return KERN_FAILURE;
}
static _malloc_lock_s crash_reporter_lock = _MALLOC_LOCK_INIT;
kern_return_t
-pgm_diagnose_fault_from_crash_reporter(vm_address_t fault_address, pgm_report_t *report,
- task_t task, vm_address_t zone_address, crash_reporter_memory_reader_t crm_reader)
+pgm_extract_report_from_corpse(vm_address_t fault_address, pgm_report_t *report, task_t task,
+ vm_address_t *zone_addresses, uint32_t zone_count, crash_reporter_memory_reader_t crm_reader)
{
_malloc_lock_lock(&crash_reporter_lock);
memory_reader_t *reader = setup_memory_reader(crm_reader);
- kern_return_t kr = diagnose_fault_from_external_process(fault_address, report, task, zone_address, reader);
- free_read_memory();
+ kern_return_t kr = extract_report_select_zone(fault_address, report, task, zone_addresses, zone_count, reader);
_malloc_lock_unlock(&crash_reporter_lock);
return kr;
-}
-
-
-#pragma mark -
-#pragma mark Signal Handler
-
-extern malloc_zone_t **malloc_zones;
-static void
-report_error_from_signal_handler(vm_address_t fault_address)
-{
- pgm_zone_t *zone = (pgm_zone_t *)malloc_zones[0];
- MALLOC_ASSERT(zone->malloc_zone.size == FN_PTR(pgm_size));
-
- if (!is_guarded(zone, fault_address)) {
- return;
- }
-
- pgm_report_t report;
- {
- trylock(zone); // Best-effort locking to avoid deadlock.
- diagnose_page_fault(zone, fault_address, &report);
- unlock(zone);
- }
- print_report(&report);
-
- MALLOC_REPORT_FATAL_ERROR(fault_address, "ProbGuard: invalid access detected");
-}
-
-static struct sigaction prev_sigaction;
-static void
-signal_handler(int sig, siginfo_t *info, void *ucontext)
-{
- MALLOC_ASSERT(sig == SIGBUS);
- report_error_from_signal_handler((vm_address_t)info->si_addr);
-
- // Delegate to previous handler.
- if (prev_sigaction.sa_flags & SA_SIGINFO) {
- prev_sigaction.sa_sigaction(sig, info, ucontext);
- } else if (prev_sigaction.sa_handler == SIG_IGN ||
- prev_sigaction.sa_handler == SIG_DFL) {
- // If the previous handler was the default handler, or was ignoring this
- // signal, install the default handler and re-raise the signal in order to
- // get a core dump and terminate this process.
- signal(SIGBUS, SIG_DFL);
- raise(SIGBUS);
- } else {
- prev_sigaction.sa_handler(sig);
- }
-}
-
-static void
-install_signal_handler(void *unused)
-{
- struct sigaction act = {
- .sa_sigaction = &signal_handler,
- .sa_flags = SA_SIGINFO
- };
- int res = sigaction(SIGBUS, &act, &prev_sigaction);
- MALLOC_ASSERT(res == 0);
}
@@ -1630,7 +1680,7 @@
kern_return_t kr = mach_vm_map(target, &address, size_rounded, mask, flags,
object, offset, copy, cur_protection, max_protection, inheritance);
MALLOC_ASSERT(kr == KERN_SUCCESS);
- return address;
+ return (vm_address_t)address;
}
static vm_address_t