Loading...
--- libmalloc/libmalloc-474.0.13/src/pgm_malloc.c
+++ libmalloc/libmalloc-646.40.3/src/pgm_malloc.c
@@ -25,7 +25,6 @@
#include <TargetConditionals.h>
#include <mach/mach_time.h> // mach_absolute_time()
-#include <sys/codesign.h> // csops()
#include "internal.h"
@@ -85,7 +84,6 @@
uint32_t max_allocations;
uint32_t max_metadata;
uint32_t sample_counter_range;
- uint32_t min_alignment;
bool debug;
uint64_t debug_log_throttle_ms;
@@ -113,8 +111,8 @@
uint64_t last_log_time;
} pgm_zone_t;
-MALLOC_STATIC_ASSERT(__offsetof(pgm_zone_t, malloc_zone) == 0,
- "pgm_zone_t instances must be usable as regular zones");
+ASSERT_WRAPPER_ZONE(pgm_zone_t);
+
MALLOC_STATIC_ASSERT(__offsetof(pgm_zone_t, padding) < PAGE_MAX_SIZE,
"First page is mapped read-only");
MALLOC_STATIC_ASSERT(__offsetof(pgm_zone_t, lock) >= PAGE_MAX_SIZE,
@@ -314,19 +312,21 @@
#pragma mark -
#pragma mark Allocator Helpers
+// Darwin ABI requires 16 byte alignment.
+static const size_t k_min_alignment = 16;
+
static bool
is_power_of_2(size_t n) {
return __builtin_popcountl(n) == 1;
}
static size_t
-block_size(size_t size, size_t min_alignment)
-{
- MALLOC_ASSERT(is_power_of_2(min_alignment));
+block_size(size_t size)
+{
if (size == 0) {
- return min_alignment;
- }
- const size_t mask = (min_alignment - 1);
+ return k_min_alignment;
+ }
+ const size_t mask = (k_min_alignment - 1);
return (size + mask) & ~mask;
}
@@ -425,14 +425,14 @@
allocate(pgm_zone_t *zone, size_t size, size_t alignment)
{
MALLOC_ASSERT(size <= PAGE_SIZE);
- MALLOC_ASSERT(zone->min_alignment <= alignment && alignment <= PAGE_SIZE);
+ MALLOC_ASSERT(k_min_alignment <= alignment && alignment <= PAGE_SIZE);
MALLOC_ASSERT(is_power_of_2(alignment));
if (is_full(zone)) {
return (vm_address_t)NULL;
}
- size = block_size(size, zone->min_alignment);
+ size = block_size(size);
uint32_t slot = choose_available_slot(zone);
uint32_t metadata = choose_metadata(zone, slot);
uint16_t offset = choose_offset_on_page(size, alignment, PAGE_SIZE);
@@ -511,7 +511,7 @@
vm_address_t new_addr;
if (sample && !is_full(zone)) {
- new_addr = allocate(zone, new_size, zone->min_alignment);
+ new_addr = allocate(zone, new_size, k_min_alignment);
MALLOC_ASSERT(new_addr);
} else {
new_addr = (vm_address_t)DELEGATE(malloc, new_size);
@@ -577,7 +577,7 @@
static void *
pgm_malloc(pgm_zone_t *zone, size_t size)
{
- SAMPLED_ALLOCATE(size, zone->min_alignment, malloc, size);
+ SAMPLED_ALLOCATE(size, k_min_alignment, malloc, size);
return ptr;
}
@@ -588,7 +588,7 @@
if (os_unlikely(os_mul_overflow(num_items, size, &total_size))) {
return DELEGATE(calloc, num_items, size);
}
- SAMPLED_ALLOCATE(total_size, zone->min_alignment, calloc, num_items, size);
+ SAMPLED_ALLOCATE(total_size, k_min_alignment, calloc, num_items, size);
memset(ptr, 0, total_size);
return ptr;
}
@@ -631,57 +631,6 @@
my_vm_deallocate((vm_address_t)zone, sizeof(pgm_zone_t));
}
-static unsigned
-pgm_batch_malloc(pgm_zone_t *zone, size_t size, void **results, unsigned count)
-{
- if (os_unlikely(count == 0)) {
- return 0;
- }
- DELEGATE_UNSAMPLED(size, batch_malloc, size, results, count);
-
- uint32_t sample_count = 1; // Sample at least one allocation.
- for (uint32_t i = 1; i < count; i++) {
- if (should_sample_counter(zone->sample_counter_range)) {
- sample_count++;
- }
- }
- // TODO(yln): Express the above with only one call to rand_uniform(). "n choose k"?
-
- for (uint32_t i = 0; i < sample_count; i++) {
- lock(zone);
- void *ptr = (void *)allocate(zone, size, zone->min_alignment);
- unlock(zone);
- if (!ptr) {
- sample_count = i;
- break; // Zone full.
- }
- results[i] = ptr;
- }
-
- void **remaining_results = results + sample_count;
- uint32_t remaining_count = count - sample_count;
- remaining_count = DELEGATE(batch_malloc, size, remaining_results, remaining_count) ;
-
- // TODO(yln): sampled allocations will always be in the beginning of the results
- // array. We could shuffle it: https://en.wikipedia.org/wiki/Fisher–Yates_shuffle
- return sample_count + remaining_count;
-}
-
-static void
-pgm_batch_free(pgm_zone_t *zone, void **to_be_freed, unsigned count)
-{
- for (uint32_t i = 0; i < count; i++) {
- vm_address_t addr = (vm_address_t)to_be_freed[i];
- if (os_unlikely(is_guarded(zone, addr))) {
- lock(zone);
- deallocate(zone, addr);
- unlock(zone);
- to_be_freed[i] = NULL;
- }
- }
- return DELEGATE(batch_free, to_be_freed, count);
-}
-
static void *
pgm_memalign(pgm_zone_t *zone, size_t alignment, size_t size)
{
@@ -689,7 +638,7 @@
if (alignment > PAGE_SIZE || !is_power_of_2(alignment) || alignment < sizeof(void *)) {
return DELEGATE(memalign, alignment, size);
}
- size_t adj_alignment = MAX(alignment, zone->min_alignment);
+ size_t adj_alignment = MAX(alignment, k_min_alignment);
SAMPLED_ALLOCATE(size, adj_alignment, memalign, alignment, size);
return ptr;
}
@@ -703,9 +652,44 @@
static boolean_t
pgm_claimed_address(pgm_zone_t *zone, void *ptr)
{
- return is_guarded(zone, (vm_address_t)ptr);
-}
-
+ DELEGATE_UNGUARDED(ptr, claimed_address, ptr);
+ return true;
+}
+
+static void *
+pgm_malloc_with_options(pgm_zone_t *zone, size_t align, size_t size,
+ uint64_t options)
+{
+ if (os_unlikely(should_sample(zone, size))) {
+ size_t adj_alignment = MAX(align, k_min_alignment);
+ lock(zone);
+ void *ptr = (void *)allocate(zone, size, adj_alignment);
+ unlock(zone);
+ if (ptr) {
+ if (options & MALLOC_NP_OPTION_CLEAR) {
+ bzero(ptr, size);
+ }
+ return ptr;
+ }
+ // If the allocation fails, fall back to the wrapped zone
+ }
+
+ // FIXME: calls wrapped_zone->malloc_with_options/memalign without NULL check
+ if (zone->wrapped_zone->version >= 15 &&
+ zone->wrapped_zone->malloc_with_options) {
+ return DELEGATE(malloc_with_options, align, size, options);
+ } else if (align) {
+ void *ptr = DELEGATE(memalign, align, size);
+ if (ptr && (options & MALLOC_NP_OPTION_CLEAR)) {
+ bzero(ptr, size);
+ }
+ return ptr;
+ } else if (options & MALLOC_NP_OPTION_CLEAR) {
+ return DELEGATE(calloc, 1, size);
+ } else {
+ return DELEGATE(malloc, size);
+ }
+}
#pragma mark -
#pragma mark Integrity Checks
@@ -720,8 +704,7 @@
(zone->max_allocations <= zone->max_metadata / 2) && // choose_metadata() relies on max_allocations << max_metadata
(zone->max_metadata <= zone->num_slots) &&
(zone->num_slots <= k_max_slots) &&
- (zone->sample_counter_range > 0) &&
- (zone->min_alignment == 1 || zone->min_alignment == 16); // strict alignment || Darwin ABI alignment
+ (zone->sample_counter_range > 0);
}
static bool
@@ -749,12 +732,6 @@
}
static bool
-check_introspection(const pgm_zone_t *zone)
-{
- return true;
-}
-
-static bool
check_slot(const pgm_zone_t *zone, const slot_t *slot)
{
if (slot->state == ss_unused) {
@@ -763,9 +740,9 @@
return (slot->state <= ss_freed) &&
(slot->metadata < zone->num_metadata) &&
(slot->size <= PAGE_SIZE) &&
- (slot->size == block_size(slot->size, zone->min_alignment)) &&
+ (slot->size == block_size(slot->size)) &&
(slot->offset <= PAGE_SIZE) &&
- (slot->offset % zone->min_alignment == 0) &&
+ (slot->offset % k_min_alignment == 0) &&
((size_t)slot->offset + slot->size <= PAGE_SIZE);
}
@@ -816,10 +793,9 @@
#pragma mark Introspection Functions
typedef enum {
- rt_zone_only = 1 << 0,
- rt_introspection = 1 << 1,
- rt_slots = 1 << 2,
- rt_metadata = 1 << 3,
+ rt_zone_only = 1 << 0,
+ rt_slots = 1 << 1,
+ rt_metadata = 1 << 2,
} read_type_t;
#define READ(remote_address, size, local_memory, checker, check_data) \
@@ -829,30 +805,15 @@
if (!checker(check_data)) return KERN_FAILURE; \
}
-// Avoid ptrauth: ptr loaded from corpse can't be authenticated in ReportCrash proccess.
-static const malloc_introspection_t *
-get_introspection_ptr(const pgm_zone_t *zone)
-{
- // return zone->malloc_zone.introspect;
- vm_address_t ptr_addr = (vm_address_t)zone + offsetof(malloc_zone_t, introspect);
- vm_address_t ptr = *(vm_address_t *)ptr_addr;
- return ptrauth_strip((malloc_introspection_t *)ptr, ptrauth_key_process_independent_data);
-}
-
static kern_return_t
read_zone(task_t task, vm_address_t zone_address, memory_reader_t reader, pgm_zone_t *zone, read_type_t read_type)
{
- if (!reader && task == mach_task_self()) {
- reader = _malloc_default_reader;
- }
+ reader = reader_or_in_memory_fallback(reader, task);
pgm_zone_t *zone_ptr;
READ(zone_address, sizeof(pgm_zone_t), &zone_ptr, check_zone, zone_ptr);
*zone = *zone_ptr; // Copy to writable memory
- if (read_type & rt_introspection) {
- READ(get_introspection_ptr(zone), sizeof(malloc_introspection_t), &zone->malloc_zone.introspect, check_introspection, zone);
- }
if (read_type & rt_slots) {
READ(zone->slots, zone->num_slots * sizeof(slot_t), &zone->slots, check_slots, zone);
}
@@ -1022,6 +983,8 @@
#pragma mark -
#pragma mark Zone Templates
+#define PGM_ZONE_VERSION 15
+
// Suppress warning: incompatible function pointer types
#define FN_PTR(fn) (void *)(&fn)
@@ -1077,20 +1040,21 @@
.destroy = FN_PTR(pgm_destroy),
// Batch operations
- .batch_malloc = FN_PTR(pgm_batch_malloc),
- .batch_free = FN_PTR(pgm_batch_free),
+ .batch_malloc = malloc_zone_batch_malloc_fallback,
+ .batch_free = malloc_zone_batch_free_fallback,
// Introspection
.zone_name = "ProbGuardMallocZone",
- .version = 14,
+ .version = PGM_ZONE_VERSION,
.introspect = (malloc_introspection_t *)&introspection_template, // Effectively const.
// Specialized operations
.memalign = FN_PTR(pgm_memalign),
.free_definite_size = FN_PTR(pgm_free_definite_size),
- .pressure_relief = NULL,
+ .pressure_relief = malloc_zone_pressure_relief_fallback,
.claimed_address = FN_PTR(pgm_claimed_address),
.try_free_default = NULL,
+ .malloc_with_options = FN_PTR(pgm_malloc_with_options),
};
@@ -1131,7 +1095,6 @@
bool internal_build;
bool MallocProbGuard_is_set;
bool MallocProbGuard;
- bool MallocProbGuardViaLaunchd;
} g_env;
void
@@ -1145,20 +1108,16 @@
g_env.MallocProbGuard_is_set = true;
g_env.MallocProbGuard = env_bool("MallocProbGuard");
}
- if (env_bool("MallocProbGuardViaLaunchd")) {
- g_env.MallocProbGuardViaLaunchd = true;
- }
}
static bool
is_platform_binary(void)
{
- uint32_t flags = 0;
- int err = csops(getpid(), CS_OPS_STATUS, &flags, sizeof(flags));
- if (err) {
- return false;
- }
- return (flags & CS_PLATFORM_BINARY);
+#if CONFIG_CHECK_PLATFORM_BINARY
+ return malloc_is_platform_binary;
+#else
+ return _malloc_is_platform_binary();
+#endif
}
extern bool main_image_has_section(const char* segname, const char *sectname);
@@ -1168,16 +1127,10 @@
if (!internal_build && !is_platform_binary()) {
return false;
}
-#if TARGET_OS_OSX
uint32_t activation_rate = (internal_build ? 250 : 1000);
if (rand_uniform(activation_rate) != 0) {
return false;
}
-#else
- if (!g_env.MallocProbGuardViaLaunchd) {
- return false;
- }
-#endif
if (main_image_has_section("__DATA", "__pgm_opt_out")) {
return false;
}
@@ -1212,8 +1165,6 @@
}
#elif PGM_ALLOW_NON_INTERNAL_ACTIVATION
return true;
-#elif TARGET_OS_DRIVERKIT
- // Never enable for DriverKit
#else
if (internal_build) {
return true;
@@ -1271,15 +1222,13 @@
uint32_t sample_rate = env_uint("MallocProbGuardSampleRate", choose_sample_rate());
// Approximate a (1 / sample_rate) chance for sampling; 1 means "always sample".
zone->sample_counter_range = (sample_rate != 1) ? (2 * sample_rate) : 1;
- bool strict_alignment = env_var("MallocProbGuardStrictAlignment") ? env_bool("MallocProbGuardStrictAlignment") : FEATURE_FLAG(ProbGuardStrictAlignment, false);
- zone->min_alignment = (strict_alignment && MALLOC_TARGET_64BIT) ? 1 : 16; // Darwin ABI requires 16 byte alignment.
zone->debug = env_bool("MallocProbGuardDebug");
zone->debug_log_throttle_ms = env_uint("MallocProbGuardDebugLogThrottleInMillis", 1000);
if (zone->debug) {
malloc_report(ASL_LEVEL_INFO,
- "ProbGuard configuration: %u kB budget, 1/%u sample rate, %u/%u/%u allocations/metadata/slots, strict alignment: %d\n",
- memory_budget_in_kb, sample_rate, zone->max_allocations, zone->max_metadata, zone->num_slots, strict_alignment);
+ "ProbGuard configuration: %u kB budget, 1/%u sample rate, %u/%u/%u allocations/metadata/slots\n",
+ memory_budget_in_kb, sample_rate, zone->max_allocations, zone->max_metadata, zone->num_slots);
}
if (!check_configuration(zone)) {
MALLOC_REPORT_FATAL_ERROR(0, "ProbGuard: bad configuration");
@@ -1298,11 +1247,21 @@
static void my_vm_protect(vm_address_t addr, size_t size, vm_prot_t protection);
static void
+disable_unsupported_apis(malloc_zone_t *pgm_zone, const malloc_zone_t *wrapped_zone)
+{
+ #define DISABLE_UNSUPPORTED(api) if (!wrapped_zone->api) pgm_zone->api = NULL;
+ DISABLE_UNSUPPORTED(memalign)
+ DISABLE_UNSUPPORTED(free_definite_size)
+ DISABLE_UNSUPPORTED(claimed_address)
+}
+
+static void
setup_zone(pgm_zone_t *zone, malloc_zone_t *wrapped_zone)
{
// Malloc zone
zone->malloc_zone = malloc_zone_template;
zone->wrapped_zone = wrapped_zone;
+ disable_unsupported_apis(&zone->malloc_zone, wrapped_zone);
// Configuration
configure_zone(zone);
@@ -1327,10 +1286,12 @@
malloc_zone_t *
pgm_create_zone(malloc_zone_t *wrapped_zone)
{
- // rdar://74948496 ([PGM] Drop all requirements for wrapped_zone)
- MALLOC_ASSERT(wrapped_zone->version >= 6);
- MALLOC_ASSERT(wrapped_zone->batch_malloc && wrapped_zone->batch_free &&
- wrapped_zone->memalign && wrapped_zone->free_definite_size);
+ // The PGM zone includes the `malloc_introspection_t::zone_type` field
+ // (version 14) so we need to support the "malloc()/calloc() set errno to
+ // ENOMEM on failure" behavior (version 13). Require wrapped zone to be at
+ // least version 13.
+ MALLOC_STATIC_ASSERT(PGM_ZONE_VERSION == 15, "PGM zone version");
+ MALLOC_ASSERT(wrapped_zone->version >= 13);
pgm_zone_t *zone = (pgm_zone_t *)my_vm_map(sizeof(pgm_zone_t), VM_PROT_READ_WRITE, VM_MEMORY_MALLOC);
setup_zone(zone, wrapped_zone);
@@ -1492,7 +1453,7 @@
MALLOC_STATIC_ASSERT(KR_NO_PGM > KERN_RETURN_MAX, "KR_NO_PGM");
static crash_reporter_memory_reader_t g_crm_reader;
-static const uint32_t k_max_read_memory = 4; // See read_zone() and read_type_t
+static const uint32_t k_max_read_memory = 3; // See read_zone() and read_type_t
static void *read_memory[k_max_read_memory];
static uint32_t num_read_memory;
static kern_return_t
@@ -1519,7 +1480,7 @@
free_read_memory()
{
for (uint32_t i = 0; i < num_read_memory; i++) {
- free(read_memory[i]);
+ _free(read_memory[i]);
}
num_read_memory = 0;
}
@@ -1527,10 +1488,12 @@
static kern_return_t
is_pgm_zone(vm_address_t zone_address, task_t task, memory_reader_t reader)
{
- READ_ZONE(zone, rt_introspection);
- if (zone->malloc_zone.version < 14)
- return KR_NO_PGM;
- unsigned zone_type = get_introspection_ptr(zone)->zone_type;
+ unsigned zone_type;
+ kern_return_t kr = get_zone_type(task, reader, zone_address, &zone_type);
+ if (kr != KERN_SUCCESS) {
+ return kr;
+ }
+
return (zone_type == MALLOC_ZONE_TYPE_PGM) ? KERN_SUCCESS : KR_NO_PGM;
}
@@ -1569,19 +1532,6 @@
memory_reader_t *reader = setup_memory_reader(crm_reader);
kern_return_t kr = extract_report_select_zone(fault_address, report, task, zone_addresses, zone_count, reader);
-
- _malloc_lock_unlock(&crash_reporter_lock);
- return kr;
-}
-
-kern_return_t
-pgm_diagnose_fault_from_crash_reporter(vm_address_t fault_address, pgm_report_t *report,
- task_t task, vm_address_t zone_address, crash_reporter_memory_reader_t crm_reader)
-{
- _malloc_lock_lock(&crash_reporter_lock);
-
- memory_reader_t *reader = setup_memory_reader(crm_reader);
- kern_return_t kr = diagnose_fault_from_external_process(fault_address, report, task, zone_address, reader);
_malloc_lock_unlock(&crash_reporter_lock);
return kr;