Loading...
--- libmalloc/libmalloc-521.100.59/src/pgm_malloc.c
+++ libmalloc/libmalloc-657.80.3/src/pgm_malloc.c
@@ -25,7 +25,6 @@
#include <TargetConditionals.h>
#include <mach/mach_time.h> // mach_absolute_time()
-#include <sys/codesign.h> // csops()
#include "internal.h"
@@ -85,7 +84,6 @@
uint32_t max_allocations;
uint32_t max_metadata;
uint32_t sample_counter_range;
- uint32_t min_alignment;
bool debug;
uint64_t debug_log_throttle_ms;
@@ -314,19 +312,21 @@
#pragma mark -
#pragma mark Allocator Helpers
+// Darwin ABI requires 16 byte alignment.
+static const size_t k_min_alignment = 16;
+
static bool
is_power_of_2(size_t n) {
return __builtin_popcountl(n) == 1;
}
static size_t
-block_size(size_t size, size_t min_alignment)
-{
- MALLOC_ASSERT(is_power_of_2(min_alignment));
+block_size(size_t size)
+{
if (size == 0) {
- return min_alignment;
- }
- const size_t mask = (min_alignment - 1);
+ return k_min_alignment;
+ }
+ const size_t mask = (k_min_alignment - 1);
return (size + mask) & ~mask;
}
@@ -425,14 +425,14 @@
allocate(pgm_zone_t *zone, size_t size, size_t alignment)
{
MALLOC_ASSERT(size <= PAGE_SIZE);
- MALLOC_ASSERT(zone->min_alignment <= alignment && alignment <= PAGE_SIZE);
+ MALLOC_ASSERT(k_min_alignment <= alignment && alignment <= PAGE_SIZE);
MALLOC_ASSERT(is_power_of_2(alignment));
if (is_full(zone)) {
return (vm_address_t)NULL;
}
- size = block_size(size, zone->min_alignment);
+ size = block_size(size);
uint32_t slot = choose_available_slot(zone);
uint32_t metadata = choose_metadata(zone, slot);
uint16_t offset = choose_offset_on_page(size, alignment, PAGE_SIZE);
@@ -511,7 +511,7 @@
vm_address_t new_addr;
if (sample && !is_full(zone)) {
- new_addr = allocate(zone, new_size, zone->min_alignment);
+ new_addr = allocate(zone, new_size, k_min_alignment);
MALLOC_ASSERT(new_addr);
} else {
new_addr = (vm_address_t)DELEGATE(malloc, new_size);
@@ -577,7 +577,14 @@
static void *
pgm_malloc(pgm_zone_t *zone, size_t size)
{
- SAMPLED_ALLOCATE(size, zone->min_alignment, malloc, size);
+ SAMPLED_ALLOCATE(size, k_min_alignment, malloc, size);
+ return ptr;
+}
+
+static void *
+pgm_malloc_type_malloc(pgm_zone_t *zone, size_t size, malloc_type_id_t type_id)
+{
+ SAMPLED_ALLOCATE(size, k_min_alignment, malloc_type_malloc, size, type_id);
return ptr;
}
@@ -588,8 +595,22 @@
if (os_unlikely(os_mul_overflow(num_items, size, &total_size))) {
return DELEGATE(calloc, num_items, size);
}
- SAMPLED_ALLOCATE(total_size, zone->min_alignment, calloc, num_items, size);
- memset(ptr, 0, total_size);
+ SAMPLED_ALLOCATE(total_size, k_min_alignment, calloc, num_items, size);
+ bzero(ptr, total_size);
+ return ptr;
+}
+
+static void *
+pgm_malloc_type_calloc(pgm_zone_t *zone, size_t num_items, size_t size,
+ malloc_type_id_t type_id)
+{
+ size_t total_size;
+ if (os_unlikely(os_mul_overflow(num_items, size, &total_size))) {
+ return DELEGATE(malloc_type_calloc, num_items, size, type_id);
+ }
+ SAMPLED_ALLOCATE(total_size, k_min_alignment, malloc_type_calloc, num_items,
+ size, type_id);
+ bzero(ptr, total_size);
return ptr;
}
@@ -622,6 +643,23 @@
return new_ptr;
}
+static void *
+pgm_malloc_type_realloc(pgm_zone_t *zone, void *ptr, size_t new_size,
+ malloc_type_id_t type_id)
+{
+ if (os_unlikely(!ptr)) {
+ return pgm_malloc_type_malloc(zone, new_size, type_id);
+ }
+ boolean_t sample = should_sample(zone, new_size);
+ if (os_likely(!sample)) {
+ DELEGATE_UNGUARDED(ptr, malloc_type_realloc, ptr, new_size, type_id);
+ }
+ lock(zone);
+ void *new_ptr = (void *)reallocate(zone, (vm_address_t)ptr, new_size, sample);
+ unlock(zone);
+ return new_ptr;
+}
+
static void my_vm_deallocate(vm_address_t addr, size_t size);
static void
pgm_destroy(pgm_zone_t *zone)
@@ -634,15 +672,27 @@
static void *
pgm_memalign(pgm_zone_t *zone, size_t alignment, size_t size)
{
- // Delegate for (alignment > page size) and invalid alignment sizes.
- if (alignment > PAGE_SIZE || !is_power_of_2(alignment) || alignment < sizeof(void *)) {
+ if (os_unlikely(alignment > PAGE_SIZE)) {
return DELEGATE(memalign, alignment, size);
}
- size_t adj_alignment = MAX(alignment, zone->min_alignment);
+ size_t adj_alignment = MAX(alignment, k_min_alignment);
SAMPLED_ALLOCATE(size, adj_alignment, memalign, alignment, size);
return ptr;
}
+static void *
+pgm_malloc_type_memalign(pgm_zone_t *zone, size_t alignment, size_t size,
+ malloc_type_id_t type_id)
+{
+ if (os_unlikely(alignment > PAGE_SIZE)) {
+ return DELEGATE(malloc_type_memalign, alignment, size, type_id);
+ }
+ size_t adj_alignment = MAX(alignment, k_min_alignment);
+ SAMPLED_ALLOCATE(size, adj_alignment, malloc_type_memalign, alignment, size,
+ type_id);
+ return ptr;
+}
+
static void
pgm_free_definite_size(pgm_zone_t *zone, void *ptr, size_t size)
{
@@ -652,40 +702,40 @@
static boolean_t
pgm_claimed_address(pgm_zone_t *zone, void *ptr)
{
- return is_guarded(zone, (vm_address_t)ptr);
+ DELEGATE_UNGUARDED(ptr, claimed_address, ptr);
+ return true;
}
static void *
pgm_malloc_with_options(pgm_zone_t *zone, size_t align, size_t size,
uint64_t options)
{
- if (os_unlikely(should_sample(zone, size))) {
- size_t adj_alignment = MAX(align, zone->min_alignment);
- lock(zone);
- void *ptr = (void *)allocate(zone, size, adj_alignment);
- unlock(zone);
- if (ptr) {
- if (options & MALLOC_NP_OPTION_CLEAR) {
- bzero(ptr, size);
- }
- return ptr;
- }
- // If the allocaiton fails, fall back to the wrapped zone
- }
-
- if (zone->wrapped_zone->version >= 15) {
+ if (os_unlikely(align > PAGE_SIZE)) {
return DELEGATE(malloc_with_options, align, size, options);
- } else if (align) {
- void *ptr = DELEGATE(memalign, align, size);
- if (ptr && (options & MALLOC_NP_OPTION_CLEAR)) {
- bzero(ptr, size);
- }
- return ptr;
- } else if (options & MALLOC_NP_OPTION_CLEAR) {
- return DELEGATE(calloc, 1, size);
- } else {
- return DELEGATE(malloc, size);
- }
+ }
+ size_t adj_alignment = MAX(align, k_min_alignment);
+ SAMPLED_ALLOCATE(size, adj_alignment, malloc_with_options, align, size, options);
+ if (options & MALLOC_NP_OPTION_CLEAR) {
+ bzero(ptr, size);
+ }
+ return ptr;
+}
+
+static void *
+pgm_malloc_type_malloc_with_options(pgm_zone_t *zone, size_t align, size_t size,
+ uint64_t options, malloc_type_id_t type_id)
+{
+ if (os_unlikely(align > PAGE_SIZE)) {
+ return DELEGATE(malloc_type_malloc_with_options, align, size, options,
+ type_id);
+ }
+ size_t adj_alignment = MAX(align, k_min_alignment);
+ SAMPLED_ALLOCATE(size, adj_alignment, malloc_type_malloc_with_options,
+ align, size, options, type_id);
+ if (options & MALLOC_NP_OPTION_CLEAR) {
+ bzero(ptr, size);
+ }
+ return ptr;
}
#pragma mark -
@@ -701,8 +751,7 @@
(zone->max_allocations <= zone->max_metadata / 2) && // choose_metadata() relies on max_allocations << max_metadata
(zone->max_metadata <= zone->num_slots) &&
(zone->num_slots <= k_max_slots) &&
- (zone->sample_counter_range > 0) &&
- (zone->min_alignment == 1 || zone->min_alignment == 16); // strict alignment || Darwin ABI alignment
+ (zone->sample_counter_range > 0);
}
static bool
@@ -738,9 +787,9 @@
return (slot->state <= ss_freed) &&
(slot->metadata < zone->num_metadata) &&
(slot->size <= PAGE_SIZE) &&
- (slot->size == block_size(slot->size, zone->min_alignment)) &&
+ (slot->size == block_size(slot->size)) &&
(slot->offset <= PAGE_SIZE) &&
- (slot->offset % zone->min_alignment == 0) &&
+ (slot->offset % k_min_alignment == 0) &&
((size_t)slot->offset + slot->size <= PAGE_SIZE);
}
@@ -981,7 +1030,8 @@
#pragma mark -
#pragma mark Zone Templates
-#define PGM_ZONE_VERSION 15
+#define PGM_ZONE_VERSION 16
+#define MIN_WRAPPED_ZONE_VERSION 16
// Suppress warning: incompatible function pointer types
#define FN_PTR(fn) (void *)(&fn)
@@ -1049,10 +1099,18 @@
// Specialized operations
.memalign = FN_PTR(pgm_memalign),
.free_definite_size = FN_PTR(pgm_free_definite_size),
- .pressure_relief = NULL,
+ .pressure_relief = malloc_zone_pressure_relief_fallback,
.claimed_address = FN_PTR(pgm_claimed_address),
.try_free_default = NULL,
.malloc_with_options = FN_PTR(pgm_malloc_with_options),
+
+ // Typed entrypoints
+ .malloc_type_malloc = FN_PTR(pgm_malloc_type_malloc),
+ .malloc_type_calloc = FN_PTR(pgm_malloc_type_calloc),
+ .malloc_type_realloc = FN_PTR(pgm_malloc_type_realloc),
+ .malloc_type_memalign = FN_PTR(pgm_malloc_type_memalign),
+ .malloc_type_malloc_with_options =
+ FN_PTR(pgm_malloc_type_malloc_with_options),
};
@@ -1093,7 +1151,7 @@
bool internal_build;
bool MallocProbGuard_is_set;
bool MallocProbGuard;
- bool MallocProbGuardViaLaunchd;
+ bool targeted_coverage;
} g_env;
void
@@ -1107,20 +1165,23 @@
g_env.MallocProbGuard_is_set = true;
g_env.MallocProbGuard = env_bool("MallocProbGuard");
}
- if (env_bool("MallocProbGuardViaLaunchd")) {
- g_env.MallocProbGuardViaLaunchd = true;
+
+ const char *all_factors = env_var("__TRIFactors");
+ const char *pgm_factor =
+ "PROBABILISTIC_GUARD_MALLOC_TARGETED_COVERAGE:enable=1";
+ if (all_factors && strstr(all_factors, pgm_factor)) {
+ g_env.targeted_coverage = true;
}
}
static bool
is_platform_binary(void)
{
- uint32_t flags = 0;
- int err = csops(getpid(), CS_OPS_STATUS, &flags, sizeof(flags));
- if (err) {
- return false;
- }
- return (flags & CS_PLATFORM_BINARY);
+#if CONFIG_CHECK_PLATFORM_BINARY
+ return malloc_is_platform_binary;
+#else
+ return _malloc_is_platform_binary();
+#endif
}
extern bool main_image_has_section(const char* segname, const char *sectname);
@@ -1130,16 +1191,10 @@
if (!internal_build && !is_platform_binary()) {
return false;
}
-#if TARGET_OS_OSX
uint32_t activation_rate = (internal_build ? 250 : 1000);
if (rand_uniform(activation_rate) != 0) {
return false;
}
-#else
- if (!g_env.MallocProbGuardViaLaunchd) {
- return false;
- }
-#endif
if (main_image_has_section("__DATA", "__pgm_opt_out")) {
return false;
}
@@ -1148,44 +1203,47 @@
#if TARGET_OS_WATCH || TARGET_OS_TV
static bool
-is_high_memory_device(void)
-{
- uint64_t high_memory = 1.2 * 1024 * 1024 * 1024; // 1.2 GB
- return platform_hw_memsize() > high_memory;
+is_low_memory_device(void)
+{
+ uint64_t low_memory = 1.2 * 1024 * 1024 * 1024; // 1.2 GB
+ return platform_hw_memsize() <= low_memory;
}
#endif
-
-#define PGM_ALLOW_NON_INTERNAL_ACTIVATION 0
-
bool
pgm_should_enable(void)
{
+ // Env var override
if (g_env.MallocProbGuard_is_set) {
return g_env.MallocProbGuard;
}
- bool internal_build = g_env.internal_build;
- if (FEATURE_FLAG(ProbGuard, true) && should_activate(internal_build)) {
-#if TARGET_OS_OSX || TARGET_OS_IOS
- return true;
-#elif TARGET_OS_WATCH || TARGET_OS_TV
- if (is_high_memory_device()) {
- return true;
- }
-#elif PGM_ALLOW_NON_INTERNAL_ACTIVATION
- return true;
-#elif TARGET_OS_DRIVERKIT
- // Never enable for DriverKit
-#else
- if (internal_build) {
- return true;
- }
-#endif
+
+ // Feature flags
+ if (!FEATURE_FLAG(ProbGuard, true)) {
+ return false;
}
if (FEATURE_FLAG(ProbGuardAllProcesses, false)) {
return true;
}
- return false;
+
+ // Excluded configurations
+#if TARGET_OS_WATCH || TARGET_OS_TV
+ if (is_low_memory_device()) {
+ return false;
+ }
+#elif TARGET_OS_BRIDGE || TARGET_OS_DRIVERKIT
+ if (!g_env.internal_build) {
+ return false;
+ }
+#endif
+
+ // Targeted coverage via Trial
+ if (g_env.targeted_coverage) {
+ return true;
+ }
+
+ // Random activation
+ return should_activate(g_env.internal_build);
}
static uint32_t
@@ -1233,15 +1291,13 @@
uint32_t sample_rate = env_uint("MallocProbGuardSampleRate", choose_sample_rate());
// Approximate a (1 / sample_rate) chance for sampling; 1 means "always sample".
zone->sample_counter_range = (sample_rate != 1) ? (2 * sample_rate) : 1;
- bool strict_alignment = env_var("MallocProbGuardStrictAlignment") ? env_bool("MallocProbGuardStrictAlignment") : FEATURE_FLAG(ProbGuardStrictAlignment, false);
- zone->min_alignment = (strict_alignment && MALLOC_TARGET_64BIT) ? 1 : 16; // Darwin ABI requires 16 byte alignment.
zone->debug = env_bool("MallocProbGuardDebug");
zone->debug_log_throttle_ms = env_uint("MallocProbGuardDebugLogThrottleInMillis", 1000);
if (zone->debug) {
malloc_report(ASL_LEVEL_INFO,
- "ProbGuard configuration: %u kB budget, 1/%u sample rate, %u/%u/%u allocations/metadata/slots, strict alignment: %d\n",
- memory_budget_in_kb, sample_rate, zone->max_allocations, zone->max_metadata, zone->num_slots, strict_alignment);
+ "ProbGuard configuration: %u kB budget, 1/%u sample rate, %u/%u/%u allocations/metadata/slots\n",
+ memory_budget_in_kb, sample_rate, zone->max_allocations, zone->max_metadata, zone->num_slots);
}
if (!check_configuration(zone)) {
MALLOC_REPORT_FATAL_ERROR(0, "ProbGuard: bad configuration");
@@ -1263,8 +1319,18 @@
disable_unsupported_apis(malloc_zone_t *pgm_zone, const malloc_zone_t *wrapped_zone)
{
#define DISABLE_UNSUPPORTED(api) if (!wrapped_zone->api) pgm_zone->api = NULL;
+
+ // In practice, there are no zones we support wrapping right now that don't
+ // have these entrypoints
DISABLE_UNSUPPORTED(memalign)
+ DISABLE_UNSUPPORTED(malloc_type_memalign)
DISABLE_UNSUPPORTED(free_definite_size)
+ DISABLE_UNSUPPORTED(claimed_address)
+
+ // These ones are actually load-bearing: the nano and scalable zones do not
+ // implement these entrypoints
+ DISABLE_UNSUPPORTED(malloc_with_options)
+ DISABLE_UNSUPPORTED(malloc_type_malloc_with_options)
}
static void
@@ -1298,12 +1364,7 @@
malloc_zone_t *
pgm_create_zone(malloc_zone_t *wrapped_zone)
{
- // The PGM zone includes the `malloc_introspection_t::zone_type` field
- // (version 14) so we need to support the "malloc()/calloc() set errno to
- // ENOMEM on failure" behavior (version 13). Require wrapped zone to be at
- // least version 13.
- MALLOC_STATIC_ASSERT(PGM_ZONE_VERSION == 15, "PGM zone version");
- MALLOC_ASSERT(wrapped_zone->version >= 13);
+ MALLOC_ASSERT(wrapped_zone->version >= MIN_WRAPPED_ZONE_VERSION);
pgm_zone_t *zone = (pgm_zone_t *)my_vm_map(sizeof(pgm_zone_t), VM_PROT_READ_WRITE, VM_MEMORY_MALLOC);
setup_zone(zone, wrapped_zone);