vm_allocation.c - tests/vm/vm_allocation.c - Xnu source code xnu-8020.101.4

/* Mach virtual memory unit tests
 *
 * The main goal of this code is to facilitate the construction,
 * running, result logging and clean up of a test suite, taking care
 * of all the scaffolding. A test suite is a sequence of very targeted
 * unit tests, each running as a separate process to isolate its
 * address space.
 * A unit test is abstracted as a unit_test_t structure, consisting of
 * a test function and a logging identifier. A test suite is a suite_t
 * structure, consisting of an unit_test_t array, fixture set up and
 * tear down functions.
 * Test suites are created dynamically. Each of its unit test runs in
 * its own fork()d process, with the fixture set up and tear down
 * running before and after each test. The parent process will log a
 * pass result if the child exits normally, and a fail result in any
 * other case (non-zero exit status, abnormal signal). The suite
 * results are then aggregated and logged after the [SUMMARY] keyword,
 * and finally the test suite is destroyed.
 * The included test suites cover the Mach memory allocators,
 * mach_vm_allocate() and mach_vm_map() with various options, and
 * mach_vm_deallocate(), mach_vm_read(), mach_vm_write(),
 * mach_vm_protect(), mach_vm_copy().
 *
 * Author: Renaud Dreyer (rdreyer@apple.com)
 *
 * Transformed to libdarwintest by Tristan Ye (tristan_ye@apple.com) */

#include <darwintest.h>

#include <stdlib.h>
#include <ctype.h>
#include <inttypes.h>
#include <stdio.h>
#include <math.h>
#include <errno.h>
#include <signal.h>
#include <getopt.h>
#include <mach/mach.h>
#include <mach/mach_init.h>
#include <mach/mach_vm.h>
#include <sys/sysctl.h>
#include <time.h>

T_GLOBAL_META(
	T_META_NAMESPACE("xnu.vm"),
	T_META_RADAR_COMPONENT_NAME("xnu"),
	T_META_RADAR_COMPONENT_VERSION("VM"));

/**************************/
/**************************/
/* Unit Testing Framework */
/**************************/
/**************************/

/*********************/
/* Private interface */
/*********************/

static const char frameworkname[] = "vm_unitester";

/* Type for test, fixture set up and fixture tear down functions. */
typedef void (*test_fn_t)();

/* Unit test structure. */
typedef struct {
	const char * name;
	test_fn_t test;
} unit_test_t;

/* Test suite structure. */
typedef struct {
	const char * name;
	int numoftests;
	test_fn_t set_up;
	unit_test_t * tests;
	test_fn_t tear_down;
} suite_t;

int _quietness        = 0;
int _expected_signal  = 0;

struct {
	uintmax_t numoftests;
	uintmax_t passed_tests;
} results = {0, 0};

#define logr(format, ...) \
	do { \
	        if (_quietness <= 1) { \
	                T_LOG(format, ## __VA_ARGS__); \
	        } \
	} while (0)

#define logv(format, ...) \
	do { \
	        if (_quietness == 0) { \
	                T_LOG(format, ## __VA_ARGS__); \
	        } \
	} while (0)

static suite_t *
create_suite(const char * name, int numoftests, test_fn_t set_up, unit_test_t * tests, test_fn_t tear_down)
{
	suite_t * suite = (suite_t *)malloc(sizeof(suite_t));
	T_QUIET; T_WITH_ERRNO; T_ASSERT_NOTNULL(suite, "malloc()");

	suite->name       = name;
	suite->numoftests = numoftests;
	suite->set_up     = set_up;
	suite->tests      = tests;
	suite->tear_down  = tear_down;
	return suite;
}

static void
destroy_suite(suite_t * suite)
{
	free(suite);
}

static void
log_suite_info(suite_t * suite)
{
	logr("[TEST] %s", suite->name);
	logr("Number of tests: %d\n", suite->numoftests);
}

static void
log_suite_results(suite_t * suite, int passed_tests)
{
	results.numoftests += (uintmax_t)suite->numoftests;
	results.passed_tests += (uintmax_t)passed_tests;
}

static void
log_test_info(unit_test_t * unit_test, unsigned test_num)
{
	logr("[BEGIN] #%04d: %s", test_num, unit_test->name);
}

static void
log_test_result(unit_test_t * unit_test, boolean_t test_passed, unsigned test_num)
{
	logr("[%s] #%04d: %s\n", test_passed ? "PASS" : "FAIL", test_num, unit_test->name);
}

/* Run a test with fixture set up and teardown, while enforcing the
 * time out constraint. */
static void
run_test(suite_t * suite, unit_test_t * unit_test, unsigned test_num)
{
	log_test_info(unit_test, test_num);

	suite->set_up();
	unit_test->test();
	suite->tear_down();
}

/* Check a child return status. */
static boolean_t
child_terminated_normally(int child_status)
{
	boolean_t normal_exit = FALSE;

	if (WIFEXITED(child_status)) {
		int exit_status = WEXITSTATUS(child_status);
		if (exit_status) {
			T_LOG("Child process unexpectedly exited with code %d.",
			    exit_status);
		} else if (!_expected_signal) {
			normal_exit = TRUE;
		}
	} else if (WIFSIGNALED(child_status)) {
		int signal = WTERMSIG(child_status);
		if (signal == _expected_signal ||
		    (_expected_signal == -1 && (signal == SIGBUS || signal == SIGSEGV))) {
			if (_quietness <= 0) {
				T_LOG("Child process died with expected signal "
				    "%d.", signal);
			}
			normal_exit = TRUE;
		} else {
			T_LOG("Child process unexpectedly died with signal %d.",
			    signal);
		}
	} else {
		T_LOG("Child process unexpectedly did not exit nor die");
	}

	return normal_exit;
}

/* Run a test in its own process, and report the result. */
static boolean_t
child_test_passed(suite_t * suite, unit_test_t * unit_test)
{
	int test_status;
	static unsigned test_num = 0;

	test_num++;

	pid_t test_pid = fork();
	T_QUIET; T_ASSERT_POSIX_SUCCESS(test_pid, "fork()");
	if (!test_pid) {
		run_test(suite, unit_test, test_num);
		exit(0);
	}
	while (waitpid(test_pid, &test_status, 0) != test_pid) {
		continue;
	}
	boolean_t test_result = child_terminated_normally(test_status);
	log_test_result(unit_test, test_result, test_num);
	return test_result;
}

/* Run each test in a suite, and report the results. */
static int
count_passed_suite_tests(suite_t * suite)
{
	int passed_tests = 0;
	int i;

	for (i = 0; i < suite->numoftests; i++) {
		passed_tests += child_test_passed(suite, &(suite->tests[i]));
	}
	return passed_tests;
}

/********************/
/* Public interface */
/********************/

#define DEFAULT_QUIETNESS    0 /* verbose */
#define RESULT_ERR_QUIETNESS 1 /* result and error */
#define ERROR_ONLY_QUIETNESS 2 /* error only */

#define run_suite(set_up, tests, tear_down, ...) \
	_run_suite((sizeof(tests) / sizeof(tests[0])), (set_up), (tests), (tear_down), __VA_ARGS__)

typedef unit_test_t UnitTests[];

void _run_suite(int numoftests, test_fn_t set_up, UnitTests tests, test_fn_t tear_down, const char * format, ...)
__printflike(5, 6);

void
_run_suite(int numoftests, test_fn_t set_up, UnitTests tests, test_fn_t tear_down, const char * format, ...)
{
	va_list ap;
	char * name;

	va_start(ap, format);
	T_QUIET; T_ASSERT_POSIX_SUCCESS(vasprintf(&name, format, ap), "vasprintf()");
	va_end(ap);
	suite_t * suite = create_suite(name, numoftests, set_up, tests, tear_down);
	log_suite_info(suite);
	log_suite_results(suite, count_passed_suite_tests(suite));
	free(name);
	destroy_suite(suite);
}

/* Setters and getters for various test framework global
 * variables. Should only be used outside of the test, set up and tear
 * down functions. */

/* Expected signal for a test, default is 0. */
void
set_expected_signal(int signal)
{
	_expected_signal = signal;
}

int
get_expected_signal()
{
	return _expected_signal;
}

/* Logging verbosity. */
void
set_quietness(int value)
{
	_quietness = value;
}

int
get_quietness()
{
	return _quietness;
}

/* For fixture set up and tear down functions, and units tests. */
void
do_nothing()
{
}

void
log_aggregated_results()
{
	T_LOG("[SUMMARY] Aggregated Test Results\n");
	T_LOG("Total: %ju", results.numoftests);
	T_LOG("Passed: %ju", results.passed_tests);
	T_LOG("Failed: %ju\n", results.numoftests - results.passed_tests);

	T_QUIET; T_ASSERT_EQ(results.passed_tests, results.numoftests,
	    "%d passed of total %d tests",
	    results.passed_tests, results.numoftests);
}

/*******************************/
/*******************************/
/* Virtual memory unit testing */
/*******************************/
/*******************************/

/* Test exit values:
 * 0: pass
 * 1: fail, generic unexpected failure
 * 2: fail, unexpected Mach return value
 * 3: fail, time out */

#define DEFAULT_VM_SIZE ((mach_vm_size_t)(1024ULL * 4096ULL))

#define POINTER(address) ((char *)(uintptr_t)(address))
#define MACH_VM_ADDRESS_T(address) (*((mach_vm_address_t *)(uintptr_t)(address)))

static int vm_address_size = sizeof(mach_vm_address_t);

static char *progname = "";

/*************************/
/* xnu version functions */
/*************************/

/* Find the xnu version string. */
char *
xnu_version_string()
{
	size_t length;
	int mib[2];
	mib[0] = CTL_KERN;
	mib[1] = KERN_VERSION;

	T_QUIET;
	T_ASSERT_POSIX_SUCCESS(sysctl(mib, 2, NULL, &length, NULL, 0), "sysctl()");
	char * version = (char *)malloc(length);
	T_QUIET;
	T_WITH_ERRNO;
	T_ASSERT_NOTNULL(version, "malloc()");
	T_QUIET;
	T_EXPECT_POSIX_SUCCESS(sysctl(mib, 2, version, &length, NULL, 0), "sysctl()");
	if (T_RESULT == T_RESULT_FAIL) {
		free(version);
		T_END;
	}
	char * xnu_string = strstr(version, "xnu-");
	free(version);
	T_QUIET;
	T_ASSERT_NOTNULL(xnu_string, "%s: error finding xnu version string.", progname);
	return xnu_string;
}

/* Find the xnu major version number. */
unsigned int
xnu_major_version()
{
	char * endptr;
	char * xnu_substring = xnu_version_string() + 4;

	errno                    = 0;
	unsigned int xnu_version = strtoul(xnu_substring, &endptr, 0);
	T_QUIET;
	T_ASSERT_TRUE((errno != ERANGE && endptr != xnu_substring),
	    "%s: error finding xnu major version number.", progname);
	return xnu_version;
}

/*************************/
/* Mach assert functions */
/*************************/

static inline void
assert_mach_return(kern_return_t kr, kern_return_t expected_kr, const char * mach_routine)
{
	T_QUIET; T_ASSERT_EQ(kr, expected_kr,
	    "%s unexpectedly returned: %s."
	    "Should have returned: %s.",
	    mach_routine, mach_error_string(kr),
	    mach_error_string(expected_kr));
}

/*******************************/
/* Arrays for test suite loops */
/*******************************/

/* Memory allocators */
typedef kern_return_t (*allocate_fn_t)(vm_map_t, mach_vm_address_t *, mach_vm_size_t, int);


/*
 * Remember any pre-reserved fixed address, which needs to be released prior to an allocation.
 */
static mach_vm_address_t fixed_vm_address = 0x0;
static mach_vm_size_t fixed_vm_size = 0;

/* forward decl */
void assert_deallocate_success(mach_vm_address_t address, mach_vm_size_t size);

/*
 * If trying to allocate at a fixed address, we need to do the delayed deallocate first.
 */
static void
check_fixed_address(mach_vm_address_t *address, mach_vm_size_t size)
{
	if (fixed_vm_address != 0 &&
	    fixed_vm_address <= *address &&
	    *address + size <= fixed_vm_address + fixed_vm_size) {
		assert_deallocate_success(fixed_vm_address, fixed_vm_size);
		fixed_vm_address = 0;
		fixed_vm_size = 0;
	}
}

kern_return_t
wrapper_mach_vm_allocate(vm_map_t map, mach_vm_address_t * address, mach_vm_size_t size, int flags)
{
	check_fixed_address(address, size);
	return mach_vm_allocate(map, address, size, flags);
}

kern_return_t
wrapper_mach_vm_map(vm_map_t map, mach_vm_address_t * address, mach_vm_size_t size, int flags)
{
	check_fixed_address(address, size);
	return mach_vm_map(map, address, size, (mach_vm_offset_t)0, flags, MACH_PORT_NULL, (memory_object_offset_t)0, FALSE,
	           VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
}

/* Should have the same behavior as when mask is zero. */
kern_return_t
wrapper_mach_vm_map_4kB(vm_map_t map, mach_vm_address_t * address, mach_vm_size_t size, int flags)
{
	check_fixed_address(address, size);
	return mach_vm_map(map, address, size, (mach_vm_offset_t)0xFFF, flags, MACH_PORT_NULL, (memory_object_offset_t)0, FALSE,
	           VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
}

kern_return_t
wrapper_mach_vm_map_2MB(vm_map_t map, mach_vm_address_t * address, mach_vm_size_t size, int flags)
{
	check_fixed_address(address, size);
	return mach_vm_map(map, address, size, (mach_vm_offset_t)0x1FFFFF, flags, MACH_PORT_NULL, (memory_object_offset_t)0, FALSE,
	           VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
}

mach_port_t
memory_entry(mach_vm_size_t * size)
{
	mach_port_t object_handle    = MACH_PORT_NULL;
	mach_vm_size_t original_size = *size;

	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_make_memory_entry_64(mach_task_self(), size, (memory_object_offset_t)0,
	    (MAP_MEM_NAMED_CREATE | VM_PROT_ALL), &object_handle, 0),
	    "mach_make_memory_entry_64()");
	T_QUIET; T_ASSERT_EQ(*size, round_page(original_size),
	    "mach_make_memory_entry_64() unexpectedly returned a named "
	    "entry of size 0x%jx (%ju).\n"
	    "Should have returned a "
	    "named entry of size 0x%jx (%ju).",
	    (uintmax_t)*size, (uintmax_t)*size, (uintmax_t)original_size, (uintmax_t)original_size);
	return object_handle;
}

kern_return_t
wrapper_mach_vm_map_named_entry(vm_map_t map, mach_vm_address_t * address, mach_vm_size_t size, int flags)
{
	mach_port_t object_handle = memory_entry(&size);
	check_fixed_address(address, size);
	kern_return_t kr = mach_vm_map(map, address, size, (mach_vm_offset_t)0, flags, object_handle, (memory_object_offset_t)0, FALSE,
	    VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_port_deallocate(mach_task_self(), object_handle), "mach_port_deallocate()");
	return kr;
}

static struct {
	allocate_fn_t allocate;
	const char * description;
} allocators[] = {
	{wrapper_mach_vm_allocate, "mach_vm_allocate()"},
	{wrapper_mach_vm_map, "mach_vm_map() (zero mask)"},
	{wrapper_mach_vm_map_4kB,
	 "mach_vm_map() "
	 "(4 kB address alignment)"},
	{wrapper_mach_vm_map_2MB,
	 "mach_vm_map() "
	 "(2 MB address alignment)"},
	{wrapper_mach_vm_map_named_entry,
	 "mach_vm_map() (named "
	 "entry, zero mask)"},
};
static int numofallocators = sizeof(allocators) / sizeof(allocators[0]);
static int allocators_idx;
enum { MACH_VM_ALLOCATE, MACH_VM_MAP, MACH_VM_MAP_4kB, MACH_VM_MAP_2MB, MACH_VM_MAP_NAMED_ENTRY };

/* VM size */
static struct {
	mach_vm_size_t size;
	const char * description;
} vm_sizes[] = {
	{DEFAULT_VM_SIZE, "default/input"},
	{0, "zero"},
	{4096ULL, "aligned"},
	{1ULL, "unaligned"},
	{4095ULL, "unaligned"},
	{4097ULL, "unaligned"},
};
static int numofsizes = sizeof(vm_sizes) / sizeof(vm_sizes[0]);
static int sizes_idx;
static int buffer_sizes_idx;
enum { DEFAULT_INPUT, ZERO_BYTES, ONE_PAGE, ONE_BYTE, ONE_PAGE_MINUS_ONE_BYTE, ONE_PAGE_AND_ONE_BYTE };

/* Unspecified/fixed address */
static struct {
	int flag;
	const char * description;
} address_flags[] = {
	{VM_FLAGS_ANYWHERE, "unspecified"}, {VM_FLAGS_FIXED, "fixed"},
};
static int numofflags = sizeof(address_flags) / sizeof(address_flags[0]);
static int flags_idx;
enum { ANYWHERE, FIXED };

/* Address alignment */
static struct {
	boolean_t alignment;
	const char * description;
} address_alignments[] = {
	{TRUE, " aligned"}, {FALSE, " unaligned"},
};
static int numofalignments = sizeof(address_alignments) / sizeof(*address_alignments);
static int alignments_idx;
enum { ALIGNED, UNALIGNED };

/* Buffer offset */
static struct {
	int offset;
	const char * description;
} buffer_offsets[] = {
	{0, ""}, {1, ""}, {2, ""},
};
static int numofoffsets = sizeof(buffer_offsets) / sizeof(buffer_offsets[0]);
static int offsets_idx;
enum { ZERO, ONE, TWO };

/* mach_vm_copy() post actions */
enum { VMCOPY_MODIFY_SRC, VMCOPY_MODIFY_DST, VMCOPY_MODIFY_SHARED_COPIED };

static struct {
	int action;
	const char * description;
} vmcopy_actions[] = {
	{VMCOPY_MODIFY_SRC, "modify vm_copy() source"},
	{VMCOPY_MODIFY_DST, "modify vm_copy() destination"},
	{VMCOPY_MODIFY_SHARED_COPIED,
	 "modify vm_copy source's shared "
	 "or copied from/to region"},
};
static int numofvmcopyactions = sizeof(vmcopy_actions) / sizeof(vmcopy_actions[0]);
static int vmcopy_action_idx;

/************************************/
/* Setters and getters for fixtures */
/************************************/

/* Allocation memory range. */
static allocate_fn_t _allocator      = wrapper_mach_vm_allocate;
static mach_vm_size_t _vm_size       = DEFAULT_VM_SIZE;
static int _address_flag             = VM_FLAGS_ANYWHERE;
static boolean_t _address_alignment  = TRUE;
static mach_vm_address_t _vm_address = 0x0;

/* Buffer for mach_vm_write(). */
static mach_vm_size_t _buffer_size       = DEFAULT_VM_SIZE;
static mach_vm_address_t _buffer_address = 0x0;
static int _buffer_offset                = 0;

/* Post action for mach_vm_copy(). */
static int _vmcopy_post_action = VMCOPY_MODIFY_SRC;

static void
set_allocator(allocate_fn_t allocate)
{
	_allocator = allocate;
}

static allocate_fn_t
get_allocator()
{
	return _allocator;
}

static void
set_vm_size(mach_vm_size_t size)
{
	_vm_size = size;
}

static mach_vm_size_t
get_vm_size()
{
	return _vm_size;
}

static void
set_address_flag(int flag)
{
	_address_flag = flag;
}

static int
get_address_flag()
{
	return _address_flag;
}

static void
set_address_alignment(boolean_t alignment)
{
	_address_alignment = alignment;
}

static boolean_t
get_address_alignment()
{
	return _address_alignment;
}

static void
set_vm_address(mach_vm_address_t address)
{
	_vm_address = address;
}

static mach_vm_address_t
get_vm_address()
{
	return _vm_address;
}

static void
set_buffer_size(mach_vm_size_t size)
{
	_buffer_size = size;
}

static mach_vm_size_t
get_buffer_size()
{
	return _buffer_size;
}

static void
set_buffer_address(mach_vm_address_t address)
{
	_buffer_address = address;
}

static mach_vm_address_t
get_buffer_address()
{
	return _buffer_address;
}

static void
set_buffer_offset(int offset)
{
	_buffer_offset = offset;
}

static int
get_buffer_offset()
{
	return _buffer_offset;
}

static void
set_vmcopy_post_action(int action)
{
	_vmcopy_post_action = action;
}

static int
get_vmcopy_post_action()
{
	return _vmcopy_post_action;
}

/*******************************/
/* Usage and option processing */
/*******************************/
static boolean_t flag_run_allocate_test = FALSE;
static boolean_t flag_run_deallocate_test = FALSE;
static boolean_t flag_run_read_test = FALSE;
static boolean_t flag_run_write_test = FALSE;
static boolean_t flag_run_protect_test = FALSE;
static boolean_t flag_run_copy_test = FALSE;

#define VM_TEST_ALLOCATE   0x00000001
#define VM_TEST_DEALLOCATE 0x00000002
#define VM_TEST_READ       0x00000004
#define VM_TEST_WRITE      0x00000008
#define VM_TEST_PROTECT    0x00000010
#define VM_TEST_COPY       0x00000020

typedef struct test_option {
	uint32_t        to_flags;
	int             to_quietness;
	mach_vm_size_t  to_vmsize;
} test_option_t;

typedef struct test_info {
	char            *ti_name;
	boolean_t       *ti_flag;
} test_info_t;

static test_option_t test_options;

enum {ALLOCATE = 0, DEALLOCATE, READ, WRITE, PROTECT, COPY};

static test_info_t test_info[] = {
	{"allocate", &flag_run_allocate_test},
	{"deallocate", &flag_run_deallocate_test},
	{"read", &flag_run_read_test},
	{"write", &flag_run_write_test},
	{"protect", &flag_run_protect_test},
	{"copy", &flag_run_copy_test},
	{NULL, NULL}
};

static void
die_on_invalid_value(int condition, const char * value_string)
{
	T_QUIET;
	T_ASSERT_EQ(condition, 0, "%s: invalid value: %s.",
	    progname, value_string);
}

static void
process_options(test_option_t options)
{
	test_info_t *tp;

	setvbuf(stdout, NULL, _IONBF, 0);

	set_vm_size(DEFAULT_VM_SIZE);
	set_quietness(DEFAULT_QUIETNESS);

	if (NULL != getenv("LTERDOS")) {
		logr("LTERDOS=YES this is LeanTestEnvironment\nIncreasing quietness by 1.");
		set_quietness(get_quietness() + 1);
	} else {
		if (options.to_quietness > 0) {
			set_quietness(options.to_quietness);
		}
	}

	if (options.to_vmsize != 0) {
		vm_sizes[0].size = options.to_vmsize;
	}

	if (options.to_flags == 0) {
		for (tp = test_info; tp->ti_name != NULL; ++tp) {
			*tp->ti_flag = TRUE;
		}
	} else {
		if (options.to_flags & VM_TEST_ALLOCATE) {
			*(test_info[ALLOCATE].ti_flag) = TRUE;
		}

		if (options.to_flags & VM_TEST_DEALLOCATE) {
			*(test_info[DEALLOCATE].ti_flag) = TRUE;
		}

		if (options.to_flags & VM_TEST_READ) {
			*(test_info[READ].ti_flag) = TRUE;
		}

		if (options.to_flags & VM_TEST_WRITE) {
			*(test_info[WRITE].ti_flag) = TRUE;
		}

		if (options.to_flags & VM_TEST_PROTECT) {
			*(test_info[PROTECT].ti_flag) = TRUE;
		}

		if (options.to_flags & VM_TEST_COPY) {
			*(test_info[COPY].ti_flag) = TRUE;
		}
	}
}

/*****************/
/* Various tools */
/*****************/

/* Find the allocator address alignment mask. */
mach_vm_address_t
get_mask()
{
	mach_vm_address_t mask;

	if (get_allocator() == wrapper_mach_vm_map_2MB) {
		mask = (mach_vm_address_t)0x1FFFFF;
	} else {
		mask = vm_page_size - 1;
	}
	return mask;
}

/* Find the size of the smallest aligned region containing a given
 * memory range. */
mach_vm_size_t
aligned_size(mach_vm_address_t address, mach_vm_size_t size)
{
	return round_page(address - mach_vm_trunc_page(address) + size);
}

/********************/
/* Assert functions */
/********************/

/* Address is aligned on allocator boundary. */
static inline void
assert_aligned_address(mach_vm_address_t address)
{
	T_QUIET; T_ASSERT_EQ((address & get_mask()), 0,
	    "Address 0x%jx is unexpectedly "
	    "unaligned.",
	    (uintmax_t)address);
}

/* Address is truncated to allocator boundary. */
static inline void
assert_trunc_address(mach_vm_address_t address, mach_vm_address_t trunc_address)
{
	T_QUIET; T_ASSERT_EQ(trunc_address, (address & ~get_mask()),
	    "Address "
	    "0x%jx is unexpectedly not truncated to address 0x%jx.",
	    (uintmax_t)address, (uintmax_t)trunc_address);
}

static inline void
assert_address_value(mach_vm_address_t address, mach_vm_address_t marker)
{
	/* this assert is used so frequently so that we simply judge on
	 * its own instead of leaving this to LD macro for efficiency
	 */
	if (MACH_VM_ADDRESS_T(address) != marker) {
		T_ASSERT_FAIL("Address 0x%jx unexpectedly has value 0x%jx, "
		    "instead of 0x%jx.", (uintmax_t)address,
		    (uintmax_t)MACH_VM_ADDRESS_T(address), (uintmax_t)marker);
	}
}

void
assert_allocate_return(mach_vm_address_t * address, mach_vm_size_t size, int address_flag, kern_return_t expected_kr)
{
	assert_mach_return(get_allocator()(mach_task_self(), address, size, address_flag), expected_kr, "Allocator");
}

void
assert_allocate_success(mach_vm_address_t * address, mach_vm_size_t size, int address_flag)
{
	assert_allocate_return(address, size, address_flag, KERN_SUCCESS);
}

void
assert_deallocate_return(mach_vm_address_t address, mach_vm_size_t size, kern_return_t expected_kr)
{
	assert_mach_return(mach_vm_deallocate(mach_task_self(), address, size), expected_kr, "mach_vm_deallocate()");
}

void
assert_deallocate_success(mach_vm_address_t address, mach_vm_size_t size)
{
	assert_deallocate_return(address, size, KERN_SUCCESS);
}

void
assert_read_return(mach_vm_address_t address,
    mach_vm_size_t size,
    vm_offset_t * data,
    mach_msg_type_number_t * data_size,
    kern_return_t expected_kr)
{
	assert_mach_return(mach_vm_read(mach_task_self(), address, size, data, data_size), expected_kr, "mach_vm_read()");
}

void
assert_read_success(mach_vm_address_t address, mach_vm_size_t size, vm_offset_t * data, mach_msg_type_number_t * data_size)
{
	assert_read_return(address, size, data, data_size, KERN_SUCCESS);
	T_QUIET; T_ASSERT_EQ(*data_size, size,
	    "Returned buffer size 0x%jx "
	    "(%ju) is unexpectedly different from source size 0x%jx "
	    "(%ju).",
	    (uintmax_t)*data_size, (uintmax_t)*data_size, (uintmax_t)size, (uintmax_t)size);
}

void
assert_write_return(mach_vm_address_t address, vm_offset_t data, mach_msg_type_number_t data_size, kern_return_t expected_kr)
{
	assert_mach_return(mach_vm_write(mach_task_self(), address, data, data_size), expected_kr, "mach_vm_write()");
}

void
assert_write_success(mach_vm_address_t address, vm_offset_t data, mach_msg_type_number_t data_size)
{
	assert_write_return(address, data, data_size, KERN_SUCCESS);
}

void
assert_allocate_copy_return(mach_vm_address_t source, mach_vm_size_t size, mach_vm_address_t * dest, kern_return_t expected_kr)
{
	assert_allocate_success(dest, size, VM_FLAGS_ANYWHERE);
	assert_mach_return(mach_vm_copy(mach_task_self(), source, size, *dest), expected_kr, "mach_vm_copy()");
}
void
assert_allocate_copy_success(mach_vm_address_t source, mach_vm_size_t size, mach_vm_address_t * dest)
{
	assert_allocate_copy_return(source, size, dest, KERN_SUCCESS);
}

void
assert_copy_return(mach_vm_address_t source, mach_vm_size_t size, mach_vm_address_t dest, kern_return_t expected_kr)
{
	assert_mach_return(mach_vm_copy(mach_task_self(), source, size, dest), expected_kr, "mach_vm_copy()");
}

void
assert_copy_success(mach_vm_address_t source, mach_vm_size_t size, mach_vm_address_t dest)
{
	assert_copy_return(source, size, dest, KERN_SUCCESS);
}

/*******************/
/* Memory patterns */
/*******************/

typedef boolean_t (*address_filter_t)(mach_vm_address_t);
typedef void (*address_action_t)(mach_vm_address_t, mach_vm_address_t);

/* Map over a memory region pattern and its complement, through a
 * (possibly reversed) boolean filter and a starting value. */
void
filter_addresses_do_else(address_filter_t filter,
    boolean_t reversed,
    mach_vm_address_t address,
    mach_vm_size_t size,
    address_action_t if_action,
    address_action_t else_action,
    mach_vm_address_t start_value)
{
	mach_vm_address_t i;
	for (i = 0; i + vm_address_size < size; i += vm_address_size) {
		if (filter(address + i) != reversed) {
			if_action(address + i, start_value + i);
		} else {
			else_action(address + i, start_value + i);
		}
	}
}

/* Various pattern actions. */
void
no_action(mach_vm_address_t i, mach_vm_address_t value)
{
}

void
read_zero(mach_vm_address_t i, mach_vm_address_t value)
{
	assert_address_value(i, 0);
}

void
verify_address(mach_vm_address_t i, mach_vm_address_t value)
{
	assert_address_value(i, value);
}

void
write_address(mach_vm_address_t i, mach_vm_address_t value)
{
	MACH_VM_ADDRESS_T(i) = value;
}

/* Various patterns. */
boolean_t
empty(mach_vm_address_t i)
{
	return FALSE;
}

boolean_t
checkerboard(mach_vm_address_t i)
{
	return !((i / vm_address_size) & 0x1);
}

boolean_t
page_ends(mach_vm_address_t i)
{
	mach_vm_address_t residue = i % vm_page_size;

	return residue == 0 || residue == vm_page_size - vm_address_size;
}

/*************************************/
/* Global variables set up functions */
/*************************************/

void
set_up_allocator()
{
	T_QUIET; T_ASSERT_TRUE(allocators_idx >= 0 && allocators_idx < numofallocators, "Invalid allocators[] index: %d.", allocators_idx);
	set_allocator(allocators[allocators_idx].allocate);
}

/* Find a fixed allocatable address by retrieving the address
 * populated by mach_vm_allocate() with VM_FLAGS_ANYWHERE. */
mach_vm_address_t
get_fixed_address(mach_vm_size_t size)
{
	/* mach_vm_map() starts looking for an address at 0x0. */
	mach_vm_address_t address = 0x0;

	/*
	 * The tests seem to have some funky off by one allocations. To avoid problems, we'll bump anything
	 * non-zero to have at least an extra couple pages.
	 */
	if (size != 0) {
		size = round_page(size + 2 * vm_page_size);
	}

	assert_allocate_success(&address, size, VM_FLAGS_ANYWHERE);

	/*
	 * Keep the memory allocated, otherwise the logv()/printf() activity sprinkled in these tests can
	 * cause malloc() to use the desired range and tests will randomly fail. The allocate routines will
	 * do the delayed vm_deallocate() to free the fixed memory just before allocation testing in the wrapper.
	 */
	T_QUIET; T_ASSERT_EQ(fixed_vm_address, 0, "previous fixed address not used");
	T_QUIET; T_ASSERT_EQ(fixed_vm_size, 0, "previous fixed size not used");
	fixed_vm_address = address;
	fixed_vm_size = size;

	assert_aligned_address(address);
	return address;
}

/* If needed, find an address at which a region of the specified size
 * can be allocated. Otherwise, set the address to 0x0. */
void
set_up_vm_address(mach_vm_size_t size)
{
	T_QUIET; T_ASSERT_TRUE(flags_idx >= 0 && flags_idx < numofflags, "Invalid address_flags[] index: %d.", flags_idx);
	T_QUIET; T_ASSERT_TRUE(alignments_idx >= 0 && alignments_idx < numofalignments, "Invalid address_alignments[] index: %d.", alignments_idx);
	set_address_flag(address_flags[flags_idx].flag);
	set_address_alignment(address_alignments[alignments_idx].alignment);

	if (!(get_address_flag() & VM_FLAGS_ANYWHERE)) {
		boolean_t aligned = get_address_alignment();
		logv(
			"Looking for fixed %saligned address for allocation "
			"of 0x%jx (%ju) byte%s...",
			aligned ? "" : "un", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s");
		mach_vm_address_t address = get_fixed_address(size);
		if (!aligned) {
			address++;
		}
		set_vm_address(address);
		logv("Found %saligned fixed address 0x%jx.", aligned ? "" : "un", (uintmax_t)address);
	} else {
		/* mach_vm_map() with VM_FLAGS_ANYWHERE starts looking for
		 *  an address at the one supplied and goes up, without
		 *  wrapping around. */
		set_vm_address(0x0);
	}
}

void
set_up_vm_size()
{
	T_QUIET; T_ASSERT_TRUE(sizes_idx >= 0 && sizes_idx < numofsizes, "Invalid vm_sizes[] index: %d.", sizes_idx);
	set_vm_size(vm_sizes[sizes_idx].size);
}

void
set_up_buffer_size()
{
	T_QUIET; T_ASSERT_TRUE(buffer_sizes_idx >= 0 && buffer_sizes_idx < numofsizes, "Invalid vm_sizes[] index: %d.", buffer_sizes_idx);
	set_buffer_size(vm_sizes[buffer_sizes_idx].size);
}

void
set_up_buffer_offset()
{
	T_QUIET; T_ASSERT_TRUE(offsets_idx >= 0 && offsets_idx < numofoffsets, "Invalid buffer_offsets[] index: %d.", offsets_idx);
	set_buffer_offset(buffer_offsets[offsets_idx].offset);
}

void
set_up_vmcopy_action()
{
	T_QUIET; T_ASSERT_TRUE(vmcopy_action_idx >= 0 && vmcopy_action_idx < numofvmcopyactions, "Invalid vmcopy_actions[] index: %d.",
	    vmcopy_action_idx);
	set_vmcopy_post_action(vmcopy_actions[vmcopy_action_idx].action);
}

void
set_up_allocator_and_vm_size()
{
	set_up_allocator();
	set_up_vm_size();
}

void
set_up_vm_variables()
{
	set_up_vm_size();
	set_up_vm_address(get_vm_size());
}

void
set_up_allocator_and_vm_variables()
{
	set_up_allocator();
	set_up_vm_variables();
}

void
set_up_buffer_variables()
{
	set_up_buffer_size();
	set_up_buffer_offset();
}

void
set_up_copy_shared_mode_variables()
{
	set_up_vmcopy_action();
}

/*******************************/
/* Allocation set up functions */
/*******************************/

/* Allocate VM region of given size. */
void
allocate(mach_vm_size_t size)
{
	mach_vm_address_t address = get_vm_address();
	int flag                  = get_address_flag();

	logv("Allocating 0x%jx (%ju) byte%s", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s");
	if (!(flag & VM_FLAGS_ANYWHERE)) {
		logv(" at address 0x%jx", (uintmax_t)address);
	}
	logv("...");
	assert_allocate_success(&address, size, flag);
	logv(
		"Memory of rounded size 0x%jx (%ju) allocated at "
		"address 0x%jx.",
		(uintmax_t)round_page(size), (uintmax_t)round_page(size), (uintmax_t)address);
	/* Fixed allocation address is truncated to the allocator
	 *  boundary. */
	if (!(flag & VM_FLAGS_ANYWHERE)) {
		mach_vm_address_t old_address = get_vm_address();
		assert_trunc_address(old_address, address);
		logv(
			"Address 0x%jx is correctly truncated to allocated "
			"address 0x%jx.",
			(uintmax_t)old_address, (uintmax_t)address);
	}
	set_vm_address(address);
}

void
allocate_buffer(mach_vm_size_t buffer_size)
{
	mach_vm_address_t data = 0x0;

	logv("Allocating 0x%jx (%ju) byte%s...", (uintmax_t)buffer_size, (uintmax_t)buffer_size, (buffer_size == 1) ? "" : "s");
	assert_allocate_success(&data, buffer_size, VM_FLAGS_ANYWHERE);
	logv(
		"Memory of rounded size 0x%jx (%ju) allocated at "
		"address 0x%jx.",
		(uintmax_t)round_page(buffer_size), (uintmax_t)round_page(buffer_size), (uintmax_t)data);
	data += get_buffer_offset();
	T_QUIET; T_ASSERT_EQ((vm_offset_t)data, data,
	    "Address 0x%jx "
	    "unexpectedly overflows to 0x%jx when cast as "
	    "vm_offset_t type.",
	    (uintmax_t)data, (uintmax_t)(vm_offset_t)data);
	set_buffer_address(data);
}

/****************************************************/
/* Global variables and allocation set up functions */
/****************************************************/

void
set_up_vm_variables_and_allocate()
{
	set_up_vm_variables();
	allocate(get_vm_size());
}

void
set_up_allocator_and_vm_variables_and_allocate()
{
	set_up_allocator();
	set_up_vm_variables_and_allocate();
}

void
set_up_vm_variables_and_allocate_extra_page()
{
	set_up_vm_size();
	/* Increment the size to insure we get an extra allocated page
	 *  for unaligned start addresses. */
	mach_vm_size_t allocation_size = get_vm_size() + 1;
	set_up_vm_address(allocation_size);

	allocate(allocation_size);
	/* In the fixed unaligned address case, restore the returned
	*  (truncated) allocation address to its unaligned value. */
	if (!get_address_alignment()) {
		set_vm_address(get_vm_address() + 1);
	}
}

void
set_up_buffer_variables_and_allocate_extra_page()
{
	set_up_buffer_variables();
	/* Increment the size to insure we get an extra allocated page
	 *  for unaligned start addresses. */
	allocate_buffer(get_buffer_size() + get_buffer_offset());
}

/* Allocate some destination and buffer memory for subsequent
 * writing, including extra pages for non-aligned start addresses. */
void
set_up_vm_and_buffer_variables_allocate_for_writing()
{
	set_up_vm_variables_and_allocate_extra_page();
	set_up_buffer_variables_and_allocate_extra_page();
}

/* Allocate some destination and source regions for subsequent
 * copying, including extra pages for non-aligned start addresses. */
void
set_up_vm_and_buffer_variables_allocate_for_copying()
{
	set_up_vm_and_buffer_variables_allocate_for_writing();
}

/************************************/
/* Deallocation tear down functions */
/************************************/

void
deallocate_range(mach_vm_address_t address, mach_vm_size_t size)
{
	logv("Deallocating 0x%jx (%ju) byte%s at address 0x%jx...", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s",
	    (uintmax_t)address);
	assert_deallocate_success(address, size);
}

void
deallocate()
{
	deallocate_range(get_vm_address(), get_vm_size());
}

/* Deallocate source memory, including the extra page for unaligned
 * start addresses. */
void
deallocate_extra_page()
{
	/* Set the address and size to their original allocation
	 *  values. */
	deallocate_range(mach_vm_trunc_page(get_vm_address()), get_vm_size() + 1);
}

/* Deallocate buffer and destination memory for mach_vm_write(),
 * including the extra page for unaligned start addresses. */
void
deallocate_vm_and_buffer()
{
	deallocate_range(mach_vm_trunc_page(get_vm_address()), get_vm_size() + 1);
	deallocate_range(mach_vm_trunc_page(get_buffer_address()), get_buffer_size() + get_buffer_offset());
}

/***********************************/
/* mach_vm_read() set up functions */
/***********************************/

/* Read the source memory into a buffer, deallocate the source, set
 * the global address and size from the buffer's. */
void
read_deallocate()
{
	mach_vm_size_t size       = get_vm_size();
	mach_vm_address_t address = get_vm_address();
	vm_offset_t read_address;
	mach_msg_type_number_t read_size;

	logv("Reading 0x%jx (%ju) byte%s at address 0x%jx...", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s",
	    (uintmax_t)address);
	assert_read_success(address, size, &read_address, &read_size);
	logv(
		"Memory of size 0x%jx (%ju) read into buffer of "
		"address 0x%jx.",
		(uintmax_t)read_size, (uintmax_t)read_size, (uintmax_t)read_address);
	/* Deallocate the originally allocated memory, including the
	 *  extra allocated page in
	 *  set_up_vm_variables_and_allocate_extra_page(). */
	deallocate_range(mach_vm_trunc_page(address), size + 1);

	/* Promoting to mach_vm types after checking for overflow, and
	 *  setting the global address from the buffer's. */
	T_QUIET; T_ASSERT_EQ((mach_vm_address_t)read_address, read_address,
	    "Address 0x%jx unexpectedly overflows to 0x%jx when cast "
	    "as mach_vm_address_t type.",
	    (uintmax_t)read_address, (uintmax_t)(mach_vm_address_t)read_address);
	T_QUIET; T_ASSERT_EQ((mach_vm_size_t)read_size, read_size,
	    "Size 0x%jx (%ju) unexpectedly overflows to 0x%jx (%ju) "
	    "when cast as mach_vm_size_t type.",
	    (uintmax_t)read_size, (uintmax_t)read_size, (uintmax_t)(mach_vm_size_t)read_size, (uintmax_t)(mach_vm_size_t)read_size);
	set_vm_address((mach_vm_address_t)read_address);
	set_vm_size((mach_vm_size_t)read_size);
}

/* Allocate some source memory, read it into a buffer, deallocate the
 * source, set the global address and size from the buffer's. */
void
set_up_vm_variables_allocate_read_deallocate()
{
	set_up_vm_variables_and_allocate_extra_page();
	read_deallocate();
}

/************************************/
/* mach_vm_write() set up functions */
/************************************/

/* Write the buffer into the destination memory. */
void
write_buffer()
{
	mach_vm_address_t address          = get_vm_address();
	vm_offset_t data                   = (vm_offset_t)get_buffer_address();
	mach_msg_type_number_t buffer_size = (mach_msg_type_number_t)get_buffer_size();

	logv(
		"Writing buffer of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)data, (uintmax_t)buffer_size, (uintmax_t)buffer_size, (uintmax_t)address);
	assert_write_success(address, data, buffer_size);
	logv("Buffer written.");
}

/* Allocate some destination and buffer memory, and write the buffer
 * into the destination memory. */
void
set_up_vm_and_buffer_variables_allocate_write()
{
	set_up_vm_and_buffer_variables_allocate_for_writing();
	write_buffer();
}

/***********************************/
/* mach_vm_copy() set up functions */
/***********************************/

void
copy_deallocate(void)
{
	mach_vm_size_t size      = get_vm_size();
	mach_vm_address_t source = get_vm_address();
	mach_vm_address_t dest   = 0;

	logv("Copying 0x%jx (%ju) byte%s at address 0x%jx...", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s",
	    (uintmax_t)source);
	assert_allocate_copy_success(source, size, &dest);
	logv(
		"Memory of size 0x%jx (%ju) copy into region of "
		"address 0x%jx.",
		(uintmax_t)size, (uintmax_t)size, (uintmax_t)dest);
	/* Deallocate the originally allocated memory, including the
	 *  extra allocated page in
	 *  set_up_vm_variables_and_allocate_extra_page(). */
	deallocate_range(mach_vm_trunc_page(source), size + 1);
	/* Promoting to mach_vm types after checking for overflow, and
	 *  setting the global address from the buffer's. */
	T_QUIET; T_ASSERT_EQ((vm_offset_t)dest, dest,
	    "Address 0x%jx unexpectedly overflows to 0x%jx when cast "
	    "as mach_vm_address_t type.",
	    (uintmax_t)dest, (uintmax_t)(vm_offset_t)dest);
	set_vm_address(dest);
	set_vm_size(size);
}

/* Copy the source region into the destination region. */
void
copy_region()
{
	mach_vm_address_t source    = get_vm_address();
	mach_vm_address_t dest      = get_buffer_address();
	mach_msg_type_number_t size = (mach_msg_type_number_t)get_buffer_size();

	logv(
		"Copying memory region of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)source, (uintmax_t)size, (uintmax_t)size, (uintmax_t)dest);
	assert_copy_success(source, size, dest);
	logv("Buffer written.");
}

/* Allocate some source memory, copy it to another region, deallocate the
* source, set the global address and size from the designation region. */
void
set_up_vm_variables_allocate_copy_deallocate()
{
	set_up_vm_variables_and_allocate_extra_page();
	copy_deallocate();
}

/* Allocate some destination and source memory, and copy the source
 * into the destination memory. */
void
set_up_source_and_dest_variables_allocate_copy()
{
	set_up_vm_and_buffer_variables_allocate_for_copying();
	copy_region();
}

/**************************************/
/* mach_vm_protect() set up functions */
/**************************************/

void
set_up_vm_variables_allocate_protect(vm_prot_t protection, const char * protection_name)
{
	set_up_vm_variables_and_allocate_extra_page();
	mach_vm_size_t size       = get_vm_size();
	mach_vm_address_t address = get_vm_address();

	logv(
		"Setting %s-protection on 0x%jx (%ju) byte%s at address "
		"0x%jx...",
		protection_name, (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s", (uintmax_t)address);
	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_protect(mach_task_self(), address, size, FALSE, protection), "mach_vm_protect()");
	logv("Region %s-protected.", protection_name);
}

void
set_up_vm_variables_allocate_readprotect()
{
	set_up_vm_variables_allocate_protect(VM_PROT_WRITE, "read");
}

void
set_up_vm_variables_allocate_writeprotect()
{
	set_up_vm_variables_allocate_protect(VM_PROT_READ, "write");
}

/*****************/
/* Address tests */
/*****************/

/* Allocated address is nonzero iff size is nonzero. */
void
test_nonzero_address_iff_nonzero_size()
{
	mach_vm_address_t address = get_vm_address();
	mach_vm_size_t size       = get_vm_size();

	T_QUIET; T_ASSERT_TRUE((address && size) || (!address && !size), "Address 0x%jx is unexpectedly %szero.", (uintmax_t)address,
	    address ? "non" : "");
	logv("Address 0x%jx is %szero as expected.", (uintmax_t)address, size ? "non" : "");
}

/* Allocated address is aligned. */
void
test_aligned_address()
{
	mach_vm_address_t address = get_vm_address();

	assert_aligned_address(address);
	logv("Address 0x%jx is aligned.", (uintmax_t)address);
}

/************************/
/* Read and write tests */
/************************/

void
verify_pattern(
	address_filter_t filter, boolean_t reversed, mach_vm_address_t address, mach_vm_size_t size, const char * pattern_name)
{
	logv(
		"Verifying %s pattern on region of address 0x%jx "
		"and size 0x%jx (%ju)...",
		pattern_name, (uintmax_t)address, (uintmax_t)size, (uintmax_t)size);
	filter_addresses_do_else(filter, reversed, address, size, verify_address, read_zero, address);
	logv("Pattern verified.");
}

void
write_pattern(
	address_filter_t filter, boolean_t reversed, mach_vm_address_t address, mach_vm_size_t size, const char * pattern_name)
{
	logv(
		"Writing %s pattern on region of address 0x%jx "
		"and size 0x%jx (%ju)...",
		pattern_name, (uintmax_t)address, (uintmax_t)size, (uintmax_t)size);
	filter_addresses_do_else(filter, reversed, address, size, write_address, no_action, address);
	logv("Pattern writen.");
}

void
write_and_verify_pattern(
	address_filter_t filter, boolean_t reversed, mach_vm_address_t address, mach_vm_size_t size, const char * pattern_name)
{
	logv(
		"Writing and verifying %s pattern on region of "
		"address 0x%jx and size 0x%jx (%ju)...",
		pattern_name, (uintmax_t)address, (uintmax_t)size, (uintmax_t)size);
	filter_addresses_do_else(filter, reversed, address, size, write_address, no_action, address);
	filter_addresses_do_else(filter, reversed, address, size, verify_address, read_zero, address);
	logv("Pattern written and verified.");
}

/* Verify that the smallest aligned region containing the
 * given range is zero-filled. */
void
test_zero_filled()
{
	verify_pattern(empty, FALSE, mach_vm_trunc_page(get_vm_address()), aligned_size(get_vm_address(), get_vm_size()),
	    "zero-filled");
}

void
test_write_address_filled()
{
	write_and_verify_pattern(empty, TRUE, get_vm_address(), round_page(get_vm_size()), "address-filled");
}

void
test_write_checkerboard()
{
	write_and_verify_pattern(checkerboard, FALSE, get_vm_address(), round_page(get_vm_size()), "checkerboard");
}

void
test_write_reverse_checkerboard()
{
	write_and_verify_pattern(checkerboard, TRUE, get_vm_address(), round_page(get_vm_size()), "reverse checkerboard");
}

void
test_write_page_ends()
{
	write_and_verify_pattern(page_ends, FALSE, get_vm_address(), round_page(get_vm_size()), "page ends");
}

void
test_write_page_interiors()
{
	write_and_verify_pattern(page_ends, TRUE, get_vm_address(), round_page(get_vm_size()), "page interiors");
}

/*********************************/
/* Allocation error return tests */
/*********************************/

/* Reallocating a page in the smallest aligned region containing the
 * given allocated range fails. */
void
test_reallocate_pages()
{
	allocate_fn_t allocator   = get_allocator();
	vm_map_t this_task        = mach_task_self();
	mach_vm_address_t address = mach_vm_trunc_page(get_vm_address());
	mach_vm_size_t size       = aligned_size(get_vm_address(), get_vm_size());
	mach_vm_address_t i;
	kern_return_t kr;

	logv(
		"Reallocating pages in allocated region of address 0x%jx "
		"and size 0x%jx (%ju)...",
		(uintmax_t)address, (uintmax_t)size, (uintmax_t)size);
	for (i = address; i < address + size; i += vm_page_size) {
		kr = allocator(this_task, &i, vm_page_size, VM_FLAGS_FIXED);
		T_QUIET; T_ASSERT_EQ(kr, KERN_NO_SPACE,
		    "Allocator "
		    "at address 0x%jx unexpectedly returned: %s.\n"
		    "Should have returned: %s.",
		    (uintmax_t)address, mach_error_string(kr), mach_error_string(KERN_NO_SPACE));
	}
	logv("Returned expected error at each page: %s.", mach_error_string(KERN_NO_SPACE));
}

/* Allocating in VM_MAP_NULL fails. */
void
test_allocate_in_null_map()
{
	mach_vm_address_t address = get_vm_address();
	mach_vm_size_t size       = get_vm_size();
	int flag                  = get_address_flag();

	logv("Allocating 0x%jx (%ju) byte%s", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s");
	if (!(flag & VM_FLAGS_ANYWHERE)) {
		logv(" at address 0x%jx", (uintmax_t)address);
	}
	logv(" in NULL VM map...");
	assert_mach_return(get_allocator()(VM_MAP_NULL, &address, size, flag), MACH_SEND_INVALID_DEST, "Allocator");
	logv("Returned expected error: %s.", mach_error_string(MACH_SEND_INVALID_DEST));
}

/* Allocating with non-user flags fails. */
void
test_allocate_with_kernel_flags()
{
	allocate_fn_t allocator   = get_allocator();
	vm_map_t this_task        = mach_task_self();
	mach_vm_address_t address = get_vm_address();
	mach_vm_size_t size       = get_vm_size();
	int flag                  = get_address_flag();
	int bad_flag, i;
	kern_return_t kr;
	int kernel_flags[] = {0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x8000, INT_MAX};
	int numofflags     = sizeof(kernel_flags) / sizeof(kernel_flags[0]);

	logv("Allocating 0x%jx (%ju) byte%s", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s");
	if (!(flag & VM_FLAGS_ANYWHERE)) {
		logv(" at address 0x%jx", (uintmax_t)address);
	}
	logv(" with various kernel flags...");
	for (i = 0; i < numofflags; i++) {
		bad_flag = kernel_flags[i] | flag;
		kr = allocator(this_task, &address, size, bad_flag);
		T_QUIET; T_ASSERT_EQ(kr, KERN_INVALID_ARGUMENT,
		    "Allocator "
		    "with kernel flag 0x%x unexpectedly returned: %s.\n"
		    "Should have returned: %s.",
		    bad_flag, mach_error_string(kr), mach_error_string(KERN_INVALID_ARGUMENT));
	}
	logv("Returned expected error with each kernel flag: %s.", mach_error_string(KERN_INVALID_ARGUMENT));
}

/*****************************/
/* mach_vm_map() error tests */
/*****************************/

/* mach_vm_map() fails with invalid protection or inheritance
 *  arguments. */
void
test_mach_vm_map_protection_inheritance_error()
{
	kern_return_t kr;
	vm_map_t my_task          = mach_task_self();
	mach_vm_address_t address = get_vm_address();
	mach_vm_size_t size       = get_vm_size();
	vm_map_offset_t mask = (get_allocator() == wrapper_mach_vm_map || get_allocator() == wrapper_mach_vm_map_named_entry)
	    ? (mach_vm_offset_t)0
	    : (mach_vm_offset_t)get_mask();
	int flag                    = get_address_flag();
	mach_port_t object_handle   = (get_allocator() == wrapper_mach_vm_map_named_entry) ? memory_entry(&size) : MACH_PORT_NULL;
	vm_prot_t cur_protections[] = {VM_PROT_DEFAULT, VM_PROT_ALL + 1, ~VM_PROT_IS_MASK, INT_MAX};
	vm_prot_t max_protections[] = {VM_PROT_ALL, VM_PROT_ALL + 1, ~VM_PROT_IS_MASK, INT_MAX};
	vm_inherit_t inheritances[] = {VM_INHERIT_DEFAULT, VM_INHERIT_LAST_VALID + 1, UINT_MAX};
	int i, j, k;

	logv("Allocating 0x%jx (%ju) byte%s", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s");
	if (!(flag & VM_FLAGS_ANYWHERE)) {
		logv(" at address 0x%jx", (uintmax_t)address);
	}
	logv(
		" with various invalid protection/inheritance "
		"arguments...");

	for (i = 0; i < 4; i++) {
		for (j = 0; j < 4; j++) {
			for (k = 0; k < 3; k++) {
				/* Skip the case with all valid arguments. */
				if (i == (j == (k == 0))) {
					continue;
				}
				kr = mach_vm_map(my_task, &address, size, mask, flag, object_handle, (memory_object_offset_t)0, FALSE,
				    cur_protections[i], max_protections[j], inheritances[k]);
				T_QUIET; T_ASSERT_EQ(kr, KERN_INVALID_ARGUMENT,
				    "mach_vm_map() "
				    "with cur_protection 0x%x, max_protection 0x%x, "
				    "inheritance 0x%x unexpectedly returned: %s.\n"
				    "Should have returned: %s.",
				    cur_protections[i], max_protections[j], inheritances[k], mach_error_string(kr),
				    mach_error_string(KERN_INVALID_ARGUMENT));
			}
		}
	}
	logv("Returned expected error in each case: %s.", mach_error_string(KERN_INVALID_ARGUMENT));
}

/* mach_vm_map() with unspecified address fails if the starting
 *  address overflows when rounded up to a boundary value. */
void
test_mach_vm_map_large_mask_overflow_error()
{
	mach_vm_address_t address = 0x1;
	mach_vm_size_t size       = get_vm_size();
	mach_vm_offset_t mask     = (mach_vm_offset_t)UINTMAX_MAX;
	/* mach_vm_map() cannot allocate 0 bytes at an unspecified
	 *  address, see 8003930. */
	kern_return_t kr_expected = size ? KERN_NO_SPACE : KERN_INVALID_ARGUMENT;

	logv(
		"Allocating 0x%jx (%ju) byte%s at an unspecified address "
		"starting at 0x%jx with mask 0x%jx...",
		(uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s", (uintmax_t)address, (uintmax_t)mask);
	assert_mach_return(mach_vm_map(mach_task_self(), &address, size, mask, VM_FLAGS_ANYWHERE, MACH_PORT_NULL,
	    (memory_object_offset_t)0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT),
	    kr_expected, "mach_vm_map()");
	logv("Returned expected error: %s.", mach_error_string(kr_expected));
}

/************************/
/* Size edge case tests */
/************************/

void
allocate_edge_size(mach_vm_address_t * address, mach_vm_size_t size, kern_return_t expected_kr)
{
	logv("Allocating 0x%jx (%ju) bytes...", (uintmax_t)size, (uintmax_t)size);
	assert_allocate_return(address, size, VM_FLAGS_ANYWHERE, expected_kr);
	logv("Returned expected value: %s.", mach_error_string(expected_kr));
}

void
test_allocate_zero_size()
{
	mach_vm_address_t address = 0x0;
	/* mach_vm_map() cannot allocate 0 bytes at an unspecified
	 *  address, see 8003930. Other allocators succeed. */
	kern_return_t kr_expected = (get_allocator() != wrapper_mach_vm_allocate) ? KERN_INVALID_ARGUMENT : KERN_SUCCESS;

	allocate_edge_size(&address, 0, kr_expected);
	if (kr_expected == KERN_SUCCESS) {
		deallocate_range(address, 0);
	}
}

/* Testing the allocation of the largest size that does not overflow
 * when rounded up to a page-aligned value. */
void
test_allocate_invalid_large_size()
{
	mach_vm_size_t size = (mach_vm_size_t)UINTMAX_MAX - vm_page_size + 1;
	if (get_allocator() != wrapper_mach_vm_map_named_entry) {
		mach_vm_address_t address = 0x0;
		allocate_edge_size(&address, size, KERN_NO_SPACE);
	} else {
		/* Named entries cannot currently be bigger than 4 GB
		 *  - 4 kb. */
		mach_port_t object_handle = MACH_PORT_NULL;
		logv("Creating named entry of 0x%jx (%ju) bytes...", (uintmax_t)size, (uintmax_t)size);
		assert_mach_return(mach_make_memory_entry_64(mach_task_self(), &size, (memory_object_offset_t)0,
		    (MAP_MEM_NAMED_CREATE | VM_PROT_ALL), &object_handle, 0),
		    KERN_FAILURE, "mach_make_memory_entry_64()");
		logv("Returned expected error: %s.", mach_error_string(KERN_FAILURE));
	}
}

/* A UINTMAX_MAX VM size will overflow to 0 when rounded up to a
 * page-aligned value. */
void
test_allocate_overflowing_size()
{
	mach_vm_address_t address = 0x0;

	allocate_edge_size(&address, (mach_vm_size_t)UINTMAX_MAX, KERN_INVALID_ARGUMENT);
}

/****************************/
/* Address allocation tests */
/****************************/

/* Allocation at address zero fails iff size is nonzero. */
void
test_allocate_at_zero()
{
	mach_vm_address_t address = 0x0;
	mach_vm_size_t size       = get_vm_size();

	kern_return_t kr_expected =
	    size ? KERN_INVALID_ADDRESS : (get_allocator() != wrapper_mach_vm_allocate) ? KERN_INVALID_ARGUMENT : KERN_SUCCESS;

	logv("Allocating 0x%jx (%ju) byte%s at address 0x0...", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s");
	assert_allocate_return(&address, size, VM_FLAGS_FIXED, kr_expected);
	logv("Returned expected value: %s.", mach_error_string(kr_expected));
	if (kr_expected == KERN_SUCCESS) {
		T_QUIET; T_ASSERT_EQ(address, 0,
		    "Address 0x%jx is unexpectedly "
		    "nonzero.\n",
		    (uintmax_t)address);
		logv("Allocated address 0x%jx is zero.", (uintmax_t)address);
		deallocate_range(address, size);
	}
}

/* Allocation at page-aligned but 2 MB boundary-unaligned address
 *  fails with KERN_NO_SPACE. */
void
test_allocate_2MB_boundary_unaligned_page_aligned_address()
{
	mach_vm_size_t size = get_vm_size();

	mach_vm_address_t address = get_fixed_address(size + vm_page_size) + vm_page_size;
	logv(
		"Found 2 MB boundary-unaligned, page aligned address "
		"0x%jx.",
		(uintmax_t)address);

	/* mach_vm_allocate() cannot allocate 0 bytes, and fails with a
	 *  fixed boundary-unaligned truncated address. */
	kern_return_t kr_expected = (!size && get_allocator() != wrapper_mach_vm_allocate)
	    ? KERN_INVALID_ARGUMENT
	    : (get_allocator() == wrapper_mach_vm_map_2MB) ? KERN_NO_SPACE : KERN_SUCCESS;
	logv("Allocating 0x%jx (%ju) byte%s at address 0x%jx...", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s",
	    (uintmax_t)address);
	assert_allocate_return(&address, size, VM_FLAGS_FIXED, kr_expected);
	logv("Returned expected value: %s.", mach_error_string(kr_expected));
	if (kr_expected == KERN_SUCCESS) {
		deallocate_range(address, size);
	}
}

/* With VM_FLAGS_ANYWHERE set, mach_vm_allocate() starts looking for
 *  an allocation address at 0x0, while mach_vm_map() starts at the
 *  supplied address and does not wrap around. See 8016663. */
void
test_allocate_page_with_highest_address_hint()
{
	/* Highest valid page-aligned address. */
	mach_vm_address_t address = (mach_vm_size_t)UINTMAX_MAX - vm_page_size + 1;

	logv(
		"Allocating one page with unspecified address, but hint at "
		"0x%jx...",
		(uintmax_t)address);
	if (get_allocator() == wrapper_mach_vm_allocate) {
		/* mach_vm_allocate() starts from 0x0 and succeeds. */
		assert_allocate_success(&address, vm_page_size, VM_FLAGS_ANYWHERE);
		logv("Memory allocated at address 0x%jx.", (uintmax_t)address);
		assert_aligned_address(address);
		deallocate_range(address, vm_page_size);
	} else {
		/* mach_vm_map() starts from the supplied address, and fails
		 *  with KERN_NO_SPACE, see 8016663. */
		assert_allocate_return(&address, vm_page_size, VM_FLAGS_ANYWHERE, KERN_NO_SPACE);
		logv("Returned expected error: %s.", mach_error_string(KERN_NO_SPACE));
	}
}

/* Allocators find an allocation address with a first fit strategy. */
void
test_allocate_first_fit_pages()
{
	allocate_fn_t allocator    = get_allocator();
	mach_vm_address_t address1 = 0x0;
	mach_vm_address_t i;
	kern_return_t kr;
	vm_map_t this_task = mach_task_self();

	logv(
		"Looking for first fit address for allocating one "
		"page...");
	assert_allocate_success(&address1, vm_page_size, VM_FLAGS_ANYWHERE);
	logv("Found address 0x%jx.", (uintmax_t)address1);
	assert_aligned_address(address1);
	mach_vm_address_t address2 = address1;
	logv(
		"Looking for next higher first fit address for allocating "
		"one page...");
	assert_allocate_success(&address2, vm_page_size, VM_FLAGS_ANYWHERE);
	logv("Found address 0x%jx.", (uintmax_t)address2);
	assert_aligned_address(address2);
	T_QUIET; T_ASSERT_GT(address2, address1,
	    "Second address 0x%jx is "
	    "unexpectedly not higher than first address 0x%jx.",
	    (uintmax_t)address2, (uintmax_t)address1);

	logv("Allocating pages between 0x%jx and 0x%jx...", (uintmax_t)address1, (uintmax_t)address2);
	for (i = address1; i <= address2; i += vm_page_size) {
		kr = allocator(this_task, &i, vm_page_size, VM_FLAGS_FIXED);
		T_QUIET; T_ASSERT_NE(kr, KERN_SUCCESS,
		    "Allocator at address 0x%jx "
		    "unexpectedly succeeded.",
		    (uintmax_t)i);
	}
	logv("Expectedly returned error at each page.");
	deallocate_range(address1, vm_page_size);
	deallocate_range(address2, vm_page_size);
}

/*******************************/
/* Deallocation segfault tests */
/*******************************/

/* mach_vm_deallocate() deallocates the smallest aligned region
 * (integral number of pages) containing the given range. */

/* Addresses in deallocated range are inaccessible. */
void
access_deallocated_range_address(mach_vm_address_t address, const char * position)
{
	logv("Will deallocate and read from %s 0x%jx of deallocated range...", position, (uintmax_t)address);
	deallocate();
	mach_vm_address_t bad_value = MACH_VM_ADDRESS_T(address);
	T_ASSERT_FAIL("Unexpectedly read value 0x%jx at address 0x%jx.\n"
	    "Should have died with signal SIGSEGV.",
	    (uintmax_t)bad_value, (uintmax_t)address);
}

/* Start of deallocated range is inaccessible. */
void
test_access_deallocated_range_start()
{
	access_deallocated_range_address(get_vm_address(), "start");
}

/* Middle of deallocated range is inaccessible. */
void
test_access_deallocated_range_middle()
{
	access_deallocated_range_address(get_vm_address() + (round_page(get_vm_size()) >> 1), "middle");
}

/* End of deallocated range is inaccessible. */
void
test_access_deallocated_range_end()
{
	access_deallocated_range_address(round_page(get_vm_size()) - vm_address_size + get_vm_address(), "end");
}

/* Deallocating almost the whole address space causes a SIGSEGV or SIGBUS. We
 * deallocate the largest valid aligned size to avoid overflowing when
 * rounding up. */
void
test_deallocate_suicide()
{
	mach_vm_address_t address = 0x0;
	mach_vm_size_t size       = (mach_vm_size_t)UINTMAX_MAX - vm_page_size + 1;

	logv("Deallocating 0x%jx (%ju) bytes at address 0x%jx...", (uintmax_t)size, (uintmax_t)size, (uintmax_t)address);
	kern_return_t kr = mach_vm_deallocate(mach_task_self(), address, size);
	T_ASSERT_FAIL("mach_vm_deallocate() with address 0x%jx and "
	    "size 0x%jx (%ju) unexpectedly returned: %s.\n"
	    "Should have died with signal SIGSEGV or SIGBUS.",
	    (uintmax_t)address, (uintmax_t)size, (uintmax_t)size, mach_error_string(kr));
}

/***************************************/
/* Deallocation and reallocation tests */
/***************************************/

/* Deallocating memory twice succeeds. */
void
test_deallocate_twice()
{
	deallocate();
	deallocate();
}

/* Deallocated and reallocated memory is zero-filled. Deallocated
 * memory is inaccessible since it can be reallocated. */
void
test_write_pattern_deallocate_reallocate_zero_filled()
{
	mach_vm_address_t address = get_vm_address();
	mach_vm_size_t size       = get_vm_size();

	write_pattern(page_ends, FALSE, address, size, "page ends");
	logv("Deallocating, then Allocating 0x%jx (%ju) byte%s at address 0x%jx...", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s",
	    (uintmax_t)address);
	deallocate();
	assert_allocate_success(&address, size, VM_FLAGS_FIXED);
	logv("Memory allocated.");
	verify_pattern(empty, FALSE, address, size, "zero-filled");
	deallocate();
}

/********************************/
/* Deallocation edge case tests */
/********************************/

/* Zero size deallocation always succeeds. */
void
test_deallocate_zero_size_ranges()
{
	int i;
	kern_return_t kr;
	vm_map_t this_task            = mach_task_self();
	mach_vm_address_t addresses[] = {0x0,
		                         0x1,
		                         vm_page_size - 1,
		                         vm_page_size,
		                         vm_page_size + 1,
		                         (mach_vm_address_t)UINT_MAX - vm_page_size + 1,
		                         (mach_vm_address_t)UINT_MAX,
		                         (mach_vm_address_t)UINTMAX_MAX - vm_page_size + 1,
		                         (mach_vm_address_t)UINTMAX_MAX};
	int numofaddresses = sizeof(addresses) / sizeof(addresses[0]);

	logv("Deallocating 0x0 (0) bytes at various addresses...");
	for (i = 0; i < numofaddresses; i++) {
		kr = mach_vm_deallocate(this_task, addresses[i], 0);
		T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "mach_vm_deallocate() at "
		    "address 0x%jx unexpectedly failed: %s.",
		    (uintmax_t)addresses[i], mach_error_string(kr));
	}
	logv("Deallocations successful.");
}

/* Deallocation succeeds if the end of the range rounds to 0x0. */
void
test_deallocate_rounded_zero_end_ranges()
{
	int i;
	kern_return_t kr;
	vm_map_t this_task = mach_task_self();
	struct {
		mach_vm_address_t address;
		mach_vm_size_t size;
	} ranges[] = {
		{0x0, (mach_vm_size_t)UINTMAX_MAX},
		{0x0, (mach_vm_size_t)UINTMAX_MAX - vm_page_size + 2},
		{0x1, (mach_vm_size_t)UINTMAX_MAX - 1},
		{0x1, (mach_vm_size_t)UINTMAX_MAX - vm_page_size + 1},
		{0x2, (mach_vm_size_t)UINTMAX_MAX - 2},
		{0x2, (mach_vm_size_t)UINTMAX_MAX - vm_page_size},
		{(mach_vm_address_t)UINTMAX_MAX - vm_page_size + 1, vm_page_size - 1},
		{(mach_vm_address_t)UINTMAX_MAX - vm_page_size + 1, 1},
		{(mach_vm_address_t)UINTMAX_MAX - 1, 1},
	};
	int numofranges = sizeof(ranges) / sizeof(ranges[0]);

	logv(
		"Deallocating various memory ranges whose end rounds to "
		"0x0...");
	for (i = 0; i < numofranges; i++) {
		kr = mach_vm_deallocate(this_task, ranges[i].address, ranges[i].size);
		T_QUIET; T_ASSERT_MACH_SUCCESS(kr,
		    "mach_vm_deallocate() with address 0x%jx and size "
		    "0x%jx (%ju) unexpectedly returned: %s.\n"
		    "Should have succeeded.",
		    (uintmax_t)ranges[i].address, (uintmax_t)ranges[i].size, (uintmax_t)ranges[i].size, mach_error_string(kr));
	}
	logv("Deallocations successful.");
}

/* Deallocating a range wrapped around the address space fails. */
void
test_deallocate_wrapped_around_ranges()
{
	int i;
	kern_return_t kr;
	vm_map_t this_task = mach_task_self();
	struct {
		mach_vm_address_t address;
		mach_vm_size_t size;
	} ranges[] = {
		{0x1, (mach_vm_size_t)UINTMAX_MAX},
		{vm_page_size, (mach_vm_size_t)UINTMAX_MAX - vm_page_size + 1},
		{(mach_vm_address_t)UINTMAX_MAX - vm_page_size + 1, vm_page_size},
		{(mach_vm_address_t)UINTMAX_MAX, 1},
	};
	int numofranges = sizeof(ranges) / sizeof(ranges[0]);

	logv(
		"Deallocating various memory ranges wrapping around the "
		"address space...");
	for (i = 0; i < numofranges; i++) {
		kr = mach_vm_deallocate(this_task, ranges[i].address, ranges[i].size);
		T_QUIET; T_ASSERT_EQ(kr, KERN_INVALID_ARGUMENT,
		    "mach_vm_deallocate() with address 0x%jx and size "
		    "0x%jx (%ju) unexpectedly returned: %s.\n"
		    "Should have returned: %s.",
		    (uintmax_t)ranges[i].address, (uintmax_t)ranges[i].size, (uintmax_t)ranges[i].size, mach_error_string(kr),
		    mach_error_string(KERN_INVALID_ARGUMENT));
	}
	logv("Returned expected error on each range: %s.", mach_error_string(KERN_INVALID_ARGUMENT));
}

/* Deallocating in VM_MAP_NULL fails. */
void
test_deallocate_in_null_map()
{
	mach_vm_address_t address = get_vm_address();
	mach_vm_size_t size       = get_vm_size();
	int flag                  = get_address_flag();

	logv("Deallocating 0x%jx (%ju) byte%s", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s");
	if (!(flag & VM_FLAGS_ANYWHERE)) {
		logv(" at address 0x%jx", (uintmax_t)address);
	}
	logv(" in NULL VM map...");
	assert_mach_return(mach_vm_deallocate(VM_MAP_NULL, address, size), MACH_SEND_INVALID_DEST, "mach_vm_deallocate()");
	logv("Returned expected error: %s.", mach_error_string(MACH_SEND_INVALID_DEST));
}

/*****************************/
/* mach_vm_read() main tests */
/*****************************/

/* Read memory of size less than a page has aligned starting
 * address. Otherwise, the destination buffer's starting address has
 * the same boundary offset as the source region's. */
void
test_read_address_offset()
{
	mach_vm_address_t address = get_vm_address();
	mach_vm_size_t size       = get_vm_size();

	if (size < vm_page_size * 2 || get_address_alignment()) {
		assert_aligned_address(address);
		logv("Buffer address 0x%jx is aligned as expected.", (uintmax_t)address);
	} else {
		T_QUIET; T_ASSERT_EQ(((address - 1) & (vm_page_size - 1)), 0,
		    "Buffer "
		    "address 0x%jx does not have the expected boundary "
		    "offset of 1.",
		    (uintmax_t)address);
		logv(
			"Buffer address 0x%jx has the expected boundary "
			"offset of 1.",
			(uintmax_t)address);
	}
}

/* Reading from VM_MAP_NULL fails. */
void
test_read_null_map()
{
	mach_vm_address_t address = get_vm_address();
	mach_vm_size_t size       = get_vm_size();
	vm_offset_t read_address;
	mach_msg_type_number_t read_size;

	logv(
		"Reading 0x%jx (%ju) byte%s at address 0x%jx in NULL VM "
		"map...",
		(uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s", (uintmax_t)address);
	assert_mach_return(mach_vm_read(VM_MAP_NULL, address, size, &read_address, &read_size), MACH_SEND_INVALID_DEST,
	    "mach_vm_read()");
	logv("Returned expected error: %s.", mach_error_string(MACH_SEND_INVALID_DEST));
}

/* Reading partially deallocated memory fails. */
void
test_read_partially_deallocated_range()
{
	mach_vm_address_t address   = get_vm_address();
	mach_vm_size_t size         = get_vm_size();
	mach_vm_address_t mid_point = mach_vm_trunc_page(address + size / 2);
	vm_offset_t read_address;
	mach_msg_type_number_t read_size;

	logv("Deallocating a mid-range page at address 0x%jx...", (uintmax_t)mid_point);
	assert_deallocate_success(mid_point, vm_page_size);
	logv("Page deallocated.");

	logv("Reading 0x%jx (%ju) byte%s at address 0x%jx...", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s",
	    (uintmax_t)address);
	assert_read_return(address, size, &read_address, &read_size, KERN_INVALID_ADDRESS);
	logv("Returned expected error: %s.", mach_error_string(KERN_INVALID_ADDRESS));
}

/* Reading partially read-protected memory fails. */
void
test_read_partially_unreadable_range()
{
	mach_vm_address_t address   = get_vm_address();
	mach_vm_size_t size         = get_vm_size();
	mach_vm_address_t mid_point = mach_vm_trunc_page(address + size / 2);
	vm_offset_t read_address;
	mach_msg_type_number_t read_size;

	/*  For sizes < msg_ool_size_small, vm_map_copyin_common() uses
	 *  vm_map_copyin_kernel_buffer() to read in the memory,
	 *  returning different errors, see 8182239. */
	kern_return_t kr_expected = (size < vm_page_size * 2) ? KERN_INVALID_ADDRESS : KERN_PROTECTION_FAILURE;

	logv("Read-protecting a mid-range page at address 0x%jx...", (uintmax_t)mid_point);
	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_protect(mach_task_self(), mid_point, vm_page_size, FALSE, VM_PROT_WRITE), "mach_vm_protect()");
	logv("Page read-protected.");

	logv("Reading 0x%jx (%ju) byte%s at address 0x%jx...", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s",
	    (uintmax_t)address);
	assert_read_return(address, size, &read_address, &read_size, kr_expected);
	logv("Returned expected error: %s.", mach_error_string(kr_expected));
}

/**********************************/
/* mach_vm_read() edge case tests */
/**********************************/

void
read_edge_size(mach_vm_size_t size, kern_return_t expected_kr)
{
	int i;
	kern_return_t kr;
	vm_map_t this_task            = mach_task_self();
	mach_vm_address_t addresses[] = {vm_page_size - 1,
		                         vm_page_size,
		                         vm_page_size + 1,
		                         (mach_vm_address_t)UINT_MAX - vm_page_size + 1,
		                         (mach_vm_address_t)UINT_MAX,
		                         (mach_vm_address_t)UINTMAX_MAX - vm_page_size + 1,
		                         (mach_vm_address_t)UINTMAX_MAX};
	int numofaddresses = sizeof(addresses) / sizeof(addresses[0]);
	vm_offset_t read_address;
	mach_msg_type_number_t read_size;

	logv("Reading 0x%jx (%ju) bytes at various addresses...", (uintmax_t)size, (uintmax_t)size);
	for (i = 0; i < numofaddresses; i++) {
		kr = mach_vm_read(this_task, addresses[i], size, &read_address, &read_size);
		T_QUIET; T_ASSERT_EQ(kr, expected_kr,
		    "mach_vm_read() at "
		    "address 0x%jx unexpectedly returned: %s.\n"
		    "Should have returned: %s.",
		    (uintmax_t)addresses[i], mach_error_string(kr), mach_error_string(expected_kr));
	}
	logv(
		"mach_vm_read() returned expected value in each case: "
		"%s.",
		mach_error_string(expected_kr));
}

/* Reading 0 bytes always succeeds. */
void
test_read_zero_size()
{
	read_edge_size(0, KERN_SUCCESS);
}

/* Reading 4GB or higher always fails. */
void
test_read_invalid_large_size()
{
	read_edge_size((mach_vm_size_t)UINT_MAX + 1, KERN_INVALID_ARGUMENT);
}

/* Reading a range wrapped around the address space fails. */
void
test_read_wrapped_around_ranges()
{
	int i;
	kern_return_t kr;
	vm_map_t this_task = mach_task_self();
	struct {
		mach_vm_address_t address;
		mach_vm_size_t size;
	} ranges[] = {
		{(mach_vm_address_t)(UINTMAX_MAX - UINT_MAX + 1), (mach_vm_size_t)UINT_MAX},
		{(mach_vm_address_t)(UINTMAX_MAX - UINT_MAX + vm_page_size), (mach_vm_size_t)(UINT_MAX - vm_page_size + 1)},
		{(mach_vm_address_t)UINTMAX_MAX - vm_page_size + 1, vm_page_size},
		{(mach_vm_address_t)UINTMAX_MAX, 1},
	};
	int numofranges = sizeof(ranges) / sizeof(ranges[0]);
	vm_offset_t read_address;
	mach_msg_type_number_t read_size;

	logv(
		"Reading various memory ranges wrapping around the "
		"address space...");
	for (i = 0; i < numofranges; i++) {
		kr = mach_vm_read(this_task, ranges[i].address, ranges[i].size, &read_address, &read_size);
		T_QUIET; T_ASSERT_EQ(kr, KERN_INVALID_ADDRESS,
		    "mach_vm_read() at address 0x%jx with size "
		    "0x%jx (%ju) unexpectedly returned: %s.\n"
		    "Should have returned: %s.",
		    (uintmax_t)ranges[i].address, (uintmax_t)ranges[i].size, (uintmax_t)ranges[i].size, mach_error_string(kr),
		    mach_error_string(KERN_INVALID_ADDRESS));
	}
	logv("Returned expected error on each range: %s.", mach_error_string(KERN_INVALID_ADDRESS));
}

/********************************/
/* mach_vm_read() pattern tests */
/********************************/

/* Write a pattern on pre-allocated memory, read into a buffer and
 * verify the pattern on the buffer. */
void
write_read_verify_pattern(address_filter_t filter, boolean_t reversed, const char * pattern_name)
{
	mach_vm_address_t address = get_vm_address();

	write_pattern(filter, reversed, address, get_vm_size(), pattern_name);
	read_deallocate();
	/* Getting the address and size of the read buffer. */
	mach_vm_address_t read_address = get_vm_address();
	mach_vm_size_t read_size = get_vm_size();
	logv(
		"Verifying %s pattern on buffer of "
		"address 0x%jx and size 0x%jx (%ju)...",
		pattern_name, (uintmax_t)read_address, (uintmax_t)read_size, (uintmax_t)read_size);
	filter_addresses_do_else(filter, reversed, read_address, read_size, verify_address, read_zero, address);
	logv("Pattern verified on destination buffer.");
}

void
test_read_address_filled()
{
	write_read_verify_pattern(empty, TRUE, "address-filled");
}

void
test_read_checkerboard()
{
	write_read_verify_pattern(checkerboard, FALSE, "checkerboard");
}

void
test_read_reverse_checkerboard()
{
	write_read_verify_pattern(checkerboard, TRUE, "reverse checkerboard");
}

/***********************************/
/* mach_vm_write() edge case tests */
/***********************************/

/* Writing in VM_MAP_NULL fails. */
void
test_write_null_map()
{
	mach_vm_address_t address          = get_vm_address();
	vm_offset_t data                   = (vm_offset_t)get_buffer_address();
	mach_msg_type_number_t buffer_size = (mach_msg_type_number_t)get_buffer_size();

	logv(
		"Writing buffer of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx in NULL VM MAP...",
		(uintmax_t)data, (uintmax_t)buffer_size, (uintmax_t)buffer_size, (uintmax_t)address);
	assert_mach_return(mach_vm_write(VM_MAP_NULL, address, data, buffer_size), MACH_SEND_INVALID_DEST, "mach_vm_write()");
	logv("Returned expected error: %s.", mach_error_string(MACH_SEND_INVALID_DEST));
}

/* Writing 0 bytes always succeeds. */
void
test_write_zero_size()
{
	set_buffer_size(0);
	write_buffer();
}

/*****************************************/
/* mach_vm_write() inaccessibility tests */
/*****************************************/

/* Writing a partially deallocated buffer fails. */
void
test_write_partially_deallocated_buffer()
{
	mach_vm_address_t address          = get_vm_address();
	vm_offset_t data                   = (vm_offset_t)get_buffer_address();
	mach_msg_type_number_t buffer_size = (mach_msg_type_number_t)get_buffer_size();
	mach_vm_address_t buffer_mid_point = (mach_vm_address_t)mach_vm_trunc_page(data + buffer_size / 2);

	logv(
		"Deallocating a mid-range buffer page at address "
		"0x%jx...",
		(uintmax_t)buffer_mid_point);
	assert_deallocate_success(buffer_mid_point, vm_page_size);
	logv("Page deallocated.");

	logv(
		"Writing buffer of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)data, (uintmax_t)buffer_size, (uintmax_t)buffer_size, (uintmax_t)address);
	assert_write_return(address, data, buffer_size, MACH_SEND_INVALID_MEMORY);
	logv("Returned expected error: %s.", mach_error_string(MACH_SEND_INVALID_MEMORY));
}

/* Writing a partially read-protected buffer fails. */
void
test_write_partially_unreadable_buffer()
{
	mach_vm_address_t address          = get_vm_address();
	vm_offset_t data                   = (vm_offset_t)get_buffer_address();
	mach_msg_type_number_t buffer_size = (mach_msg_type_number_t)get_buffer_size();
	mach_vm_address_t buffer_mid_point = (mach_vm_address_t)mach_vm_trunc_page(data + buffer_size / 2);

	logv(
		"Read-protecting a mid-range buffer page at address "
		"0x%jx...",
		(uintmax_t)buffer_mid_point);
	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_protect(mach_task_self(), buffer_mid_point, vm_page_size, FALSE, VM_PROT_WRITE),
	    "mach_vm_protect()");
	logv("Page read-protected.");

	logv(
		"Writing buffer of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)data, (uintmax_t)buffer_size, (uintmax_t)buffer_size, (uintmax_t)address);
	assert_write_return(address, data, buffer_size, MACH_SEND_INVALID_MEMORY);
	logv("Returned expected error: %s.", mach_error_string(MACH_SEND_INVALID_MEMORY));
}

/* Writing on partially deallocated memory fails. */
void
test_write_on_partially_deallocated_range()
{
	mach_vm_address_t address          = get_vm_address();
	mach_vm_address_t start            = mach_vm_trunc_page(address);
	vm_offset_t data                   = (vm_offset_t)get_buffer_address();
	mach_msg_type_number_t buffer_size = (mach_msg_type_number_t)get_buffer_size();

	logv(
		"Deallocating the first destination page at address "
		"0x%jx...",
		(uintmax_t)start);
	assert_deallocate_success(start, vm_page_size);
	logv("Page deallocated.");

	logv(
		"Writing buffer of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)data, (uintmax_t)buffer_size, (uintmax_t)buffer_size, (uintmax_t)address);
	assert_write_return(address, data, buffer_size, KERN_INVALID_ADDRESS);
	logv("Returned expected error: %s.", mach_error_string(KERN_INVALID_ADDRESS));
}

/* Writing on partially unwritable memory fails. */
void
test_write_on_partially_unwritable_range()
{
	mach_vm_address_t address          = get_vm_address();
	mach_vm_address_t start            = mach_vm_trunc_page(address);
	vm_offset_t data                   = (vm_offset_t)get_buffer_address();
	mach_msg_type_number_t buffer_size = (mach_msg_type_number_t)get_buffer_size();

	/*  For sizes < msg_ool_size_small,
	 *  vm_map_copy_overwrite_nested() uses
	 *  vm_map_copyout_kernel_buffer() to read in the memory,
	 *  returning different errors, see 8217123. */
	kern_return_t kr_expected = (buffer_size < vm_page_size * 2) ? KERN_INVALID_ADDRESS : KERN_PROTECTION_FAILURE;

	logv(
		"Write-protecting the first destination page at address "
		"0x%jx...",
		(uintmax_t)start);
	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_protect(mach_task_self(), start, vm_page_size, FALSE, VM_PROT_READ), "mach_vm_protect()");
	logv("Page write-protected.");

	logv(
		"Writing buffer of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)data, (uintmax_t)buffer_size, (uintmax_t)buffer_size, (uintmax_t)address);
	assert_write_return(address, data, buffer_size, kr_expected);
	logv("Returned expected error: %s.", mach_error_string(kr_expected));
}

/*********************************/
/* mach_vm_write() pattern tests */
/*********************************/

/* Verify that a zero-filled buffer and destination memory are still
 * zero-filled after writing. */
void
test_zero_filled_write()
{
	verify_pattern(empty, FALSE, mach_vm_trunc_page(get_vm_address()), round_page(get_vm_size() + 1), "zero-filled");
	verify_pattern(empty, FALSE, mach_vm_trunc_page(get_buffer_address()),
	    round_page(get_buffer_size() + get_buffer_offset()), "zero-filled");
}

/* Write a pattern on a buffer, write the buffer into some destination
 * memory, and verify the pattern on both buffer and destination. */
void
pattern_write(address_filter_t filter, boolean_t reversed, const char * pattern_name)
{
	mach_vm_address_t address        = get_vm_address();
	mach_vm_size_t size              = get_vm_size();
	mach_vm_address_t buffer_address = get_buffer_address();
	mach_vm_size_t buffer_size       = get_buffer_size();

	write_pattern(filter, reversed, buffer_address, buffer_size, pattern_name);
	write_buffer();
	verify_pattern(filter, reversed, buffer_address, buffer_size, pattern_name);
	logv(
		"Verifying %s pattern on destination of "
		"address 0x%jx and size 0x%jx (%ju)...",
		pattern_name, (uintmax_t)address, (uintmax_t)buffer_size, (uintmax_t)size);
	filter_addresses_do_else(filter, reversed, address, buffer_size, verify_address, read_zero, buffer_address);
	logv("Pattern verified on destination.");
}

void
test_address_filled_write()
{
	pattern_write(empty, TRUE, "address-filled");
}

void
test_checkerboard_write()
{
	pattern_write(checkerboard, FALSE, "checkerboard");
}

void
test_reverse_checkerboard_write()
{
	pattern_write(checkerboard, TRUE, "reverse checkerboard");
}

/**********************************/
/* mach_vm_copy() edge case tests */
/**********************************/

/* Copying in VM_MAP_NULL fails. */
void
test_copy_null_map()
{
	mach_vm_address_t source    = get_vm_address();
	mach_vm_address_t dest      = get_buffer_address();
	mach_msg_type_number_t size = (mach_msg_type_number_t)get_buffer_size();

	logv(
		"Copying buffer of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx in NULL VM MAP...",
		(uintmax_t)dest, (uintmax_t)size, (uintmax_t)size, (uintmax_t)source);
	assert_mach_return(mach_vm_copy(VM_MAP_NULL, source, size, dest), MACH_SEND_INVALID_DEST, "mach_vm_copy()");
	logv("Returned expected error: %s.", mach_error_string(MACH_SEND_INVALID_DEST));
}

void
copy_edge_size(mach_vm_size_t size, kern_return_t expected_kr)
{
	int i;
	kern_return_t kr;
	vm_map_t this_task            = mach_task_self();
	mach_vm_address_t addresses[] = {0x0,
		                         0x1,
		                         vm_page_size - 1,
		                         vm_page_size,
		                         vm_page_size + 1,
		                         (mach_vm_address_t)UINT_MAX - vm_page_size + 1,
		                         (mach_vm_address_t)UINT_MAX,
		                         (mach_vm_address_t)UINTMAX_MAX - vm_page_size + 1,
		                         (mach_vm_address_t)UINTMAX_MAX};
	int numofaddresses     = sizeof(addresses) / sizeof(addresses[0]);
	mach_vm_address_t dest = 0;

	logv("Allocating 0x%jx (%ju) byte%s...", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s");
	assert_allocate_success(&dest, 4096, VM_FLAGS_ANYWHERE);
	logv("Copying 0x%jx (%ju) bytes at various addresses...", (uintmax_t)size, (uintmax_t)size);
	for (i = 0; i < numofaddresses; i++) {
		kr = mach_vm_copy(this_task, addresses[i], size, dest);
		T_QUIET; T_ASSERT_EQ(kr, expected_kr,
		    "mach_vm_copy() at "
		    "address 0x%jx unexpectedly returned: %s.\n"
		    "Should have returned: %s.",
		    (uintmax_t)addresses[i], mach_error_string(kr), mach_error_string(expected_kr));
	}
	logv(
		"mach_vm_copy() returned expected value in each case: "
		"%s.",
		mach_error_string(expected_kr));

	deallocate_range(dest, 4096);
}

/* Copying 0 bytes always succeeds. */
void
test_copy_zero_size()
{
	copy_edge_size(0, KERN_SUCCESS);
}

/* Copying 4GB or higher always fails. */
void
test_copy_invalid_large_size()
{
	copy_edge_size((mach_vm_size_t)UINT_MAX - 1, KERN_INVALID_ADDRESS);
}

/* Reading a range wrapped around the address space fails. */
void
test_copy_wrapped_around_ranges()
{
	int i;
	kern_return_t kr;
	vm_map_t this_task = mach_task_self();
	struct {
		mach_vm_address_t address;
		mach_vm_size_t size;
	} ranges[] = {
		{(mach_vm_address_t)(UINTMAX_MAX - UINT_MAX + 1), (mach_vm_size_t)UINT_MAX},
		{(mach_vm_address_t)(UINTMAX_MAX - UINT_MAX + vm_page_size), (mach_vm_size_t)(UINT_MAX - vm_page_size + 1)},
		{(mach_vm_address_t)UINTMAX_MAX - vm_page_size + 1, vm_page_size},
		{(mach_vm_address_t)UINTMAX_MAX, 1},
	};
	int numofranges        = sizeof(ranges) / sizeof(ranges[0]);
	mach_vm_address_t dest = 0;

	logv("Allocating 0x1000 (4096) bytes...");
	assert_allocate_success(&dest, 4096, VM_FLAGS_ANYWHERE);

	logv(
		"Copying various memory ranges wrapping around the "
		"address space...");
	for (i = 0; i < numofranges; i++) {
		kr = mach_vm_copy(this_task, ranges[i].address, ranges[i].size, dest);
		T_QUIET; T_ASSERT_EQ(kr, KERN_INVALID_ADDRESS,
		    "mach_vm_copy() at address 0x%jx with size "
		    "0x%jx (%ju) unexpectedly returned: %s.\n"
		    "Should have returned: %s.",
		    (uintmax_t)ranges[i].address, (uintmax_t)ranges[i].size, (uintmax_t)ranges[i].size, mach_error_string(kr),
		    mach_error_string(KERN_INVALID_ADDRESS));
	}
	logv("Returned expected error on each range: %s.", mach_error_string(KERN_INVALID_ADDRESS));

	deallocate_range(dest, 4096);
}

/********************************/
/* mach_vm_copy() pattern tests */
/********************************/

/* Write a pattern on pre-allocated region, copy into another region
 * and verify the pattern in the region. */
void
write_copy_verify_pattern(address_filter_t filter, boolean_t reversed, const char * pattern_name)
{
	mach_vm_address_t source = get_vm_address();
	mach_vm_size_t src_size = get_vm_size();
	write_pattern(filter, reversed, source, src_size, pattern_name);
	/* Getting the address and size of the dest region */
	mach_vm_address_t dest  = get_buffer_address();
	mach_vm_size_t dst_size = get_buffer_size();

	logv(
		"Copying memory region of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)source, (uintmax_t)dst_size, (uintmax_t)dst_size, (uintmax_t)dest);
	assert_copy_success(source, dst_size, dest);
	logv(
		"Verifying %s pattern in region of "
		"address 0x%jx and size 0x%jx (%ju)...",
		pattern_name, (uintmax_t)dest, (uintmax_t)dst_size, (uintmax_t)dst_size);
	filter_addresses_do_else(filter, reversed, dest, dst_size, verify_address, read_zero, source);
	logv("Pattern verified on destination region.");
}

void
test_copy_address_filled()
{
	write_copy_verify_pattern(empty, TRUE, "address-filled");
}

void
test_copy_checkerboard()
{
	write_copy_verify_pattern(checkerboard, FALSE, "checkerboard");
}

void
test_copy_reverse_checkerboard()
{
	write_copy_verify_pattern(checkerboard, TRUE, "reverse checkerboard");
}

/* Verify that a zero-filled source and destination memory are still
 * zero-filled after writing. */
void
test_zero_filled_copy_dest()
{
	verify_pattern(empty, FALSE, mach_vm_trunc_page(get_vm_address()), round_page(get_vm_size() + 1), "zero-filled");
	verify_pattern(empty, FALSE, mach_vm_trunc_page(get_buffer_address()),
	    round_page(get_buffer_size() + get_buffer_offset()), "zero-filled");
}

/****************************************/
/* mach_vm_copy() inaccessibility tests */
/****************************************/

/* Copying partially deallocated memory fails. */
void
test_copy_partially_deallocated_range()
{
	mach_vm_address_t source    = get_vm_address();
	mach_vm_size_t size         = get_vm_size();
	mach_vm_address_t mid_point = mach_vm_trunc_page(source + size / 2);
	mach_vm_address_t dest      = 0;

	logv("Deallocating a mid-range page at address 0x%jx...", (uintmax_t)mid_point);
	assert_deallocate_success(mid_point, vm_page_size);
	logv("Page deallocated.");

	logv("Copying 0x%jx (%ju) byte%s at address 0x%jx...", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s",
	    (uintmax_t)source);

	assert_allocate_copy_return(source, size, &dest, KERN_INVALID_ADDRESS);

	logv("Returned expected error: %s.", mach_error_string(KERN_INVALID_ADDRESS));

	deallocate_range(dest, size);
}

/* Copy partially read-protected memory fails. */
void
test_copy_partially_unreadable_range()
{
	mach_vm_address_t source    = get_vm_address();
	mach_vm_size_t size         = get_vm_size();
	mach_vm_address_t mid_point = mach_vm_trunc_page(source + size / 2);
	mach_vm_address_t dest      = 0;

	/*  For sizes < 1 page, vm_map_copyin_common() uses
	 *  vm_map_copyin_kernel_buffer() to read in the memory,
	 *  returning different errors, see 8182239. */
	kern_return_t kr_expected = (size < vm_page_size) ? KERN_INVALID_ADDRESS : KERN_PROTECTION_FAILURE;

	logv("Read-protecting a mid-range page at address 0x%jx...", (uintmax_t)mid_point);
	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_protect(mach_task_self(), mid_point, vm_page_size, FALSE, VM_PROT_WRITE), "mach_vm_protect()");
	logv("Page read-protected.");

	logv("Copying 0x%jx (%ju) byte%s at address 0x%jx...", (uintmax_t)size, (uintmax_t)size, (size == 1) ? "" : "s",
	    (uintmax_t)source);
	assert_allocate_copy_return(source, size, &dest, kr_expected);
	logv("Returned expected error: %s.", mach_error_string(kr_expected));

	deallocate_range(dest, size);
}

/* Copying to a partially deallocated region fails. */
void
test_copy_dest_partially_deallocated_region()
{
	mach_vm_address_t dest             = get_vm_address();
	mach_vm_address_t source           = get_buffer_address();
	mach_msg_type_number_t size        = (mach_msg_type_number_t)get_buffer_size();
	mach_vm_address_t source_mid_point = (mach_vm_address_t)mach_vm_trunc_page(dest + size / 2);
#if __MAC_OX_X_VERSION_MIN_REQUIRED > 1080
	logv(
		"Deallocating a mid-range source page at address "
		"0x%jx...",
		(uintmax_t)source_mid_point);
	assert_deallocate_success(source_mid_point, vm_page_size);
	logv("Page deallocated.");

	logv(
		"Copying region of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)source, (uintmax_t)size, (uintmax_t)size, (uintmax_t)dest);
	assert_copy_return(source, size, dest, KERN_INVALID_ADDRESS);
	logv("Returned expected error: %s.", mach_error_string(KERN_INVALID_ADDRESS));
#else
	logv(
		"Bypassing partially deallocated region test "
		"(See <rdar://problem/12190999>)");
#endif /* __MAC_OX_X_VERSION_MIN_REQUIRED > 1080 */
}

/* Copying from a partially deallocated region fails. */
void
test_copy_source_partially_deallocated_region()
{
	mach_vm_address_t source           = get_vm_address();
	mach_vm_address_t dest             = get_buffer_address();
	mach_msg_type_number_t size        = (mach_msg_type_number_t)get_buffer_size();
	mach_vm_address_t source_mid_point = (mach_vm_address_t)mach_vm_trunc_page(source + size / 2);

	logv(
		"Deallocating a mid-range source page at address "
		"0x%jx...",
		(uintmax_t)source_mid_point);
	assert_deallocate_success(source_mid_point, vm_page_size);
	logv("Page deallocated.");

	logv(
		"Copying region of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)source, (uintmax_t)size, (uintmax_t)size, (uintmax_t)dest);
	assert_copy_return(source, size, dest, KERN_INVALID_ADDRESS);
	logv("Returned expected error: %s.", mach_error_string(KERN_INVALID_ADDRESS));
}

/* Copying from a partially read-protected region fails. */
void
test_copy_source_partially_unreadable_region()
{
	mach_vm_address_t source    = get_vm_address();
	mach_vm_address_t dest      = get_buffer_address();
	mach_msg_type_number_t size = (mach_msg_type_number_t)get_buffer_size();
	mach_vm_address_t mid_point = (mach_vm_address_t)mach_vm_trunc_page(source + size / 2);
	kern_return_t kr            = (size < vm_page_size * 2) ? KERN_INVALID_ADDRESS : KERN_PROTECTION_FAILURE;

	logv(
		"Read-protecting a mid-range buffer page at address "
		"0x%jx...",
		(uintmax_t)mid_point);
	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_protect(mach_task_self(), mid_point, vm_page_size, FALSE, VM_PROT_WRITE), "mach_vm_protect()");
	logv("Page read-protected.");

	logv(
		"Copying region at address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)source, (uintmax_t)size, (uintmax_t)size, (uintmax_t)dest);

	assert_copy_return(source, size, dest, kr);
	logv("Returned expected error: %s.", mach_error_string(kr));
}

/* Copying to a partially write-protected region fails. */
void
test_copy_dest_partially_unwriteable_region()
{
	kern_return_t kr;
	mach_vm_address_t dest      = get_vm_address();
	mach_vm_address_t source    = get_buffer_address();
	mach_msg_type_number_t size = (mach_msg_type_number_t)get_buffer_size();
	mach_vm_address_t mid_point = (mach_vm_address_t)mach_vm_trunc_page(dest + size / 2);

#if __MAC_OX_X_VERSION_MIN_REQUIRED > 1080
	logv(
		"Read-protecting a mid-range buffer page at address "
		"0x%jx...",
		(uintmax_t)mid_point);
	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_protect(mach_task_self(), mid_point, vm_page_size, FALSE, VM_PROT_READ), "mach_vm_protect()");
	logv("Page read-protected.");
	logv(
		"Copying region at address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)source, (uintmax_t)size, (uintmax_t)size, (uintmax_t)dest);
	if (size >= vm_page_size) {
		kr = KERN_PROTECTION_FAILURE;
	} else {
		kr = KERN_INVALID_ADDRESS;
	}
	assert_copy_return(source, size, dest, kr);
	logv("Returned expected error: %s.", mach_error_string(kr));
#else
	logv(
		"Bypassing partially unwriteable region test "
		"(See <rdar://problem/12190999>)");
#endif /* __MAC_OX_X_VERSION_MIN_REQUIRED > 1080 */
}

/* Copying on partially deallocated memory fails. */
void
test_copy_source_on_partially_deallocated_range()
{
	mach_vm_address_t source    = get_vm_address();
	mach_vm_address_t dest      = get_buffer_address();
	mach_vm_address_t start     = mach_vm_trunc_page(source);
	mach_msg_type_number_t size = (mach_msg_type_number_t)get_buffer_size();

	logv(
		"Deallocating the first source page at address "
		"0x%jx...",
		(uintmax_t)start);
	assert_deallocate_success(start, vm_page_size);
	logv("Page deallocated.");

	logv(
		"Writing buffer of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)dest, (uintmax_t)size, (uintmax_t)size, (uintmax_t)source);
	assert_copy_return(source, size, dest, KERN_INVALID_ADDRESS);
	logv("Returned expected error: %s.", mach_error_string(KERN_INVALID_ADDRESS));
}

/* Copying on partially deallocated memory fails. */
void
test_copy_dest_on_partially_deallocated_range()
{
	mach_vm_address_t source    = get_vm_address();
	mach_vm_address_t dest      = get_buffer_address();
	mach_vm_address_t start     = mach_vm_trunc_page(dest);
	mach_msg_type_number_t size = (mach_msg_type_number_t)get_buffer_size();

	logv(
		"Deallocating the first destination page at address "
		"0x%jx...",
		(uintmax_t)start);
	assert_deallocate_success(start, vm_page_size);
	logv("Page deallocated.");

	logv(
		"Writing buffer of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)dest, (uintmax_t)size, (uintmax_t)size, (uintmax_t)source);
	assert_copy_return(source, size, dest, KERN_INVALID_ADDRESS);
	logv("Returned expected error: %s.", mach_error_string(KERN_INVALID_ADDRESS));
}

/* Copying on partially unwritable memory fails. */
void
test_copy_dest_on_partially_unwritable_range()
{
	mach_vm_address_t source    = get_vm_address();
	mach_vm_address_t dest      = get_buffer_address();
	mach_vm_address_t start     = mach_vm_trunc_page(dest);
	mach_msg_type_number_t size = (mach_msg_type_number_t)get_buffer_size();

	/*  For sizes < msg_ool_size_small,
	 *  vm_map_copy_overwrite_nested() uses
	 *  vm_map_copyout_kernel_buffer() to read in the memory,
	 *  returning different errors, see 8217123. */
	kern_return_t kr_expected = (size < vm_page_size * 2) ? KERN_INVALID_ADDRESS : KERN_PROTECTION_FAILURE;

	logv(
		"Write-protecting the first destination page at address "
		"0x%jx...",
		(uintmax_t)start);
	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_protect(mach_task_self(), start, vm_page_size, FALSE, VM_PROT_READ), "mach_vm_protect()");
	logv("Page write-protected.");

	logv(
		"Writing buffer of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)dest, (uintmax_t)size, (uintmax_t)size, (uintmax_t)source);
	assert_copy_return(source, size, dest, kr_expected);
	logv("Returned expected error: %s.", mach_error_string(kr_expected));
}

/* Copying on partially unreadable memory fails. */
void
test_copy_source_on_partially_unreadable_range()
{
	mach_vm_address_t source    = get_vm_address();
	mach_vm_address_t dest      = get_buffer_address();
	mach_vm_address_t start     = mach_vm_trunc_page(source);
	mach_msg_type_number_t size = (mach_msg_type_number_t)get_buffer_size();

	/*  For sizes < msg_ool_size_small,
	 *  vm_map_copy_overwrite_nested() uses
	 *  vm_map_copyout_kernel_buffer() to read in the memory,
	 *  returning different errors, see 8217123. */
	kern_return_t kr_expected = (size < vm_page_size * 2) ? KERN_INVALID_ADDRESS : KERN_PROTECTION_FAILURE;

	logv(
		"Read-protecting the first destination page at address "
		"0x%jx...",
		(uintmax_t)start);
	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_protect(mach_task_self(), start, vm_page_size, FALSE, VM_PROT_WRITE), "mach_vm_protect()");
	logv("Page read-protected.");

	logv(
		"Writing buffer of address 0x%jx and size 0x%jx (%ju), on "
		"memory at address 0x%jx...",
		(uintmax_t)dest, (uintmax_t)size, (uintmax_t)size, (uintmax_t)source);
	assert_copy_return(source, size, dest, kr_expected);
	logv("Returned expected error: %s.", mach_error_string(kr_expected));
}

/********************************/
/* mach_vm_protect() main tests */
/********************************/

void
test_zero_filled_extended()
{
	verify_pattern(empty, FALSE, mach_vm_trunc_page(get_vm_address()), round_page(get_vm_size() + 1), "zero-filled");
}

/* Allocated region is still zero-filled after read-protecting it and
 * then restoring read-access. */
void
test_zero_filled_readprotect()
{
	mach_vm_address_t address = get_vm_address();
	mach_vm_size_t size       = get_vm_size();

	logv("Setting read access on 0x%jx (%ju) byte%s at address 0x%jx...", (uintmax_t)size, (uintmax_t)size,
	    (size == 1) ? "" : "s", (uintmax_t)address);
	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_protect(mach_task_self(), address, size, FALSE, VM_PROT_DEFAULT), "mach_vm_protect()");
	logv("Region has read access.");
	test_zero_filled_extended();
}

void
verify_protection(vm_prot_t protection, const char * protection_name)
{
	mach_vm_address_t address    = get_vm_address();
	mach_vm_size_t size          = get_vm_size();
	mach_vm_size_t original_size = size;
	vm_region_basic_info_data_64_t info;
	mach_msg_type_number_t count = VM_REGION_BASIC_INFO_COUNT_64;
	mach_port_t unused;

	logv(
		"Verifying %s-protection on region of address 0x%jx and "
		"size 0x%jx (%ju) with mach_vm_region()...",
		protection_name, (uintmax_t)address, (uintmax_t)size, (uintmax_t)size);
	T_QUIET; T_ASSERT_MACH_SUCCESS(
		mach_vm_region(mach_task_self(), &address, &size, VM_REGION_BASIC_INFO_64, (vm_region_info_t)&info, &count, &unused),
		"mach_vm_region()");
	if (original_size) {
		T_QUIET; T_ASSERT_EQ((info.protection & protection), 0,
		    "Region "
		    "is unexpectedly %s-unprotected.",
		    protection_name);
		logv("Region is %s-protected as expected.", protection_name);
	} else {
		T_QUIET; T_ASSERT_NE(info.protection & protection, 0,
		    "Region is "
		    "unexpectedly %s-protected.",
		    protection_name);
		logv("Region is %s-unprotected as expected.", protection_name);
	}
}

void
test_verify_readprotection()
{
	verify_protection(VM_PROT_READ, "read");
}

void
test_verify_writeprotection()
{
	verify_protection(VM_PROT_WRITE, "write");
}

/******************************/
/* Protection bus error tests */
/******************************/

/* mach_vm_protect() affects the smallest aligned region (integral
 * number of pages) containing the given range. */

/* Addresses in read-protected range are inaccessible. */
void
access_readprotected_range_address(mach_vm_address_t address, const char * position)
{
	logv("Reading from %s 0x%jx of read-protected range...", position, (uintmax_t)address);
	mach_vm_address_t bad_value = MACH_VM_ADDRESS_T(address);
	T_ASSERT_FAIL("Unexpectedly read value 0x%jx at address 0x%jx."
	    "Should have died with signal SIGBUS.",
	    (uintmax_t)bad_value, (uintmax_t)address);
}

/* Start of read-protected range is inaccessible. */
void
test_access_readprotected_range_start()
{
	access_readprotected_range_address(mach_vm_trunc_page(get_vm_address()), "start");
}

/* Middle of read-protected range is inaccessible. */
void
test_access_readprotected_range_middle()
{
	mach_vm_address_t address = get_vm_address();
	access_readprotected_range_address(mach_vm_trunc_page(address) + (aligned_size(address, get_vm_size()) >> 1), "middle");
}

/* End of read-protected range is inaccessible. */
void
test_access_readprotected_range_end()
{
	access_readprotected_range_address(round_page(get_vm_address() + get_vm_size()) - vm_address_size, "end");
}

/* Addresses in write-protected range are unwritable. */
void
write_writeprotected_range_address(mach_vm_address_t address, const char * position)
{
	logv("Writing on %s 0x%jx of write-protected range...", position, (uintmax_t)address);
	MACH_VM_ADDRESS_T(address) = 0x0;
	T_ASSERT_FAIL("Unexpectedly wrote value 0x0 value at address 0x%jx."
	    "Should have died with signal SIGBUS.",
	    (uintmax_t)address);
}

/* Start of write-protected range is unwritable. */
void
test_write_writeprotected_range_start()
{
	write_writeprotected_range_address(mach_vm_trunc_page(get_vm_address()), "start");
}

/* Middle of write-protected range is unwritable. */
void
test_write_writeprotected_range_middle()
{
	mach_vm_address_t address = get_vm_address();
	write_writeprotected_range_address(mach_vm_trunc_page(address) + (aligned_size(address, get_vm_size()) >> 1), "middle");
}

/* End of write-protected range is unwritable. */
void
test_write_writeprotected_range_end()
{
	write_writeprotected_range_address(round_page(get_vm_address() + get_vm_size()) - vm_address_size, "end");
}

/*************************************/
/* mach_vm_protect() edge case tests */
/*************************************/

void
protect_zero_size(vm_prot_t protection, const char * protection_name)
{
	int i;
	kern_return_t kr;
	vm_map_t this_task            = mach_task_self();
	mach_vm_address_t addresses[] = {0x0,
		                         0x1,
		                         vm_page_size - 1,
		                         vm_page_size,
		                         vm_page_size + 1,
		                         (mach_vm_address_t)UINT_MAX - vm_page_size + 1,
		                         (mach_vm_address_t)UINT_MAX,
		                         (mach_vm_address_t)UINTMAX_MAX - vm_page_size + 1,
		                         (mach_vm_address_t)UINTMAX_MAX};
	int numofaddresses = sizeof(addresses) / sizeof(addresses[0]);

	logv("%s-protecting 0x0 (0) bytes at various addresses...", protection_name);
	for (i = 0; i < numofaddresses; i++) {
		kr = mach_vm_protect(this_task, addresses[i], 0, FALSE, protection);
		T_QUIET; T_ASSERT_MACH_SUCCESS(kr,
		    "mach_vm_protect() at "
		    "address 0x%jx unexpectedly failed: %s.",
		    (uintmax_t)addresses[i], mach_error_string(kr));
	}
	logv("Protection successful.");
}

void
test_readprotect_zero_size()
{
	protect_zero_size(VM_PROT_READ, "Read");
}

void
test_writeprotect_zero_size()
{
	protect_zero_size(VM_PROT_WRITE, "Write");
}

/* Protecting a range wrapped around the address space fails. */
void
protect_wrapped_around_ranges(vm_prot_t protection, const char * protection_name)
{
	int i;
	kern_return_t kr;
	vm_map_t this_task = mach_task_self();
	struct {
		mach_vm_address_t address;
		mach_vm_size_t size;
	} ranges[] = {
		{0x1, (mach_vm_size_t)UINTMAX_MAX},
		{vm_page_size, (mach_vm_size_t)UINTMAX_MAX - vm_page_size + 1},
		{(mach_vm_address_t)UINTMAX_MAX - vm_page_size + 1, vm_page_size},
		{(mach_vm_address_t)UINTMAX_MAX, 1},
	};
	int numofranges = sizeof(ranges) / sizeof(ranges[0]);

	logv(
		"%s-protecting various memory ranges wrapping around the "
		"address space...",
		protection_name);
	for (i = 0; i < numofranges; i++) {
		kr = mach_vm_protect(this_task, ranges[i].address, ranges[i].size, FALSE, protection);
		T_QUIET; T_ASSERT_EQ(kr, KERN_INVALID_ARGUMENT,
		    "mach_vm_protect() with address 0x%jx and size "
		    "0x%jx (%ju) unexpectedly returned: %s.\n"
		    "Should have returned: %s.",
		    (uintmax_t)ranges[i].address, (uintmax_t)ranges[i].size, (uintmax_t)ranges[i].size, mach_error_string(kr),
		    mach_error_string(KERN_INVALID_ARGUMENT));
	}
	logv("Returned expected error on each range: %s.", mach_error_string(KERN_INVALID_ARGUMENT));
}

void
test_readprotect_wrapped_around_ranges()
{
	protect_wrapped_around_ranges(VM_PROT_READ, "Read");
}

void
test_writeprotect_wrapped_around_ranges()
{
	protect_wrapped_around_ranges(VM_PROT_WRITE, "Write");
}

/*******************/
/* vm_copy() tests */
/*******************/

/* Verify the address space is being shared. */
void
assert_share_mode(mach_vm_address_t address, unsigned share_mode, const char * share_mode_name)
{
	mach_vm_size_t size = get_vm_size();
	vm_region_extended_info_data_t info;
	mach_msg_type_number_t count = VM_REGION_EXTENDED_INFO_COUNT;
	mach_port_t unused;

/*
 * XXX Fails on UVM kernel.  See <rdar://problem/12164664>
 */
#if notyet /* __MAC_OS_X_VERSION_MIN_REQUIRED < 1090 */
	logv(
		"Verifying %s share mode on region of address 0x%jx and "
		"size 0x%jx (%ju)...",
		share_mode_name, (uintmax_t)address, (uintmax_t)size, (uintmax_t)size);
	T_QUIET; T_ASSERT_MACH_SUCCESS(
		mach_vm_region(mach_task_self(), &address, &size, VM_REGION_EXTENDED_INFO, (vm_region_info_t)&info, &count, &unused),
		"mach_vm_region()");
	T_QUIET; T_ASSERT_EQ(info.share_mode, share_mode,
	    "Region's share mode "
	    " unexpectedly is not %s but %d.",
	    share_mode_name, info.share_mode);
	logv("Region has a share mode of %s as expected.", share_mode_name);
#else
	logv("Bypassing share_mode verification (See <rdar://problem/12164664>)");
#endif /* __MAC_OS_X_VERSION_MIN_REQUIRED < 1090 */
}

/* Do the vm_copy() and verify its success. */
void
assert_vmcopy_success(vm_address_t src, vm_address_t dst, const char * source_name)
{
	kern_return_t kr;
	mach_vm_size_t size = get_vm_size();

	logv("Copying (using mach_vm_copy()) from a %s source...", source_name);
	kr = mach_vm_copy(mach_task_self(), src, size, dst);
	T_QUIET; T_ASSERT_MACH_SUCCESS(kr,
	    "mach_vm_copy() with the source address "
	    "0x%jx, designation address 0x%jx, and size 0x%jx (%ju) unexpectly "
	    "returned %s.\n  Should have returned: %s.",
	    (uintmax_t)src, (uintmax_t)dst, (uintmax_t)size, (uintmax_t)size, mach_error_string(kr),
	    mach_error_string(KERN_SUCCESS));
	logv("Copy (mach_vm_copy()) was successful as expected.");
}

void
write_region(mach_vm_address_t address, mach_vm_size_t start)
{
	mach_vm_size_t size = get_vm_size();

	filter_addresses_do_else(empty, FALSE, address, size, write_address, write_address, start);
}

void
verify_region(mach_vm_address_t address, mach_vm_address_t start)
{
	mach_vm_size_t size = get_vm_size();

	filter_addresses_do_else(empty, FALSE, address, size, verify_address, verify_address, start);
}

/* Perform the post vm_copy() action and verify its results. */
void
modify_one_and_verify_all_regions(vm_address_t src, vm_address_t dst, vm_address_t shared_copied, boolean_t shared)
{
	mach_vm_size_t size = get_vm_size();
	int action          = get_vmcopy_post_action();

	/* Do the post vm_copy() action. */
	switch (action) {
	case VMCOPY_MODIFY_SRC:
		logv("Modifying: source%s...", shared ? " (shared with other region)" : "");
		write_region(src, 1);
		break;

	case VMCOPY_MODIFY_DST:
		logv("Modifying: destination...");
		write_region(dst, 1);
		break;

	case VMCOPY_MODIFY_SHARED_COPIED:
		/* If no shared_copied then no need to verify (nothing changed). */
		if (!shared_copied) {
			return;
		}
		logv("Modifying: shared/copied%s...", shared ? " (shared with source region)" : "");
		write_region(shared_copied, 1);
		break;

	default:
		T_ASSERT_FAIL("Unknown post vm_copy() action (%d)", action);
	}
	logv("Modification was successful as expected.");

	/* Verify all the regions with what is expected. */
	logv("Verifying: source... ");
	verify_region(src, (VMCOPY_MODIFY_SRC == action || (shared && VMCOPY_MODIFY_SHARED_COPIED == action)) ? 1 : 0);
	logv("destination... ");
	verify_region(dst, (VMCOPY_MODIFY_DST == action) ? 1 : 0);
	if (shared_copied) {
		logv("shared/copied... ");
		verify_region(shared_copied, (VMCOPY_MODIFY_SHARED_COPIED == action || (shared && VMCOPY_MODIFY_SRC == action)) ? 1 : 0);
	}
	logv("Verification was successful as expected.");
}

/* Test source being a simple fresh region. */
void
test_vmcopy_fresh_source()
{
	mach_vm_size_t size = get_vm_size();
	mach_vm_address_t src, dst;

	if (get_vmcopy_post_action() == VMCOPY_MODIFY_SHARED_COPIED) {
		/* No shared/copied region to modify so just return. */
		logv("No shared/copied region as expected.");
		return;
	}

	assert_allocate_success(&src, size, TRUE);

	assert_share_mode(src, SM_EMPTY, "SM_EMPTY");

	write_region(src, 0);

	assert_allocate_success(&dst, size, TRUE);

	assert_vmcopy_success(src, dst, "freshly allocated");

	modify_one_and_verify_all_regions(src, dst, 0, FALSE);

	assert_deallocate_success(src, size);
	assert_deallocate_success(dst, size);
}

/* Test source copied from a shared region. */
void
test_vmcopy_shared_source()
{
	mach_vm_size_t size = get_vm_size();
	mach_vm_address_t src, dst, shared;
	int action = get_vmcopy_post_action();
	int pid, status;

	assert_allocate_success(&src, size, TRUE);

	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_inherit(mach_task_self(), src, size, VM_INHERIT_SHARE), "mach_vm_inherit()");

	write_region(src, 0);

	pid = fork();
	if (pid == 0) {
		/* Verify that the child's 'src' is shared with the
		 *  parent's src */
		assert_share_mode(src, SM_SHARED, "SM_SHARED");
		assert_allocate_success(&dst, size, TRUE);
		assert_vmcopy_success(src, dst, "shared");
		if (VMCOPY_MODIFY_SHARED_COPIED == action) {
			logv("Modifying: shared...");
			write_region(src, 1);
			logv("Modification was successsful as expected.");
			logv("Verifying: source... ");
			verify_region(src, 1);
			logv("destination...");
			verify_region(dst, (VMCOPY_MODIFY_DST == action) ? 1 : 0);
			logv("Verification was successful as expected.");
		} else {
			modify_one_and_verify_all_regions(src, dst, 0, TRUE);
		}
		assert_deallocate_success(dst, size);
		exit(0);
	} else if (pid > 0) {
		/* In the parent the src becomes the shared */
		shared = src;
		wait(&status);
		if (WEXITSTATUS(status) != 0) {
			exit(status);
		}
		/* verify shared (shared with child's src) */
		logv("Verifying: shared...");
		verify_region(shared, (VMCOPY_MODIFY_SHARED_COPIED == action || VMCOPY_MODIFY_SRC == action) ? 1 : 0);
		logv("Verification was successful as expected.");
	} else {
		T_WITH_ERRNO; T_ASSERT_FAIL("fork failed");
	}

	assert_deallocate_success(src, size);
}

/* Test source copied from another mapping. */
void
test_vmcopy_copied_from_source()
{
	mach_vm_size_t size = get_vm_size();
	mach_vm_address_t src, dst, copied;

	assert_allocate_success(&copied, size, TRUE);
	write_region(copied, 0);

	assert_allocate_success(&src, size, TRUE);

	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_copy(mach_task_self(), copied, size, src), "mach_vm_copy()");

	assert_share_mode(src, SM_COW, "SM_COW");

	assert_allocate_success(&dst, size, TRUE);

	assert_vmcopy_success(src, dst, "copied from");

	modify_one_and_verify_all_regions(src, dst, copied, FALSE);

	assert_deallocate_success(src, size);
	assert_deallocate_success(dst, size);
	assert_deallocate_success(copied, size);
}

/* Test source copied to another mapping. */
void
test_vmcopy_copied_to_source()
{
	mach_vm_size_t size = get_vm_size();
	mach_vm_address_t src, dst, copied;

	assert_allocate_success(&src, size, TRUE);
	write_region(src, 0);

	assert_allocate_success(&copied, size, TRUE);

	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_copy(mach_task_self(), src, size, copied), "mach_vm_copy()");

	assert_share_mode(src, SM_COW, "SM_COW");

	assert_allocate_success(&dst, size, TRUE);

	assert_vmcopy_success(src, dst, "copied to");

	modify_one_and_verify_all_regions(src, dst, copied, FALSE);

	assert_deallocate_success(src, size);
	assert_deallocate_success(dst, size);
	assert_deallocate_success(copied, size);
}

/* Test a truedshared source copied. */
void
test_vmcopy_trueshared_source()
{
	mach_vm_size_t size   = get_vm_size();
	mach_vm_address_t src = 0x0, dst, shared;
	vm_prot_t cur_protect = (VM_PROT_READ | VM_PROT_WRITE);
	vm_prot_t max_protect = (VM_PROT_READ | VM_PROT_WRITE);
	mem_entry_name_port_t mem_obj;

	assert_allocate_success(&shared, size, TRUE);
	write_region(shared, 0);

	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_make_memory_entry_64(mach_task_self(), &size, (memory_object_offset_t)shared, cur_protect, &mem_obj,
	    (mem_entry_name_port_t)NULL),
	    "mach_make_memory_entry_64()");
	T_QUIET; T_ASSERT_MACH_SUCCESS(
		mach_vm_map(mach_task_self(), &src, size, 0, TRUE, mem_obj, 0, FALSE, cur_protect, max_protect, VM_INHERIT_NONE),
		"mach_vm_map()");

	assert_share_mode(src, SM_TRUESHARED, "SM_TRUESHARED");

	assert_allocate_success(&dst, size, TRUE);

	assert_vmcopy_success(src, dst, "true shared");

	modify_one_and_verify_all_regions(src, dst, shared, TRUE);

	assert_deallocate_success(src, size);
	assert_deallocate_success(dst, size);
	assert_deallocate_success(shared, size);
}

/* Test a private aliazed source copied. */
void
test_vmcopy_private_aliased_source()
{
	mach_vm_size_t size   = get_vm_size();
	mach_vm_address_t src = 0x0, dst, shared;
	vm_prot_t cur_protect = (VM_PROT_READ | VM_PROT_WRITE);
	vm_prot_t max_protect = (VM_PROT_READ | VM_PROT_WRITE);

	assert_allocate_success(&shared, size, TRUE);
	write_region(shared, 0);

	T_QUIET; T_ASSERT_MACH_SUCCESS(mach_vm_remap(mach_task_self(), &src, size, 0, TRUE, mach_task_self(), shared, FALSE, &cur_protect,
	    &max_protect, VM_INHERIT_NONE),
	    "mach_vm_remap()");

	assert_share_mode(src, SM_PRIVATE_ALIASED, "SM_PRIVATE_ALIASED");

	assert_allocate_success(&dst, size, TRUE);

	assert_vmcopy_success(src, dst, "true shared");

	modify_one_and_verify_all_regions(src, dst, shared, TRUE);

	assert_deallocate_success(src, size);
	assert_deallocate_success(dst, size);
	assert_deallocate_success(shared, size);
}

/*************/
/* VM Suites */
/*************/

void
run_allocate_test_suites()
{
	/* <rdar://problem/10304215> CoreOSZin 12Z30: VMUnitTest fails:
	 * error finding xnu major version number. */
	/* unsigned int xnu_version = xnu_major_version(); */

	UnitTests allocate_main_tests = {
		{"Allocated address is nonzero iff size is nonzero", test_nonzero_address_iff_nonzero_size},
		{"Allocated address is page-aligned", test_aligned_address},
		{"Allocated memory is zero-filled", test_zero_filled},
		{"Write and verify address-filled pattern", test_write_address_filled},
		{"Write and verify checkerboard pattern", test_write_checkerboard},
		{"Write and verify reverse checkerboard pattern", test_write_reverse_checkerboard},
		{"Write and verify page ends pattern", test_write_page_ends},
		{"Write and verify page interiors pattern", test_write_page_interiors},
		{"Reallocate allocated pages", test_reallocate_pages},
	};
	UnitTests allocate_address_error_tests = {
		{"Allocate at address zero", test_allocate_at_zero},
		{"Allocate at a 2 MB boundary-unaligned, page-aligned "
		 "address",
		 test_allocate_2MB_boundary_unaligned_page_aligned_address},
	};
	UnitTests allocate_argument_error_tests = {
		{"Allocate in NULL VM map", test_allocate_in_null_map}, {"Allocate with kernel flags", test_allocate_with_kernel_flags},
	};
	UnitTests allocate_fixed_size_tests = {
		{"Allocate zero size", test_allocate_zero_size},
		{"Allocate overflowing size", test_allocate_overflowing_size},
		{"Allocate a page with highest address hint", test_allocate_page_with_highest_address_hint},
		{"Allocate two pages and verify first fit strategy", test_allocate_first_fit_pages},
	};
	UnitTests allocate_invalid_large_size_test = {
		{"Allocate invalid large size", test_allocate_invalid_large_size},
	};
	UnitTests mach_vm_map_protection_inheritance_error_test = {
		{"mach_vm_map() with invalid protection/inheritance "
		 "arguments",
		 test_mach_vm_map_protection_inheritance_error},
	};
	UnitTests mach_vm_map_large_mask_overflow_error_test = {
		{"mach_vm_map() with large address mask", test_mach_vm_map_large_mask_overflow_error},
	};

	/* Run the test suites with various allocators and VM sizes, and
	 *  unspecified or fixed (page-aligned or page-unaligned),
	 *  addresses. */
	for (allocators_idx = 0; allocators_idx < numofallocators; allocators_idx++) {
		for (sizes_idx = 0; sizes_idx < numofsizes; sizes_idx++) {
			for (flags_idx = 0; flags_idx < numofflags; flags_idx++) {
				for (alignments_idx = 0; alignments_idx < numofalignments; alignments_idx++) {
					/* An allocated address will be page-aligned. */
					/* Only run the zero size mach_vm_map() error tests in the
					 *  unspecified address case, since we won't be able to retrieve a
					 *  fixed address for allocation. See 8003930. */
					if ((flags_idx == ANYWHERE && alignments_idx == UNALIGNED) ||
					    (allocators_idx != MACH_VM_ALLOCATE && sizes_idx == ZERO_BYTES && flags_idx == FIXED)) {
						continue;
					}
					run_suite(set_up_allocator_and_vm_variables, allocate_argument_error_tests, do_nothing,
					    "%s argument error tests, %s%s address, "
					    "%s size: 0x%jx (%ju)",
					    allocators[allocators_idx].description, address_flags[flags_idx].description,
					    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
					    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
					    (uintmax_t)vm_sizes[sizes_idx].size);
					/* mach_vm_map() only protection and inheritance error
					 *  tests. */
					if (allocators_idx != MACH_VM_ALLOCATE) {
						run_suite(set_up_allocator_and_vm_variables, mach_vm_map_protection_inheritance_error_test, do_nothing,
						    "%s protection and inheritance "
						    "error test, %s%s address, %s size: 0x%jx "
						    "(%ju)",
						    allocators[allocators_idx].description, address_flags[flags_idx].description,
						    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
						    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
						    (uintmax_t)vm_sizes[sizes_idx].size);
					}
					/* mach_vm_map() cannot allocate 0 bytes, see 8003930. */
					if (allocators_idx == MACH_VM_ALLOCATE || sizes_idx != ZERO_BYTES) {
						run_suite(set_up_allocator_and_vm_variables_and_allocate, allocate_main_tests, deallocate,
						    "%s main "
						    "allocation tests, %s%s address, %s size: 0x%jx "
						    "(%ju)",
						    allocators[allocators_idx].description, address_flags[flags_idx].description,
						    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
						    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
						    (uintmax_t)vm_sizes[sizes_idx].size);
					}
				}
			}
			run_suite(set_up_allocator_and_vm_size, allocate_address_error_tests, do_nothing,
			    "%s address "
			    "error allocation tests, %s size: 0x%jx (%ju)",
			    allocators[allocators_idx].description, vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
			    (uintmax_t)vm_sizes[sizes_idx].size);
		}
		run_suite(set_up_allocator, allocate_fixed_size_tests, do_nothing, "%s fixed size allocation tests",
		    allocators[allocators_idx].description);
		/* <rdar://problem/10304215> CoreOSZin 12Z30: VMUnitTest fails:
		 * error finding xnu major version number. */
		/* mach_vm_map() with a named entry triggers a panic with this test
		 *  unless under xnu-1598 or later, see 8048580. */
		/* if (allocators_idx != MACH_VM_MAP_NAMED_ENTRY
		|| xnu_version >= 1598) { */
		if (allocators_idx != MACH_VM_MAP_NAMED_ENTRY) {
			run_suite(set_up_allocator, allocate_invalid_large_size_test, do_nothing, "%s invalid large size allocation test",
			    allocators[allocators_idx].description);
		}
	}
	/* mach_vm_map() only large mask overflow tests. */
	for (sizes_idx = 0; sizes_idx < numofsizes; sizes_idx++) {
		run_suite(set_up_vm_size, mach_vm_map_large_mask_overflow_error_test, do_nothing,
		    "mach_vm_map() large mask overflow "
		    "error test, size: 0x%jx (%ju)",
		    (uintmax_t)vm_sizes[sizes_idx].size, (uintmax_t)vm_sizes[sizes_idx].size);
	}
}

void
run_deallocate_test_suites()
{
	UnitTests access_deallocated_memory_tests = {
		{"Read start of deallocated range", test_access_deallocated_range_start},
		{"Read middle of deallocated range", test_access_deallocated_range_middle},
		{"Read end of deallocated range", test_access_deallocated_range_end},
	};
	UnitTests deallocate_reallocate_tests = {
		{"Deallocate twice", test_deallocate_twice},
		{"Write pattern, deallocate, reallocate (deallocated "
		 "memory is inaccessible), and verify memory is "
		 "zero-filled",
		 test_write_pattern_deallocate_reallocate_zero_filled},
	};
	UnitTests deallocate_null_map_test = {
		{"Deallocate in NULL VM map", test_deallocate_in_null_map},
	};
	UnitTests deallocate_edge_case_tests = {
		{"Deallocate zero size ranges", test_deallocate_zero_size_ranges},
		{"Deallocate memory ranges whose end rounds to 0x0", test_deallocate_rounded_zero_end_ranges},
		{"Deallocate wrapped around memory ranges", test_deallocate_wrapped_around_ranges},
	};
	UnitTests deallocate_suicide_test = {
		{"Deallocate whole address space", test_deallocate_suicide},
	};

	/* All allocations done with mach_vm_allocate(). */
	set_allocator(wrapper_mach_vm_allocate);

	/* Run the test suites with various VM sizes, and unspecified or
	 *  fixed (page-aligned or page-unaligned), addresses. */
	for (sizes_idx = 0; sizes_idx < numofsizes; sizes_idx++) {
		for (flags_idx = 0; flags_idx < numofflags; flags_idx++) {
			for (alignments_idx = 0; alignments_idx < numofalignments; alignments_idx++) {
				/* An allocated address will be page-aligned. */
				if (flags_idx == ANYWHERE && alignments_idx == UNALIGNED) {
					continue;
				}
				/* Accessing deallocated memory should cause a segmentation
				 *  fault. */
				/* Nothing gets deallocated if size is zero. */
				if (sizes_idx != ZERO_BYTES) {
					set_expected_signal(SIGSEGV);
					run_suite(set_up_vm_variables_and_allocate, access_deallocated_memory_tests, do_nothing,
					    "Deallocated memory access tests, "
					    "%s%s address, %s size: 0x%jx (%ju)",
					    address_flags[flags_idx].description,
					    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
					    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
					    (uintmax_t)vm_sizes[sizes_idx].size);
					set_expected_signal(0);
				}
				run_suite(set_up_vm_variables_and_allocate, deallocate_reallocate_tests, do_nothing,
				    "Deallocation and reallocation tests, %s%s "
				    "address, %s size: 0x%jx (%ju)",
				    address_flags[flags_idx].description,
				    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
				    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
				    (uintmax_t)vm_sizes[sizes_idx].size);
				run_suite(set_up_vm_variables, deallocate_null_map_test, do_nothing,
				    "mach_vm_deallocate() null map test, "
				    "%s%s address, %s size: 0x%jx (%ju)",
				    address_flags[flags_idx].description,
				    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
				    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
				    (uintmax_t)vm_sizes[sizes_idx].size);
			}
		}
	}
	run_suite(do_nothing, deallocate_edge_case_tests, do_nothing, "Edge case deallocation tests");

	set_expected_signal(-1);        /* SIGSEGV or SIGBUS */
	run_suite(do_nothing, deallocate_suicide_test, do_nothing, "Whole address space deallocation test");
	set_expected_signal(0);
}

void
run_read_test_suites()
{
	UnitTests read_main_tests = {
		{"Read address is nonzero iff size is nonzero", test_nonzero_address_iff_nonzero_size},
		{"Read address has the correct boundary offset", test_read_address_offset},
		{"Reallocate read pages", test_reallocate_pages},
		{"Read and verify zero-filled memory", test_zero_filled},
	};
	UnitTests read_pattern_tests = {
		{"Read address-filled pattern", test_read_address_filled},
		{"Read checkerboard pattern", test_read_checkerboard},
		{"Read reverse checkerboard pattern", test_read_reverse_checkerboard},
	};
	UnitTests read_null_map_test = {
		{"Read from NULL VM map", test_read_null_map},
	};
	UnitTests read_edge_case_tests = {
		{"Read zero size", test_read_zero_size},
		{"Read invalid large size", test_read_invalid_large_size},
		{"Read wrapped around memory ranges", test_read_wrapped_around_ranges},
	};
	UnitTests read_inaccessible_tests = {
		{"Read partially decallocated memory", test_read_partially_deallocated_range},
		{"Read partially read-protected memory", test_read_partially_unreadable_range},
	};

	/* All allocations done with mach_vm_allocate(). */
	set_allocator(wrapper_mach_vm_allocate);

	/* Run the test suites with various VM sizes, and unspecified or
	 *  fixed (page-aligned or page-unaligned) addresses. */
	for (sizes_idx = 0; sizes_idx < numofsizes; sizes_idx++) {
		for (flags_idx = 0; flags_idx < numofflags; flags_idx++) {
			for (alignments_idx = 0; alignments_idx < numofalignments; alignments_idx++) {
				/* An allocated address will be page-aligned. */
				if (flags_idx == ANYWHERE && alignments_idx == UNALIGNED) {
					continue;
				}
				run_suite(set_up_vm_variables_allocate_read_deallocate, read_main_tests, deallocate,
				    "mach_vm_read() "
				    "main tests, %s%s address, %s size: 0x%jx (%ju)",
				    address_flags[flags_idx].description,
				    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
				    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
				    (uintmax_t)vm_sizes[sizes_idx].size);
				run_suite(set_up_vm_variables_and_allocate_extra_page, read_pattern_tests, deallocate,
				    "mach_vm_read() pattern tests, %s%s address, %s "
				    "size: 0x%jx (%ju)",
				    address_flags[flags_idx].description,
				    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
				    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
				    (uintmax_t)vm_sizes[sizes_idx].size);
				run_suite(set_up_vm_variables_and_allocate_extra_page, read_null_map_test, deallocate_extra_page,
				    "mach_vm_read() null map test, "
				    "%s%s address, %s size: 0x%jx (%ju)",
				    address_flags[flags_idx].description,
				    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
				    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
				    (uintmax_t)vm_sizes[sizes_idx].size);
				/* A zero size range is always accessible. */
				if (sizes_idx != ZERO_BYTES) {
					run_suite(set_up_vm_variables_and_allocate_extra_page, read_inaccessible_tests, deallocate_extra_page,
					    "mach_vm_read() inaccessibility tests, %s%s "
					    "address, %s size: 0x%jx (%ju)",
					    address_flags[flags_idx].description,
					    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
					    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
					    (uintmax_t)vm_sizes[sizes_idx].size);
				}
			}
		}
	}
	run_suite(do_nothing, read_edge_case_tests, do_nothing, "mach_vm_read() fixed size tests");
}

void
run_write_test_suites()
{
	UnitTests write_main_tests = {
		{"Write and verify zero-filled memory", test_zero_filled_write},
	};
	UnitTests write_pattern_tests = {
		{"Write address-filled pattern", test_address_filled_write},
		{"Write checkerboard pattern", test_checkerboard_write},
		{"Write reverse checkerboard pattern", test_reverse_checkerboard_write},
	};
	UnitTests write_edge_case_tests = {
		{"Write into NULL VM map", test_write_null_map}, {"Write zero size", test_write_zero_size},
	};
	UnitTests write_inaccessible_tests = {
		{"Write partially decallocated buffer", test_write_partially_deallocated_buffer},
		{"Write partially read-protected buffer", test_write_partially_unreadable_buffer},
		{"Write on partially deallocated range", test_write_on_partially_deallocated_range},
		{"Write on partially write-protected range", test_write_on_partially_unwritable_range},
	};

	/* All allocations done with mach_vm_allocate(). */
	set_allocator(wrapper_mach_vm_allocate);

	/* Run the test suites with various destination sizes and
	 *  unspecified or fixed (page-aligned or page-unaligned)
	 *  addresses, and various buffer sizes and boundary offsets. */
	for (sizes_idx = 0; sizes_idx < numofsizes; sizes_idx++) {
		for (flags_idx = 0; flags_idx < numofflags; flags_idx++) {
			for (alignments_idx = 0; alignments_idx < numofalignments; alignments_idx++) {
				for (buffer_sizes_idx = 0; buffer_sizes_idx < numofsizes; buffer_sizes_idx++) {
					for (offsets_idx = 0; offsets_idx < numofoffsets; offsets_idx++) {
						/* An allocated address will be page-aligned. */
						if ((flags_idx == ANYWHERE && alignments_idx == UNALIGNED)) {
							continue;
						}
						run_suite(set_up_vm_and_buffer_variables_allocate_for_writing, write_edge_case_tests,
						    deallocate_vm_and_buffer,
						    "mach_vm_write() edge case tests, %s%s address, %s "
						    "size: 0x%jx (%ju), buffer %s size: 0x%jx (%ju), "
						    "buffer boundary offset: %d",
						    address_flags[flags_idx].description,
						    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
						    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
						    (uintmax_t)vm_sizes[sizes_idx].size, vm_sizes[buffer_sizes_idx].description,
						    (uintmax_t)vm_sizes[buffer_sizes_idx].size, (uintmax_t)vm_sizes[buffer_sizes_idx].size,
						    buffer_offsets[offsets_idx].offset);
						/* A zero size buffer is always accessible. */
						if (buffer_sizes_idx != ZERO_BYTES) {
							run_suite(set_up_vm_and_buffer_variables_allocate_for_writing, write_inaccessible_tests,
							    deallocate_vm_and_buffer,
							    "mach_vm_write() inaccessibility tests, "
							    "%s%s address, %s size: 0x%jx (%ju), buffer "
							    "%s size: 0x%jx (%ju), buffer boundary "
							    "offset: %d",
							    address_flags[flags_idx].description,
							    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
							    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
							    (uintmax_t)vm_sizes[sizes_idx].size, vm_sizes[buffer_sizes_idx].description,
							    (uintmax_t)vm_sizes[buffer_sizes_idx].size, (uintmax_t)vm_sizes[buffer_sizes_idx].size,
							    buffer_offsets[offsets_idx].offset);
						}
						/* The buffer cannot be larger than the destination. */
						if (vm_sizes[sizes_idx].size < vm_sizes[buffer_sizes_idx].size) {
							continue;
						}
						run_suite(set_up_vm_and_buffer_variables_allocate_write, write_main_tests, deallocate_vm_and_buffer,
						    "mach_vm_write() main tests, %s%s address, %s "
						    "size: 0x%jx (%ju), buffer %s size: 0x%jx (%ju), "
						    "buffer boundary offset: %d",
						    address_flags[flags_idx].description,
						    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
						    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
						    (uintmax_t)vm_sizes[sizes_idx].size, vm_sizes[buffer_sizes_idx].description,
						    (uintmax_t)vm_sizes[buffer_sizes_idx].size, (uintmax_t)vm_sizes[buffer_sizes_idx].size,
						    buffer_offsets[offsets_idx].offset);
						run_suite(set_up_vm_and_buffer_variables_allocate_for_writing, write_pattern_tests,
						    deallocate_vm_and_buffer,
						    "mach_vm_write() pattern tests, %s%s address, %s "
						    "size: 0x%jx (%ju), buffer %s size: 0x%jx (%ju), "
						    "buffer boundary offset: %d",
						    address_flags[flags_idx].description,
						    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
						    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
						    (uintmax_t)vm_sizes[sizes_idx].size, vm_sizes[buffer_sizes_idx].description,
						    (uintmax_t)vm_sizes[buffer_sizes_idx].size, (uintmax_t)vm_sizes[buffer_sizes_idx].size,
						    buffer_offsets[offsets_idx].offset);
					}
				}
			}
		}
	}
}

void
run_protect_test_suites()
{
	UnitTests readprotection_main_tests = {
		{"Read-protect, read-allow and verify zero-filled memory", test_zero_filled_readprotect},
		{"Verify that region is read-protected iff size is "
		 "nonzero",
		 test_verify_readprotection},
	};
	UnitTests access_readprotected_memory_tests = {
		{"Read start of read-protected range", test_access_readprotected_range_start},
		{"Read middle of read-protected range", test_access_readprotected_range_middle},
		{"Read end of read-protected range", test_access_readprotected_range_end},
	};
	UnitTests writeprotection_main_tests = {
		{"Write-protect and verify zero-filled memory", test_zero_filled_extended},
		{"Verify that region is write-protected iff size is "
		 "nonzero",
		 test_verify_writeprotection},
	};
	UnitTests write_writeprotected_memory_tests = {
		{"Write at start of write-protected range", test_write_writeprotected_range_start},
		{"Write in middle of write-protected range", test_write_writeprotected_range_middle},
		{"Write at end of write-protected range", test_write_writeprotected_range_end},
	};
	UnitTests protect_edge_case_tests = {
		{"Read-protect zero size ranges", test_readprotect_zero_size},
		{"Write-protect zero size ranges", test_writeprotect_zero_size},
		{"Read-protect wrapped around memory ranges", test_readprotect_wrapped_around_ranges},
		{"Write-protect wrapped around memory ranges", test_writeprotect_wrapped_around_ranges},
	};

	/* All allocations done with mach_vm_allocate(). */
	set_allocator(wrapper_mach_vm_allocate);

	/* Run the test suites with various VM sizes, and unspecified or
	 *  fixed (page-aligned or page-unaligned), addresses. */
	for (sizes_idx = 0; sizes_idx < numofsizes; sizes_idx++) {
		for (flags_idx = 0; flags_idx < numofflags; flags_idx++) {
			for (alignments_idx = 0; alignments_idx < numofalignments; alignments_idx++) {
				/* An allocated address will be page-aligned. */
				if (flags_idx == ANYWHERE && alignments_idx == UNALIGNED) {
					continue;
				}
				run_suite(set_up_vm_variables_allocate_readprotect, readprotection_main_tests, deallocate_extra_page,
				    "Main read-protection tests, %s%s address, %s "
				    "size: 0x%jx (%ju)",
				    address_flags[flags_idx].description,
				    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
				    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
				    (uintmax_t)vm_sizes[sizes_idx].size);
				run_suite(set_up_vm_variables_allocate_writeprotect, writeprotection_main_tests, deallocate_extra_page,
				    "Main write-protection tests, %s%s address, %s "
				    "size: 0x%jx (%ju)",
				    address_flags[flags_idx].description,
				    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
				    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
				    (uintmax_t)vm_sizes[sizes_idx].size);
				/* Nothing gets protected if size is zero. */
				if (sizes_idx != ZERO_BYTES) {
					set_expected_signal(SIGBUS);
					/* Accessing read-protected memory should cause a bus
					 *  error. */
					run_suite(set_up_vm_variables_allocate_readprotect, access_readprotected_memory_tests, deallocate_extra_page,
					    "Read-protected memory access tests, %s%s "
					    "address, %s size: 0x%jx (%ju)",
					    address_flags[flags_idx].description,
					    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
					    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
					    (uintmax_t)vm_sizes[sizes_idx].size);
					/* Writing on write-protected memory should cause a bus
					 *  error. */
					run_suite(set_up_vm_variables_allocate_writeprotect, write_writeprotected_memory_tests, deallocate_extra_page,
					    "Write-protected memory writing tests, %s%s "
					    "address, %s size: 0x%jx (%ju)",
					    address_flags[flags_idx].description,
					    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
					    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
					    (uintmax_t)vm_sizes[sizes_idx].size);
					set_expected_signal(0);
				}
			}
		}
	}
	run_suite(do_nothing, protect_edge_case_tests, do_nothing, "Edge case protection tests");
}

void
run_copy_test_suites()
{
	/* Copy tests */
	UnitTests copy_main_tests = {
		{"Copy and verify zero-filled memory", test_zero_filled_copy_dest},
	};
	UnitTests copy_pattern_tests = {
		{"Copy address-filled pattern", test_copy_address_filled},
		{"Copy checkerboard pattern", test_copy_checkerboard},
		{"Copy reverse checkerboard pattern", test_copy_reverse_checkerboard},
	};
	UnitTests copy_edge_case_tests = {
		{"Copy with NULL VM map", test_copy_null_map},
		{"Copy zero size", test_copy_zero_size},
		{"Copy invalid large size", test_copy_invalid_large_size},
		{"Read wrapped around memory ranges", test_copy_wrapped_around_ranges},
	};
	UnitTests copy_inaccessible_tests = {
		{"Copy source partially decallocated region", test_copy_source_partially_deallocated_region},
		/* XXX */
		{"Copy destination partially decallocated region", test_copy_dest_partially_deallocated_region},
		{"Copy source partially read-protected region", test_copy_source_partially_unreadable_region},
		/* XXX */
		{"Copy destination partially write-protected region", test_copy_dest_partially_unwriteable_region},
		{"Copy source on partially deallocated range", test_copy_source_on_partially_deallocated_range},
		{"Copy destination on partially deallocated range", test_copy_dest_on_partially_deallocated_range},
		{"Copy source on partially read-protected range", test_copy_source_on_partially_unreadable_range},
		{"Copy destination on partially write-protected range", test_copy_dest_on_partially_unwritable_range},
	};

	UnitTests copy_shared_mode_tests = {
		{"Copy using freshly allocated source", test_vmcopy_fresh_source},
		{"Copy using shared source", test_vmcopy_shared_source},
		{"Copy using a \'copied from\' source", test_vmcopy_copied_from_source},
		{"Copy using a \'copied to\' source", test_vmcopy_copied_to_source},
		{"Copy using a true shared source", test_vmcopy_trueshared_source},
		{"Copy using a private aliased source", test_vmcopy_private_aliased_source},
	};

	/* All allocations done with mach_vm_allocate(). */
	set_allocator(wrapper_mach_vm_allocate);

	/* All the tests are done with page size regions. */
	set_vm_size(vm_page_size);

	/* Run the test suites with various shared modes for source */
	for (vmcopy_action_idx = 0; vmcopy_action_idx < numofvmcopyactions; vmcopy_action_idx++) {
		run_suite(set_up_copy_shared_mode_variables, copy_shared_mode_tests, do_nothing, "Copy shared mode tests, %s",
		    vmcopy_actions[vmcopy_action_idx].description);
	}

	for (sizes_idx = 0; sizes_idx < numofsizes; sizes_idx++) {
		for (flags_idx = 0; flags_idx < numofflags; flags_idx++) {
			for (alignments_idx = 0; alignments_idx < numofalignments; alignments_idx++) {
				for (buffer_sizes_idx = 0; buffer_sizes_idx < numofsizes; buffer_sizes_idx++) {
					for (offsets_idx = 0; offsets_idx < numofoffsets; offsets_idx++) {
						/* An allocated address will be page-aligned. */
						if ((flags_idx == ANYWHERE && alignments_idx == UNALIGNED)) {
							continue;
						}
						run_suite(set_up_vm_and_buffer_variables_allocate_for_copying, copy_edge_case_tests,
						    deallocate_vm_and_buffer,
						    "mach_vm_copy() edge case tests, %s%s address, %s "
						    "size: 0x%jx (%ju), buffer %s size: 0x%jx (%ju), "
						    "buffer boundary offset: %d",
						    address_flags[flags_idx].description,
						    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
						    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
						    (uintmax_t)vm_sizes[sizes_idx].size, vm_sizes[buffer_sizes_idx].description,
						    (uintmax_t)vm_sizes[buffer_sizes_idx].size, (uintmax_t)vm_sizes[buffer_sizes_idx].size,
						    buffer_offsets[offsets_idx].offset);
						/* The buffer cannot be larger than the destination. */
						if (vm_sizes[sizes_idx].size < vm_sizes[buffer_sizes_idx].size) {
							continue;
						}

						/* A zero size buffer is always accessible. */
						if (buffer_sizes_idx != ZERO_BYTES) {
							run_suite(set_up_vm_and_buffer_variables_allocate_for_copying, copy_inaccessible_tests,
							    deallocate_vm_and_buffer,
							    "mach_vm_copy() inaccessibility tests, "
							    "%s%s address, %s size: 0x%jx (%ju), buffer "
							    "%s size: 0x%jx (%ju), buffer boundary "
							    "offset: %d",
							    address_flags[flags_idx].description,
							    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
							    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
							    (uintmax_t)vm_sizes[sizes_idx].size, vm_sizes[buffer_sizes_idx].description,
							    (uintmax_t)vm_sizes[buffer_sizes_idx].size, (uintmax_t)vm_sizes[buffer_sizes_idx].size,
							    buffer_offsets[offsets_idx].offset);
						}
						run_suite(set_up_source_and_dest_variables_allocate_copy, copy_main_tests, deallocate_vm_and_buffer,
						    "mach_vm_copy() main tests, %s%s address, %s "
						    "size: 0x%jx (%ju), destination %s size: 0x%jx (%ju), "
						    "destination boundary offset: %d",
						    address_flags[flags_idx].description,
						    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
						    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
						    (uintmax_t)vm_sizes[sizes_idx].size, vm_sizes[buffer_sizes_idx].description,
						    (uintmax_t)vm_sizes[buffer_sizes_idx].size, (uintmax_t)vm_sizes[buffer_sizes_idx].size,
						    buffer_offsets[offsets_idx].offset);
						run_suite(set_up_source_and_dest_variables_allocate_copy, copy_pattern_tests, deallocate_vm_and_buffer,
						    "mach_vm_copy() pattern tests, %s%s address, %s "
						    "size: 0x%jx (%ju) destination %s size: 0x%jx (%ju), "
						    "destination boundary offset: %d",
						    address_flags[flags_idx].description,
						    (flags_idx == ANYWHERE) ? "" : address_alignments[alignments_idx].description,
						    vm_sizes[sizes_idx].description, (uintmax_t)vm_sizes[sizes_idx].size,
						    (uintmax_t)vm_sizes[sizes_idx].size, vm_sizes[buffer_sizes_idx].description,
						    (uintmax_t)vm_sizes[buffer_sizes_idx].size, (uintmax_t)vm_sizes[buffer_sizes_idx].size,
						    buffer_offsets[offsets_idx].offset);
					}
				}
			}
		}
	}
}

void
perform_test_with_options(test_option_t options)
{
	process_options(options);

	/* <rdar://problem/10304215> CoreOSZin 12Z30: VMUnitTest fails:
	 * error finding xnu major version number. */
	/* printf("xnu version is %s.\n\n", xnu_version_string()); */

	if (flag_run_allocate_test) {
		run_allocate_test_suites();
	}

	if (flag_run_deallocate_test) {
		run_deallocate_test_suites();
	}

	if (flag_run_read_test) {
		run_read_test_suites();
	}

	if (flag_run_write_test) {
		run_write_test_suites();
	}

	if (flag_run_protect_test) {
		run_protect_test_suites();
	}

	if (flag_run_copy_test) {
		run_copy_test_suites();
	}

	log_aggregated_results();
}

T_DECL(vm_test_allocate, "Allocate VM unit test")
{
	test_options.to_flags = VM_TEST_ALLOCATE;
	test_options.to_vmsize = 0;
	test_options.to_quietness = ERROR_ONLY_QUIETNESS;

	perform_test_with_options(test_options);
}

T_DECL(vm_test_deallocate, "Deallocate VM unit test",
    T_META_IGNORECRASHES(".*vm_allocation.*"))
{
	test_options.to_flags = VM_TEST_DEALLOCATE;
	test_options.to_vmsize = 0;
	test_options.to_quietness = ERROR_ONLY_QUIETNESS;

	perform_test_with_options(test_options);
}

T_DECL(vm_test_read, "Read VM unit test")
{
	test_options.to_flags = VM_TEST_READ;
	test_options.to_vmsize = 0;
	test_options.to_quietness = ERROR_ONLY_QUIETNESS;

	perform_test_with_options(test_options);
}

T_DECL(vm_test_write, "Write VM unit test")
{
	test_options.to_flags = VM_TEST_WRITE;
	test_options.to_vmsize = 0;
	test_options.to_quietness = ERROR_ONLY_QUIETNESS;

	perform_test_with_options(test_options);
}

T_DECL(vm_test_protect, "Protect VM unit test",
    T_META_IGNORECRASHES(".*vm_allocation.*"))
{
	test_options.to_flags = VM_TEST_PROTECT;
	test_options.to_vmsize = 0;
	test_options.to_quietness = ERROR_ONLY_QUIETNESS;

	perform_test_with_options(test_options);
}

T_DECL(vm_test_copy, "Copy VM unit test")
{
	test_options.to_flags = VM_TEST_COPY;
	test_options.to_vmsize = 0;
	test_options.to_quietness = ERROR_ONLY_QUIETNESS;

	perform_test_with_options(test_options);
}