/*
 * mach eventlink: Tests mach eventlink kernel synchronization primitive.
 */

#include <dispatch/dispatch.h>
#include <mach/mach.h>
#include <mach/mach_eventlink.h>
#include <mach/semaphore.h>
#include <os/atomic_private.h>
#include <pthread.h>
#include <sys/sysctl.h>

#include <darwintest.h>
#include "sched/sched_test_utils.h"
#include "test_utils.h"

T_GLOBAL_META(T_META_NAMESPACE("xnu.mach_eventlink"));

static int g_loop_iterations = 100000;
static semaphore_t g_sem_done = SEMAPHORE_NULL;

static kern_return_t
test_eventlink_create(mach_port_t *port_pair)
{
	kern_return_t kr;

	kr = mach_eventlink_create(mach_task_self(), MELC_OPTION_NO_COPYIN, port_pair);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_create");

	return kr;
}

static pthread_t
thread_create_for_test(void * (*function)(void *), void *arg)
{
	pthread_t pthread;
	pthread_attr_t attr;

	pthread_attr_init(&attr);
	pthread_create(&pthread, &attr, function, arg);

	T_LOG("pthread created\n");
	return pthread;
}

static void *
while1loop(__unused void *arg)
{
	while (1) {
		;
	}
	return NULL;
}

static void *
test_eventlink_wait_with_timeout(void *arg)
{
	kern_return_t kr;
	mach_port_t eventlink_port = (mach_port_t) (uintptr_t)arg;
	mach_port_t self = mach_thread_self();
	uint64_t ticks = mach_absolute_time();
	uint64_t count = 1;

	/* Associate thread with eventlink port */
	kr = mach_eventlink_associate(eventlink_port, self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate");

	/* Wait on the eventlink with timeout */
	kr = mach_eventlink_wait_until(eventlink_port, &count, MELSW_OPTION_NONE,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, ticks + 5000);

	T_EXPECT_MACH_ERROR(kr, KERN_OPERATION_TIMED_OUT, "mach_eventlink_wait_until returned expected error");
	T_EXPECT_EQ(count, (uint64_t)0, "mach_eventlink_wait_until returned correct count value");

	return NULL;
}

static void *
test_eventlink_wait_no_wait(void *arg)
{
	kern_return_t kr;
	mach_port_t eventlink_port = (mach_port_t) (uintptr_t)arg;
	mach_port_t self = mach_thread_self();
	uint64_t count = 1;

	/* Associate thread with eventlink port */
	kr = mach_eventlink_associate(eventlink_port, self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate");

	/* Wait on the eventlink */
	kr = mach_eventlink_wait_until(eventlink_port, &count, MELSW_OPTION_NO_WAIT,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_EXPECT_MACH_ERROR(kr, KERN_OPERATION_TIMED_OUT, "mach_eventlink_wait_until returned expected error");
	T_EXPECT_EQ(count, (uint64_t)0, "mach_eventlink_wait_until returned correct count value");

	return NULL;
}

static void *
test_eventlink_wait_destroy(void *arg)
{
	kern_return_t kr;
	mach_port_t eventlink_port = (mach_port_t) (uintptr_t)arg;
	mach_port_t self = mach_thread_self();
	uint64_t count = 1;

	/* Associate thread with eventlink port */
	kr = mach_eventlink_associate(eventlink_port, self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate");

	/* Wait on the eventlink */
	kr = mach_eventlink_wait_until(eventlink_port, &count, MELSW_OPTION_NONE,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_EXPECT_MACH_ERROR(kr, KERN_TERMINATED, "mach_eventlink_wait_until returned expected error");

	return NULL;
}

static void *
test_eventlink_wait_for_signal(void *arg)
{
	kern_return_t kr;
	mach_port_t eventlink_port = (mach_port_t) (uintptr_t)arg;
	mach_port_t self = mach_thread_self();
	uint64_t count = 0;

	/* Associate thread with eventlink port */
	kr = mach_eventlink_associate(eventlink_port, self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate");

	/* Wait on the eventlink */
	kr = mach_eventlink_wait_until(eventlink_port, &count, MELSW_OPTION_NONE,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_wait_until");
	T_EXPECT_EQ(count, (uint64_t)1, "mach_eventlink_wait_until returned correct count value");

	return NULL;
}

static void *
test_eventlink_wait_then_signal(void *arg)
{
	kern_return_t kr;
	mach_port_t eventlink_port = (mach_port_t) (uintptr_t)arg;
	mach_port_t self = mach_thread_self();
	uint64_t count = 0;

	/* Associate thread with eventlink port */
	kr = mach_eventlink_associate(eventlink_port, self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate");

	/* Wait on the eventlink */
	kr = mach_eventlink_wait_until(eventlink_port, &count, MELSW_OPTION_NONE,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_wait_until");
	T_EXPECT_EQ(count, (uint64_t)1, "mach_eventlink_wait_until returned correct count value");

	/* Signal the eventlink to wakeup other side */
	kr = mach_eventlink_signal(eventlink_port, 0);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_signal");

	return NULL;
}

static void *
test_eventlink_wait_then_wait_signal_with_no_wait(void *arg)
{
	kern_return_t kr;
	mach_port_t eventlink_port = (mach_port_t) (uintptr_t)arg;
	mach_port_t self = mach_thread_self();
	uint64_t count = 0;

	/* Associate thread with eventlink port */
	kr = mach_eventlink_associate(eventlink_port, self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate");

	/* Wait on the eventlink */
	kr = mach_eventlink_wait_until(eventlink_port, &count, MELSW_OPTION_NONE,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_wait_until");
	T_EXPECT_EQ(count, (uint64_t)1, "mach_eventlink_wait_until returned correct count value");

	/* Signal wait the eventlink */
	kr = mach_eventlink_signal_wait_until(eventlink_port, &count, 0, MELSW_OPTION_NO_WAIT,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_EXPECT_MACH_ERROR(kr, KERN_OPERATION_TIMED_OUT, "mach_eventlink_wait_until returned expected error");
	T_EXPECT_EQ(count, (uint64_t)1, "mach_eventlink_wait_until returned correct count value");

	return NULL;
}

static void *
test_eventlink_wait_then_wait_signal_with_prepost(void *arg)
{
	kern_return_t kr;
	mach_port_t eventlink_port = (mach_port_t) (uintptr_t)arg;
	mach_port_t self = mach_thread_self();
	uint64_t count = 0;

	/* Associate thread with eventlink port */
	kr = mach_eventlink_associate(eventlink_port, self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate");

	/* Wait on the eventlink */
	kr = mach_eventlink_wait_until(eventlink_port, &count, MELSW_OPTION_NONE,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_wait_until");
	T_EXPECT_EQ(count, (uint64_t)1, "mach_eventlink_wait_until returned correct count value");

	/* Signal wait the eventlink with stale counter value */
	count = 0;
	kr = mach_eventlink_signal_wait_until(eventlink_port, &count, 0, MELSW_OPTION_NONE,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_signal_wait_until");
	T_EXPECT_EQ(count, (uint64_t)1, "mach_eventlink_wait_until returned correct count value");

	return NULL;
}

static void *
test_eventlink_wait_then_signal_loop(void *arg)
{
	kern_return_t kr;
	mach_port_t eventlink_port = (mach_port_t) (uintptr_t)arg;
	mach_port_t self = mach_thread_self();
	uint64_t count = 0;
	int i;

	/* Associate thread with eventlink port */
	kr = mach_eventlink_associate(eventlink_port, self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate");

	/* Wait on the eventlink */
	kr = mach_eventlink_wait_until(eventlink_port, &count, MELSW_OPTION_NONE,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_wait_until");
	T_EXPECT_EQ(count, (uint64_t)1, "mach_eventlink_wait_until returned correct count value");

	for (i = 1; i < g_loop_iterations; i++) {
		/* Signal wait the eventlink */
		kr = mach_eventlink_signal_wait_until(eventlink_port, &count, 0, MELSW_OPTION_NONE,
		    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

		if (kr != KERN_SUCCESS) {
			T_ASSERT_MACH_SUCCESS(kr,
			    "mach_eventlink_signal_wait_until");
		}
		if (count != (uint64_t)(i + 1)) {
			T_EXPECT_EQ(count, (uint64_t)(i + 1),
			    "mach_eventlink_wait_until returned correct count value");
		}
	}

	/* Signal the eventlink to wakeup other side */
	kr = mach_eventlink_signal(eventlink_port, 0);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_signal");

	if (g_sem_done != SEMAPHORE_NULL) {
		kr = semaphore_wait(g_sem_done);
		T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_wait(g_sem_done)");
	}

	return NULL;
}

/*
 * Test 1: Create ipc eventlink kernel object.
 *
 * Calls eventlink creates which returns a pair of eventlink port objects.
 */
T_DECL(test_eventlink_create, "eventlink create test",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];

	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	mach_port_deallocate(mach_task_self(), port_pair[0]);
	mach_port_deallocate(mach_task_self(), port_pair[1]);
}

/*
 * Test 2: Create ipc eventlink kernel object and call eventlink destroy
 *
 * Calls eventlink creates which returns a pair of eventlink port objects.
 * Calls eventlink destroy on eventlink port pair.
 */
T_DECL(test_eventlink_destroy, "eventlink destroy test",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];

	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	kr = mach_eventlink_destroy(port_pair[0]);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_destroy");
	kr = mach_eventlink_destroy(port_pair[1]);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_destroy");
}

/*
 * Test 3: Associate threads to eventlink object.
 *
 * Create eventlink object pair and associate threads to each side and then
 * disassociate threads and check for error conditions.
 */
T_DECL(test_eventlink_associate, "eventlink associate test",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];
	mach_port_t self = mach_thread_self();
	mach_port_t other_thread = MACH_PORT_NULL;
	pthread_t pthread;

	/* eventlink associate to NULL eventlink object */
	kr = mach_eventlink_associate(MACH_PORT_NULL, self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_EXPECT_MACH_ERROR(kr, MACH_SEND_INVALID_DEST, "mach_eventlink_associate with null eventlink returned expected error");

	/* eventlink disassociate to NULL eventlink object */
	kr = mach_eventlink_disassociate(MACH_PORT_NULL, MELD_OPTION_NONE);
	T_EXPECT_MACH_ERROR(kr, MACH_SEND_INVALID_DEST, "mach_eventlink_disassociate with null eventlink returned expected error");

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(while1loop, NULL);
	other_thread = pthread_mach_thread_np(pthread);

	for (int i = 0; i < 3; i++) {
		/* Associate thread to eventlink objects */
		kr = mach_eventlink_associate(port_pair[0], self, 0, 0, 0, 0, MELA_OPTION_NONE);
		T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate for object 1");

		kr = mach_eventlink_associate(port_pair[1], other_thread, 0, 0, 0, 0, MELA_OPTION_NONE);
		T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate for object 2");

		/* Try to associate again with diff threads, expect failure */
		kr = mach_eventlink_associate(port_pair[0], other_thread, 0, 0, 0, 0, MELA_OPTION_NONE);
		T_EXPECT_MACH_ERROR(kr, KERN_NAME_EXISTS, "mach_eventlink_associate for associated "
		    "objects returned expected error");

		kr = mach_eventlink_associate(port_pair[1], self, 0, 0, 0, 0, MELA_OPTION_NONE);
		T_EXPECT_MACH_ERROR(kr, KERN_NAME_EXISTS, "mach_eventlink_associate for associated "
		    "objects return expected error");

		/* Try to disassociate the threads */
		kr = mach_eventlink_disassociate(port_pair[0], MELD_OPTION_NONE);
		T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_disassociate for object 1");

		kr = mach_eventlink_disassociate(port_pair[1], MELD_OPTION_NONE);
		T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_disassociate for object 2");

		/* Try to disassociate the threads again, expect failure */
		kr = mach_eventlink_disassociate(port_pair[0], MELD_OPTION_NONE);
		T_EXPECT_MACH_ERROR(kr, KERN_INVALID_ARGUMENT, "mach_eventlink_disassociate for "
		    "disassociated objects returned expected error");

		kr = mach_eventlink_disassociate(port_pair[1], MELD_OPTION_NONE);
		T_EXPECT_MACH_ERROR(kr, KERN_INVALID_ARGUMENT, "mach_eventlink_disassociate for "
		    "disassociated objects returned expected error");
	}

	kr = mach_eventlink_destroy(port_pair[0]);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_destroy");

	/* Try disassociate on other end of destoryed eventlink pair */
	kr = mach_eventlink_disassociate(port_pair[1], MELD_OPTION_NONE);
	T_EXPECT_MACH_ERROR(kr, KERN_TERMINATED, "mach_eventlink_disassociate for "
	    "terminated object returned expected error");

	kr = mach_eventlink_destroy(port_pair[1]);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_destroy");
}

/*
 * Test 4: Test eventlink wait with timeout.
 *
 * Create an eventlink object, associate threads and test eventlink wait with timeout.
 */
T_DECL(test_eventlink_wait_timeout, "eventlink wait timeout test",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];
	pthread_t pthread;

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(test_eventlink_wait_with_timeout, (void *)(uintptr_t)port_pair[0]);
	sleep(10);

	/* destroy the eventlink object, the wake status of thread will check if the test passsed or failed */
	mach_port_deallocate(mach_task_self(), port_pair[0]);
	mach_port_deallocate(mach_task_self(), port_pair[1]);

	pthread_join(pthread, NULL);
}

/*
 * Test 5: Test eventlink wait with no wait.
 *
 * Create an eventlink object, associate threads and test eventlink wait with no wait flag.
 */
T_DECL(test_eventlink_wait_no_wait, "eventlink wait no wait test",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];
	pthread_t pthread;

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(test_eventlink_wait_no_wait, (void *)(uintptr_t)port_pair[0]);
	pthread_join(pthread, NULL);

	mach_port_deallocate(mach_task_self(), port_pair[0]);
	mach_port_deallocate(mach_task_self(), port_pair[1]);
}

/*
 * Test 6: Test eventlink wait and destroy.
 *
 * Create an eventlink object, associate threads and destroy the port.
 */
T_DECL(test_eventlink_wait_and_destroy, "eventlink wait and destroy",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];
	pthread_t pthread;

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(test_eventlink_wait_destroy, (void *)(uintptr_t)port_pair[0]);

	sleep(5);

	/* Increase the send right count for port before destroy to make sure no sender does not fire on destroy */
	kr = mach_port_mod_refs(mach_task_self(), port_pair[0], MACH_PORT_RIGHT_SEND, 2);
	T_ASSERT_MACH_SUCCESS(kr, "mach_port_mod_refs");

	/* Destroy the port for thread to wakeup */
	kr = mach_eventlink_destroy(port_pair[0]);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_destroy");

	pthread_join(pthread, NULL);
	mach_port_deallocate(mach_task_self(), port_pair[1]);
}


/*
 * Test 7: Test eventlink wait and destroy remote side.
 *
 * Create an eventlink object, associate threads, wait and destroy the remote eventlink port.
 */
T_DECL(test_eventlink_wait_and_destroy_remote, "eventlink wait and remote destroy",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];
	pthread_t pthread;

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(test_eventlink_wait_destroy, (void *)(uintptr_t)port_pair[0]);

	sleep(5);

	/* Increase the send right count for port before destroy to make sure no sender does not fire on destroy */
	kr = mach_port_mod_refs(mach_task_self(), port_pair[1], MACH_PORT_RIGHT_SEND, 2);
	T_ASSERT_MACH_SUCCESS(kr, "mach_port_mod_refs");

	/* Destroy the port for thread to wakeup */
	kr = mach_eventlink_destroy(port_pair[1]);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_destroy");

	pthread_join(pthread, NULL);
	mach_port_deallocate(mach_task_self(), port_pair[0]);
}

/*
 * Test 8: Test eventlink wait and deallocate port.
 *
 * Create an eventlink object, associate threads, wait and deallocate the eventlink port.
 */
T_DECL(test_eventlink_wait_and_deallocate, "eventlink wait and deallocate",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];
	pthread_t pthread;

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(test_eventlink_wait_destroy, (void *)(uintptr_t)port_pair[0]);

	sleep(5);

	/* Destroy the port for thread to wakeup */
	mach_port_deallocate(mach_task_self(), port_pair[0]);

	pthread_join(pthread, NULL);
	mach_port_deallocate(mach_task_self(), port_pair[1]);
}

/*
 * Test 9: Test eventlink wait and disassociate.
 *
 * Create an eventlink object, associate threads, wait and disassociate thread from the eventlink port.
 */
T_DECL(test_eventlink_wait_and_disassociate, "eventlink wait and disassociate",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];
	pthread_t pthread;

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(test_eventlink_wait_destroy, (void *)(uintptr_t)port_pair[0]);

	sleep(5);

	/* Disassociate thread from eventlink for thread to wakeup */
	kr = mach_eventlink_disassociate(port_pair[0], MELD_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_disassociate");

	pthread_join(pthread, NULL);
	mach_port_deallocate(mach_task_self(), port_pair[1]);
	mach_port_deallocate(mach_task_self(), port_pair[0]);
}

/*
 * Test 10: Test eventlink wait and signal.
 *
 * Create an eventlink object, associate threads and test wait signal.
 */
T_DECL(test_eventlink_wait_and_signal, "eventlink wait and signal",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];
	pthread_t pthread;
	mach_port_t self = mach_thread_self();

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(test_eventlink_wait_for_signal, (void *)(uintptr_t)port_pair[0]);

	sleep(5);

	/* Associate thread and signal the eventlink */
	kr = mach_eventlink_associate(port_pair[1], self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate for object 2");

	kr = mach_eventlink_signal(port_pair[1], 0);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_signal for object 2");

	pthread_join(pthread, NULL);

	mach_port_deallocate(mach_task_self(), port_pair[0]);
	mach_port_deallocate(mach_task_self(), port_pair[1]);
}

/*
 * Test 11: Test eventlink wait_signal.
 *
 * Create an eventlink object, associate threads and test wait_signal.
 */
T_DECL(test_eventlink_wait_signal, "eventlink wait_signal",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];
	pthread_t pthread;
	mach_port_t self = mach_thread_self();
	uint64_t count = 0;

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(test_eventlink_wait_then_signal, (void *)(uintptr_t)port_pair[0]);

	sleep(5);

	/* Associate thread and wait_signal the eventlink */
	kr = mach_eventlink_associate(port_pair[1], self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate for object 2");

	/* Wait on the eventlink with timeout */
	kr = mach_eventlink_signal_wait_until(port_pair[1], &count, 0, MELSW_OPTION_NONE,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_signal_wait_until");
	T_EXPECT_EQ(count, (uint64_t)1, "mach_eventlink_signal_wait_until returned correct count value");

	pthread_join(pthread, NULL);

	mach_port_deallocate(mach_task_self(), port_pair[0]);
	mach_port_deallocate(mach_task_self(), port_pair[1]);
}

/*
 * Test 12: Test eventlink wait_signal with no wait.
 *
 * Create an eventlink object, associate threads and test wait_signal with no wait.
 */
T_DECL(test_eventlink_wait_signal_no_wait, "eventlink wait_signal with no wait",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];
	pthread_t pthread;
	mach_port_t self = mach_thread_self();
	uint64_t count = 0;

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(test_eventlink_wait_then_wait_signal_with_no_wait, (void *)(uintptr_t)port_pair[0]);

	sleep(5);

	/* Associate thread and wait_signal the eventlink */
	kr = mach_eventlink_associate(port_pair[1], self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate for object 2");

	/* Wait on the eventlink with timeout */
	kr = mach_eventlink_signal_wait_until(port_pair[1], &count, 0, MELSW_OPTION_NONE,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_signal_wait_until");
	T_EXPECT_EQ(count, (uint64_t)1, "mach_eventlink_signal_wait_until returned correct count value");

	pthread_join(pthread, NULL);

	mach_port_deallocate(mach_task_self(), port_pair[0]);
	mach_port_deallocate(mach_task_self(), port_pair[1]);
}

/*
 * Test 13: Test eventlink wait_signal with prepost.
 *
 * Create an eventlink object, associate threads and test wait_signal with prepost.
 */
T_DECL(test_eventlink_wait_signal_prepost, "eventlink wait_signal with prepost",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];
	pthread_t pthread;
	mach_port_t self = mach_thread_self();
	uint64_t count = 0;

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(test_eventlink_wait_then_wait_signal_with_prepost, (void *)(uintptr_t)port_pair[0]);

	sleep(5);

	/* Associate thread and wait_signal the eventlink */
	kr = mach_eventlink_associate(port_pair[1], self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate for object 2");

	/* Wait on the eventlink with timeout */
	kr = mach_eventlink_signal_wait_until(port_pair[1], &count, 0, MELSW_OPTION_NONE,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_signal_wait_until");
	T_EXPECT_EQ(count, (uint64_t)1, "mach_eventlink_signal_wait_until returned correct count value");

	pthread_join(pthread, NULL);

	mach_port_deallocate(mach_task_self(), port_pair[0]);
	mach_port_deallocate(mach_task_self(), port_pair[1]);
}

/*
 * Test 14: Test eventlink wait_signal with associate on wait option.
 *
 * Create an eventlink object, set associate on wait on one side and test wait_signal.
 */
T_DECL(test_eventlink_wait_signal_associate_on_wait, "eventlink wait_signal associate on wait",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];
	pthread_t pthread;
	uint64_t count = 0;

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(test_eventlink_wait_then_signal, (void *)(uintptr_t)port_pair[0]);

	sleep(5);

	/* Set associate on wait and wait_signal the eventlink */
	kr = mach_eventlink_associate(port_pair[1], MACH_PORT_NULL, 0, 0, 0, 0, MELA_OPTION_ASSOCIATE_ON_WAIT);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate with associate on wait for object 2");

	/* Wait on the eventlink with timeout */
	kr = mach_eventlink_signal_wait_until(port_pair[1], &count, 0, MELSW_OPTION_NONE,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_signal_wait_until");
	T_EXPECT_EQ(count, (uint64_t)1, "mach_eventlink_signal_wait_until returned correct count value");

	/* Remove associate on wait option */
	kr = mach_eventlink_disassociate(port_pair[1], MELD_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_disassociate");

	/* Wait on the eventlink with timeout */
	kr = mach_eventlink_signal_wait_until(port_pair[1], &count, 0, MELSW_OPTION_NONE,
	    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

	T_EXPECT_MACH_ERROR(kr, KERN_INVALID_ARGUMENT, "mach_eventlink_wait_until returned expected error");

	pthread_join(pthread, NULL);

	mach_port_deallocate(mach_task_self(), port_pair[0]);
	mach_port_deallocate(mach_task_self(), port_pair[1]);
}

/*
 * Test 15: Test eventlink wait_signal_loop.
 *
 * Create an eventlink object, associate threads and test wait_signal in a loop.
 */
T_DECL(test_eventlink_wait_signal_loop, "eventlink wait_signal in loop",
    T_META_ASROOT(true), T_META_RUN_CONCURRENTLY(true), T_META_TAG_VM_PREFERRED)
{
	kern_return_t kr;
	mach_port_t port_pair[2];
	pthread_t pthread;
	mach_port_t self = mach_thread_self();
	uint64_t count = 0;
	int i;

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(test_eventlink_wait_then_signal_loop, (void *)(uintptr_t)port_pair[0]);

	/* Associate thread and wait_signal the eventlink */
	kr = mach_eventlink_associate(port_pair[1], self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate for object 2");

	for (i = 0; i < g_loop_iterations; i++) {
		/* Wait on the eventlink with timeout */
		kr = mach_eventlink_signal_wait_until(port_pair[1], &count, 0, MELSW_OPTION_NONE,
		    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

		T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "main thread: mach_eventlink_signal_wait_until");
		T_QUIET; T_EXPECT_EQ(count, (uint64_t)(i + 1), "main thread: mach_eventlink_signal_wait_until returned correct count value");
	}

	pthread_join(pthread, NULL);

	mach_port_deallocate(mach_task_self(), port_pair[0]);
	mach_port_deallocate(mach_task_self(), port_pair[1]);
}


static const uint64_t DEFAULT_INTERVAL_NS = 15000000; // 15 ms

static void
set_realtime(pthread_t thread, uint64_t interval_nanos)
{
	kern_return_t kr;
	thread_time_constraint_policy_data_t pol;

	mach_port_t target_thread = pthread_mach_thread_np(thread);
	T_QUIET; T_ASSERT_GT(target_thread, 0, "pthread_mach_thread_np");

	/* 1s 100ms 10ms */
	pol.period      = (uint32_t)nanos_to_abs(interval_nanos);
	pol.constraint  = (uint32_t)nanos_to_abs(interval_nanos);
	pol.computation = (uint32_t)nanos_to_abs(interval_nanos - 1000000); // 1 ms of leeway

	pol.preemptible = 0; /* Ignored by OS */
	kr = thread_policy_set(target_thread, THREAD_TIME_CONSTRAINT_POLICY, (thread_policy_t) &pol,
	    THREAD_TIME_CONSTRAINT_POLICY_COUNT);
	T_ASSERT_MACH_SUCCESS(kr, "thread_policy_set(THREAD_TIME_CONSTRAINT_POLICY)");
}


static _Atomic bool suspend_resume_thread_stop = false;

static void *
test_suspend_resume_thread(void *arg)
{
	T_ASSERT_NE(g_sem_done, SEMAPHORE_NULL, "g_sem_done should be initialized");

	uint64_t count = 0;
	mach_port_t suspend_resume_other_thread_port = (mach_port_t) (uintptr_t)arg;
	kern_return_t kr1 = KERN_SUCCESS, kr2 = KERN_SUCCESS;

	while (!os_atomic_load(&suspend_resume_thread_stop, relaxed)
	    && kr1 == KERN_SUCCESS && kr2 == KERN_SUCCESS) {
		kr1 = thread_suspend(suspend_resume_other_thread_port);
		kr2 = thread_resume(suspend_resume_other_thread_port);
		count++;

		if (count % 100 == 0) {
			/*
			 * A tight loop may entirely prevent forward progress
			 * for the targeted thread.  Back off a bit.
			 */
			usleep(100);
		}
	}

	T_LOG("thread suspend/resume count: %llu", count);
	T_EXPECT_MACH_SUCCESS(kr1, "thread_suspend");
	T_EXPECT_MACH_SUCCESS(kr2, "thread_resume");

	/* Signal that it is now safe to exit the thread under test. */
	int kr = semaphore_signal(g_sem_done);
	T_QUIET; T_EXPECT_MACH_SUCCESS(kr, "semaphore_signal(g_sem_done)");

	return NULL;
}

static void
log_handoff_sysctl(void)
{
	if (!is_development_kernel()) {
		/* kern.direct_handoff does not exist on release kernel */
		return;
	}

	int direct_handoff = 0;
	size_t size = sizeof(direct_handoff);
	T_QUIET; T_ASSERT_POSIX_ZERO(sysctlbyname("kern.direct_handoff", &direct_handoff,
	    &size, NULL, 0), "sysctlbyname kern.direct_handoff");
	T_LOG("kern.direct_handoff: %d", direct_handoff);
}

static uint64_t
handoff_success_count_sysctl(void)
{
	if (!is_development_kernel()) {
		/* kern.mach_eventlink_handoff_success_count does not exist on release kernel */
		return 0;
	}
	uint64_t success_count = 0;
	size_t size = sizeof(success_count);
	T_QUIET; T_ASSERT_POSIX_ZERO(sysctlbyname("kern.mach_eventlink_handoff_success_count",
	    &success_count,
	    &size, NULL, 0), "sysctlbyname kern.mach_eventlink_handoff_success_count");
	return success_count;
}

/*
 * Test 16: Test suspension of a thread in the middle of a wait-signal operation
 * Also tests that the eventlink signal leads to a real handoff.
 * rdar://120761588 rdar://123887338 rdar://138657435
 */
T_DECL(test_eventlink_wait_signal_suspend_loop, "eventlink wait_signal + thread_suspend/resume in loop",
    T_META_RUN_CONCURRENTLY(false), /* Test uses global handoff counter */
    T_META_TAG_VM_PREFERRED,
    T_META_ENABLED(!TARGET_OS_VISION) /* <rdar://143331046> */
    ) {
	kern_return_t kr;
	mach_port_t port_pair[2];
	pthread_t pthread, suspend_thread;
	mach_port_t self = mach_thread_self();
	uint64_t count = 0;
	int i;

	bool development = is_development_kernel();
	T_LOG("kern.development: %d", development);

	log_handoff_sysctl();

	uint64_t handoffs_start = handoff_success_count_sysctl();;
	T_LOG("handoffs_start: %llu", handoffs_start);

	kr = semaphore_create(mach_task_self(), &g_sem_done, SYNC_POLICY_FIFO, 0);
	T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_create");

	/* Create an eventlink and associate threads to it */
	kr = test_eventlink_create(port_pair);
	if (kr != KERN_SUCCESS) {
		return;
	}

	pthread = thread_create_for_test(test_eventlink_wait_then_signal_loop, (void *)(uintptr_t)port_pair[0]);
	mach_port_t suspend_resume_other_thread_port = pthread_mach_thread_np(pthread);

	/*
	 * Threads must be RT to get direct handoff mode.
	 */
	set_realtime(pthread_self(), DEFAULT_INTERVAL_NS);
	set_realtime(pthread, DEFAULT_INTERVAL_NS);

	suspend_thread = thread_create_for_test(test_suspend_resume_thread,
	    (void *)(uintptr_t)suspend_resume_other_thread_port);

	/* Associate thread and wait_signal the eventlink */
	kr = mach_eventlink_associate(port_pair[1], self, 0, 0, 0, 0, MELA_OPTION_NONE);
	T_ASSERT_MACH_SUCCESS(kr, "mach_eventlink_associate for object 2");

	for (i = 0; i < g_loop_iterations; i++) {
		/* Wait on the eventlink with timeout */
		kr = mach_eventlink_signal_wait_until(port_pair[1], &count, 0, MELSW_OPTION_NONE,
		    KERN_CLOCK_MACH_ABSOLUTE_TIME, 0);

		if (kr != KERN_SUCCESS) {
			T_ASSERT_MACH_SUCCESS(kr,
			    "main thread: mach_eventlink_signal_wait_until");
		}
		if (count != (uint64_t)(i + 1)) {
			T_EXPECT_EQ(count, (uint64_t)(i + 1),
			    "main thread: mach_eventlink_signal_wait_until returned correct count value");
		}
	}

	os_atomic_store(&suspend_resume_thread_stop, true, relaxed);

	pthread_join(suspend_thread, NULL);
	pthread_join(pthread, NULL);

	kr = mach_port_deallocate(mach_task_self(), port_pair[0]);
	T_ASSERT_MACH_SUCCESS(kr, "mach_port_deallocate");
	kr = mach_port_deallocate(mach_task_self(), port_pair[1]);
	T_ASSERT_MACH_SUCCESS(kr, "mach_port_deallocate");
	kr = semaphore_destroy(mach_task_self(), g_sem_done);
	T_QUIET; T_ASSERT_MACH_SUCCESS(kr, "semaphore_destroy");

	uint64_t handoffs_end = handoff_success_count_sysctl();
	T_LOG("handoffs_end: %llu", handoffs_end);

	if (development) {
		T_QUIET; T_ASSERT_GE(handoffs_end, handoffs_start, "kern.mach_eventlink_handoff_success_count did not overflow");
		const uint64_t successful_handoffs = handoffs_end - handoffs_start;
		const uint64_t min_handoffs = MAX(2 * g_loop_iterations, 2) - 2;
		T_EXPECT_GE(successful_handoffs, min_handoffs, "found at least %llu handoffs", min_handoffs);
	}
}