/*
 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
 *
 * This file contains the low-level serial drivers used on ARM/ARM64 devices.
 * The generic serial console code in osfmk/console/serial_console.c will call
 * into this code to transmit and receive serial data.
 *
 * Logging can be performed on multiple serial interfaces at once through a
 * method called serial multiplexing. This is implemented by enumerating which
 * serial interfaces are available on boot and registering them into a linked
 * list of interfaces pointed to by gPESF. When outputting or receiving
 * characters, each interface is queried in turn.
 *
 * Please view doc/arm_serial.md for an in-depth description of these drivers.
 */
#include <kern/clock.h>
#include <kern/debug.h>
#include <libkern/OSBase.h>
#include <libkern/section_keywords.h>
#include <mach/mach_time.h>
#include <machine/atomic.h>
#include <machine/machine_routines.h>
#include <pexpert/pexpert.h>
#include <pexpert/protos.h>
#include <pexpert/device_tree.h>
#if defined __arm__
#include <arm/caches_internal.h>
#include <arm/machine_routines.h>
#include <arm/proc_reg.h>
#include <pexpert/arm/board_config.h>
#include <vm/pmap.h>
#elif defined __arm64__
#include <pexpert/arm/consistent_debug.h>
#include <pexpert/arm64/board_config.h>
#include <arm64/proc_reg.h>
#endif
#include <pexpert/arm/protos.h>
#include <kern/sched_prim.h>
#if HIBERNATION
#include <machine/pal_hibernate.h>
#endif /* HIBERNATION */

struct pe_serial_functions {
	/* Initialize the underlying serial hardware. */
	void (*init) (void);

	/* Return a non-zero value if the serial interface is ready to send more data. */
	unsigned int (*transmit_ready) (void);

	/* Write a single byte of data to serial. */
	void (*transmit_data) (uint8_t c);

	/* Return a non-zero value if there's a byte of data available. */
	unsigned int (*receive_ready) (void);

	/* Read a single byte from serial. */
	uint8_t (*receive_data) (void);

	/* Enables IRQs from this device. */
	void (*enable_irq) (void);

	/* Disables IRQs from this device. */
	void (*disable_irq) (void);

	/* Clears IRQs from this device. */
	bool (*acknowledge_irq) (void);

	/**
	 * Whether this serial driver can handle irqs. This value should be set by
	 * querying the device tree to see if the serial device has interrupts
	 * associated with it.
	 *
	 * For a device to support IRQs:
	 *   - enable_irq, disable_irq, and acknowledge_irq must be non-null
	 *   - The AppleSerialShim kext must be able to match to the serial device
	 *     in the IORegistry and call serial_enable_irq with the proper
	 *     serial_device_t
	 *   - The device tree entry for the serial device should have an interrupt
	 *     associated with it.
	 */
	bool has_irq;

	/* enum identifying which serial device these functions belong to. */
	serial_device_t device;

	/* Pointer to the next serial interface in the linked-list. */
	struct pe_serial_functions *next;
};

MARK_AS_HIBERNATE_DATA_CONST_LATE static struct pe_serial_functions* gPESF = NULL;

/**
 * Whether uart has been initialized already. This value is kept across a
 * sleep/wake cycle so we know we need to reinitialize when serial_init is
 * called again after wake.
 */
MARK_AS_HIBERNATE_DATA static bool uart_initted = false;

/* Whether uart should run in simple mode that works during hibernation resume. */
MARK_AS_HIBERNATE_DATA static bool uart_hibernation = false;

/**
 * Used to track if all IRQs have been initialized. Each bit of this variable
 * represents whether or not a serial device that reports supporting IRQs has
 * been initialized yet (1 -> not initialized, 0 -> initialized)
 */
static uint32_t serial_irq_status = 0;

/**
 * Set by the 'disable-uart-irq' boot-arg to force serial IRQs into polling mode
 * by preventing the serial driver shim kext from registering itself with
 * serial_enable_irq.
 */
static bool disable_uart_irq = 0;

/**
 * Indicates whether or not a given device's irqs have been set up by calling
 * serial_enable_irq for that particular device.
 *
 * @param device_fns Serial functions for the device that is being checked
 * @return Whether or not the irqs have been initialized for that device
 */
static bool
irq_initialized(struct pe_serial_functions *device_fns)
{
	return (serial_irq_status & device_fns->device) == 0;
}

/**
 * Indicates whether or not a given device supports irqs and if they are ready
 * to be used.
 *
 * @param device_fns Serial functions for the device that is being checked
 * @return Whether or not the device can and will send IRQs.
 */
static bool
irq_available_and_ready(struct pe_serial_functions *device_fns)
{
	return device_fns->has_irq && irq_initialized(device_fns);
}

/**
 * Searches through the global serial functions list and returns the serial function for a particular device
 *
 * @param device The device identifier to search for
 * @return Serial functions for the specified device
 */
static struct pe_serial_functions *
get_serial_functions(serial_device_t device)
{
	struct pe_serial_functions *fns = gPESF;
	while (fns != NULL) {
		if (fns->device == device) {
			return fns;
		}
		fns = fns->next;
	}
	return NULL;
}

/**
 * The action to take when polling and waiting for a serial device to be ready
 * for output. On ARM64, takes a WFE because the WFE timeout will wake us up in
 * the worst case. On ARMv7 devices, we need to hot poll.
 */
static void
serial_poll(void)
{
	#if __arm64__
	__builtin_arm_wfe();
	#endif
}

/**
 * This ensures that if we have a future product that supports hibernation, but
 * doesn't support either UART serial or dock-channels, then hibernation will
 * gracefully fall back to the serial method that is supported.
 */
#if HIBERNATION || defined(APPLE_UART)
MARK_AS_HIBERNATE_DATA static vm_offset_t uart_base = 0;
#endif /* HIBERNATION || defined(APPLE_UART) */

#if HIBERNATION || defined(DOCKCHANNEL_UART)
MARK_AS_HIBERNATE_DATA static vm_offset_t dockchannel_uart_base = 0;
#endif /* HIBERNATION || defined(DOCKCHANNEL_UART) */

/*****************************************************************************/

#ifdef APPLE_UART

static int32_t dt_pclk      = -1;
static int32_t dt_sampling  = -1;
static int32_t dt_ubrdiv    = -1;

static void apple_uart_set_baud_rate(uint32_t baud_rate);

static void
apple_uart_init(void)
{
	uint32_t ucon0 = 0x405; /* NCLK, No interrupts, No DMA - just polled */

	rULCON0 = 0x03;         /* 81N, not IR */

	// Override with pclk dt entry
	if (dt_pclk != -1) {
		ucon0 = ucon0 & ~0x400;
	}

	rUCON0 = ucon0;
	rUMCON0 = 0x00;         /* Clear Flow Control */

	apple_uart_set_baud_rate(115200);

	rUFCON0 = 0x07;         /* Clear & Enable FIFOs */
	rUMCON0 = 0x01;         /* Assert RTS on UART0 */
}

static void
apple_uart_enable_irq(void)
{
	/* sets Tx FIFO watermark to 0 bytes so interrupt is sent when FIFO empty */
	rUFCON0 &= ~(0xC0);

	/* Enables Tx interrupt */
	rUCON0 |= 0x2000;
}

static void
apple_uart_disable_irq(void)
{
	/* Disables Tx interrupts */
	rUCON0 &= ~(0x2000);
}

static bool
apple_uart_ack_irq(void)
{
	rUTRSTAT0 |= 0x20;
	return true;
}

static void
apple_uart_drain_fifo(void)
{
	/* wait while Tx FIFO is full or the FIFO count != 0 */
	while ((rUFSTAT0 & 0x2F0)) {
		serial_poll();
	}
}

static void
apple_uart_set_baud_rate(uint32_t baud_rate)
{
	uint32_t div = 0;
	uint32_t uart_clock = 0;
	uint32_t sample_rate = 16;

	if (baud_rate < 300) {
		baud_rate = 9600;
	}

	if (rUCON0 & 0x400) {
		// NCLK
		uart_clock = (uint32_t)gPEClockFrequencyInfo.fix_frequency_hz;
	} else {
		// PCLK
		uart_clock = (uint32_t)gPEClockFrequencyInfo.prf_frequency_hz;
	}

	if (dt_sampling != -1) {
		// Use the sampling rate specified in the Device Tree
		sample_rate = dt_sampling & 0xf;
	}

	if (dt_ubrdiv != -1) {
		// Use the ubrdiv specified in the Device Tree
		div = dt_ubrdiv & 0xffff;
	} else {
		// Calculate ubrdiv. UBRDIV = (SourceClock / (BPS * Sample Rate)) - 1
		div = uart_clock / (baud_rate * sample_rate);

		uint32_t actual_baud = uart_clock / ((div + 0) * sample_rate);
		uint32_t baud_low    = uart_clock / ((div + 1) * sample_rate);

		// Adjust div to get the closest target baudrate
		if ((baud_rate - baud_low) > (actual_baud - baud_rate)) {
			div--;
		}
	}

	// Sample Rate [19:16], UBRDIV [15:0]
	rUBRDIV0 = ((16 - sample_rate) << 16) | div;
}

MARK_AS_HIBERNATE_TEXT static unsigned int
apple_uart_tr0(void)
{
	/* UART is ready unless the FIFO is full. */
	return (rUFSTAT0 & 0x200) == 0;
}

MARK_AS_HIBERNATE_TEXT static void
apple_uart_td0(uint8_t c)
{
	rUTXH0 = c;
}

static unsigned int
apple_uart_rr0(void)
{
	/* Receive is ready when there are >0 bytes in the receive FIFO */
	return rUFSTAT0 & 0xF;
}

static uint8_t
apple_uart_rd0(void)
{
	return (uint8_t)rURXH0;
}

MARK_AS_HIBERNATE_DATA_CONST_LATE
static struct pe_serial_functions apple_serial_functions =
{
	.init = apple_uart_init,
	.transmit_ready = apple_uart_tr0,
	.transmit_data = apple_uart_td0,
	.receive_ready = apple_uart_rr0,
	.receive_data = apple_uart_rd0,
	.enable_irq = apple_uart_enable_irq,
	.disable_irq = apple_uart_disable_irq,
	.acknowledge_irq = apple_uart_ack_irq,
	.device = SERIAL_APPLE_UART
};

#endif /* APPLE_UART */

/*****************************************************************************/

static void
dcc_uart_init(void)
{
}

static uint8_t
read_dtr(void)
{
#ifdef __arm__
	uint8_t c;
	__asm__ volatile (
                 "mrc p14, 0, %0, c0, c5\n"
 :               "=r"(c));
	return c;
#else
	panic_unimplemented();
	return 0;
#endif
}
static void
write_dtr(uint8_t c)
{
#ifdef __arm__
	__asm__ volatile (
                 "mcr p14, 0, %0, c0, c5\n"
                 :
                 :"r"(c));
#else
	(void)c;
	panic_unimplemented();
#endif
}

static unsigned int
dcc_tr0(void)
{
#ifdef __arm__
	return !(arm_debug_read_dscr() & ARM_DBGDSCR_TXFULL);
#else
	panic_unimplemented();
	return 0;
#endif
}

static void
dcc_td0(uint8_t c)
{
	write_dtr(c);
}

static unsigned int
dcc_rr0(void)
{
#ifdef __arm__
	return arm_debug_read_dscr() & ARM_DBGDSCR_RXFULL;
#else
	panic_unimplemented();
	return 0;
#endif
}

static uint8_t
dcc_rd0(void)
{
	return read_dtr();
}

SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) dcc_serial_functions =
{
	.init = dcc_uart_init,
	.transmit_ready = dcc_tr0,
	.transmit_data = dcc_td0,
	.receive_ready = dcc_rr0,
	.receive_data = dcc_rd0,
	.device = SERIAL_DCC_UART
};

/*****************************************************************************/

#ifdef DOCKCHANNEL_UART
#define DOCKCHANNEL_WR_MAX_STALL_US (30*1000)

static vm_offset_t      dock_agent_base;
static uint32_t         max_dockchannel_drain_period;
static uint64_t         dockchannel_drain_deadline;  // Deadline for external agent to drain before a software drain occurs
static bool             use_sw_drain;
static uint32_t         dock_wstat_mask;
static uint64_t         prev_dockchannel_spaces;        // Previous w_stat level of the DockChannel.
static uint64_t         dockchannel_stall_grace;
MARK_AS_HIBERNATE_DATA static bool     use_sw_drain;
MARK_AS_HIBERNATE_DATA static uint32_t dock_wstat_mask;

// forward reference
static struct pe_serial_functions dockchannel_serial_functions;

//=======================
// Local funtions
//=======================

static int
dockchannel_drain_on_stall()
{
	// Called when DockChannel runs out of spaces.
	// Check if the DockChannel reader has stalled. If so, empty the DockChannel ourselves.
	// Return number of bytes drained.

	if (mach_absolute_time() >= dockchannel_drain_deadline) {
		// It's been more than DOCKCHANEL_WR_MAX_STALL_US and nobody read from the FIFO
		// Drop a character.
		(void)rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
		os_atomic_inc(&prev_dockchannel_spaces, relaxed);
		return 1;
	}
	return 0;
}

static void
dockchannel_clear_intr(void)
{
	rDOCKCHANNELS_AGENT_AP_INTR_CTRL &= ~(0x3);
	rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x3;
	rDOCKCHANNELS_AGENT_AP_ERR_INTR_CTRL &= ~(0x3);
	rDOCKCHANNELS_AGENT_AP_ERR_INTR_STATUS |= 0x3;
}

static void
dockchannel_disable_irq(void)
{
	rDOCKCHANNELS_AGENT_AP_INTR_CTRL &= ~(0x1);
}

static void
dockchannel_enable_irq(void)
{
	// set interrupt to be when fifo has 255 empty
	rDOCKCHANNELS_DEV_WR_WATERMARK(DOCKCHANNEL_UART_CHANNEL) = 0xFF;
	rDOCKCHANNELS_AGENT_AP_INTR_CTRL |= 0x1;
}

static bool
dockchannel_ack_irq(void)
{
	/* First check if the IRQ is for the kernel */
	if (rDOCKCHANNELS_AGENT_AP_INTR_STATUS & 0x1) {
		rDOCKCHANNELS_AGENT_AP_INTR_STATUS |= 0x1;
		return true;
	}
	return false;
}

MARK_AS_HIBERNATE_TEXT static void
dockchannel_transmit_data(uint8_t c)
{
	rDOCKCHANNELS_DEV_WDATA1(DOCKCHANNEL_UART_CHANNEL) = (unsigned)c;

	if (use_sw_drain && !uart_hibernation) {
		os_atomic_dec(&prev_dockchannel_spaces, relaxed); // After writing a byte we have one fewer space than previously expected.
	}
}

static unsigned int
dockchannel_receive_ready(void)
{
	return rDOCKCHANNELS_DEV_RDATA0(DOCKCHANNEL_UART_CHANNEL) & 0x7f;
}

static uint8_t
dockchannel_receive_data(void)
{
	return (uint8_t)((rDOCKCHANNELS_DEV_RDATA1(DOCKCHANNEL_UART_CHANNEL) >> 8) & 0xff);
}

MARK_AS_HIBERNATE_TEXT static unsigned int
dockchannel_transmit_ready(void)
{
	uint32_t spaces = rDOCKCHANNELS_DEV_WSTAT(DOCKCHANNEL_UART_CHANNEL) & dock_wstat_mask;

	if (!uart_hibernation) {
		if (use_sw_drain) {
			if (spaces > prev_dockchannel_spaces) {
				// More spaces showed up. That can only mean someone read the FIFO.
				// Note that if the DockFIFO is empty we cannot tell if someone is listening,
				// we can only give them the benefit of the doubt.
				dockchannel_drain_deadline = mach_absolute_time() + dockchannel_stall_grace;
			}
			prev_dockchannel_spaces = spaces;
			return spaces || dockchannel_drain_on_stall();
		}
	}

	return spaces;
}

static void
dockchannel_init(void)
{
	if (use_sw_drain) {
		nanoseconds_to_absolutetime(DOCKCHANNEL_WR_MAX_STALL_US * NSEC_PER_USEC, &dockchannel_stall_grace);
	}

	// Clear all interrupt enable and status bits
	dockchannel_clear_intr();

	// Setup DRAIN timer
	rDOCKCHANNELS_DEV_DRAIN_CFG(DOCKCHANNEL_UART_CHANNEL) = max_dockchannel_drain_period;

	// Drain timer doesn't get loaded with value from drain period register if fifo
	// is already full. Drop a character from the fifo.
	rDOCKCHANNELS_DOCK_RDATA1(DOCKCHANNEL_UART_CHANNEL);
}

MARK_AS_HIBERNATE_DATA_CONST_LATE
static struct pe_serial_functions dockchannel_serial_functions =
{
	.init = dockchannel_init,
	.transmit_ready = dockchannel_transmit_ready,
	.transmit_data = dockchannel_transmit_data,
	.receive_ready = dockchannel_receive_ready,
	.receive_data = dockchannel_receive_data,
	.enable_irq = dockchannel_enable_irq,
	.disable_irq = dockchannel_disable_irq,
	.acknowledge_irq = dockchannel_ack_irq,
	.device = SERIAL_DOCKCHANNEL
};

#endif /* DOCKCHANNEL_UART */

/****************************************************************************/
#ifdef PI3_UART
vm_offset_t pi3_gpio_base_vaddr = 0;
vm_offset_t pi3_aux_base_vaddr = 0;
static unsigned int
pi3_uart_tr0(void)
{
	return (unsigned int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x20;
}

static void
pi3_uart_td0(uint8_t c)
{
	BCM2837_PUT32(BCM2837_AUX_MU_IO_REG_V, (uint32_t) c);
}

static unsigned int
pi3_uart_rr0(void)
{
	return (unsigned int) BCM2837_GET32(BCM2837_AUX_MU_LSR_REG_V) & 0x01;
}

static uint8_t
pi3_uart_rd0(void)
{
	return (uint8_t) BCM2837_GET32(BCM2837_AUX_MU_IO_REG_V);
}

static void
pi3_uart_init(void)
{
	// Scratch variable
	uint32_t i;

	// Reset mini uart registers
	BCM2837_PUT32(BCM2837_AUX_ENABLES_V, 1);
	BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 0);
	BCM2837_PUT32(BCM2837_AUX_MU_LCR_REG_V, 3);
	BCM2837_PUT32(BCM2837_AUX_MU_MCR_REG_V, 0);
	BCM2837_PUT32(BCM2837_AUX_MU_IER_REG_V, 0);
	BCM2837_PUT32(BCM2837_AUX_MU_IIR_REG_V, 0xC6);
	BCM2837_PUT32(BCM2837_AUX_MU_BAUD_REG_V, 270);

	i = (uint32_t)BCM2837_FSEL_REG(14);
	// Configure GPIOs 14 & 15 for alternate function 5
	i &= ~(BCM2837_FSEL_MASK(14));
	i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(14));
	i &= ~(BCM2837_FSEL_MASK(15));
	i |= (BCM2837_FSEL_ALT5 << BCM2837_FSEL_OFFS(15));

	BCM2837_PUT32(BCM2837_FSEL_REG(14), i);

	BCM2837_PUT32(BCM2837_GPPUD_V, 0);

	// Barrier before AP spinning for 150 cycles
	__builtin_arm_isb(ISB_SY);

	for (i = 0; i < 150; i++) {
		asm volatile ("add x0, x0, xzr");
	}

	__builtin_arm_isb(ISB_SY);

	BCM2837_PUT32(BCM2837_GPPUDCLK0_V, (1 << 14) | (1 << 15));

	__builtin_arm_isb(ISB_SY);

	for (i = 0; i < 150; i++) {
		asm volatile ("add x0, x0, xzr");
	}

	__builtin_arm_isb(ISB_SY);

	BCM2837_PUT32(BCM2837_GPPUDCLK0_V, 0);

	BCM2837_PUT32(BCM2837_AUX_MU_CNTL_REG_V, 3);
}

SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) pi3_uart_serial_functions =
{
	.init = pi3_uart_init,
	.transmit_ready = pi3_uart_tr0,
	.transmit_data = pi3_uart_td0,
	.receive_ready = pi3_uart_rr0,
	.receive_data = pi3_uart_rd0,
	.device = SERIAL_PI3_UART
};

#endif /* PI3_UART */

/*****************************************************************************/

#ifdef VMAPPLE_UART

static vm_offset_t vmapple_uart0_base_vaddr = 0;

#define PL011_LCR_WORD_LENGTH_8  0x60u
#define PL011_LCR_FIFO_DISABLE   0x00u

#define PL011_LCR_FIFO_ENABLE    0x10u

#define PL011_LCR_ONE_STOP_BIT   0x00u
#define PL011_LCR_PARITY_DISABLE 0x00u
#define PL011_LCR_BREAK_DISABLE  0x00u
#define PL011_IBRD_DIV_38400     0x27u
#define PL011_FBRD_DIV_38400     0x09u
#define PL011_ICR_CLR_ALL_IRQS   0x07ffu
#define PL011_CR_UART_ENABLE     0x01u
#define PL011_CR_TX_ENABLE       0x100u
#define PL011_CR_RX_ENABLE       0x200u

#define VMAPPLE_UART0_DR         *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x00))
#define VMAPPLE_UART0_ECR        *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x04))
#define VMAPPLE_UART0_FR         *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x18))
#define VMAPPLE_UART0_IBRD       *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x24))
#define VMAPPLE_UART0_FBRD       *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x28))
#define VMAPPLE_UART0_LCR_H      *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x2c))
#define VMAPPLE_UART0_CR         *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x30))
#define VMAPPLE_UART0_TIMSC      *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x38))
#define VMAPPLE_UART0_ICR        *((volatile uint32_t *) (vmapple_uart0_base_vaddr + 0x44))

static unsigned int
vmapple_uart_transmit_ready(void)
{
	return (unsigned int) !(VMAPPLE_UART0_FR & 0x20);
}

static void
vmapple_uart_transmit_data(uint8_t c)
{
	VMAPPLE_UART0_DR = (uint32_t) c;
}

static unsigned int
vmapple_uart_receive_ready(void)
{
	return (unsigned int) !(VMAPPLE_UART0_FR & 0x10);
}

static uint8_t
vmapple_uart_receive_data(void)
{
	return (uint8_t) (VMAPPLE_UART0_DR & 0xff);
}

static void
vmapple_uart_init(void)
{
	VMAPPLE_UART0_CR = 0x0;
	VMAPPLE_UART0_ECR = 0x0;
	VMAPPLE_UART0_LCR_H = (
		PL011_LCR_WORD_LENGTH_8 |
		PL011_LCR_FIFO_ENABLE |
		PL011_LCR_ONE_STOP_BIT |
		PL011_LCR_PARITY_DISABLE |
		PL011_LCR_BREAK_DISABLE
		);
	VMAPPLE_UART0_IBRD = PL011_IBRD_DIV_38400;
	VMAPPLE_UART0_FBRD = PL011_FBRD_DIV_38400;
	VMAPPLE_UART0_TIMSC = 0x0;
	VMAPPLE_UART0_ICR = PL011_ICR_CLR_ALL_IRQS;
	VMAPPLE_UART0_CR = (
		PL011_CR_UART_ENABLE |
		PL011_CR_TX_ENABLE |
		PL011_CR_RX_ENABLE
		);
}

SECURITY_READ_ONLY_LATE(static struct pe_serial_functions) vmapple_uart_serial_functions =
{
	.init = vmapple_uart_init,
	.transmit_ready = vmapple_uart_transmit_ready,
	.transmit_data = vmapple_uart_transmit_data,
	.receive_ready = vmapple_uart_receive_ready,
	.receive_data = vmapple_uart_receive_data,
	.device = SERIAL_VMAPPLE_UART
};

#endif /* VMAPPLE_UART */

/*****************************************************************************/

static void
register_serial_functions(struct pe_serial_functions *fns)
{
	fns->next = gPESF;
	gPESF = fns;
}

#if HIBERNATION
/**
 * Transitions the serial driver into a mode that can be run in the hibernation
 * resume context. In this mode, the serial driver runs at a barebones level
 * without making sure the serial devices are properly initialized or utilizing
 * features such as the software drain timer for dockchannels.
 *
 * Upon the next call to serial_init (once the hibernation image has been
 * loaded), this mode is exited and we return to the normal operation of the
 * driver.
 */
MARK_AS_HIBERNATE_TEXT void
serial_hibernation_init(void)
{
	uart_hibernation = true;
#if defined(APPLE_UART)
	uart_base = gHibernateGlobals.hibUartRegPhysBase;
#endif /* defined(APPLE_UART) */
#if defined(DOCKCHANNEL_UART)
	dockchannel_uart_base = gHibernateGlobals.dockChannelRegPhysBase;
#endif /* defined(DOCKCHANNEL_UART) */
}

/**
 * Transitions the serial driver back to non-hibernation mode so it can resume
 * normal operations. Should only be called from serial_init on a hibernation
 * resume.
 */
MARK_AS_HIBERNATE_TEXT static void
serial_hibernation_cleanup(void)
{
	uart_hibernation = false;
#if defined(APPLE_UART)
	uart_base = gHibernateGlobals.hibUartRegVirtBase;
#endif /* defined(APPLE_UART) */
#if defined(DOCKCHANNEL_UART)
	dockchannel_uart_base = gHibernateGlobals.dockChannelRegVirtBase;
#endif /* defined(DOCKCHANNEL_UART) */
}
#endif /* HIBERNATION */

int
serial_init(void)
{
	DTEntry         entryP = NULL;
	uint32_t        prop_size;
	vm_offset_t     soc_base;
	uintptr_t const *reg_prop;
	uint32_t const  *prop_value __unused = NULL;
	uint32_t        dccmode;

	struct pe_serial_functions *fns = gPESF;

	/**
	 * Even if the serial devices have already been initialized on cold boot,
	 * when coming out of a sleep/wake, they'll need to be re-initialized. Since
	 * the uart_initted value is kept across a sleep/wake, always re-initialize
	 * to be safe.
	 */
	if (uart_initted) {
#if HIBERNATION
		if (uart_hibernation) {
			serial_hibernation_cleanup();
		}
#endif /* HIBERNATION */
		while (fns != NULL) {
			fns->init();
			fns = fns->next;
		}

		return 1;
	}

	dccmode = 0;
	if (PE_parse_boot_argn("dcc", &dccmode, sizeof(dccmode))) {
		register_serial_functions(&dcc_serial_functions);
	}

	soc_base = pe_arm_get_soc_base_phys();

	if (soc_base == 0) {
		return 0;
	}

	PE_parse_boot_argn("disable-uart-irq", &disable_uart_irq, sizeof(disable_uart_irq));

#ifdef PI3_UART
	if (SecureDTFindEntry("name", "gpio", &entryP) == kSuccess) {
		SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
		pi3_gpio_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
	}
	if (SecureDTFindEntry("name", "aux", &entryP) == kSuccess) {
		SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
		pi3_aux_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
	}
	if ((pi3_gpio_base_vaddr != 0) && (pi3_aux_base_vaddr != 0)) {
		register_serial_functions(&pi3_uart_serial_functions);
	}
#endif /* PI3_UART */

#ifdef VMAPPLE_UART
	if (SecureDTFindEntry("name", "uart0", &entryP) == kSuccess) {
		SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
		vmapple_uart0_base_vaddr = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
	}

	if (vmapple_uart0_base_vaddr != 0) {
		register_serial_functions(&vmapple_uart_serial_functions);
	}
#endif /* VMAPPLE_UART */

#ifdef DOCKCHANNEL_UART
	uint32_t no_dockchannel_uart = 0;
	if (SecureDTFindEntry("name", "dockchannel-uart", &entryP) == kSuccess) {
		SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
		// Should be two reg entries
		if (prop_size / sizeof(uintptr_t) != 4) {
			panic("Malformed dockchannel-uart property");
		}
		dockchannel_uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
		dock_agent_base = ml_io_map(soc_base + *(reg_prop + 2), *(reg_prop + 3));
		PE_parse_boot_argn("no-dockfifo-uart", &no_dockchannel_uart, sizeof(no_dockchannel_uart));
		// Keep the old name for boot-arg
		if (no_dockchannel_uart == 0) {
			register_serial_functions(&dockchannel_serial_functions);
			SecureDTGetProperty(entryP, "max-aop-clk", (void const **)&prop_value, &prop_size);
			max_dockchannel_drain_period = (uint32_t)((prop_value)?  (*prop_value * 0.03) : DOCKCHANNEL_DRAIN_PERIOD);
			prop_value = NULL;
			SecureDTGetProperty(entryP, "enable-sw-drain", (void const **)&prop_value, &prop_size);
			use_sw_drain = (prop_value)?  *prop_value : 0;
			prop_value = NULL;
			SecureDTGetProperty(entryP, "dock-wstat-mask", (void const **)&prop_value, &prop_size);
			dock_wstat_mask = (prop_value)?  *prop_value : 0x1ff;
			prop_value = NULL;
			SecureDTGetProperty(entryP, "interrupts", (void const **)&prop_value, &prop_size);
			if (prop_value) {
				dockchannel_serial_functions.has_irq = true;
			}
		} else {
			dockchannel_clear_intr();
		}
		// If no dockchannel-uart is found in the device tree, fall back
		// to looking for the traditional UART serial console.
	}

#endif /* DOCKCHANNEL_UART */

#ifdef APPLE_UART
	char const *serial_compat = 0;

	/*
	 * The boot serial port should have a property named "boot-console".
	 * If we don't find it there, look for "uart0" and "uart1".
	 */
	if (SecureDTFindEntry("boot-console", NULL, &entryP) == kSuccess) {
		SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
		uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
		SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
	} else if (SecureDTFindEntry("name", "uart0", &entryP) == kSuccess) {
		SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
		uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
		SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
	} else if (SecureDTFindEntry("name", "uart1", &entryP) == kSuccess) {
		SecureDTGetProperty(entryP, "reg", (void const **)&reg_prop, &prop_size);
		uart_base = ml_io_map(soc_base + *reg_prop, *(reg_prop + 1));
		SecureDTGetProperty(entryP, "compatible", (void const **)&serial_compat, &prop_size);
	}

	if (NULL != entryP) {
		SecureDTGetProperty(entryP, "pclk", (void const **)&prop_value, &prop_size);
		if (prop_value) {
			dt_pclk = *prop_value;
		}

		prop_value = NULL;
		SecureDTGetProperty(entryP, "sampling", (void const **)&prop_value, &prop_size);
		if (prop_value) {
			dt_sampling = *prop_value;
		}

		prop_value = NULL;
		SecureDTGetProperty(entryP, "ubrdiv", (void const **)&prop_value, &prop_size);
		if (prop_value) {
			dt_ubrdiv = *prop_value;
		}

		SecureDTGetProperty(entryP, "interrupts", (void const **)&prop_value, &prop_size);
		if (prop_value) {
			apple_serial_functions.has_irq = true;
		}
	}

	if (serial_compat && !strcmp(serial_compat, "uart-1,samsung")) {
		register_serial_functions(&apple_serial_functions);
	}
#endif /* APPLE_UART */

	if (gPESF == NULL) {
		return 0;
	}

	fns = gPESF;
	while (fns != NULL) {
		fns->init();
		if (fns->has_irq) {
			serial_irq_status |= fns->device; // serial_device_t is one-hot
		}
		fns = fns->next;
	}

#if HIBERNATION
	/* hibernation needs to know the UART register addresses since it can't directly use this serial driver */
	if (dockchannel_uart_base) {
		gHibernateGlobals.dockChannelRegPhysBase = ml_vtophys(dockchannel_uart_base);
		gHibernateGlobals.dockChannelRegVirtBase = dockchannel_uart_base;
		gHibernateGlobals.dockChannelWstatMask = dock_wstat_mask;
	}
	if (uart_base) {
		gHibernateGlobals.hibUartRegPhysBase = ml_vtophys(uart_base);
		gHibernateGlobals.hibUartRegVirtBase = uart_base;
	}
#endif /* HIBERNATION */

	uart_initted = true;

	return 1;
}

/**
 * Returns a deadline for the longest time the serial driver should wait for an
 * interrupt for. This serves as a timeout for the IRQ to allow for the software
 * drain timer that dockchannels supports.
 *
 * @param fns serial functions representing the device to find the deadline for
 *
 * @returns absolutetime deadline for this device's IRQ.
 */
static uint64_t
serial_interrupt_deadline(__unused struct pe_serial_functions *fns)
{
#if defined(DOCKCHANNEL_UART)
	if (fns->device == SERIAL_DOCKCHANNEL && use_sw_drain) {
		return dockchannel_drain_deadline;
	}
#endif

	/**
	 *  Default to 1.5ms for all other devices. 1.5ms was chosen as the baudrate
	 * of the AppleSerialDevice is 115200, meaning that it should only take
	 * ~1.5ms to drain the 16 character buffer completely.
	 */
	uint64_t timeout_interval;
	nanoseconds_to_absolutetime(1500 * NSEC_PER_USEC, &timeout_interval);
	return mach_absolute_time() + timeout_interval;
}

/**
 * Goes to sleep waiting for an interrupt from a specificed serial device.
 *
 * @param fns serial functions representing the device to wait for
 */
static void
serial_wait_for_interrupt(struct pe_serial_functions *fns)
{
	assert_wait_deadline(fns, THREAD_UNINT, serial_interrupt_deadline(fns));
	if (!fns->transmit_ready()) {
		fns->enable_irq();
		thread_block(THREAD_CONTINUE_NULL);
	} else {
		clear_wait(current_thread(), THREAD_AWAKENED);
	}
}

/**
 * Output a character onto every registered serial interface.
 *
 * @param c The character to output.
 * @param poll Whether the driver should poll to send the character or if it can
 *             wait for an interrupt
 */
MARK_AS_HIBERNATE_TEXT void
uart_putc_options(char c, bool poll)
{
	struct pe_serial_functions *fns = gPESF;

	while (fns != NULL) {
		while (!fns->transmit_ready()) {
			if (!uart_hibernation) {
				if (!poll && irq_available_and_ready(fns)) {
					serial_wait_for_interrupt(fns);
				} else {
					serial_poll();
				}
			}
		}
		fns->transmit_data((uint8_t)c);
		fns = fns->next;
	}
}

/**
 * Output a character onto every registered serial interface by polling.
 *
 * @param c The character to output.
 */
void
uart_putc(char c)
{
	uart_putc_options(c, true);
}

/**
 * Read a character from the first registered serial interface that has data
 * available.
 *
 * @return The character if any interfaces have data available, otherwise -1.
 */
int
uart_getc(void)
{
	struct pe_serial_functions *fns = gPESF;
	while (fns != NULL) {
		if (fns->receive_ready()) {
			return (int)fns->receive_data();
		}
		fns = fns->next;
	}
	return -1;
}

/**
 * Enables IRQs for a specific serial device and returns whether or not IRQs for
 * that device where enabled successfully. For a serial driver to have irqs
 * enabled, it must have the enable_irq, disable_irq, and acknowledge_irq
 * functions defined and the has_irq flag set.
 *
 * @param device Serial device to enable irqs on
 * @note This function should only be called from the AppleSerialShim kext
 */
kern_return_t
serial_enable_irq(serial_device_t device)
{
	struct pe_serial_functions *fns = get_serial_functions(device);

	if (!fns || !fns->has_irq || disable_uart_irq) {
		return KERN_FAILURE;
	}

	serial_irq_status &= ~device;

	return KERN_SUCCESS;
}

/**
 * Acknowledges an irq for a specific serial device and wakes up the thread
 * waiting on the interrupt if one exists.
 *
 * @param device Serial device to enable irqs for
 * @note This function should only be called from the AppleSerialShim kext
 */
kern_return_t
serial_ack_irq(serial_device_t device)
{
	struct pe_serial_functions *fns = get_serial_functions(device);

	if (!fns || !fns->has_irq) {
		return KERN_FAILURE;
	}

	/* Disable IRQs until next time a thread waits for an interrupt */
	fns->disable_irq();

	/**
	 * Because IRQs are enabled only when we know a thread is about to sleep, we
	 * can call wake up and reasonably expect there to be a thread waiting.
	 */
	thread_wakeup(fns);

	return KERN_SUCCESS;
}

/**
 * Returns true if the pending IRQ for device is one that can be handled by the
 * platform serial driver.
 *
 * @param device Serial device to enable irqs for
 * @note This function is called from a primary interrupt context and should be
 *       kept lightweight.
 * @note This function should only be called from the AppleSerialShim kext
 */
bool
serial_filter_irq(serial_device_t device)
{
	struct pe_serial_functions *fns = get_serial_functions(device);

	if (!fns || !fns->has_irq) {
		return false;
	}

	return fns->acknowledge_irq();
}

/**
 * Prepares all serial devices to go to sleep by draining the hardware FIFOs
 * and disabling interrupts.
 */
void
serial_go_to_sleep(void)
{
	struct pe_serial_functions *fns = gPESF;
	while (fns != NULL) {
		if (irq_available_and_ready(fns)) {
			fns->disable_irq();
		}
		fns = fns->next;
	}

#ifdef APPLE_UART
	/* APPLE_UART needs to drain FIFO before sleeping */
	if (get_serial_functions(SERIAL_APPLE_UART)) {
		apple_uart_drain_fifo();
	}
#endif /* APPLE_UART */
}