/*
* Copyright (c) 2007 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. The rights granted to you under the License
* may not be used to create, or enable the creation or redistribution of,
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
*
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
*
*/
#ifndef ARM_CPU_DATA_INTERNAL
#define ARM_CPU_DATA_INTERNAL
#include <mach_assert.h>
#include <kern/assert.h>
#include <kern/kern_types.h>
#include <kern/percpu.h>
#include <kern/processor.h>
#include <pexpert/pexpert.h>
#include <arm/dbgwrap.h>
#include <arm/machine_routines.h>
#include <arm/proc_reg.h>
#include <arm/thread.h>
#include <arm/pmap.h>
#if MONOTONIC
#include <machine/monotonic.h>
#endif /* MONOTONIC */
#define NSEC_PER_HZ (NSEC_PER_SEC / 100)
typedef struct reset_handler_data {
vm_offset_t assist_reset_handler; /* Assist handler phys address */
vm_offset_t cpu_data_entries; /* CpuDataEntries phys address */
#if !__arm64__
vm_offset_t boot_args; /* BootArgs phys address */
#endif
} reset_handler_data_t;
extern reset_handler_data_t ResetHandlerData;
/* Put the static check for cpumap_t here as it's defined in <kern/processor.h> */
static_assert(sizeof(cpumap_t) * CHAR_BIT >= MAX_CPUS, "cpumap_t bitvector is too small for current MAX_CPUS value");
#ifdef __arm__
#define CPUWINDOWS_BASE_MASK 0xFFF00000UL
#else
#define CPUWINDOWS_BASE_MASK 0xFFFFFFFFFFE00000UL
#endif
#define CPUWINDOWS_BASE (VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK)
#define CPUWINDOWS_TOP (CPUWINDOWS_BASE + (MAX_CPUS * CPUWINDOWS_MAX * ARM_PGBYTES))
static_assert((CPUWINDOWS_BASE >= VM_MIN_KERNEL_ADDRESS) && (CPUWINDOWS_TOP <= VM_MAX_KERNEL_ADDRESS),
"CPU copy windows too large for CPUWINDOWS_BASE_MASK value");
typedef struct cpu_data_entry {
void *cpu_data_paddr; /* Cpu data physical address */
struct cpu_data *cpu_data_vaddr; /* Cpu data virtual address */
#if __arm__
uint32_t cpu_data_offset_8;
uint32_t cpu_data_offset_12;
#elif __arm64__
#else
#error Check cpu_data_entry padding for this architecture
#endif
} cpu_data_entry_t;
typedef struct rtclock_timer {
mpqueue_head_t queue;
uint64_t deadline;
uint32_t is_set:1,
has_expired:1,
:0;
} rtclock_timer_t;
typedef struct {
/*
* The wake variants of these counters are reset to 0 when the CPU wakes.
*/
uint64_t irq_ex_cnt;
uint64_t irq_ex_cnt_wake;
uint64_t ipi_cnt;
uint64_t ipi_cnt_wake;
uint64_t timer_cnt;
#if MONOTONIC
uint64_t pmi_cnt_wake;
#endif /* MONOTONIC */
uint64_t undef_ex_cnt;
uint64_t unaligned_cnt;
uint64_t vfp_cnt;
uint64_t data_ex_cnt;
uint64_t instr_ex_cnt;
} cpu_stat_t;
typedef struct cpu_data {
unsigned short cpu_number;
unsigned short cpu_flags;
int cpu_type;
int cpu_subtype;
int cpu_threadtype;
vm_offset_t istackptr;
vm_offset_t intstack_top;
#if __arm64__
vm_offset_t excepstackptr;
vm_offset_t excepstack_top;
#else
vm_offset_t fiqstackptr;
vm_offset_t fiqstack_top;
#endif
thread_t cpu_active_thread;
vm_offset_t cpu_active_stack;
cpu_id_t cpu_id;
unsigned volatile int cpu_signal;
ast_t cpu_pending_ast;
cache_dispatch_t cpu_cache_dispatch;
#if __arm64__
uint64_t cpu_base_timebase;
uint64_t cpu_timebase;
#else
union {
struct {
uint32_t low;
uint32_t high;
} split;
struct {
uint64_t val;
} raw;
} cbtb;
#define cpu_base_timebase_low cbtb.split.low
#define cpu_base_timebase_high cbtb.split.high
union {
struct {
uint32_t low;
uint32_t high;
} split;
struct {
uint64_t val;
} raw;
} ctb;
#define cpu_timebase_low ctb.split.low
#define cpu_timebase_high ctb.split.high
#endif
bool cpu_hibernate; /* This cpu is currently hibernating the system */
bool cpu_running;
bool cluster_master;
#if __ARM_ARCH_8_5__
bool sync_on_cswitch;
#endif /* __ARM_ARCH_8_5__ */
/* true if processor_start() or processor_exit() is operating on this CPU */
bool in_state_transition;
uint32_t cpu_decrementer;
get_decrementer_t cpu_get_decrementer_func;
set_decrementer_t cpu_set_decrementer_func;
fiq_handler_t cpu_get_fiq_handler;
void *cpu_tbd_hardware_addr;
void *cpu_tbd_hardware_val;
void *cpu_console_buf;
processor_idle_t cpu_idle_notify;
uint64_t cpu_idle_latency;
uint64_t cpu_idle_pop;
#if __arm__ || __ARM_KERNEL_PROTECT__
vm_offset_t cpu_exc_vectors;
#endif /* __ARM_KERNEL_PROTECT__ */
vm_offset_t cpu_reset_handler;
uintptr_t cpu_reset_assist;
uint32_t cpu_reset_type;
unsigned int interrupt_source;
void *cpu_int_state;
IOInterruptHandler interrupt_handler;
void *interrupt_nub;
void *interrupt_target;
void *interrupt_refCon;
idle_timer_t idle_timer_notify;
void *idle_timer_refcon;
uint64_t idle_timer_deadline;
uint64_t rtcPop;
rtclock_timer_t rtclock_timer;
struct _rtclock_data_ *rtclock_datap;
arm_debug_state_t *cpu_user_debug; /* Current debug state */
vm_offset_t cpu_debug_interface_map;
volatile int debugger_active;
volatile int PAB_active; /* Tells the console if we are dumping backtraces */
void *cpu_xcall_p0;
void *cpu_xcall_p1;
void *cpu_imm_xcall_p0;
void *cpu_imm_xcall_p1;
#if defined(ARMA7)
volatile uint32_t cpu_CLW_active;
volatile uint64_t cpu_CLWFlush_req;
volatile uint64_t cpu_CLWFlush_last;
volatile uint64_t cpu_CLWClean_req;
volatile uint64_t cpu_CLWClean_last;
#endif
#if __arm64__
vm_offset_t coresight_base[CORESIGHT_REGIONS];
#endif
/* CCC ARMv8 registers */
uint64_t cpu_regmap_paddr;
uint32_t cpu_phys_id;
uint32_t cpu_l2_access_penalty;
platform_error_handler_t platform_error_handler;
int cpu_mcount_off;
#define ARM_CPU_ON_SLEEP_PATH 0x50535553UL
volatile unsigned int cpu_sleep_token;
unsigned int cpu_sleep_token_last;
cluster_type_t cpu_cluster_type;
uint32_t cpu_cluster_id;
uint32_t cpu_l2_id;
uint32_t cpu_l2_size;
uint32_t cpu_l3_id;
uint32_t cpu_l3_size;
enum {
CPU_NOT_HALTED = 0,
CPU_HALTED,
CPU_HALTED_WITH_STATE
} halt_status;
#if defined(HAS_APPLE_PAC)
uint64_t rop_key;
uint64_t jop_key;
#endif /* defined(HAS_APPLE_PAC) */
/* large structs with large alignment requirements */
#if KPC
/* double-buffered performance counter data */
uint64_t *cpu_kpc_buf[2];
/* PMC shadow and reload value buffers */
uint64_t *cpu_kpc_shadow;
uint64_t *cpu_kpc_reload;
#endif
#if MONOTONIC
struct mt_cpu cpu_monotonic;
#endif /* MONOTONIC */
cpu_stat_t cpu_stat;
#if !XNU_MONITOR
struct pmap_cpu_data cpu_pmap_cpu_data;
#endif
dbgwrap_thread_state_t halt_state;
#if DEVELOPMENT || DEBUG
uint64_t wfe_count;
uint64_t wfe_deadline_checks;
uint64_t wfe_terminations;
#endif
} cpu_data_t;
/*
* cpu_flags
*/
#define SleepState 0x0800
#define StartedState 0x1000
extern cpu_data_entry_t CpuDataEntries[MAX_CPUS];
PERCPU_DECL(cpu_data_t, cpu_data);
#define BootCpuData __PERCPU_NAME(cpu_data)
extern boot_args *BootArgs;
#if __arm__
extern unsigned int *ExceptionLowVectorsBase;
extern unsigned int *ExceptionVectorsTable;
#elif __arm64__
extern unsigned int LowResetVectorBase;
extern unsigned int LowResetVectorEnd;
#if WITH_CLASSIC_S2R
extern uint8_t SleepToken[8];
#endif
extern unsigned int LowExceptionVectorBase;
#else
#error Unknown arch
#endif
extern cpu_data_t *cpu_datap(int cpu);
extern cpu_data_t *cpu_data_alloc(boolean_t is_boot);
extern void cpu_stack_alloc(cpu_data_t*);
extern void cpu_data_init(cpu_data_t *cpu_data_ptr);
extern void cpu_data_free(cpu_data_t *cpu_data_ptr);
extern kern_return_t cpu_data_register(cpu_data_t *cpu_data_ptr);
extern cpu_data_t *processor_to_cpu_datap( processor_t processor);
#if __arm64__
typedef struct sysreg_restore {
uint64_t tcr_el1;
} sysreg_restore_t;
extern sysreg_restore_t sysreg_restore;
#endif /* __arm64__ */
#endif /* ARM_CPU_DATA_INTERNAL */