/*
 * Copyright (c) 2022 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */


#ifndef _KERN_LOCK_PTR_H_
#define _KERN_LOCK_PTR_H_

#include <kern/lock_types.h>
#include <kern/lock_group.h>
#include <kern/lock_attr.h>
#include <vm/vm_memtag.h>

__BEGIN_DECLS
#pragma GCC visibility push(hidden)

#if CONFIG_KERNEL_TAGGING
#define HW_LCK_PTR_BITS         (64 - 4 - 1 - 1 - 16)
#else /* !CONFIG_KERNEL_TAGGING */
#define HW_LCK_PTR_BITS         (64 - 1 - 1 - 16)
#endif /* !CONFIG_KERNEL_TAGGING */

/*!
 * @typedef hw_lck_ptr_t
 *
 * @brief
 * Low level lock that can share the slot of a pointer value.
 *
 * @discussion
 * An @c hw_lck_ptr_t is a fair spinlock fitting in a single
 * pointer sized word, and allows for that word to retain
 * its original pointer value without any loss.
 *
 * It uses the top bits of the pointer which in the kernel
 * aren't significant to store an MCS queue as well as various
 * state bits.
 *
 * The pointer can't be PAC signed, but the MTESAN tag
 * is preserved (when the feature is on).
 *
 * No assumption is made on the alignment of the pointer,
 * and clients may use the low bits of the pointer to encode
 * state as they see fit.
 *
 * It is intended to be used for fine grained locking
 * in hash table bucket heads or similar structures.
 *
 * The pointer value can be changed by taking the lock,
 * (which returns the current value of the pointer),
 * and then unlock with the new pointer value.
 */
typedef union hw_lck_ptr {
	struct {
		intptr_t        lck_ptr_bits    : HW_LCK_PTR_BITS;
#if CONFIG_KERNEL_TAGGING
		uintptr_t       lck_ptr_tag     : 4;
#endif /* CONFIG_KERNEL_TAGGING */
		uintptr_t       lck_ptr_locked  : 1;
		uintptr_t       lck_ptr_stats   : 1;
		uint16_t        lck_ptr_mcs_tail __kernel_ptr_semantics;
	};
	uintptr_t               lck_ptr_value;
} hw_lck_ptr_t;


/* init/destroy */

#if !LCK_GRP_USE_ARG
#define hw_lck_ptr_init(lck, val, grp)        hw_lck_ptr_init(lck, val)
#define hw_lck_ptr_destroy(lck, grp)          hw_lck_ptr_destroy(lck)
#endif /* !LCK_GRP_USE_ARG */

/*!
 * @function hw_lck_ptr_init()
 *
 * @brief
 * Initializes an hw_lck_ptr_t with the specified pointer value.
 *
 * @discussion
 * hw_lck_ptr_destroy() must be called to destroy this lock.
 *
 * @param lck           the lock to initialize
 * @param val           the pointer value to store in the lock
 * @param grp           the lock group associated with this lock
 */
extern void hw_lck_ptr_init(
	hw_lck_ptr_t           *lck,
	void                   *val,
	lck_grp_t              *grp);

/*!
 * @function hw_lck_ptr_destroy()
 *
 * @brief
 * Detroys an hw_lck_ptr_t initialized with hw_lck_ptr_init().
 *
 * @param lck           the lock to destroy
 * @param grp           the lock group associated with this lock
 */
extern void hw_lck_ptr_destroy(
	hw_lck_ptr_t           *lck,
	lck_grp_t              *grp);

/*!
 * @function hw_lck_ptr_held()
 *
 * @brief
 * Returns whether the pointer lock is currently held by anyone.
 *
 * @param lck           the lock to check.
 */
extern bool hw_lck_ptr_held(
	hw_lck_ptr_t           *lck) __result_use_check;


/* lock */

#if !LCK_GRP_USE_ARG
#define hw_lck_ptr_lock(lck, grp)             hw_lck_ptr_lock(lck)
#define hw_lck_ptr_lock_nopreempt(lck, grp)   hw_lck_ptr_lock_nopreempt(lck)
#endif /* !LCK_GRP_USE_ARG */


/*!
 * @function hw_lck_ptr_lock()
 *
 * @brief
 * Locks a pointer lock, and returns its current value.
 *
 * @discussion
 * This call will disable preemption.
 *
 * @param lck           the lock to lock
 * @param grp           the lock group associated with this lock
 */
extern void *hw_lck_ptr_lock(
	hw_lck_ptr_t           *lck,
	lck_grp_t              *grp) __result_use_check;

/*!
 * @function hw_lck_ptr_lock_nopreempt()
 *
 * @brief
 * Locks a pointer lock, and returns its current value.
 *
 * @discussion
 * Preemption must be disabled to make this call,
 * and must stay disabled until @c hw_lck_ptr_unlock()
 * or @c hw_lck_ptr_unlock_nopreempt() is called
 *
 * @param lck           the lock to lock
 * @param grp           the lock group associated with this lock
 */
extern void *hw_lck_ptr_lock_nopreempt(
	hw_lck_ptr_t           *lck,
	lck_grp_t              *grp) __result_use_check;


/* unlock */

#if !LCK_GRP_USE_ARG
#define hw_lck_ptr_unlock(lck, val, grp) \
	hw_lck_ptr_unlock(lck, val)
#define hw_lck_ptr_unlock_nopreempt(lck, val, grp) \
	hw_lck_ptr_unlock_nopreempt(lck, val)
#endif /* !LCK_GRP_USE_ARG */


/*!
 * @function hw_lck_ptr_unlock()
 *
 * @brief
 * Unlocks a pointer lock, and update its currently held value.
 *
 * @discussion
 * This call will reenable preemption.
 *
 * @param lck           the lock to unlock
 * @param val           the value to update the pointer to
 * @param grp           the lock group associated with this lock
 */
extern void hw_lck_ptr_unlock(
	hw_lck_ptr_t           *lck,
	void                   *val,
	lck_grp_t              *grp);

/*!
 * @function hw_lck_ptr_unlock_nopreempt()
 *
 * @brief
 * Unlocks a pointer lock, and update its currently held value.
 *
 * @discussion
 * This call will not reenable preemption.
 *
 * @param lck           the lock to unlock
 * @param val           the value to update the pointer to
 * @param grp           the lock group associated with this lock
 */
extern void hw_lck_ptr_unlock_nopreempt(
	hw_lck_ptr_t           *lck,
	void                   *val,
	lck_grp_t              *grp);


/* wait_for_value */

#if !LCK_GRP_USE_ARG
#define hw_lck_ptr_wait_for_value(lck, val, grp) \
	hw_lck_ptr_wait_for_value(lck, val)
#endif /* !LCK_GRP_USE_ARG */

static inline void *
__hw_lck_ptr_value(hw_lck_ptr_t val)
{
	vm_offset_t ptr = val.lck_ptr_bits;

#if CONFIG_KERNEL_TAGGING
	if (ptr) {
		ptr = vm_memtag_add_ptr_tag(ptr, val.lck_ptr_tag);
	}
#endif /* CONFIG_KERNEL_TAGGING */
	return (void *)ptr;
}


/*!
 * @function hw_lck_ptr_value()
 *
 * @brief
 * Returns the pointer value currently held by this lock.
 *
 * @param lck           the lock to get the pointer value of.
 */
static inline void *
hw_lck_ptr_value(hw_lck_ptr_t *lck)
{
	hw_lck_ptr_t tmp;

	tmp = atomic_load_explicit((hw_lck_ptr_t _Atomic *)lck,
	    memory_order_relaxed);

	return __hw_lck_ptr_value(tmp);
}

/*!
 * @function hw_lck_ptr_wait_for_value()
 *
 * @brief
 * Spins until the pointer in the lock has the specified value.
 *
 * @discussion
 * This function has an implicit acquire barrier pairing
 * with the hw_lck_ptr_unlock() which sets the observed value.
 *
 * @param lck           the lock to spin on
 * @param val           the value to wait for
 * @param grp           the lock group associated with this lock
 */
extern void hw_lck_ptr_wait_for_value(
	hw_lck_ptr_t           *lck,
	void                   *val,
	lck_grp_t              *grp);


#pragma GCC visibility pop
__END_DECLS

#endif /* _KERN_LOCK_PTR_H_ */