Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 | /* * Copyright (c) 2016-2022 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _SKYWALK_MEM_SKMEMREGIONVAR_H #define _SKYWALK_MEM_SKMEMREGIONVAR_H #ifdef BSD_KERNEL_PRIVATE #include <skywalk/core/skywalk_var.h> #include <skywalk/os_nexus_private.h> /* * Segment types. */ typedef enum { SKSEG_TYPE_INVALID = 0, SKSEG_TYPE_ALLOC, /* segment is in skr_hash_table */ SKSEG_TYPE_FREE, /* segment is in skr_segfree */ SKSEG_TYPE_DESTROYED /* in process of being destroyed */ } sksegment_type_t; /* * Segment memory states. */ typedef enum { SKSEG_STATE_INVALID = 0, SKSEG_STATE_DETACHED, /* not backed by a IOBMD */ SKSEG_STATE_MAPPED, /* mapped (IOBMD non-volatile) */ SKSEG_STATE_MAPPED_WIRED, /* mapped (IOBMD non-volatile+wired) */ } sksegment_state_t; struct skmem_region; /* * Segment. * * Segments that are available for use can be found in the doubly-linked * list (skr_seg_free) as well as the red-black tree (skr_seg_tfree). * The latter is used to faciliate finding a segment by its index, which * is required when allocating a segment from a mirrored region. * * Allocated segments are inserted into the allocated-address hash chain; * they don't exist in any tree at that point. */ struct sksegment { TAILQ_ENTRY(sksegment) sg_link; /* sksegment linkage */ RB_ENTRY(sksegment) sg_node; /* sksegment node in tree */ struct skmem_region *sg_region; /* controlling region */ /* * If attached to a IOBMD, sg_{start,end} will be valid. */ IOSKMemoryBufferRef sg_md; /* backing IOBMD */ mach_vm_address_t sg_start; /* start address (inclusive) */ mach_vm_address_t sg_end; /* end address (exclusive) */ uint32_t sg_index; /* index in skr_seg[] */ sksegment_type_t sg_type; /* segment type */ sksegment_state_t sg_state; /* segment state */ }; #define SKSEGMENT_IN_FREELIST(_sg) \ ((_sg)->sg_link.tqe_next != NULL || \ (_sg)->sg_link.tqe_prev != NULL) /* * Segment hash bucket. */ struct sksegment_bkt { TAILQ_HEAD(, sksegment) sgb_head; /* sksegment allocated list */ }; /* * Region IDs. * * When adding or removing regions, adjust the templates in skmem.c * accordingly. Do not reorder regions without making the appropriate * changes in the code that relies on the existing arena layout. */ typedef enum { /* * The following are user task mappable. * * XXX: When adding new ones, ensure that they get added before * SKMEM_REGION_GUARD_TAIL, and make the appropriate changes in * skmem_region_init(). */ SKMEM_REGION_GUARD_HEAD = 0, /* leading guard page(s) */ SKMEM_REGION_SCHEMA, /* channel layout */ SKMEM_REGION_RING, /* rings */ SKMEM_REGION_BUF_DEF, /* Default rx/tx buffer */ SKMEM_REGION_BUF_LARGE, /* Large rx/tx buffer */ SKMEM_REGION_RXBUF_DEF, /* Default rx only buffers */ SKMEM_REGION_RXBUF_LARGE, /* Large rx only buffers */ SKMEM_REGION_TXBUF_DEF, /* Default tx only buffers */ SKMEM_REGION_TXBUF_LARGE, /* Large tx only buffers */ SKMEM_REGION_UMD, /* userland metadata */ SKMEM_REGION_TXAUSD, /* tx/alloc/event user slot descriptors */ SKMEM_REGION_RXFUSD, /* rx/free user slot descriptors */ SKMEM_REGION_UBFT, /* userland buflet metadata */ SKMEM_REGION_USTATS, /* statistics */ SKMEM_REGION_FLOWADV, /* flow advisories */ SKMEM_REGION_NEXUSADV, /* nexus advisories */ SKMEM_REGION_SYSCTLS, /* sysctl */ SKMEM_REGION_GUARD_TAIL, /* trailing guard page(s) */ /* * The following are NOT user task mappable. */ SKMEM_REGION_KMD, /* rx/tx kernel metadata */ SKMEM_REGION_RXKMD, /* rx only kernel metadata */ SKMEM_REGION_TXKMD, /* tx only kernel metadata */ SKMEM_REGION_KBFT, /* rx/tx kernel buflet metadata */ SKMEM_REGION_RXKBFT, /* rx only kernel buflet metadata */ SKMEM_REGION_TXKBFT, /* tx only kernel buflet metadata */ SKMEM_REGION_TXAKSD, /* tx/alloc/event kernel slot descriptors */ SKMEM_REGION_RXFKSD, /* rx/free kernel slot descriptors */ SKMEM_REGION_KSTATS, /* kernel statistics snapshot */ SKMEM_REGION_INTRINSIC, /* intrinsic objects */ SKMEM_REGIONS /* max */ } skmem_region_id_t; #define SKMEM_PP_REGIONS 14 extern const skmem_region_id_t skmem_pp_region_ids[SKMEM_PP_REGIONS]; /* * Region parameters structure. Based on requested object parameters, * skmem_region_params_config() will compute the segment parameters as * well as the configured object parameters. */ struct skmem_region_params { /* * Region parameters. */ const char *srp_name; /* (i) region name */ skmem_region_id_t srp_id; /* (i) region identifier */ uint32_t srp_cflags; /* (i) region creation flags */ uint32_t srp_r_seg_size; /* (i) requested seg size */ uint32_t srp_c_seg_size; /* (o) configured seg size */ uint32_t srp_seg_cnt; /* (o) number of segments */ /* * Object parameters. */ uint32_t srp_r_obj_size; /* (i) requested obj size */ uint32_t srp_r_obj_cnt; /* (i) requested obj count */ uint32_t srp_c_obj_size; /* (o) configured obj size */ uint32_t srp_c_obj_cnt; /* (o) configured obj count */ size_t srp_align; /* (i) object alignment */ /* * SKMEM_REGION_{UMD,KMD} specific parameters. */ nexus_meta_type_t srp_md_type; /* (i) metadata type */ nexus_meta_subtype_t srp_md_subtype; /* (i) metadata subtype */ uint16_t srp_max_frags; /* (i) max frags per packet */ }; typedef void (*sksegment_ctor_fn_t)(struct sksegment *, IOSKMemoryBufferRef, void *); typedef void (*sksegment_dtor_fn_t)(struct sksegment *, IOSKMemoryBufferRef, void *); /* * Region. */ #define SKR_MAX_CACHES 2 /* max # of caches allowed on a region */ struct skmem_region { decl_lck_mtx_data(, skr_lock); /* region lock */ /* * Statistics. */ uint64_t skr_meminuse; /* memory in use */ uint64_t skr_w_meminuse; /* wired memory in use */ uint64_t skr_memtotal; /* total memory in region */ uint64_t skr_alloc; /* number of allocations */ uint64_t skr_free; /* number of frees */ uint32_t skr_seginuse; /* total unfreed segments */ uint32_t skr_rescale; /* # of hash table rescales */ /* * Region properties. */ struct skmem_region_params skr_params; /* region parameters */ #define skr_id skr_params.srp_id /* region ID */ #define skr_cflags skr_params.srp_cflags /* creation flags */ TAILQ_ENTRY(skmem_region) skr_link; /* skmem_region linkage */ char skr_name[64]; /* region name */ uuid_t skr_uuid; /* region uuid */ uint32_t skr_mode; /* skmem_region mode flags */ uint32_t skr_size; /* total region size */ IOSKMemoryBufferSpec skr_bufspec; /* IOSKMemoryBuffer spec */ IOSKRegionSpec skr_regspec; /* IOSKRegion spec */ IOSKRegionRef skr_reg; /* backing IOSKRegion */ struct zone *skr_zreg; /* backing zone (pseudo mode) */ void *skr_private; /* opaque arg to callbacks */ struct skmem_cache *skr_cache[SKR_MAX_CACHES]; /* client slab/cache layer */ /* * Objects. */ #define skr_r_obj_size skr_params.srp_r_obj_size /* requested obj size */ #define skr_r_obj_cnt skr_params.srp_r_obj_cnt /* requested obj count */ #define skr_c_obj_size skr_params.srp_c_obj_size /* configured obj size */ #define skr_c_obj_cnt skr_params.srp_c_obj_cnt /* configured obj count */ #define skr_align skr_params.srp_align /* object alignment */ #define skr_md_type skr_params.srp_md_type /* metadata type */ #define skr_md_subtype skr_params.srp_md_subtype /* metadata subtype */ #define skr_max_frags skr_params.srp_max_frags /* max number of buflets */ /* * Segment. */ sksegment_ctor_fn_t skr_seg_ctor; /* segment constructor */ sksegment_dtor_fn_t skr_seg_dtor; /* segment destructor */ uint32_t skr_seg_objs; /* # of objects per segment */ #define skr_seg_size skr_params.srp_c_seg_size /* configured segment size */ #define skr_seg_max_cnt skr_params.srp_seg_cnt /* max # of segments */ uint32_t skr_seg_bmap_len; /* # of skr_seg_bmap */ size_t skr_seg_bmap_size; bitmap_t *__sized_by(skr_seg_bmap_size) skr_seg_bmap; /* segment bitmaps */ uint32_t skr_seg_free_cnt; /* # of free segments */ uint32_t skr_hash_initial; /* initial hash table size */ uint32_t skr_hash_limit; /* hash table size limit */ uint32_t skr_hash_shift; /* get to interesting bits */ uint32_t skr_hash_mask; /* hash table mask */ size_t skr_hash_size; struct sksegment_bkt *__counted_by(skr_hash_size) skr_hash_table; /* alloc'd segment htable */ TAILQ_HEAD(segfreehead, sksegment) skr_seg_free; /* free segment list */ RB_HEAD(segtfreehead, sksegment) skr_seg_tfree; /* free tree */ uint32_t skr_seg_waiters; /* # of waiter threads */ /* * Region. */ uint32_t skr_refcnt; /* reference count */ /* * Mirror. */ struct skmem_region *skr_mirror; }; #define SKR_LOCK(_skr) \ lck_mtx_lock(&(_skr)->skr_lock) #define SKR_LOCK_ASSERT_HELD(_skr) \ LCK_MTX_ASSERT(&(_skr)->skr_lock, LCK_MTX_ASSERT_OWNED) #define SKR_LOCK_ASSERT_NOTHELD(_skr) \ LCK_MTX_ASSERT(&(_skr)->skr_lock, LCK_MTX_ASSERT_NOTOWNED) #define SKR_UNLOCK(_skr) \ lck_mtx_unlock(&(_skr)->skr_lock) /* valid values for skr_mode */ #define SKR_MODE_NOREDIRECT 0x1 /* unaffect by defunct */ #define SKR_MODE_MMAPOK 0x2 /* can be mapped to user task */ #define SKR_MODE_KREADONLY 0x4 /* kernel read only */ #define SKR_MODE_UREADONLY 0x8 /* if user map, map it read-only */ #define SKR_MODE_PERSISTENT 0x10 /* memory stays non-volatile */ #define SKR_MODE_MONOLITHIC 0x20 /* monolithic region */ #define SKR_MODE_NOMAGAZINES 0x40 /* disable magazines layer */ #define SKR_MODE_NOCACHE 0x80 /* caching-inhibited */ #define SKR_MODE_SEGPHYSCONTIG 0x100 /* phys. contiguous segment */ #define SKR_MODE_SHAREOK 0x200 /* allow object sharing */ #define SKR_MODE_IODIR_IN 0x400 /* I/O direction In */ #define SKR_MODE_IODIR_OUT 0x800 /* I/O direction Out */ #define SKR_MODE_GUARD 0x1000 /* guard pages region */ #define SKR_MODE_PUREDATA 0x2000 /* purely data; no pointers */ #define SKR_MODE_PSEUDO 0x4000 /* external backing store */ #define SKR_MODE_THREADSAFE 0x8000 /* thread safe */ #define SKR_MODE_MEMTAG 0x10000 /* enable memory tagging in this region */ #define SKR_MODE_SLAB (1U << 30) /* backend for slab layer */ #define SKR_MODE_MIRRORED (1U << 31) /* controlled by another region */ #define SKR_MODE_BITS \ "\020\01NOREDIRECT\02MMAPOK\03KREADONLY\04UREADONLY" \ "\05PERSISTENT\06MONOLITHIC\07NOMAGAZINES\10NOCACHE" \ "\11SEGPHYSCONTIG\012SHAREOK\013IODIR_IN\014IODIR_OUT" \ "\015GUARD\016PUREDATA\017PSEUDO\020THREADSAFE\021MEMTAG\037SLAB" \ "\040MIRRORED" /* valid values for skmem_region_create() */ #define SKMEM_REGION_CR_NOREDIRECT 0x1 /* unaffected by defunct */ #define SKMEM_REGION_CR_MMAPOK 0x2 /* can be mapped to user task */ #define SKMEM_REGION_CR_KREADONLY 0x4 /* kernel space readonly */ #define SKMEM_REGION_CR_UREADONLY 0x8 /* if user map, map it RO */ #define SKMEM_REGION_CR_PERSISTENT 0x10 /* memory stays non-volatile */ #define SKMEM_REGION_CR_MONOLITHIC 0x20 /* monolithic region */ #define SKMEM_REGION_CR_NOMAGAZINES 0x40 /* disable magazines layer */ #define SKMEM_REGION_CR_NOCACHE 0x80 /* caching-inhibited */ #define SKMEM_REGION_CR_SEGPHYSCONTIG 0x100 /* phys. contiguous segment */ #define SKMEM_REGION_CR_SHAREOK 0x200 /* allow object sharing */ #define SKMEM_REGION_CR_IODIR_IN 0x400 /* I/O direction in */ #define SKMEM_REGION_CR_IODIR_OUT 0x800 /* I/O direction out */ #define SKMEM_REGION_CR_GUARD 0x1000 /* guard pages region */ #define SKMEM_REGION_CR_PUREDATA 0x2000 /* purely data; no pointers */ #define SKMEM_REGION_CR_PSEUDO 0x4000 /* external backing store */ #define SKMEM_REGION_CR_THREADSAFE 0x8000 /* thread safe */ #define SKMEM_REGION_CR_MEMTAG 0x10000 /* enable memory tagging in this region */ #define SKMEM_REGION_CR_BITS \ "\021\01NOREDIRECT\02MMAPOK\03KREADONLY\04UREADONLY" \ "\05PERSISTENT\06MONOLITHIC\07NOMAGAZINES\10NOCACHE" \ "\11SEGPHYSCONTIG\012SHAREOK\013IODIR_IN\014IODIR_OUT" \ "\015GUARD\016PUREDATA\017PSEUDO\020THREADSAFE\021MEMTAG" __BEGIN_DECLS extern void skmem_region_init(void); extern void skmem_region_fini(void); extern void skmem_region_reap_caches(boolean_t); extern void skmem_region_params_config(struct skmem_region_params *); extern struct skmem_region *skmem_region_create(const char *, struct skmem_region_params *, sksegment_ctor_fn_t, sksegment_dtor_fn_t, void *); extern void skmem_region_mirror(struct skmem_region *, struct skmem_region *); extern void skmem_region_slab_config(struct skmem_region *, struct skmem_cache *, bool); extern void *__sized_by(objsize) skmem_region_alloc(struct skmem_region *, void *__sized_by(*msize) *, struct sksegment **, struct sksegment **, uint32_t, uint32_t objsize, uint32_t *msize); extern void skmem_region_free(struct skmem_region *, void *, void *); extern void skmem_region_retain(struct skmem_region *); extern boolean_t skmem_region_release(struct skmem_region *); extern mach_vm_address_t skmem_region_obj_lookup(struct skmem_region *, uint32_t); extern int skmem_region_get_info(struct skmem_region *, uint32_t *, struct sksegment **); extern boolean_t skmem_region_for_pp(skmem_region_id_t); extern void skmem_region_get_stats(struct skmem_region *, struct sk_stats_region *); #if (DEVELOPMENT || DEBUG) extern uint64_t skmem_region_get_mtbf(void); /* * Reasonable boundaries for MTBF that would make sense for testing, * in milliseconds; why not pick a couple of Mersenne p numbers? */ #define SKMEM_REGION_MTBF_MIN 2 /* almost 2 msec */ #define SKMEM_REGION_MTBF_MAX 3021377 /* almost 1 hour */ extern void skmem_region_set_mtbf(uint64_t); #endif /* (DEVELOPMENT || DEBUG) */ #if SK_LOG extern const char *skmem_region_id2name(skmem_region_id_t); #endif /* SK_LOG */ __END_DECLS #endif /* BSD_KERNEL_PRIVATE */ #endif /* _SKYWALK_MEM_SKMEMVAR_H */ |