Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 | /* * Copyright (c) 2000-2019 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_FREE_COPYRIGHT@ */ /* * Mach Operating System * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University * All Rights Reserved. * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ /* */ /* * File: thread.h * Author: Avadis Tevanian, Jr. * * This file contains the structure definitions for threads. * */ /* * Copyright (c) 1993 The University of Utah and * the Computer Systems Laboratory (CSL). All rights reserved. * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS * IS" CONDITION. THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * CSL requests users of this software to return to csl-dist@cs.utah.edu any * improvements that they make and grant CSL redistribution rights. * */ #ifndef _KERN_THREAD_H_ #define _KERN_THREAD_H_ #include <mach/kern_return.h> #include <mach/mach_types.h> #include <mach/mach_param.h> #include <mach/message.h> #include <mach/boolean.h> #include <mach/vm_param.h> #include <mach/thread_info.h> #include <mach/thread_status.h> #include <mach/exception_types.h> #include <kern/kern_types.h> #include <vm/vm_kern.h> #include <sys/cdefs.h> #include <sys/_types/_size_t.h> #ifdef MACH_KERNEL_PRIVATE #include <mach_assert.h> #include <mach_ldebug.h> #include <ipc/ipc_types.h> #include <mach/port.h> #include <kern/cpu_number.h> #include <kern/smp.h> #include <kern/smr_types.h> #include <kern/queue.h> #include <kern/timer.h> #include <kern/simple_lock.h> #include <kern/locks.h> #include <kern/sched.h> #include <kern/sched_prim.h> #include <mach/sfi_class.h> #include <kern/thread_call.h> #include <kern/thread_group.h> #include <kern/timer_call.h> #include <kern/task.h> #include <kern/exception.h> #include <kern/affinity.h> #include <kern/debug.h> #include <kern/block_hint.h> #include <kern/recount.h> #include <kern/turnstile.h> #include <kern/mpsc_queue.h> #if CONFIG_EXCLAVES #include <mach/exclaves.h> #endif /* CONFIG_EXCLAVES */ #include <kern/waitq.h> #include <san/kasan.h> #include <san/kcov_data.h> #include <os/refcnt.h> #include <ipc/ipc_kmsg.h> #include <ipc/ipc_policy.h> #include <machine/atomic.h> #include <machine/cpu_data.h> #include <machine/thread.h> #endif /* MACH_KERNEL_PRIVATE */ #ifdef XNU_KERNEL_PRIVATE /* priority queue static asserts fail for __ARM64_ARCH_8_32__ kext builds */ #include <kern/priority_queue.h> #endif /* XNU_KERNEL_PRIVATE */ __BEGIN_DECLS #ifdef XNU_KERNEL_PRIVATE #if CONFIG_TASKWATCH /* Taskwatch related. TODO: find this a better home */ typedef struct task_watcher task_watch_t; #endif /* CONFIG_TASKWATCH */ /* Thread tags; for easy identification. */ __options_closed_decl(thread_tag_t, uint16_t, { THREAD_TAG_MAINTHREAD = 0x01, THREAD_TAG_CALLOUT = 0x02, THREAD_TAG_IOWORKLOOP = 0x04, THREAD_TAG_PTHREAD = 0x10, THREAD_TAG_WORKQUEUE = 0x20, THREAD_TAG_USER_JOIN = 0x40, THREAD_TAG_AIO_WORKQUEUE = 0x80, }); typedef struct thread_ro *thread_ro_t; /*! * @struct thread_ro * * @brief * A structure allocated in a read only zone that safely * represents the linkages of a thread to its cred, proc, task, ... * * @discussion * The lifetime of a @c thread_ro structure is 1:1 with that * of a @c thread_t or a @c uthread_t and holding a thread reference * always allows to dereference this structure safely. */ struct thread_ro { struct thread *tro_owner; #if MACH_BSD __xnu_struct_group(thread_ro_creds, tro_creds, { /* * @c tro_cred holds the current thread credentials. * * For most threads, this is a cache of the proc's * credentials that has been updated at the last * syscall boundary via current_cached_proc_cred_update(). * * If the thread assumed a different identity using settid(), * then the proc cached credential lives in @c tro_realcred * instead. */ struct ucred *tro_cred; struct ucred *tro_realcred; }); struct proc *tro_proc; struct proc_ro *tro_proc_ro; #endif struct task *tro_task; struct exception_action *tro_exc_actions; }; /* * Flags for `thread set status`. */ __options_decl(thread_set_status_flags_t, uint32_t, { TSSF_FLAGS_NONE = 0, /* Translate the state to user. */ TSSF_TRANSLATE_TO_USER = 0x01, /* Translate the state to user. Preserve flags */ TSSF_PRESERVE_FLAGS = 0x02, /* Check kernel signed flag */ TSSF_CHECK_USER_FLAGS = 0x04, /* Allow only user state PTRS */ TSSF_ALLOW_ONLY_USER_PTRS = 0x08, /* Generate random diversifier and stash it */ TSSF_RANDOM_USER_DIV = 0x10, /* Stash sigreturn token */ TSSF_STASH_SIGRETURN_TOKEN = 0x20, /* Check sigreturn token */ TSSF_CHECK_SIGRETURN_TOKEN = 0x40, /* Allow only matching sigreturn token */ TSSF_ALLOW_ONLY_MATCHING_TOKEN = 0x80, /* Stash diversifier from thread */ TSSF_THREAD_USER_DIV = 0x100, /* Check for entitlement */ TSSF_CHECK_ENTITLEMENT = 0x200, /* Stash diversifier from task */ TSSF_TASK_USER_DIV = 0x400, /* Only take the PC from the new thread state */ TSSF_ONLY_PC = 0x800, }); /* * Size in bits of compact thread id (ctid). */ #define CTID_SIZE_BIT 20 typedef uint32_t ctid_t; #define THREAD_BOUND_CLUSTER_NONE UINT32_MAX #define THREAD_BOUND_PSET_NONE PSET_ID_INVALID /* Thread resume modes */ __options_closed_decl(thread_suspend_mode, uint8_t, { /* calling from thread_resume */ THREAD_SUSPEND_NORMAL = 0, /* calling from thread_resume */ THREAD_SUSPEND_LEGACY = 1, /* calling from thread_suspension_no_senders */ THREAD_SUSPEND_ALL = 2, }); #endif /* XNU_KERNEL_PRIVATE */ #ifdef MACH_KERNEL_PRIVATE extern zone_t thread_ro_zone; __options_decl(thread_work_interval_flags_t, uint32_t, { TH_WORK_INTERVAL_FLAGS_NONE = 0x0, #if CONFIG_SCHED_AUTO_JOIN /* Flags to indicate status about work interval thread is currently part of */ TH_WORK_INTERVAL_FLAGS_AUTO_JOIN_LEAK = 0x1, #endif /* CONFIG_SCHED_AUTO_JOIN */ TH_WORK_INTERVAL_FLAGS_HAS_WORKLOAD_ID = 0x2, TH_WORK_INTERVAL_FLAGS_RT_ALLOWED = 0x4, }); #if CONFIG_EXCLAVES /* Thread exclaves interrupt-safe state bits (ORd) */ __options_decl(thread_exclaves_intstate_flags_t, uint32_t, { /* Thread is currently executing in secure kernel or exclaves userspace * or was interrupted/preempted while doing so. */ TH_EXCLAVES_EXECUTION = 0x1, }); __options_decl(thread_exclaves_state_flags_t, uint16_t, { /* Thread exclaves state bits (ORd) */ /* Thread is handling RPC from a client in xnu or Darwin userspace (but * may have returned to xnu due to an exclaves scheduler request or having * upcalled). Must not re-enter exclaves via RPC or return to Darwin * userspace. */ TH_EXCLAVES_RPC = 0x1, /* Thread has made an upcall RPC request back into xnu while handling RPC * into exclaves from a client in xnu or Darwin userspace. Must not * re-enter exclaves via RPC or return to Darwin userspace. */ TH_EXCLAVES_UPCALL = 0x2, /* Thread has made an exclaves scheduler request (such as a wait or wake) * from the xnu scheduler while handling RPC into exclaves from a client in * xnu or Darwin userspace. Must not re-enter exclaves via RPC or return to * Darwin userspace. */ TH_EXCLAVES_SCHEDULER_REQUEST = 0x4, /* Thread is calling into xnu proxy server directly (but may have * returned to xnu due to an exclaves scheduler request or having * upcalled). Must not re-enter exclaves or return to Darwin userspace. */ TH_EXCLAVES_XNUPROXY = 0x8, /* Thread is calling into the exclaves scheduler directly. * Must not re-enter exclaves or return to Darwin userspace. */ TH_EXCLAVES_SCHEDULER_CALL = 0x10, /* Thread has called the stop upcall and once the thread returns from * downcall, exit_with_reason needs to be called on the task. */ TH_EXCLAVES_STOP_UPCALL_PENDING = 0x20, /* Thread is expecting that an exclaves-side thread may be spawned. */ TH_EXCLAVES_SPAWN_EXPECTED = 0x40, /* Thread is resuming the panic thread. * Must not re-enter exclaves or return to Darwin userspace. */ TH_EXCLAVES_RESUME_PANIC_THREAD = 0x80, /* Thread is an AOE exclave thread */ TH_EXCLAVES_AOE = 0x100, }); #define TH_EXCLAVES_STATE_ANY ( \ TH_EXCLAVES_RPC | \ TH_EXCLAVES_UPCALL | \ TH_EXCLAVES_SCHEDULER_REQUEST | \ TH_EXCLAVES_XNUPROXY | \ TH_EXCLAVES_SCHEDULER_CALL | \ TH_EXCLAVES_RESUME_PANIC_THREAD) __options_decl(thread_exclaves_inspection_flags_t, uint16_t, { /* Thread is on Stackshot's inspection queue */ TH_EXCLAVES_INSPECTION_STACKSHOT = 0x1, /* Thread is on Kperf's inspection queue */ TH_EXCLAVES_INSPECTION_KPERF = 0x2, /* Thread must not be inspected (may deadlock, etc.) - set by collector thread*/ TH_EXCLAVES_INSPECTION_NOINSPECT = 0x8000, }); #endif /* CONFIG_EXCLAVES */ typedef union thread_rr_state { uint32_t trr_value; struct { #define TRR_FAULT_NONE 0 #define TRR_FAULT_PENDING 1 #define TRR_FAULT_OBSERVED 2 /* * Set to TRR_FAULT_PENDING with interrupts disabled * by the thread when it is entering a user fault codepath. * * Moved to TRR_FAULT_OBSERVED from TRR_FAULT_PENDING: * - by the thread if at IPI time, * - or by task_restartable_ranges_synchronize() if the thread * is interrupted (under the thread lock) * * Cleared by the thread when returning from a user fault * codepath. */ uint8_t trr_fault_state; /* * Set by task_restartable_ranges_synchronize() * if trr_fault_state is TRR_FAULT_OBSERVED * and a rendez vous at the AST is required. * * Set atomically if trr_fault_state == TRR_FAULT_OBSERVED, * and trr_ipi_ack_pending == 0 */ uint8_t trr_sync_waiting; /* * Updated under the thread_lock(), * set by task_restartable_ranges_synchronize() * when the thread was IPIed and the caller is waiting * for an ACK. */ uint16_t trr_ipi_ack_pending; }; } thread_rr_state_t; struct vm_map_lock_ctx; struct thread { #if MACH_ASSERT #define THREAD_MAGIC 0x1fc01fc01fc01fc0ULL /* materializes with a single mov immediate */ /* Ensure nothing uses &thread as a queue entry */ uint64_t thread_magic; #endif /* MACH_ASSERT */ /* * NOTE: The runq field in the thread structure has an unusual * locking protocol. If its value is PROCESSOR_NULL, then it is * locked by the thread_lock, but if its value is something else * then it is locked by the associated run queue lock. It is * set to PROCESSOR_NULL without holding the thread lock, but the * transition from PROCESSOR_NULL to non-null must be done * under the thread lock and the run queue lock. To enforce the * protocol, runq should only be accessed using the * thread_get/set/clear_runq functions and locked variants below. * * New waitq APIs allow the 'links' and '__runq' fields to be * anywhere in the thread structure. */ union { queue_chain_t runq_links; /* run queue links */ queue_chain_t wait_links; /* wait queue links */ struct mpsc_queue_chain mpsc_links; /* thread daemon mpsc links */ struct priority_queue_entry_sched wait_prioq_links; /* priority ordered waitq links */ }; event64_t wait_event; /* wait queue event */ struct { processor_t runq; } __runq; /* internally managed run queue assignment, see above comment */ waitq_t waitq; /* wait queue this thread is enqueued on */ struct turnstile *turnstile; /* thread's turnstile, protected by primitives interlock */ void *inheritor; /* inheritor of the primitive the thread will block on */ struct priority_queue_sched_max sched_inheritor_queue; /* Inheritor queue for kernel promotion */ struct priority_queue_sched_max base_inheritor_queue; /* Inheritor queue for user promotion */ #if CONFIG_SCHED_EDGE bool th_bound_pset_enqueued; bool th_shared_rsrc_enqueued[CLUSTER_SHARED_RSRC_TYPE_COUNT]; bool th_shared_rsrc_heavy_user[CLUSTER_SHARED_RSRC_TYPE_COUNT]; bool th_shared_rsrc_heavy_perf_control[CLUSTER_SHARED_RSRC_TYPE_COUNT]; /* * Caution! These bits should only be written/read by the current, * just previous, or thread_setrun()-ing processor, before this * thread can be picked up to run by its next processor. */ uint8_t th_expired_quantum_on_lower_core:1, th_expired_quantum_on_higher_core:1, :0; #endif /* CONFIG_SCHED_EDGE */ #if CONFIG_SCHED_CLUTCH /* * In the clutch scheduler, the threads are maintained in runqs at the clutch_bucket * level (clutch_bucket defines a unique thread group and scheduling bucket pair). The * thread is linked via a couple of linkages in the clutch bucket: * * - A stable priority queue linkage which is the main runqueue (based on sched_pri) for the clutch bucket * - A regular priority queue linkage which is based on thread's base/promoted pri (used for clutch bucket priority calculation) * - A queue linkage used for timesharing operations of threads at the scheduler tick */ struct priority_queue_entry_stable th_clutch_runq_link; struct priority_queue_entry_sched th_clutch_pri_link; queue_chain_t th_clutch_timeshare_link; #endif /* CONFIG_SCHED_CLUTCH */ /* Data updated during assert_wait/thread_wakeup */ decl_simple_lock_data(, sched_lock); /* scheduling lock (thread_lock()) */ decl_simple_lock_data(, wake_lock); /* for thread stop / wait (wake_lock()) */ uint16_t options; /* options set by thread itself */ #define TH_OPT_INTMASK 0x0003 /* interrupt / abort level */ #define TH_OPT_VMPRIV 0x0004 /* may allocate reserved memory */ #define TH_OPT_SYSTEM_CRITICAL 0x0010 /* Thread must always be allowed to run - even under heavy load */ #define TH_OPT_PROC_CPULIMIT 0x0020 /* Thread has a task-wide CPU limit applied to it */ #define TH_OPT_PRVT_CPULIMIT 0x0040 /* Thread has a thread-private CPU limit applied to it */ #define TH_OPT_IDLE_THREAD 0x0080 /* Thread is a per-processor idle thread */ #define TH_OPT_GLOBAL_FORCED_IDLE 0x0100 /* Thread performs forced idle for thermal control */ #define TH_OPT_SCHED_VM_GROUP 0x0200 /* Thread belongs to special scheduler VM group */ #define TH_OPT_HONOR_QLIMIT 0x0400 /* Thread will honor qlimit while sending mach_msg, regardless of MACH_SEND_ALWAYS */ #define TH_OPT_SEND_IMPORTANCE 0x0800 /* Thread will allow importance donation from kernel rpc */ #define TH_OPT_ZONE_PRIV 0x1000 /* Thread may use the zone replenish reserve */ #define TH_OPT_IPC_TG_BLOCKED 0x2000 /* Thread blocked in sync IPC and has made the thread group blocked callout */ #define TH_OPT_FORCED_LEDGER 0x4000 /* Thread has a forced CPU limit */ #define TH_IN_MACH_EXCEPTION 0x8000 /* Thread is currently handling a mach exception */ bool wake_active; /* wake event on stop */ bool at_safe_point; /* thread_abort_safely allowed */ uint8_t sched_saved_run_weight; #if DEVELOPMENT || DEBUG bool pmap_footprint_suspended; #endif /* DEVELOPMENT || DEBUG */ ast_t reason; /* why we blocked */ uint32_t quantum_remaining; wait_result_t wait_result; /* outcome of wait - * may be examined by this thread * WITHOUT locking */ thread_rr_state_t t_rr_state; /* state for restartable ranges */ thread_continue_t continuation; /* continue here next dispatch */ void *parameter; /* continuation parameter */ /* Data updated/used in thread_invoke */ vm_offset_t kernel_stack; /* current kernel stack */ vm_offset_t reserved_stack; /* reserved kernel stack */ /*** Machine-dependent state ***/ struct machine_thread machine; #if KASAN struct kasan_thread_data kasan_data; #endif #if CONFIG_KCOV kcov_thread_data_t kcov_data; #endif /* Thread state: */ int state; /* * Thread states [bits or'ed] * All but TH_WAIT_REPORT are encoded in SS_TH_FLAGS * All are encoded in kcdata.py ('ths_state') */ #define TH_WAIT 0x01 /* queued for waiting */ #define TH_SUSP 0x02 /* stopped or requested to stop */ #define TH_RUN 0x04 /* running or on runq */ #define TH_UNINT 0x08 /* waiting uninteruptibly */ #define TH_TERMINATE 0x10 /* halted at termination */ #define TH_TERMINATE2 0x20 /* added to termination queue */ #define TH_WAIT_REPORT 0x40 /* the wait is using the sched_call, * only set if TH_WAIT is also set */ #define TH_IDLE 0x80 /* idling processor */ #define TH_WAKING 0x100 /* between waitq remove and thread_go */ /* Scheduling information */ sched_mode_t sched_mode; /* scheduling mode */ sched_mode_t saved_mode; /* saved mode during forced mode demotion */ /* This thread's contribution to global sched counters */ sched_bucket_t th_sched_bucket; sfi_class_id_t sfi_class; /* SFI class (XXX Updated on CSW/QE/AST) */ sfi_class_id_t sfi_wait_class; /* Currently in SFI wait for this class, protected by sfi_lock */ uint32_t sched_flags; /* current flag bits */ #if CONFIG_SCHED_SMT #define TH_SFLAG_NO_SMT 0x0001 /* On an SMT CPU, this thread must be scheduled alone */ #endif /* CONFIG_SCHED_SMT */ #define TH_SFLAG_FAILSAFE 0x0002 /* fail-safe has tripped */ #define TH_SFLAG_THROTTLED 0x0004 /* throttled thread forced to timeshare mode (may be applied in addition to failsafe) */ /* unused TH_SFLAG_PROMOTED 0x0008 */ #define TH_SFLAG_ABORT 0x0010 /* abort interruptible waits */ #define TH_SFLAG_ABORTSAFELY 0x0020 /* ... but only those at safe point */ #define TH_SFLAG_ABORTED_MASK (TH_SFLAG_ABORT | TH_SFLAG_ABORTSAFELY) #define TH_SFLAG_DEPRESS 0x0040 /* normal depress yield */ #define TH_SFLAG_POLLDEPRESS 0x0080 /* polled depress yield */ #define TH_SFLAG_DEPRESSED_MASK (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS) /* unused TH_SFLAG_PRI_UPDATE 0x0100 */ #define TH_SFLAG_EAGERPREEMPT 0x0200 /* Any preemption of this thread should be treated as if AST_URGENT applied */ #define TH_SFLAG_RW_PROMOTED 0x0400 /* promote reason: blocking with RW lock held */ #define TH_SFLAG_BASE_PRI_FROZEN 0x0800 /* (effective) base_pri is frozen */ #define TH_SFLAG_WAITQ_PROMOTED 0x1000 /* promote reason: waitq wakeup (generally for IPC receive) */ #if __AMP__ /* unused TH_SFLAG_ECORE_ONLY 0x2000 */ /* unused TH_SFLAG_PCORE_ONLY 0x4000 */ #endif #define TH_SFLAG_EXEC_PROMOTED 0x8000 /* promote reason: thread is in an exec */ #define TH_SFLAG_THREAD_GROUP_AUTO_JOIN 0x10000 /* thread has been auto-joined to thread group */ #if __AMP__ /* unused TH_SFLAG_BOUND_SOFT 0x20000 */ #endif /* __AMP__ */ #if CONFIG_PREADOPT_TG #define TH_SFLAG_REEVALUTE_TG_HIERARCHY_LATER 0x40000 /* thread needs to reevaluate its TG hierarchy */ #endif #define TH_SFLAG_FLOOR_PROMOTED 0x80000 /* promote reason: boost requested */ /* 'promote reasons' that request a priority floor only, not a custom priority */ #define TH_SFLAG_PROMOTE_REASON_MASK (TH_SFLAG_RW_PROMOTED | TH_SFLAG_WAITQ_PROMOTED | TH_SFLAG_EXEC_PROMOTED | TH_SFLAG_FLOOR_PROMOTED) #define TH_SFLAG_RT_DISALLOWED 0x100000 /* thread wants RT but may not have joined a work interval that allows it */ #define TH_SFLAG_DEMOTED_MASK (TH_SFLAG_THROTTLED | TH_SFLAG_FAILSAFE | TH_SFLAG_RT_DISALLOWED) /* saved_mode contains previous sched_mode */ #define TH_SFLAG_RT_CPULIMIT 0x200000 /* thread should have a CPU limit applied. */ #define TH_SFLAG_FAILSAFE_REPORTED 0x400000 /* whether the kernel has already logged that thread triggered the failsafe (to prevent log spam) */ #define TH_SFLAG_AUTO_RESUMED 0x800000 /* thread is auto resumed after the suspending task exited */ int16_t sched_pri; /* scheduled (current) priority */ int16_t base_pri; /* effective base priority (equal to req_base_pri unless TH_SFLAG_BASE_PRI_FROZEN) */ int16_t req_base_pri; /* requested base priority */ int16_t max_priority; /* copy of max base priority */ int16_t task_priority; /* copy of task base priority */ uint16_t priority_floor_count; /* number of push to boost the floor priority */ int16_t suspend_count; /* Kernel holds on this thread */ #if HAS_MTE uint8_t page_wait_class; /* The type of VM page this thread will wait on next */ #endif /* HAS_MTE */ int iotier_override; /* atomic operations to set, cleared on ret to user */ os_ref_atomic_t ref_count; /* number of references to me */ uint32_t rwlock_count; /* Number of lck_rw_t locks held by thread */ struct smrq_slist_head smr_stack; #if MACH_ASSERT struct rw_lock_debug rw_lock_held; /* rw_locks currently held by the thread */ #endif /* MACH_ASSERT */ struct vm_map_lock_ctx *vm_map_lock_ctx_held; /* range lock currently declared by the thread (may or may not be locked) */ integer_t importance; /* task-relative importance */ /* Priority depression expiration */ integer_t depress_timer_active; timer_call_t depress_timer; /* real-time parameters */ struct { /* see mach/thread_policy.h */ uint32_t period; uint32_t computation; uint32_t constraint; bool preemptible; uint8_t priority_offset; /* base_pri = BASEPRI_RTQUEUES + priority_offset */ uint64_t deadline; } realtime; uint64_t last_run_time; /* time when thread was switched away from */ uint64_t last_made_runnable_time; /* time when thread was unblocked or preempted */ uint64_t last_basepri_change_time; /* time when thread was last changed in basepri while runnable */ uint64_t same_pri_latency; /* * workq_quantum_deadline is the workq thread's next runtime deadline. This * value is set to 0 if the thread has no such deadline applicable to it. * * The synchronization for this field is due to how this field is modified * 1) This field is always modified on the thread by itself or on the thread * when it is not running/runnable * 2) Change of this field is immediately followed by a * corresponding change to the AST_KEVENT to either set or clear the * AST_KEVENT_WORKQ_QUANTUM_EXPIRED bit * * workq_quantum_deadline can be modified by the thread on itself during * interrupt context. However, due to (2) and due to the fact that the * change to the AST_KEVENT is volatile, this forces the compiler to * guarantee the order between the write to workq_quantum_deadline and the * kevent field and therefore guarantees the correct synchronization. */ uint64_t workq_quantum_deadline; #if WORKQ_QUANTUM_HISTORY_DEBUG #define WORKQ_QUANTUM_HISTORY_COUNT 16 struct workq_quantum_history { uint64_t time; uint64_t deadline; bool arm; } workq_quantum_history[WORKQ_QUANTUM_HISTORY_COUNT]; uint64_t workq_quantum_history_index; #define WORKQ_QUANTUM_HISTORY_WRITE_ENTRY(thread, ...) ({\ thread_t __th = (thread); \ uint64_t __index = os_atomic_inc_orig(&thread->workq_quantum_history_index, relaxed); \ struct workq_quantum_history _wq_quantum_history = { mach_approximate_time(), __VA_ARGS__}; \ __th->workq_quantum_history[__index % WORKQ_QUANTUM_HISTORY_COUNT] = \ (struct workq_quantum_history) _wq_quantum_history; \ }) #else /* WORKQ_QUANTUM_HISTORY_DEBUG */ #define WORKQ_QUANTUM_HISTORY_WRITE_ENTRY(thread, ...) #endif /* WORKQ_QUANTUM_HISTORY_DEBUG */ #define THREAD_NOT_RUNNABLE (~0ULL) #if CONFIG_THREAD_GROUPS struct thread_group *thread_group; #endif /* Data used during setrun/dispatch */ processor_t bound_processor; /* bound to a processor? */ processor_t last_processor; /* processor last dispatched on */ processor_t chosen_processor; /* Where we want to run this thread */ /* Fail-safe computation since last unblock or qualifying yield */ uint64_t computation_metered; uint64_t computation_epoch; uint64_t computation_interrupt_epoch; uint64_t safe_release; /* when to release fail-safe */ /* Call out from scheduler */ void (*sched_call)(int type, thread_t thread); /* Statistics and timesharing calculations */ #if defined(CONFIG_SCHED_TIMESHARE_CORE) natural_t sched_stamp; /* last scheduler tick */ natural_t sched_usage; /* timesharing cpu usage [sched] */ natural_t pri_shift; /* usage -> priority from pset */ natural_t cpu_usage; /* instrumented cpu usage [%cpu] */ natural_t cpu_delta; /* accumulated cpu_usage delta */ #endif /* CONFIG_SCHED_TIMESHARE_CORE */ uint32_t c_switch; /* total context switches */ uint32_t p_switch; /* total processor switches */ uint32_t ps_switch; /* total pset switches */ /* Timing data structures */ uint64_t sched_time_save; /* saved time for scheduler tick */ uint64_t vtimer_user_save; /* saved values for vtimers */ uint64_t vtimer_prof_save; uint64_t vtimer_rlim_save; uint64_t vtimer_qos_save; timer_data_t runnable_timer; /* time the thread is runnable (including running) */ struct recount_thread th_recount; /* resource accounting */ #if CONFIG_SCHED_SFI /* Timing for wait state */ uint64_t wait_sfi_begin_time; /* start time for thread waiting in SFI */ #endif /* * Processor/cache affinity * - affinity_threads links task threads with the same affinity set */ queue_chain_t affinity_threads; affinity_set_t affinity_set; #if CONFIG_TASKWATCH task_watch_t *taskwatch; /* task watch */ #endif /* CONFIG_TASKWATCH */ /* Various bits of state to stash across a continuation, exclusive to the current thread block point */ union { struct { /* set before ipc_mqueue_receive() as implicit arguments */ mach_msg_recv_bufs_t recv_bufs; /* receive context */ mach_msg_option64_t option; /* 64 bits options for receive */ ipc_object_t object; /* object received on */ /* set by ipc_mqueue_receive() as implicit results */ mach_msg_return_t state; /* receive state */ mach_port_seqno_t seqno; /* seqno of recvd message */ mach_msg_size_t msize; /* actual size for the msg */ mach_msg_size_t asize; /* actual size for aux data */ mach_port_name_t receiver_name; /* the receive port name */ struct ipc_kmsg *XNU_PTRAUTH_SIGNED_PTR("thread.ith_kmsg") kmsg; /* received message */ } receive; struct { struct semaphore *waitsemaphore; /* semaphore ref */ struct semaphore *signalsemaphore; /* semaphore ref */ int options; /* semaphore options */ kern_return_t result; /* primary result */ mach_msg_continue_t continuation; } sema; struct { #define THREAD_SAVE_IOKIT_TLS_COUNT 8 void *tls[THREAD_SAVE_IOKIT_TLS_COUNT]; } iokit; } saved; /* Only user threads can cause Mach exceptions, only kernel threads can be thread call threads */ union { /* Thread call thread's state structure, stored on its stack */ struct thread_call_thread_state *thc_state; /* Structure to save information about Mach exception */ struct { int os_reason; exception_type_t exception_type; mach_exception_code_t code; mach_exception_subcode_t subcode; } mach_exc_info; }; /* User level suspensions */ int32_t user_stop_count; int32_t legacy_user_stop_count; /* IPC data structures */ #if IMPORTANCE_INHERITANCE natural_t ith_assertions; /* assertions pending drop */ #endif circle_queue_head_t ith_messages; /* messages to reap */ mach_port_t ith_kernel_reply_port; /* reply port for kernel RPCs */ /* VM Fault Tolerance */ bool th_vm_faults_disabled; /* Ast/Halt data structures */ bool recover; /* True if page faulted in recoverable IO */ #if DEBUG || DEVELOPMENT struct thread_test_context *th_test_ctx; /* thread-specific data for kernel tests */ #endif queue_chain_t threads; /* global list of all threads */ /* Activation */ queue_chain_t task_threads; /* Task membership */ #if __x86_64__ || __arm__ struct task *t_task; #endif struct thread_ro *t_tro; vm_map_t map; thread_t handoff_thread; /* Timed wait expiration */ timer_call_t wait_timer; uint16_t wait_timer_active; /* is the call running */ bool wait_timer_armed; /* should the wait be cleared */ /* Miscellaneous bits guarded by mutex */ uint32_t active:1, /* Thread is active and has not been terminated */ ipc_active:1, /* IPC with the thread ports is allowed */ started:1, /* Thread has been started after creation */ static_param:1, /* Disallow policy parameter changes */ inspection:1, /* TRUE when task is being inspected by crash reporter */ policy_reset:1, /* Disallow policy parameter changes on terminating threads */ suspend_parked:1, /* thread parked in thread_suspended */ corpse_dup:1, /* TRUE when thread is an inactive duplicate in a corpse */ :0; /* Pending thread ast(s) */ os_atomic(ast_t) ast; decl_lck_mtx_data(, mutex); struct ipc_port * XNU_PTRAUTH_SIGNED_PTR("thread.ith_special_reply_port") ith_special_reply_port; /* ref to special reply port */ struct ipc_port * XNU_PTRAUTH_SIGNED_PTR("thread.thread_ports") thread_ports[THREAD_SELF_PORT_COUNT]; /* no right */ #if CONFIG_CSR struct ipc_port * XNU_PTRAUTH_SIGNED_PTR("thread.thread_settable_self_port") thread_settable_self_port; /* send right */ #endif /* CONFIG_CSR */ struct ipc_port * XNU_PTRAUTH_SIGNED_PTR("thread.thread_resume_port") thread_resume_port; #if CONFIG_DTRACE uint16_t t_dtrace_flags; /* DTrace thread states */ #define TH_DTRACE_EXECSUCCESS 0x01 uint16_t t_dtrace_inprobe; /* Executing under dtrace_probe */ uint32_t t_dtrace_predcache; /* DTrace per thread predicate value hint */ int64_t t_dtrace_tracing; /* Thread time under dtrace_probe() */ int64_t t_dtrace_vtime; #endif clock_sec_t t_page_creation_time; uint32_t t_page_creation_count; uint32_t t_page_creation_throttled; #if (DEVELOPMENT || DEBUG) uint64_t t_page_creation_throttled_hard; uint64_t t_page_creation_throttled_soft; #endif /* DEVELOPMENT || DEBUG */ int t_pagein_error; /* for vm_fault(), holds error from vnop_pagein() */ mach_port_name_t ith_voucher_name; ipc_voucher_t ith_voucher; #ifdef KPERF /* The high 8 bits are the number of frames to sample of a user callstack. */ #define T_KPERF_CALLSTACK_DEPTH_OFFSET (24) #define T_KPERF_SET_CALLSTACK_DEPTH(DEPTH) (((uint32_t)(DEPTH)) << T_KPERF_CALLSTACK_DEPTH_OFFSET) #define T_KPERF_GET_CALLSTACK_DEPTH(FLAGS) ((FLAGS) >> T_KPERF_CALLSTACK_DEPTH_OFFSET) #define T_KPERF_ACTIONID_OFFSET (18) #define T_KPERF_SET_ACTIONID(AID) (((uint32_t)(AID)) << T_KPERF_ACTIONID_OFFSET) #define T_KPERF_GET_ACTIONID(FLAGS) ((FLAGS) >> T_KPERF_ACTIONID_OFFSET) #endif #define T_KPERF_AST_CALLSTACK 0x1 /* dump a callstack on thread's next AST */ #define T_KPERF_AST_DISPATCH 0x2 /* dump a name on thread's next AST */ #define T_KPC_ALLOC 0x4 /* thread needs a kpc_buf allocated */ #define T_KPERF_AST_ALL \ (T_KPERF_AST_CALLSTACK | T_KPERF_AST_DISPATCH | T_KPC_ALLOC) /* only go up to T_KPERF_ACTIONID_OFFSET - 1 */ #ifdef KPERF uint32_t kperf_ast; uint32_t kperf_pet_gen; /* last generation of PET that sampled this thread*/ uint32_t kperf_c_switch; /* last dispatch detection */ uint32_t kperf_pet_cnt; /* how many times a thread has been sampled by PET */ #if CONFIG_EXCLAVES uint32_t kperf_exclaves_ast; #endif /* CONFIG_EXCLAVES */ #endif /* KPERF */ _Atomic uint16_t t_telemetry_ast; #if CONFIG_MEMORY_MICROSTACKSHOT /* * Metadata for a page grab sample emitted in microstackshots. */ struct { /* * The number of UPL grabs generating samples on this thread. */ uint8_t tpgi_upl_count; /* * The number of IOPL grabs generating samples on this thread. */ uint8_t tpgi_iopl_count; /* * The VM tag for the pages being grabbed. */ uint16_t tpgi_tag; } t_page_grab_info; /* * Metadata for a VM fault sample emitted in microstackshots. */ struct { /* * The virtual address of the VM fault. */ uint64_t tvfi_va:48; /* * How the fault was resolved, what source it came from (e.g. page * page, demand I/O, speculative, etc.). */ uint64_t tvfi_type:8; /* * Any flags related to the fault; see `telemetry.h`'s * `telemetry_vm_fault_flags_t`. */ uint64_t tvfi_flags:8; } t_vm_fault_info; #endif /* CONFIG_MEMORY_MICROSTACKSHOT */ #ifdef CONFIG_CPU_COUNTERS /* accumulated performance counters for this thread */ uint64_t *kpc_buf; #endif /* CONFIG_CPU_COUNTERS */ #if HYPERVISOR /* hypervisor virtual CPU object associated with this thread */ void *hv_thread_target; #endif /* HYPERVISOR */ /* Statistics accumulated per-thread and aggregated per-task */ uint32_t syscalls_unix; uint32_t syscalls_mach; ledger_t t_ledger; /* shortcut to task ledger */ ledger_t t_threadledger; /* per thread ledger */ ledger_t t_bankledger; /* ledger to charge someone */ uint64_t t_deduct_bank_ledger_time; /* cpu time to be deducted from bank ledger */ uint64_t t_deduct_bank_ledger_energy; /* energy to be deducted from bank ledger */ uint64_t thread_id; /* system wide unique thread-id */ uint32_t ctid; /* system wide compact thread-id */ uint32_t ctsid; /* this thread ts ID */ /* policy is protected by the thread mutex */ struct thread_requested_policy requested_policy; struct thread_effective_policy effective_policy; /* usynch override is protected by the task lock, eventually will be thread mutex */ struct thread_qos_override { struct thread_qos_override *override_next; uint32_t override_contended_resource_count; int16_t override_qos; int16_t override_resource_type; user_addr_t override_resource; } *overrides; uint32_t kevent_overrides; uint8_t user_promotion_basepri; uint8_t kern_promotion_schedpri; _Atomic uint16_t kevent_ast_bits; io_stat_info_t thread_io_stats; /* per-thread I/O statistics */ uint32_t thread_callout_interrupt_wakeups; uint32_t thread_callout_platform_idle_wakeups; uint32_t thread_timer_wakeups_bin_1; uint32_t thread_timer_wakeups_bin_2; thread_tag_t thread_tag; /* * callout_* fields are only set for thread call threads whereas mach_exc_* are set * by user threads on themselves while taking a Mach exception. So it's okay for them to * share this bitfield. */ uint16_t callout_woken_from_icontext:1, callout_woken_from_platform_idle:1, callout_woke_thread:1, /* * If a pending exception does not have this bit set, and a new exception * is raised that does have this bit set, the new exception will overwrite * the prior one. * If the prior and new exceptions both have this bit set, the exception raised * first will be retained. */ mach_exc_sticky:1, mach_exc_ktriage:1, thread_bitfield_unused:11; /* * Pset to which the thread is soft-bound for scheduling. The thread will * always run on its bound pset unless that pset has been derecommended for * scheduling. */ pset_id_t th_bound_pset_id; #if CONFIG_THREAD_GROUPS #if CONFIG_PREADOPT_TG /* The preadopt thread group is set on the thread * * a) By another thread when it is a creator and it is scheduled with the * thread group on the TR * b) On itself when it binds a thread request and becomes a * servicer or when it rebinds to the thread request * c) On itself when it processes knotes and finds the first * EVFILT_MACHPORT event to deliver to userspace * * Note that this is a full reference owned by the thread_t and not a * borrowed reference. * * This reference is cleared from the thread_t by the thread itself at the * following times: * a) When it explicitly adopts a work interval or a bank voucher * b) If it still exists on the thread, after it has unbound and is about * to park * c) During thread termination if one still exists * d) When a different preadoption thread group is set on the thread * * It is modified under the thread lock. */ struct thread_group *preadopt_thread_group; /* This field here is present in order to make sure that the t->thread_group * is always pointing to a valid thread group and isn't a dangling pointer. * * Consider the following scenario: * a) t->thread_group points to the preadoption thread group * b) The preadoption thread group is modified on the thread but we are * unable to resolve the hierarchy immediately due to the current state of * the thread * * In order to make sure that t->thread_group points to a valid thread * group until we can resolve the hierarchy again, we save the existing * thread_group it points to in old_preadopt_thread_group. The next time a * hierarchy resolution is done, we know that t->thread_group will not point * to this field anymore so we can clear it. * * This field is always going to take the reference that was previously in * preadopt_thread_group so it will have a full +1 */ struct thread_group *old_preadopt_thread_group; #endif /* CONFIG_PREADOPT_TG */ /* This is a borrowed reference to the TG from the ith_voucher and is saved * here since we may not always be in the right context to able to do the * lookups. * * It is set always set on self under the thread lock */ struct thread_group *bank_thread_group; /* Whether this is the autojoin thread group or the work interval thread * group depends on whether the thread's sched_flags has the * TH_SFLAG_THREAD_GROUP_AUTO_JOIN bit set */ union { /* This is a borrowed reference to the auto join thread group from the * work_interval. It is set with the thread lock held */ struct thread_group *auto_join_thread_group; /* This is a borrowed reference to the explicit work_interval thread group * and is always set on self */ struct thread_group *work_interval_thread_group; }; #endif /* CONFIG_THREAD_GROUPS */ /* work interval (if any) associated with the thread. Only modified by * current thread on itself or when another thread when the thread is held * off of runq */ struct work_interval *th_work_interval; thread_work_interval_flags_t th_work_interval_flags; #if SCHED_TRACE_THREAD_WAKEUPS uintptr_t thread_wakeup_bt[64]; #endif turnstile_update_flags_t inheritor_flags; /* inheritor flags for inheritor field */ block_hint_t pending_block_hint; block_hint_t block_hint; /* What type of primitive last caused us to block. */ uint32_t decompressions; /* Per-thread decompressions counter to be added to per-task decompressions counter */ int thread_region_page_shift; /* Page shift that this thread would like to use when */ /* introspecting a task. This is currently being used */ /* by footprint which uses a thread for each task being inspected. */ #if CONFIG_SCHED_RT_ALLOW /* Used when a thread is requested to set/clear its own CPU limit */ uint32_t t_ledger_req_action:2, t_ledger_req_percentage:7, t_ledger_req_interval_ms:16, :0; #endif /* CONFIG_SCHED_RT_ALLOW */ #if CONFIG_IOSCHED void *decmp_upl; #endif /* CONFIG_IOSCHED */ struct knote *ith_knote; /* knote fired for rcv */ #if CONFIG_SPTM /* TXM thread stack associated with this thread */ uintptr_t txm_thread_stack; #endif #if CONFIG_EXCLAVES /* Per-thread IPC context for exclaves communication. Only modified by the * current thread on itself. */ exclaves_ctx_t th_exclaves_ipc_ctx; /* Thread exclaves interrupt-safe state. Only mutated by the current thread * on itself with interrupts disabled, and only ever read by the current * thread (with no locking), including from interrupt context, or during * debug/stackshot. */ thread_exclaves_intstate_flags_t th_exclaves_intstate; /* Thread exclaves state. Only mutated by the current thread on itself, and * only ever read by the current thread (with no locking). Unsafe to read * from interrupt context. */ thread_exclaves_state_flags_t th_exclaves_state; /* Thread stackshot state. Prevents returning to Exclave world until after * an external agent has triggered inspection (likely via Exclave stackshot), * and woken this thread. */ thread_exclaves_inspection_flags_t _Atomic th_exclaves_inspection_state; /* Task for which conclave teardown is being called by this thread. Used * for context by conclave crash info upcall to find the task for appending * the conclave crash info. */ task_t conclave_stop_task; /* Queue of threads being inspected by Stackshot. * Modified under exclaves_collect_mtx. */ queue_chain_t th_exclaves_inspection_queue_stackshot; /* Queue of threads being inspected by kperf. * Modified under exclaves_collect_mtx. */ queue_chain_t th_exclaves_inspection_queue_kperf; #endif /* CONFIG_EXCLAVES */ #if HAS_MTE /* * Temporary context used while performing faultable accesess on IOMD memory. * This holds the task that originally provided the backing memory for the IOMD. * This value being non-NULL signifies that we're in the critical faultable access window. */ task_t iomd_faultable_buffer_provider; #endif /* HAS_MTE */ /* * This accounting is similar in spirit to `mach_exc_info` et al: this field stashes metadata * about an in-flight IPC policy violation for which we've raised an AST but have not yet processed it. */ ipc_sec_policy_in_flight_violation_info_t pending_ipc_sec_policy_violation_info; }; #define ith_receive saved.receive /* arguments */ #define ith_recv_bufs saved.receive.recv_bufs #define ith_object saved.receive.object #define ith_option saved.receive.option /* results */ #define ith_state saved.receive.state #define ith_seqno saved.receive.seqno #define ith_msize saved.receive.msize #define ith_asize saved.receive.asize #define ith_receiver_name saved.receive.receiver_name #define ith_kmsg saved.receive.kmsg #define sth_waitsemaphore saved.sema.waitsemaphore #define sth_signalsemaphore saved.sema.signalsemaphore #define sth_options saved.sema.options #define sth_result saved.sema.result #define sth_continuation saved.sema.continuation #define ITH_KNOTE_NULL ((void *)NULL) #define ITH_KNOTE_PSEUDO ((void *)0xdeadbeef) /* * The ith_knote is used during message delivery, and can safely be interpreted * only when used for one of these codepaths, which the test for the msgt_name * being RECEIVE or SEND_ONCE is about. */ #define ITH_KNOTE_VALID(kn, msgt_name) \ (((kn) != ITH_KNOTE_NULL && (kn) != ITH_KNOTE_PSEUDO) && \ ((msgt_name) == MACH_MSG_TYPE_PORT_RECEIVE || \ (msgt_name) == MACH_MSG_TYPE_PORT_SEND_ONCE)) #if MACH_ASSERT #define assert_thread_magic(thread) assert3u((thread)->thread_magic, ==, THREAD_MAGIC) #else #define assert_thread_magic(thread) do { (void)(thread); } while (0) #endif extern thread_t thread_bootstrap(void); extern void thread_machine_init_template(void); extern void thread_init(void); extern void thread_daemon_init(void); extern void thread_reference( thread_t thread); extern void thread_deallocate( thread_t thread); extern void thread_inspect_deallocate( thread_inspect_t thread); extern void thread_read_deallocate( thread_read_t thread); extern kern_return_t thread_terminate( thread_t thread); extern void thread_terminate_self(void); extern kern_return_t thread_terminate_internal( thread_t thread); extern void thread_start( thread_t thread) __attribute__ ((noinline)); extern void thread_start_in_assert_wait( thread_t thread, struct waitq *waitq, event64_t event, wait_interrupt_t interruptible) __attribute__ ((noinline)); extern void thread_terminate_enqueue( thread_t thread); extern void thread_exception_enqueue( task_t task, thread_t thread, exception_type_t etype); extern void thread_backtrace_enqueue( kcdata_object_t obj, exception_port_t ports[static BT_EXC_PORTS_COUNT], exception_type_t etype); extern void thread_copy_resource_info( thread_t dst_thread, thread_t src_thread); extern void thread_terminate_crashed_threads(void); extern void thread_stack_enqueue( thread_t thread); extern void thread_hold( thread_t thread); extern void thread_release( thread_t thread); extern void thread_corpse_continue(void) __dead2; extern boolean_t thread_is_active(thread_t thread); extern lck_grp_t thread_lck_grp; /* Locking for scheduler state, always acquired with interrupts disabled (splsched()) */ #define thread_lock_init(th) simple_lock_init(&(th)->sched_lock, 0) #define thread_lock(th) simple_lock(&(th)->sched_lock, &thread_lck_grp) #define thread_unlock(th) simple_unlock(&(th)->sched_lock) #define thread_lock_assert(th, x) simple_lock_assert(&(th)->sched_lock, (x)) #define wake_lock_init(th) simple_lock_init(&(th)->wake_lock, 0) #define wake_lock(th) simple_lock(&(th)->wake_lock, &thread_lck_grp) #define wake_unlock(th) simple_unlock(&(th)->wake_lock) #define thread_should_halt_fast(thread) (!(thread)->active) extern void stack_alloc( thread_t thread); extern void stack_handoff( thread_t from, thread_t to); extern void stack_free( thread_t thread); extern void stack_free_reserved( thread_t thread); extern boolean_t stack_alloc_try( thread_t thread); extern void stack_collect(void); extern kern_return_t thread_info_internal( thread_t thread, thread_flavor_t flavor, thread_info_t thread_info_out, mach_msg_type_number_t *thread_info_count); extern kern_return_t kernel_thread_create( thread_continue_t continuation, void *parameter, integer_t priority, thread_t *new_thread); extern kern_return_t kernel_thread_start_priority( thread_continue_t continuation, void *parameter, integer_t priority, thread_t *new_thread); extern void machine_stack_attach( thread_t thread, vm_offset_t stack); extern vm_offset_t machine_stack_detach( thread_t thread); extern void machine_stack_handoff( thread_t old, thread_t new); extern thread_t machine_switch_context( thread_t old_thread, thread_continue_t continuation, thread_t new_thread); extern void machine_load_context( thread_t thread) __attribute__((noreturn)); extern void machine_thread_state_initialize( thread_t thread); extern kern_return_t machine_thread_set_state( thread_t thread, thread_flavor_t flavor, thread_state_t state, mach_msg_type_number_t count); extern mach_vm_address_t machine_thread_pc( thread_t thread); extern void machine_thread_reset_pc( thread_t thread, mach_vm_address_t pc); extern boolean_t machine_thread_on_core( thread_t thread); extern boolean_t machine_thread_on_core_allow_invalid( thread_t thread); extern kern_return_t machine_thread_get_state( thread_t thread, thread_flavor_t flavor, thread_state_t state, mach_msg_type_number_t *count); extern kern_return_t machine_thread_state_convert_from_user( thread_t thread, thread_flavor_t flavor, thread_state_t tstate, mach_msg_type_number_t count, thread_state_t old_tstate, mach_msg_type_number_t old_count, thread_set_status_flags_t tssf_flags); extern kern_return_t machine_thread_state_convert_to_user( thread_t thread, thread_flavor_t flavor, thread_state_t tstate, mach_msg_type_number_t *count, thread_set_status_flags_t tssf_flags); extern kern_return_t machine_thread_dup( thread_t self, thread_t target, boolean_t is_corpse); extern void machine_thread_init(void); extern void machine_thread_template_init(thread_t thr_template); #if __has_feature(ptrauth_calls) extern bool machine_thread_state_is_debug_flavor(int flavor); #endif /* __has_feature(ptrauth_calls) */ extern void machine_thread_create( thread_t thread, task_t task, bool first_thread); extern kern_return_t machine_thread_process_signature( thread_t thread, task_t task); extern void machine_thread_switch_addrmode( thread_t thread); extern void machine_thread_destroy( thread_t thread); extern void machine_set_current_thread( thread_t thread); extern kern_return_t machine_thread_get_kern_state( thread_t thread, thread_flavor_t flavor, thread_state_t tstate, mach_msg_type_number_t *count); extern kern_return_t machine_thread_inherit_taskwide( thread_t thread, task_t parent_task); extern kern_return_t machine_thread_set_tsd_base( thread_t thread, mach_vm_offset_t tsd_base); #define thread_mtx_try(thread) lck_mtx_try_lock(&(thread)->mutex) #define thread_mtx_held(thread) LCK_MTX_ASSERT_DEBUG(&(thread)->mutex, LCK_MTX_ASSERT_OWNED) extern void thread_apc_ast(thread_t thread); extern void thread_update_qos_cpu_time(thread_t thread); void act_machine_sv_free(thread_t, int); vm_offset_t min_valid_stack_address(void); vm_offset_t max_valid_stack_address(void); #if CONFIG_SCHED_SMT extern bool thread_no_smt(thread_t thread); extern bool processor_active_thread_no_smt(processor_t processor); #endif /* CONFIG_SCHED_SMT */ extern void thread_set_options(uint32_t thopt); #if CONFIG_THREAD_GROUPS struct thread_group *thread_get_current_voucher_thread_group(thread_t thread); #endif /* CONFIG_THREAD_GROUPS */ #if CONFIG_COALITIONS uint64_t thread_get_current_voucher_resource_coalition_id(thread_t thread); #endif /* CONFIG_COALITIONS */ #endif /* MACH_KERNEL_PRIVATE */ #if BSD_KERNEL_PRIVATE /* Duplicated from osfmk/kern/ipc_tt.h */ __options_decl(port_intrans_options_t, uint32_t, { PORT_INTRANS_OPTIONS_NONE = 0x0000, PORT_INTRANS_THREAD_IN_CURRENT_TASK = 0x0001, PORT_INTRANS_THREAD_NOT_CURRENT_THREAD = 0x0002, PORT_INTRANS_SKIP_TASK_EVAL = 0x0004, PORT_INTRANS_ALLOW_CORPSE_TASK = 0x0008, }); extern thread_t port_name_to_thread( mach_port_name_t port_name, port_intrans_options_t options); #endif /* BSD_KERNEL_PRIVATE */ #ifdef XNU_KERNEL_PRIVATE extern void thread_require( thread_t thread); extern void thread_deallocate_safe( thread_t thread); extern uint64_t thread_rettokern_addr( thread_t thread); extern uint64_t thread_wqquantum_addr( thread_t thread); extern integer_t thread_kern_get_pri(thread_t thr) __pure2; extern void thread_kern_set_pri(thread_t thr, integer_t pri); extern integer_t thread_kern_get_kernel_maxpri(void) __pure2; uint16_t thread_set_tag(thread_t thread, uint16_t tag); uint16_t thread_get_tag(thread_t thread); __options_decl(shared_rsrc_policy_agent_t, uint32_t, { SHARED_RSRC_POLICY_AGENT_DISPATCH = 0, SHARED_RSRC_POLICY_AGENT_SYSCTL = 1, SHARED_RSRC_POLICY_AGENT_PERFCTL_CSW = 2, SHARED_RSRC_POLICY_AGENT_PERFCTL_QUANTUM = 3, }); boolean_t thread_shared_rsrc_policy_get(thread_t thread, cluster_shared_rsrc_type_t type); kern_return_t thread_shared_rsrc_policy_set(thread_t thread, uint32_t index, cluster_shared_rsrc_type_t type, shared_rsrc_policy_agent_t agent); kern_return_t thread_shared_rsrc_policy_clear(thread_t thread, cluster_shared_rsrc_type_t type, shared_rsrc_policy_agent_t agent); #ifdef MACH_KERNEL_PRIVATE static inline thread_tag_t thread_set_tag_internal(thread_t thread, thread_tag_t tag) { return os_atomic_or_orig(&thread->thread_tag, tag, relaxed); } static inline thread_tag_t thread_get_tag_internal(thread_t thread) { return thread->thread_tag; } #endif /* MACH_KERNEL_PRIVATE */ extern uint64_t thread_last_run_time( thread_t thread); extern kern_return_t thread_state_initialize( thread_t thread); extern kern_return_t thread_setstatus( thread_t thread, int flavor, thread_state_t tstate, mach_msg_type_number_t count); extern kern_return_t thread_setstatus_from_user( thread_t thread, int flavor, thread_state_t tstate, mach_msg_type_number_t count, thread_state_t old_tstate, mach_msg_type_number_t old_count, thread_set_status_flags_t flags, audit_token_t *audit); extern kern_return_t thread_getstatus( thread_t thread, int flavor, thread_state_t tstate, mach_msg_type_number_t *count); extern kern_return_t thread_getstatus_to_user( thread_t thread, int flavor, thread_state_t tstate, mach_msg_type_number_t *count, thread_set_status_flags_t flags); extern kern_return_t thread_create_with_continuation( task_t task, thread_t *new_thread, thread_continue_t continuation); extern kern_return_t main_thread_create_waiting( task_t task, thread_continue_t continuation, event_t event, thread_t *new_thread); extern kern_return_t thread_create_workq_waiting( task_t task, thread_continue_t thread_return, thread_t *new_thread, bool is_permanently_bound); extern kern_return_t thread_create_aio_workq_waiting( task_t task, thread_continue_t thread_return, thread_t *new_thread); extern void thread_yield_internal( mach_msg_timeout_t interval); extern void thread_yield_to_preemption(void); extern void thread_depress_timer_setup( thread_t self); /* * Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are: * * 1) Block. Prevent CPU consumption of the thread from exceeding the limit. * 2) Exception. Generate a resource consumption exception when the limit is exceeded. * 3) Disable. Remove any existing CPU limit. */ #define THREAD_CPULIMIT_BLOCK 0x1 #define THREAD_CPULIMIT_EXCEPTION 0x2 #define THREAD_CPULIMIT_DISABLE 0x3 struct _thread_ledger_indices { ledger_entry_id_t cpu_time; }; extern struct _thread_ledger_indices thread_ledgers; extern int thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns); extern int thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns); extern uint64_t thread_cpulimit_remaining(uint64_t now); extern bool thread_cpulimit_interval_has_expired(uint64_t now); extern void thread_cpulimit_restart(uint64_t now); extern void thread_read_times( thread_t thread, time_value_t *user_time, time_value_t *system_time, time_value_t *runnable_time); extern void thread_read_times_unsafe( thread_t thread, time_value_t *user_time, time_value_t *system_time, time_value_t *runnable_time); extern uint64_t thread_get_runtime_self(void); extern void thread_setuserstack( thread_t thread, mach_vm_offset_t user_stack); extern user_addr_t thread_adjuserstack( thread_t thread, int adjust); extern void thread_setentrypoint( thread_t thread, mach_vm_offset_t entry); extern kern_return_t thread_set_tsd_base( thread_t thread, mach_vm_offset_t tsd_base); extern kern_return_t thread_setsinglestep( thread_t thread, int on); extern kern_return_t thread_userstack( thread_t, int, thread_state_t, unsigned int, mach_vm_offset_t *, int *, boolean_t); extern kern_return_t thread_entrypoint( thread_t, int, thread_state_t, unsigned int, mach_vm_offset_t *); extern kern_return_t thread_userstackdefault( mach_vm_offset_t *, boolean_t); extern kern_return_t thread_wire_internal( host_priv_t host_priv, thread_t thread, boolean_t wired, boolean_t *prev_state); extern kern_return_t thread_dup(thread_t); extern kern_return_t thread_dup2(thread_t, thread_t); #if !defined(_SCHED_CALL_T_DEFINED) #define _SCHED_CALL_T_DEFINED typedef void (*sched_call_t)( int type, thread_t thread); #endif #define SCHED_CALL_BLOCK 0x1 #define SCHED_CALL_UNBLOCK 0x2 extern void thread_sched_call( thread_t thread, sched_call_t call); extern boolean_t thread_is_static_param( thread_t thread); extern task_t get_threadtask(thread_t) __pure2; extern task_t get_threadtask_early(thread_t) __pure2; /* * Thread is running within a 64-bit address space. */ #define thread_is_64bit_addr(thd) \ task_has_64Bit_addr(get_threadtask(thd)) /* * Thread is using 64-bit machine state. */ #define thread_is_64bit_data(thd) \ task_has_64Bit_data(get_threadtask(thd)) struct uthread; #if defined(__x86_64__) extern int thread_task_has_ldt(thread_t); #endif extern void set_thread_pagein_error(thread_t, int); extern event_t workq_thread_init_and_wq_lock(task_t, thread_t); // bsd/pthread/ extern event_t aio_workq_thread_init_and_wq_lock(task_t, thread_t); // bsd/aio/ struct proc; struct uthread; struct image_params; extern const size_t uthread_size; extern thread_ro_t get_thread_ro_unchecked(thread_t) __pure2; extern thread_ro_t get_thread_ro(thread_t) __pure2; extern thread_ro_t current_thread_ro_unchecked(void) __pure2; extern thread_ro_t current_thread_ro(void) __pure2; extern void clear_thread_ro_proc(thread_t); extern struct uthread *get_bsdthread_info(thread_t) __pure2; extern thread_t get_machthread(struct uthread *) __pure2; extern uint64_t uthread_tid(struct uthread *) __pure2; extern user_addr_t thread_get_sigreturn_token(thread_t thread); extern uint32_t thread_get_sigreturn_diversifier(thread_t thread); extern void uthread_init(task_t, struct uthread *, thread_ro_t, int); extern void uthread_cleanup_name(struct uthread *uthread); extern void uthread_cleanup(struct uthread *, thread_ro_t); extern void uthread_cred_ref(struct ucred *); extern void uthread_cred_free(struct ucred *); extern void uthread_destroy(struct uthread *); extern void uthread_reset_proc_refcount(struct uthread *); extern void uthread_set_exec_data(struct uthread *uth, struct image_params *imgp); extern bool uthread_is64bit(struct uthread *uth) __pure2; #if PROC_REF_DEBUG extern void uthread_init_proc_refcount(struct uthread *); extern void uthread_destroy_proc_refcount(struct uthread *); extern void uthread_assert_zero_proc_refcount(struct uthread *); #else #define uthread_init_proc_refcount(uth) ((void)(uth)) #define uthread_destroy_proc_refcount(uth) ((void)(uth)) #define uthread_assert_zero_proc_refcount(uth) ((void)(uth)) #endif #if CONFIG_DEBUG_SYSCALL_REJECTION extern uint64_t uthread_get_syscall_rejection_flags(void *); extern uint64_t *uthread_get_syscall_rejection_mask(void *); extern uint64_t *uthread_get_syscall_rejection_once_mask(void *); extern bool uthread_syscall_rejection_is_enabled(void *); #endif /* CONFIG_DEBUG_SYSCALL_REJECTION */ extern mach_port_name_t uthread_joiner_port(struct uthread *); extern user_addr_t uthread_joiner_address(struct uthread *); extern void uthread_joiner_wake(task_t task, struct uthread *); extern boolean_t thread_should_halt( thread_t thread); extern boolean_t thread_should_abort( thread_t); extern bool current_thread_in_kernel_fault(void); extern int is_64signalregset(void); extern void act_set_kperf(thread_t); extern void act_set_astledger(thread_t thread); /** * Options for how to sample for telemetry, if @code AST_TELEMETRY is set in * @code ast_bits on a {@code struct thread}. */ __options_decl(telemetry_ast_t, uint16_t, { /** * Telemetry triggered by a Performance Monitor Interrupt (PMI), like from * the CPU cycle counter. Only available if @code CONFIG_CPU_COUNTERS * is configured. */ TELEMETRY_AST_PMI = 0x0001, /** * Telemetry triggered by logical disk writes. */ TELEMETRY_AST_IO = 0x0002, /** * Telemetry triggered by a hook invoked by the Mandatory Access Control * Framework (MACF). Only available if @code CONFIG_MACF is configured. */ TELEMETRY_AST_MACF = 0x0004, /** * Telemetry triggered by a VM fault. Only available if * @code CONFIG_MEMORY_MICROSTACKSHOTS is configured. * * Must use @code asct_set_telemetry_ast_vm_fault to set. */ TELEMETRY_AST_VM_FAULT = 0x0008, /** * Telemetry triggered by a page grab. Only available if * @code CONFIG_MEMORY_MICROSTACKSHOTS is configured. * * Must use @code asct_set_telemetry_ast_page_grab to set. */ TELEMETRY_AST_PAGE_GRAB = 0x0010, /* * Add additional triggers above this point. */ /** * Telemetry was triggered while the thread was running in user space. */ TELEMETRY_AST_USER = 0x1000, /** * Telemetry was triggered while the thread was running in the kernel. */ TELEMETRY_AST_KERNEL = 0x2000, }); extern void act_set_telemetry_ast(thread_t, telemetry_ast_t); extern telemetry_ast_t act_clear_telemetry_ast(thread_t thread); extern void act_set_telemetry_ast_vm_fault(thread_t, uint64_t, int, uint16_t); extern void act_set_telemetry_ast_page_grab(thread_t, bool, uint16_t); extern void act_set_astproc_resource(thread_t); extern vm_offset_t thread_get_kernel_stack(thread_t); extern kern_return_t thread_process_signature(thread_t thread, task_t task); extern uint32_t dtrace_get_thread_predcache(thread_t); extern int64_t dtrace_get_thread_vtime(thread_t); extern int64_t dtrace_get_thread_tracing(thread_t); extern uint16_t dtrace_get_thread_inprobe(thread_t); extern int dtrace_get_thread_last_cpu_id(thread_t); extern vm_offset_t dtrace_get_kernel_stack(thread_t); #define dtrace_get_kernel_stack thread_get_kernel_stack extern void dtrace_set_thread_predcache(thread_t, uint32_t); extern void dtrace_set_thread_vtime(thread_t, int64_t); extern void dtrace_set_thread_tracing(thread_t, int64_t); extern void dtrace_set_thread_inprobe(thread_t, uint16_t); extern void dtrace_thread_bootstrap(void); extern void dtrace_thread_didexec(thread_t); extern int64_t dtrace_calc_thread_recent_vtime(thread_t); extern kern_return_t thread_set_wq_state32( thread_t thread, thread_state_t tstate); extern kern_return_t thread_set_wq_state64( thread_t thread, thread_state_t tstate); extern vm_offset_t kernel_stack_mask; extern vm_offset_t kernel_stack_size; extern vm_offset_t kernel_stack_depth_max; extern void mach_exception_ast(thread_t); extern void fd_guard_ast(thread_t, mach_exception_code_t, mach_exception_subcode_t); extern void vn_guard_ast(thread_t, mach_exception_code_t, mach_exception_subcode_t); extern void mach_port_guard_ast(thread_t, mach_exception_code_t, mach_exception_subcode_t); extern void virt_memory_guard_ast(thread_t, mach_exception_code_t, mach_exception_subcode_t); #if CONFIG_FREEZE extern void frozen_swapin_guard_ast(thread_t, mach_exception_code_t, mach_exception_subcode_t); #endif /* CONFIG_FREEZE */ extern void thread_ast_mach_exception(thread_t, int, exception_type_t, mach_exception_code_t, mach_exception_subcode_t, bool, bool); extern void thread_guard_violation(thread_t, mach_exception_code_t, mach_exception_subcode_t, bool); extern void thread_update_io_stats(thread_t, int size, int io_flags); extern kern_return_t thread_set_voucher_name(mach_port_name_t name); extern kern_return_t thread_get_voucher_origin_pid(thread_t thread, int32_t *pid); extern kern_return_t thread_get_voucher_origin_proximate_pid(thread_t thread, int32_t *origin_pid, int32_t *proximate_pid); extern kern_return_t thread_get_current_voucher_origin_pid(int32_t *pid); extern void thread_enable_send_importance(thread_t thread, boolean_t enable); /* * Translate signal context data pointer to userspace representation */ extern kern_return_t machine_thread_siguctx_pointer_convert_to_user( thread_t thread, user_addr_t *uctxp); extern void machine_tecs(thread_t thr); typedef enum cpuvn { CPUVN_CI = 1 } cpuvn_e; extern int machine_csv(cpuvn_e cve); #if defined(__x86_64__) extern void machine_thread_set_insn_copy_optout(thread_t thr); #endif /* * Translate array of function pointer syscall arguments from userspace representation */ extern kern_return_t machine_thread_function_pointers_convert_from_user( thread_t thread, user_addr_t *fptrs, uint32_t count); /* * Get the duration of the given thread's last wait. */ uint64_t thread_get_last_wait_duration(thread_t thread); #if CONFIG_SCHED_SMT extern bool thread_get_no_smt(void); #endif /* CONFIG_SCHED_SMT */ #if defined(__x86_64__) extern bool curtask_get_insn_copy_optout(void); extern void curtask_set_insn_copy_optout(void); #endif /* defined(__x86_64__) */ /*! @function ctid_get_thread * @abstract translates a ctid_t to thread_t * @discussion ctid are system wide compact thread-id * associated to thread_t at thread creation * and recycled at thread termination. If a ctid is * referenced past the corresponding thread termination, * it is considered stale, and the behavior is not defined. * Note that this call does not acquire a reference on the thread, * so as soon as the matching thread terminates, the ctid * will become stale, and it could be re-used and associated with * another thread. You must externally guarantee that the thread * will not exit while you are using its ctid. * @result thread_t corresponding to ctid */ extern thread_t ctid_get_thread(ctid_t ctid); /*! @function ctid_get_thread * @abstract translates a ctid_t to thread_t * @discussion Unsafe variant of ctid_get_thread() to be used * when the caller can't guarantee the liveness of this ctid_t. * may return NULL or a freed thread_t. */ extern thread_t ctid_get_thread_unsafe(ctid_t ctid); /*! * @function thread_get_ctid * @abstract returns the ctid of thread. * @param thread to find the corresponding ctid. * @discussion the ctid provided will become stale after the matching thread * terminates. * @result uint32_t ctid. */ extern ctid_t thread_get_ctid(thread_t thread); #endif /* XNU_KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE typedef struct thread_pri_floor { thread_t thread; } thread_pri_floor_t; /* Accessor functions for thread block hint information */ extern uint32_t thread_get_block_hint(thread_t thread); #ifdef MACH_KERNEL_PRIVATE extern void thread_floor_boost_ast(thread_t thread); extern void thread_floor_boost_set_promotion_locked(thread_t thread); #endif /* MACH_KERNEL_PRIVATE */ /*! @function thread_priority_floor_start * @abstract boost the current thread priority to floor. * @discussion Increase the priority of the current thread to at least MINPRI_FLOOR. * The boost will be mantained until a corresponding thread_priority_floor_end() * is called. Every call of thread_priority_floor_start() needs to have a corresponding * call to thread_priority_floor_end() from the same thread. * No thread can return to userspace before calling thread_priority_floor_end(). * * NOTE: avoid to use this function. Try to use gate_t or sleep_with_inheritor() * instead. * @result a token to be given to the corresponding thread_priority_floor_end() */ extern thread_pri_floor_t thread_priority_floor_start(void); /*! @function thread_priority_floor_end * @abstract ends the floor boost. * @param token the token obtained from thread_priority_floor_start() * @discussion ends the priority floor boost started with thread_priority_floor_start() */ extern void thread_priority_floor_end(thread_pri_floor_t *token); #if defined(__x86_64__) extern void thread_set_no_smt(bool set); #endif /* __x86_64__ */ extern void thread_mtx_lock(thread_t thread); extern void thread_mtx_unlock(thread_t thread); extern uint64_t thread_dispatchqaddr( thread_t thread); bool thread_is_eager_preempt(thread_t thread); void thread_set_eager_preempt(thread_t thread); void thread_clear_eager_preempt(thread_t thread); void thread_set_honor_qlimit(thread_t thread); void thread_clear_honor_qlimit(thread_t thread); extern ipc_port_t convert_thread_to_port(thread_t); extern ipc_port_t convert_thread_to_port_immovable(thread_t); extern ipc_port_t convert_thread_inspect_to_port(thread_inspect_t); extern ipc_port_t convert_thread_read_to_port(thread_read_t); extern void convert_thread_array_to_ports(thread_act_array_t, size_t, mach_thread_flavor_t); extern boolean_t is_external_pageout_thread(void); extern boolean_t is_vm_privileged(void); extern boolean_t set_vm_privilege(boolean_t); extern kern_allocation_name_t thread_set_allocation_name(kern_allocation_name_t new_name); extern void *thread_iokit_tls_get(uint32_t index); extern void thread_iokit_tls_set(uint32_t index, void * data); extern int thread_self_region_page_shift(void); extern void thread_self_region_page_shift_set(int pgshift); extern kern_return_t thread_create_immovable(task_t task, thread_t *new_thread); extern kern_return_t thread_terminate_immovable(thread_t thread); struct thread_attr_for_ipc_propagation; extern kern_return_t thread_get_ipc_propagate_attr(thread_t thread, struct thread_attr_for_ipc_propagation *attr); extern size_t thread_get_current_exec_path(char *path, size_t size); #endif /* KERNEL_PRIVATE */ #ifdef XNU_KERNEL_PRIVATE extern void thread_get_thread_name(thread_t th, char* name); /* Read the runq assignment, under the thread lock. */ extern processor_t thread_get_runq(thread_t thread); /* * Read the runq assignment, under both the thread lock and * the pset lock corresponding to the last non-null assignment. */ extern processor_t thread_get_runq_locked(thread_t thread); /* * Set the runq assignment to a non-null value, under both the * thread lock and the pset lock corresponding to the new * assignment. */ extern void thread_set_runq_locked(thread_t thread, processor_t new_runq); /* * Set the runq assignment to PROCESSOR_NULL, under the pset * lock corresponding to the current non-null assignment. */ extern void thread_clear_runq(thread_t thread); /* * Set the runq assignment to PROCESSOR_NULL, under both the * thread lock and the pset lock corresponding to the current * non-null assignment. */ extern void thread_clear_runq_locked(thread_t thread); /* * Assert the runq assignment to be PROCESSOR_NULL, under * some guarantee that the runq will not change from null to * non-null, such as holding the thread lock. */ extern void thread_assert_runq_null(thread_t thread); /* * Assert the runq assignment to be non-null, under the pset * lock corresponding to the current non-null assignment. */ extern void thread_assert_runq_nonnull(thread_t thread); extern bool thread_supports_cooperative_workqueue(thread_t thread); extern void thread_arm_workqueue_quantum(thread_t thread); extern void thread_disarm_workqueue_quantum(thread_t thread); extern void thread_evaluate_workqueue_quantum_expiry(thread_t thread); extern bool thread_has_expired_workqueue_quantum(thread_t thread, bool should_trace); #if CONFIG_SPTM extern void thread_associate_txm_thread_stack(uintptr_t thread_stack); extern void thread_disassociate_txm_thread_stack(uintptr_t thread_stack); extern uintptr_t thread_get_txm_thread_stack(void); #endif /* CONFIG_SPTM */ /* Kernel side prototypes for MIG routines */ extern kern_return_t thread_get_exception_ports( thread_t thread, exception_mask_t exception_mask, exception_mask_array_t masks, mach_msg_type_number_t *CountCnt, exception_port_array_t ports, exception_behavior_array_t behaviors, thread_state_flavor_array_t flavors); extern kern_return_t thread_get_special_port( thread_inspect_t thread, int which, ipc_port_t *portp); extern uint64_t thread_c_switch(thread_t thread); #endif /* XNU_KERNEL_PRIVATE */ /*! @function thread_has_thread_name * @abstract Checks if a thread has a name. * @discussion This function takes one input, a thread, and returns * a boolean value indicating if that thread already has a name associated * with it. * @param th The thread to inspect. * @result TRUE if the thread has a name, FALSE otherwise. */ extern boolean_t thread_has_thread_name(thread_t th); /*! @function thread_set_thread_name * @abstract Set a thread's name. * @discussion This function takes two input parameters: a thread to name, * and the name to apply to the thread. The name will be copied over to * the thread in order to better identify the thread. If the name is * longer than MAXTHREADNAMESIZE - 1, it will be truncated. * @param th The thread to be named. * @param name The name to apply to the thread. */ extern void thread_set_thread_name(thread_t th, const char* name); #if !MACH_KERNEL_PRIVATE || !defined(current_thread) extern thread_t current_thread(void) __pure2; #endif #if HAS_MTE /*! @function current_thread_enter_iomd_faultable_access_with_buffer_provider * @abstract Store context for a critical faultable region while accessing tagged memory. * @discussion We have paths in which the kernel is holding a tagged mapping which * has a true share with userspace. Therefore, userspace can induce the kernel * to take a TCF by changing the tag after handing the memory to the kernel. * To deal with this, we enter a critical region while accessing this sort of * memory, during which we'll recognize the fault as being 'caused by' the * userspace task. * @param provider The task to whom tag mismatches should be attributed. */ void current_thread_enter_iomd_faultable_access_with_buffer_provider(task_t provider); /*! @function current_thread_exit_iomd_faultable_access * @abstract Exit a critical region of accessing uncontrolled tagged memory. * @discussion See current_thread_enter_iomd_faultable_access_with_buffer_provider */ void current_thread_exit_iomd_faultable_access(void); /*! @function current_thread_get_iomd_faultable_access_buffer_provider * @abstract Retrieve the task pointer storing the current faultable buffer provider. * @discussion See current_thread_enter_iomd_faultable_access_with_buffer_provider */ task_t current_thread_get_iomd_faultable_access_buffer_provider(void); #endif /* HAS_MTE */ extern uint64_t thread_tid(thread_t thread) __pure2; extern void thread_reference( thread_t thread); extern void thread_deallocate( thread_t thread); #ifdef XNU_KERNEL_PRIVATE extern kern_return_t thread_suspend_internal( thread_t thread, thread_suspend_mode mode); extern kern_return_t thread_resume_internal( thread_t thread, thread_suspend_mode mode); #endif /* XNU_KERNEL_PRIVATE */ /*! @function kernel_thread_start * @abstract Create a kernel thread. * @discussion This function takes three input parameters, namely reference * to the function that the thread should execute, caller specified data * and a reference which is used to return the newly created kernel * thread. The function returns KERN_SUCCESS on success or an appropriate * kernel code type indicating the error. It may be noted that the caller * is responsible for explicitly releasing the reference to the created * thread when no longer needed. This should be done by calling * thread_deallocate(new_thread). * @param continuation A C-function pointer where the thread will begin execution. * @param parameter Caller specified data to be passed to the new thread. * @param new_thread Reference to the new thread is returned in this parameter. * @result Returns KERN_SUCCESS on success or an appropriate kernel code type. */ extern kern_return_t kernel_thread_start( thread_continue_t continuation, void *parameter, thread_t *new_thread); __END_DECLS #endif /* _KERN_THREAD_H_ */ |