Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 | /* * Copyright (c) 1999-2024 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */ /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ /* * Mach Operating System * Copyright (c) 1987 Carnegie-Mellon University * All rights reserved. The CMU software License Agreement specifies * the terms and conditions for use and redistribution. */ /* * Copyright (c) 1994 NeXT Computer, Inc. All rights reserved. * * Copyright (c) 1982, 1986, 1988 Regents of the University of California. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)mbuf.h 8.3 (Berkeley) 1/21/94 */ /* * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce * support for mandatory and extensible security protections. This notice * is included in support of clause 2.2 (b) of the Apple Public License, * Version 2.0. */ #ifndef _SYS_MBUF_H_ #define _SYS_MBUF_H_ #include <sys/appleapiopts.h> #include <sys/cdefs.h> #include <sys/_types/_u_int32_t.h> /* u_int32_t */ #include <sys/_types/_u_int64_t.h> /* u_int64_t */ #include <sys/_types/_u_short.h> /* u_short */ #ifdef KERNEL #include <sys/kpi_mbuf.h> #endif #ifdef XNU_KERNEL_PRIVATE #include <sys/lock.h> #include <sys/queue.h> #include <machine/endian.h> /* * Mbufs are of a single size, which includes overhead. * An mbuf may add a single "mbuf cluster" of size * MCLBYTES/MBIGCLBYTES/M16KCLBYTES (also in machine/param.h), which has * no additional overhead and is used instead of the internal data area; * this is done when at least MINCLSIZE of data must be stored. */ #if CONFIG_MBUF_MCACHE #include <sys/mcache.h> #define _MSIZESHIFT 8 /* 256 */ #define _MSIZE (1 << _MSIZESHIFT) /* size of an mbuf */ #else /* CONFIG_MBUF_MCACHE */ #define _MSIZE 512 #endif /* CONFIG_MBUF_MCACHE */ #define NCLPGSHIFT (PAGE_SHIFT - MCLSHIFT) #define NCLPG (1 << NCLPGSHIFT) /* # of cl per page */ #define NBCLPGSHIFT (PAGE_SHIFT - MBIGCLSHIFT) #define NBCLPG (1 << NBCLPGSHIFT) /* # of big cl per page */ #define NMBPCL (MCLBYTES / _MSIZE) #define NCLPJCLSHIFT (M16KCLSHIFT - MCLSHIFT) #define NCLPJCL (1 << NCLPJCLSHIFT) /* # of cl per jumbo cl */ #define NCLPBGSHIFT (MBIGCLSHIFT - MCLSHIFT) #define NCLPBG (1 << NCLPBGSHIFT) /* # of cl per big cl */ /* * Macros for type conversion * mtod(m,t) - convert mbuf pointer to data pointer of correct type * mtodo(m, o) -- Same as above but with offset 'o' into data. */ #define mtod(m, t) ((t)(void *)m_mtod_current(m)) #define mtodo(m, o) ((void *)(mtod(m, uint8_t *) + (o))) /* header at beginning of each mbuf: */ struct m_hdr { struct mbuf *mh_next; /* next buffer in chain */ struct mbuf *mh_nextpkt; /* next chain in queue/record */ uintptr_t mh_data; /* location of data */ int32_t mh_len; /* amount of data in this mbuf */ u_int16_t mh_type; /* type of data in this mbuf */ u_int16_t mh_flags; /* flags; see below */ #if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) /* This is needed because of how _MLEN is defined and used. Ideally, _MLEN * should be defined using the offsetof(struct mbuf, M_dat), since there is * no guarantee that mbuf.M_dat will start where mbuf.m_hdr ends. The compiler * may (and does in the armv7k case) insert padding between m_hdr and M_dat in * mbuf. We cannot easily use offsetof, however, since _MLEN is referenced * in the definition of mbuf. */ } __attribute__((aligned(8))); #else }; #endif /* * Packet tag structure (see below for details). */ struct m_tag { uint64_t m_tag_cookie; /* Error checking */ SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */ void *__sized_by(m_tag_len) m_tag_data; uint16_t m_tag_type; /* Module specific type */ uint16_t m_tag_len; /* Length of data */ uint32_t m_tag_id; /* Module ID */ void *m_tag_mb_cl; /* pointer to mbuf or cluster container */ #ifndef __LP64__ u_int32_t m_tag_pad; #endif /* !__LP64__ */ }; #define M_TAG_ALIGN(len) \ (P2ROUNDUP(len, sizeof (u_int64_t)) + sizeof (struct m_tag)) #define M_TAG_INIT(tag, id, type, len, data, mb_cl) { \ VERIFY(IS_P2ALIGNED((tag), sizeof(u_int64_t))); \ (tag)->m_tag_type = (type); \ (tag)->m_tag_len = (uint16_t)(len); \ (tag)->m_tag_id = (id); \ (tag)->m_tag_data = (data); \ (tag)->m_tag_mb_cl = (mb_cl); \ m_tag_create_cookie(tag); \ } #define M_TAG_VALID_PATTERN 0xfeedfacefeedfaceULL #define M_TAG_FREE_PATTERN 0xdeadbeefdeadbeefULL /* * Packet tag header structure at the top of mbuf whe mbufs are use for m_tag * Pointers are 32-bit in ILP32; m_tag needs 64-bit alignment, hence padded. */ struct m_taghdr { #ifndef __LP64__ u_int32_t pad; /* For structure alignment */ #endif /* !__LP64__ */ u_int64_t mth_refcnt; /* Number of tags in this mbuf */ }; /* * Driver auxiliary metadata tag (KERNEL_TAG_TYPE_DRVAUX). */ struct m_drvaux_tag { u_int32_t da_family; /* IFNET_FAMILY values */ u_int32_t da_subfamily; /* IFNET_SUBFAMILY values */ u_int32_t da_reserved; /* for future */ u_int32_t da_length; /* length of following data */ }; /* Values for pftag_flags (16-bit wide) */ #define PF_TAG_GENERATED 0x1 /* pkt generated by PF */ #define PF_TAG_FRAGCACHE 0x2 #define PF_TAG_TRANSLATE_LOCALHOST 0x4 #if PF_ECN #define PF_TAG_HDR_INET 0x8 /* hdr points to IPv4 */ #define PF_TAG_HDR_INET6 0x10 /* hdr points to IPv6 */ #endif /* PF_ECN */ #define PF_TAG_REASSEMBLED 0x20 /* pkt reassembled by PF */ #define PF_TAG_REFRAGMENTED 0x40 /* pkt refragmented by PF */ /* * PF mbuf tag */ struct pf_mtag { u_int16_t pftag_flags; /* PF_TAG flags */ u_int16_t pftag_rtableid; /* alternate routing table id */ u_int16_t pftag_tag; u_int16_t pftag_routed; #if PF_ECN void *pftag_hdr; /* saved hdr pos in mbuf, for ECN */ #endif /* PF_ECN */ }; /* System reserved PF tags */ #define PF_TAG_ID_SYSTEM_SERVICE 0xff00 #define PF_TAG_ID_STACK_DROP 0xff01 /* * PF fragment tag */ struct pf_fragment_tag { uint32_t ft_id; /* fragment id */ uint16_t ft_hdrlen; /* header length of reassembled pkt */ uint16_t ft_unfragpartlen; /* length of the per-fragment headers */ uint16_t ft_extoff; /* last extension header offset or 0 */ uint16_t ft_maxlen; /* maximum fragment payload length */ }; /* * TCP mbuf tag */ struct tcp_pktinfo { union { struct { uint16_t seg_size; /* segment size (actual MSS) */ uint16_t hdr_len; /* size of IP+TCP header, might be zero */ uint32_t start_seq; /* start seq of this packet */ pid_t pid; pid_t e_pid; } __tx; struct { uint8_t seg_cnt; /* # of coalesced TCP pkts */ } __rx; } __offload; #define tx_seg_size proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.seg_size #define tso_segsz tx_seg_size #define tx_hdr_len proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.hdr_len #define tx_start_seq proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.start_seq #define tx_tcp_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.pid #define tx_tcp_e_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.e_pid #define rx_seg_cnt proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.seg_cnt }; /* * MPTCP mbuf tag */ struct mptcp_pktinfo { uint64_t mtpi_dsn; /* MPTCP Data Sequence Number */ uint32_t mtpi_rel_seq; /* Relative Seq Number */ uint16_t mtpi_length; /* Length of mapping */ uint16_t mtpi_csum; #define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn #define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_rel_seq #define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_length #define mp_csum proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_csum }; /* * TCP specific mbuf tag. Note that the current implementation uses * MPTCP metadata strictly between MPTCP and the TCP subflow layers, * hence tm_tcp and tm_mptcp are mutually exclusive. This also means * that TCP messages functionality is currently incompatible with MPTCP. */ struct tcp_mtag { union { struct tcp_pktinfo tm_tcp; /* TCP and below */ struct mptcp_pktinfo tm_mptcp; /* MPTCP-TCP only */ }; }; struct udp_mtag { pid_t _pid; pid_t _e_pid; #define tx_udp_pid proto_mtag.__pr_u.udp._pid #define tx_udp_e_pid proto_mtag.__pr_u.udp._e_pid }; struct rawip_mtag { pid_t _pid; pid_t _e_pid; #define tx_rawip_pid proto_mtag.__pr_u.rawip._pid #define tx_rawip_e_pid proto_mtag.__pr_u.rawip._e_pid }; struct driver_mtag_ { uintptr_t _drv_tx_compl_arg; uintptr_t _drv_tx_compl_data; kern_return_t _drv_tx_status; uint16_t _drv_flowid; #define drv_tx_compl_arg builtin_mtag._drv_mtag._drv_tx_compl_arg #define drv_tx_compl_data builtin_mtag._drv_mtag._drv_tx_compl_data #define drv_tx_status builtin_mtag._drv_mtag._drv_tx_status #define drv_flowid builtin_mtag._drv_mtag._drv_flowid }; /* * Protocol specific mbuf tag (at most one protocol metadata per mbuf). * * Care must be taken to ensure that they are mutually exclusive, e.g. * IPsec policy ID implies no TCP segment offload (which is fine given * that the former is used on the virtual ipsec interface that does * not advertise the TSO capability.) */ struct proto_mtag_ { union { struct tcp_mtag tcp; /* TCP specific */ struct udp_mtag udp; /* UDP specific */ struct rawip_mtag rawip; /* raw IPv4/IPv6 specific */ } __pr_u; }; /* * NECP specific mbuf tag. */ struct necp_mtag_ { u_int32_t necp_policy_id; u_int32_t necp_skip_policy_id; u_int32_t necp_route_rule_id; u_int16_t necp_last_interface_index; u_int16_t necp_app_id; }; union builtin_mtag { struct { struct proto_mtag_ _proto_mtag; /* built-in protocol-specific tag */ struct pf_mtag _pf_mtag; /* built-in PF tag */ struct necp_mtag_ _necp_mtag; /* built-in NECP tag */ } _net_mtag; struct driver_mtag_ _drv_mtag; #define necp_mtag builtin_mtag._net_mtag._necp_mtag #define proto_mtag builtin_mtag._net_mtag._proto_mtag #define driver_mtag builtin_mtag._drv_mtag }; /* * Record/packet header in first mbuf of chain; valid only if M_PKTHDR set. */ struct pkthdr { struct ifnet *rcvif; /* rcv interface */ /* variables for ip and tcp reassembly */ void *pkt_hdr; /* pointer to packet header */ int32_t len; /* total packet length */ /* variables for hardware checksum */ /* Note: csum_flags is used for hardware checksum and VLAN */ u_int32_t csum_flags; /* flags regarding checksum */ union { struct { u_int16_t val; /* checksum value */ u_int16_t start; /* checksum start offset */ } _csum_rx; #define csum_rx_val _csum_rx.val #define csum_rx_start _csum_rx.start struct { u_int16_t start; /* checksum start offset */ u_int16_t stuff; /* checksum stuff offset */ } _csum_tx; #define csum_tx_start _csum_tx.start #define csum_tx_stuff _csum_tx.stuff /* * Generic data field used by csum routines. * It gets used differently in different contexts. */ u_int32_t csum_data; }; u_int16_t vlan_tag; /* VLAN tag, host byte order */ /* * Packet classifier info * * PKTF_FLOW_ID set means valid flow ID. A non-zero flow ID value * means the packet has been classified by one of the flow sources. * It is also a prerequisite for flow control advisory, which is * enabled by additionally setting PKTF_FLOW_ADV. * * The protocol value is a best-effort representation of the payload. * It is opportunistically updated and used only for optimization. * It is not a substitute for parsing the protocol header(s); use it * only as a hint. * * If PKTF_IFAINFO is set, pkt_ifainfo contains one or both of the * indices of interfaces which own the source and/or destination * addresses of the packet. For the local/loopback case (PKTF_LOOP), * both should be valid, and thus allows for the receiving end to * quickly determine the actual interfaces used by the the addresses; * they may not necessarily be the same or refer to the loopback * interface. Otherwise, in the non-local/loopback case, the indices * are opportunistically set, and because of that only one may be set * (0 means the index has not been determined.) In addition, the * interface address flags are also recorded. This allows us to avoid * storing the corresponding {in,in6}_ifaddr in an mbuf tag. Ideally * this would be a superset of {ia,ia6}_flags, but the namespaces are * overlapping at present, so we'll need a new set of values in future * to achieve this. For now, we will just rely on the address family * related code paths examining this mbuf to interpret the flags. */ u_int8_t pkt_proto; /* IPPROTO value */ u_int8_t pkt_flowsrc; /* FLOWSRC values */ u_int32_t pkt_flowid; /* flow ID */ u_int32_t pkt_flags; /* PKTF flags (see below) */ u_int32_t pkt_svc; /* MBUF_SVC value */ u_int32_t pkt_compl_context; /* Packet completion context */ union { struct { u_int16_t src; /* ifindex of src addr i/f */ u_int16_t src_flags; /* src PKT_IFAIFF flags */ u_int16_t dst; /* ifindex of dst addr i/f */ u_int16_t dst_flags; /* dst PKT_IFAIFF flags */ } _pkt_iaif; #define src_ifindex _pkt_iaif.src #define src_iff _pkt_iaif.src_flags #define dst_ifindex _pkt_iaif.dst #define dst_iff _pkt_iaif.dst_flags u_int64_t pkt_ifainfo; /* data field used by ifainfo */ struct { u_int32_t if_data; /* bytes in interface queue */ u_int32_t sndbuf_data; /* bytes in socket buffer */ } _pkt_bsr; /* Buffer status report used by cellular interface */ #define bufstatus_if _pkt_bsr.if_data #define bufstatus_sndbuf _pkt_bsr.sndbuf_data }; u_int64_t pkt_timestamp; /* TX: enqueue time, RX: receive timestamp */ u_int64_t pkt_deadline; /* In Mach time. */ /* * Tags (external and built-in) */ SLIST_HEAD(packet_tags, m_tag) tags; /* list of external tags */ union builtin_mtag builtin_mtag; uint32_t comp_gencnt; uint32_t pkt_crumbs:16, pkt_compl_callbacks:8, pkt_ext_flags:6, pkt_unused:2; /* Currently unused - feel free to grab those 2 bits */ /* * Module private scratch space (32-bit aligned), currently 16-bytes * large. Anything stored here is not guaranteed to survive across * modules. The AQM layer (outbound) uses all 16-bytes for both * packet scheduling and flow advisory information. */ struct { union { u_int8_t __mpriv8[16]; u_int16_t __mpriv16[8]; struct { union { u_int8_t __val8[4]; u_int16_t __val16[2]; u_int32_t __val32; } __mpriv32_u; } __mpriv32[4]; u_int64_t __mpriv64[2]; } __mpriv_u; } pkt_mpriv __attribute__((aligned(4))); /* * While qset_id takes 64 bits here, as upper 32 bits of qset_id are reserved * currently, there is a scope to limit to 32 bits if other use cases need * pkt_mpriv */ #define pkt_mpriv_qsetid pkt_mpriv.__mpriv_u.__mpriv64[0] #define pkt_mpriv_srcid pkt_mpriv.__mpriv_u.__mpriv32[2].__mpriv32_u.__val32 #define pkt_mpriv_fidx pkt_mpriv.__mpriv_u.__mpriv32[3].__mpriv32_u.__val32 }; /* * Flow data source type. A data source module is responsible for generating * a unique flow ID and associating it to each data flow as pkt_flowid. * This is required for flow control/advisory, as it allows the output queue * to identify the data source object and inform that it can resume its * transmission (in the event it was flow controlled.) */ #define FLOWSRC_INPCB 1 /* flow ID generated by INPCB */ #define FLOWSRC_IFNET 2 /* flow ID generated by interface */ #define FLOWSRC_PF 3 /* flow ID generated by PF */ #define FLOWSRC_CHANNEL 4 /* flow ID generated by channel */ /* * FLOWSRC_MPKL_INPUT is not a true flow data source and is used for * multi-layer packet logging. We're usurping the pkt_flowsrc field because * the mbuf packet header ran out of space and pkt_flowsrc is normally * used on output so we assume we can safely overwrite the normal semantic of * the field. * This value is meant to be used on incoming packet from a lower level protocol * to pass information to some upper level protocol. When FLOWSRC_MPKL_INPUT * is set, the following fields are used: * - pkt_proto: the IP protocol ID of the lower level protocol * - pkt_flowid: the identifier of the packet at the lower protocol. * For example ESP would set pkt_proto to IPPROTO_ESP and pkt_flowid to the SPI. */ /* * Packet flags. Unlike m_flags, all packet flags are copied along when * copying m_pkthdr, i.e. no equivalent of M_COPYFLAGS here. These flags * (and other classifier info) will be cleared during DLIL input. * * Some notes about M_LOOP and PKTF_LOOP: * * - M_LOOP flag is overloaded, and its use is discouraged. Historically, * that flag was used by the KAME implementation for allowing certain * certain exceptions to be made in the IP6_EXTHDR_CHECK() logic; this * was originally meant to be set as the packet is looped back to the * system, and in some circumstances temporarily set in ip6_output(). * Over time, this flag was used by the pre-output routines to indicate * to the DLIL frameout and output routines, that the packet may be * looped back to the system under the right conditions. In addition, * this is an mbuf flag rather than an mbuf packet header flag. * * - PKTF_LOOP is an mbuf packet header flag, which is set if and only * if the packet was looped back to the system. This flag should be * used instead for newer code. */ #define PKTF_FLOW_ID 0x1 /* pkt has valid flowid value */ #define PKTF_FLOW_ADV 0x2 /* pkt triggers local flow advisory */ #define PKTF_FLOW_LOCALSRC 0x4 /* pkt is locally originated */ #define PKTF_FLOW_RAWSOCK 0x8 /* pkt locally generated by raw sock */ #define PKTF_PRIO_PRIVILEGED 0x10 /* packet priority is privileged */ #define PKTF_PROXY_DST 0x20 /* processed but not locally destined */ #define PKTF_INET_RESOLVE 0x40 /* IPv4 resolver packet */ #define PKTF_INET6_RESOLVE 0x80 /* IPv6 resolver packet */ #define PKTF_RESOLVE_RTR 0x100 /* pkt is for resolving router */ #define PKTF_SKIP_PKTAP 0x200 /* pkt has already passed through pktap */ #define PKTF_WAKE_PKT 0x400 /* packet caused system to wake from sleep */ #define PKTF_MPTCP 0x800 /* TCP with MPTCP metadata */ #define PKTF_MPSO 0x1000 /* MPTCP socket meta data */ #define PKTF_LOOP 0x2000 /* loopbacked packet */ #define PKTF_IFAINFO 0x4000 /* pkt has valid interface addr info */ #define PKTF_SO_BACKGROUND 0x8000 /* data is from background source */ #define PKTF_FORWARDED 0x10000 /* pkt was forwarded from another i/f */ #define PKTF_PRIV_GUARDED 0x20000 /* pkt_mpriv area guard enabled */ #define PKTF_KEEPALIVE 0x40000 /* pkt is kernel-generated keepalive */ #define PKTF_SO_REALTIME 0x80000 /* data is realtime traffic */ #define PKTF_VALID_UNSENT_DATA 0x100000 /* unsent data is valid */ #define PKTF_TCP_REXMT 0x200000 /* packet is TCP retransmission */ #define PKTF_REASSEMBLED 0x400000 /* Packet was reassembled */ #define PKTF_TX_COMPL_TS_REQ 0x800000 /* tx completion timestamp requested */ #define PKTF_TS_VALID 0x1000000 /* pkt timestamp is valid */ #define PKTF_DRIVER_MTAG 0x2000000 /* driver mbuf tags fields inited */ #define PKTF_NEW_FLOW 0x4000000 /* Data from a new flow */ #define PKTF_START_SEQ 0x8000000 /* valid start sequence */ #define PKTF_LAST_PKT 0x10000000 /* last packet in the flow */ #define PKTF_MPTCP_REINJ 0x20000000 /* Packet has been reinjected for MPTCP */ #define PKTF_MPTCP_DFIN 0x40000000 /* Packet is a data-fin */ #define PKTF_HBH_CHKED 0x80000000 /* HBH option is checked */ #define PKTF_EXT_OUTPUT_SCOPE 0x1 /* outgoing packet has ipv6 address scope id */ #define PKTF_EXT_L4S 0x2 /* pkts is from a L4S connection */ #define PKTF_EXT_QUIC 0x4 /* flag to denote a QUIC packet */ #define PKTF_EXT_QSET_ID_VALID 0x8 /* flag to denote if traffic rules are run */ #define PKTF_EXT_ULPN 0x10 /* packet transitted coprocessor */ #define PKTF_EXT_LPW 0x20 /* packet received in low power wake */ #define PKT_CRUMB_TS_COMP_REQ 0x0001 /* timestamp completion requested */ #define PKT_CRUMB_TS_COMP_CB 0x0002 /* timestamp callback called */ #define PKT_CRUMB_DLIL_OUTPUT 0x0004 /* dlil_output called */ #define PKT_CRUMB_FLOW_TX 0x0008 /* dp_flow_tx_process called */ #define PKT_CRUMB_FQ_ENQUEUE 0x0010 /* fq_enqueue called */ #define PKT_CRUMB_FQ_DEQUEUE 0x0020 /* fq_dequeue called */ #define PKT_CRUMB_SK_PKT_COPY 0x0040 /* copy from mbuf to skywalk packet */ #define PKT_CRUMB_TCP_OUTPUT 0x0080 #define PKT_CRUMB_UDP_OUTPUT 0x0100 #define PKT_CRUMB_SOSEND 0x0200 #define PKT_CRUMB_DLIL_INPUT 0x0400 #define PKT_CRUMB_IP_INPUT 0x0800 #define PKT_CRUMB_TCP_INPUT 0x1000 #define PKT_CRUMB_UDP_INPUT 0x2000 /* m_hdr_common crumbs flags */ #define CRUMB_INPUT_FLAG 0x0000000000010000 #define CRUMB_INTERFACE_FLAG 0x000000000001ffff /* flags related to flow control/advisory and identification */ #define PKTF_FLOW_MASK \ (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK) /* * Description of external storage mapped into mbuf, valid only if M_EXT set. */ typedef void (*__single m_ext_free_func_t)(caddr_t, u_int, caddr_t); struct m_ext { caddr_t __counted_by(ext_size) ext_buf; /* start of buffer */ m_ext_free_func_t ext_free; /* free routine (plain-text), if not the usual */ u_int ext_size; /* size of the external buffer */ caddr_t ext_arg; /* additional ext_free argument (plain-text) */ struct ext_ref { struct mbuf *paired; u_int16_t minref; u_int16_t refcnt; u_int16_t prefcnt; u_int16_t flags; u_int32_t priv; } *ext_refflags; }; /* define m_ext to a type since it gets redefined below */ typedef struct m_ext _m_ext_t; #if CONFIG_MBUF_MCACHE /* * The following _MLEN and _MHLEN macros are private to xnu. Private code * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since * the sizes of the structures are dependent upon specific xnu configs. */ #define _MLEN (_MSIZE - sizeof(struct m_hdr)) /* normal data len */ #define _MHLEN (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */ #define NMBPGSHIFT (PAGE_SHIFT - _MSIZESHIFT) #define NMBPG (1 << NMBPGSHIFT) /* # of mbufs per page */ #define NMBPCLSHIFT (MCLSHIFT - _MSIZESHIFT) /* * The mbuf object */ struct mbuf { struct m_hdr m_hdr; union { struct { struct pkthdr MH_pkthdr; /* M_PKTHDR set */ union { struct m_ext MH_ext; /* M_EXT set */ char MH_databuf[_MHLEN]; } MH_dat; } MH; char M_databuf[_MLEN]; /* !M_PKTHDR, !M_EXT */ } M_dat; }; #define m_next m_hdr.mh_next #define m_len m_hdr.mh_len #define m_data m_hdr.mh_data #define m_type m_hdr.mh_type #define m_flags m_hdr.mh_flags #define m_nextpkt m_hdr.mh_nextpkt #define m_act m_nextpkt #define m_ext M_dat.MH.MH_dat.MH_ext #define m_pkthdr M_dat.MH.MH_pkthdr #define m_pktdat M_dat.MH.MH_dat.MH_databuf #else /* !CONFIG_MBUF_MCACHE */ /* * The following _MLEN and _MHLEN macros are private to xnu. Private code * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since * the sizes of the structures are dependent upon specific xnu configs. */ #define _MLEN (_MSIZE - sizeof(struct m_hdr_common)) /* normal data len */ #define _MHLEN (_MLEN) /* data len w/pkthdr */ struct m_hdr_common { struct m_hdr M_hdr; struct m_ext M_ext __attribute__((aligned(16))); /* M_EXT set */ #if defined(__arm64__) uint64_t m_hdr_crumbs; #endif struct pkthdr M_pkthdr __attribute__((aligned(16))); /* M_PKTHDR set */ }; _Static_assert(sizeof(struct m_hdr_common) == 224, "Crumbs effecting size of struct"); #if defined(__arm64__) _Static_assert(sizeof(struct m_hdr_common) == 224, "Crumbs effecting size of struct"); #endif /* * The mbuf object */ struct mbuf { struct m_hdr_common M_hdr_common; union { char MH_databuf[_MHLEN]; char M_databuf[_MLEN]; /* !M_PKTHDR, !M_EXT */ } M_dat __attribute__((aligned(16))); }; #define m_next M_hdr_common.M_hdr.mh_next #define m_len M_hdr_common.M_hdr.mh_len #define m_data M_hdr_common.M_hdr.mh_data #define m_type M_hdr_common.M_hdr.mh_type #define m_flags M_hdr_common.M_hdr.mh_flags #define m_nextpkt M_hdr_common.M_hdr.mh_nextpkt #define m_ext M_hdr_common.M_ext #define m_pkthdr M_hdr_common.M_pkthdr #define m_pktdat M_dat.MH_databuf #if defined(__arm64__) #define m_mhdrcommon_crumbs M_hdr_common.m_hdr_crumbs #endif /* __arm64__ */ #endif /* CONFIG_MBUF_MCACHE */ #define m_act m_nextpkt #define m_dat M_dat.M_databuf #define m_pktlen(_m) ((_m)->m_pkthdr.len) #define m_pftag(_m) (&(_m)->m_pkthdr.builtin_mtag._net_mtag._pf_mtag) #define m_necptag(_m) (&(_m)->m_pkthdr.builtin_mtag._net_mtag._necp_mtag) /* mbuf flags (private) */ #define M_EXT 0x0001 /* has associated external storage */ #define M_PKTHDR 0x0002 /* start of record */ #define M_EOR 0x0004 /* end of record */ #define M_PROTO1 0x0008 /* protocol-specific */ #define M_PROTO2 0x0010 /* protocol-specific */ #define M_PROTO3 0x0020 /* protocol-specific */ #define M_LOOP 0x0040 /* packet is looped back (also see PKTF_LOOP) */ #define M_PROTO5 0x0080 /* protocol-specific */ /* mbuf pkthdr flags, also in m_flags (private) */ #define M_BCAST 0x0100 /* send/received as link-level broadcast */ #define M_MCAST 0x0200 /* send/received as link-level multicast */ #define M_FRAG 0x0400 /* packet is a fragment of a larger packet */ #define M_FIRSTFRAG 0x0800 /* packet is first fragment */ #define M_LASTFRAG 0x1000 /* packet is last fragment */ #define M_PROMISC 0x2000 /* packet is promiscuous (shouldn't go to stack) */ #define M_HASFCS 0x4000 /* packet has FCS */ #define M_TAGHDR 0x8000 /* m_tag hdr structure at top of mbuf data */ /* * Flags to purge when crossing layers. */ #define M_PROTOFLAGS \ (M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO5) /* flags copied when copying m_pkthdr */ #define M_COPYFLAGS \ (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 | \ M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG | \ M_FIRSTFRAG|M_LASTFRAG|M_PROMISC|M_HASFCS) /* flags indicating hw checksum support and sw checksum requirements */ #define CSUM_IP 0x0001 /* will csum IP */ #define CSUM_TCP 0x0002 /* will csum TCP */ #define CSUM_UDP 0x0004 /* will csum UDP */ #define CSUM_IP_FRAGS 0x0008 /* will csum IP fragments */ #define CSUM_FRAGMENT 0x0010 /* will do IP fragmentation */ #define CSUM_TCPIPV6 0x0020 /* will csum TCP for IPv6 */ #define CSUM_UDPIPV6 0x0040 /* will csum UDP for IPv6 */ #define CSUM_FRAGMENT_IPV6 0x0080 /* will do IPv6 fragmentation */ #define CSUM_IP_CHECKED 0x0100 /* did csum IP */ #define CSUM_IP_VALID 0x0200 /* ... the csum is valid */ #define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */ #define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */ #define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */ #define CSUM_ZERO_INVERT 0x2000 /* invert 0 to -0 (0xffff) */ #define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP) #define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */ #define CSUM_DELAY_IPV6_DATA (CSUM_TCPIPV6 | CSUM_UDPIPV6) #define CSUM_DATA_IPV6_VALID CSUM_DATA_VALID /* csum_data field is valid */ #define CSUM_TX_FLAGS \ (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \ CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_ZERO_INVERT) #define CSUM_RX_FULL_FLAGS \ (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \ CSUM_DATA_VALID) #define CSUM_RX_FLAGS \ (CSUM_RX_FULL_FLAGS | CSUM_PARTIAL) /* * Note: see also IF_HWASSIST_CSUM defined in <net/if_var.h> */ /* VLAN tag present */ #define CSUM_VLAN_TAG_VALID 0x00010000 /* vlan_tag field is valid */ /* checksum start adjustment has been done */ #define CSUM_ADJUST_DONE 0x00020000 /* VLAN encapsulation present */ #define CSUM_VLAN_ENCAP_PRESENT 0x00040000 /* mbuf has vlan encapsulation */ /* TCP Segment Offloading requested on this mbuf */ #define CSUM_TSO_IPV4 0x00100000 /* This mbuf needs to be segmented by the NIC */ #define CSUM_TSO_IPV6 0x00200000 /* This mbuf needs to be segmented by the NIC */ #define TSO_IPV4_OK(_ifp, _m) \ (((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \ ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \ #define TSO_IPV4_NOTOK(_ifp, _m) \ (!((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \ ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \ #define TSO_IPV6_OK(_ifp, _m) \ (((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \ ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \ #define TSO_IPV6_NOTOK(_ifp, _m) \ (!((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \ ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \ #endif /* XNU_KERNEL_PRIVATE */ /* mbuf types */ #define MT_FREE 0 /* should be on free list */ #define MT_DATA 1 /* dynamic (data) allocation */ #define MT_HEADER 2 /* packet header */ #define MT_SOCKET 3 /* socket structure */ #define MT_PCB 4 /* protocol control block */ #define MT_RTABLE 5 /* routing tables */ #define MT_HTABLE 6 /* IMP host tables */ #define MT_ATABLE 7 /* address resolution tables */ #define MT_SONAME 8 /* socket name */ #define MT_SOOPTS 10 /* socket options */ #define MT_FTABLE 11 /* fragment reassembly header */ #define MT_RIGHTS 12 /* access rights */ #define MT_IFADDR 13 /* interface address */ #define MT_CONTROL 14 /* extra-data protocol message */ #define MT_OOBDATA 15 /* expedited data */ #define MT_TAG 16 /* volatile metadata associated to pkts */ #define MT_MAX 32 /* enough? */ enum { MTF_FREE = (1 << MT_FREE), MTF_DATA = (1 << MT_DATA), MTF_HEADER = (1 << MT_HEADER), MTF_SOCKET = (1 << MT_SOCKET), MTF_PCB = (1 << MT_PCB), MTF_RTABLE = (1 << MT_RTABLE), MTF_HTABLE = (1 << MT_HTABLE), MTF_ATABLE = (1 << MT_ATABLE), MTF_SONAME = (1 << MT_SONAME), MTF_SOOPTS = (1 << MT_SOOPTS), MTF_FTABLE = (1 << MT_FTABLE), MTF_RIGHTS = (1 << MT_RIGHTS), MTF_IFADDR = (1 << MT_IFADDR), MTF_CONTROL = (1 << MT_CONTROL), MTF_OOBDATA = (1 << MT_OOBDATA), MTF_TAG = (1 << MT_TAG), }; #ifdef XNU_KERNEL_PRIVATE /* * mbuf allocation/deallocation macros: * * MGET(struct mbuf *m, int how, int type) * allocates an mbuf and initializes it to contain internal data. * * MGETHDR(struct mbuf *m, int how, int type) * allocates an mbuf and initializes it to contain a packet header * and internal data. */ #define MGET(m, how, type) ((m) = m_get((how), (type))) #define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type))) /* * Mbuf cluster macros. * MCLALLOC(caddr_t p, int how) allocates an mbuf cluster. * MCLGET adds such clusters to a normal mbuf; * the flag M_EXT is set upon success. * MCLFREE releases a reference to a cluster allocated by MCLALLOC, * freeing the cluster if the reference count has reached 0. * * Normal mbuf clusters are normally treated as character arrays * after allocation, but use the first word of the buffer as a free list * pointer while on the free list. */ union mcluster { union mcluster *mcl_next; char mcl_buf[MCLBYTES]; }; #define MCLGET(m, how) ((m) = m_mclget(m, how)) /* * Mbuf big cluster */ union mbigcluster { union mbigcluster *mbc_next; char mbc_buf[MBIGCLBYTES]; }; /* * Mbuf jumbo cluster */ union m16kcluster { union m16kcluster *m16kcl_next; char m16kcl_buf[M16KCLBYTES]; }; #define MCLHASREFERENCE(m) m_mclhasreference(m) /* * MFREE(struct mbuf *m, struct mbuf *n) * Free a single mbuf and associated external storage. * Place the successor, if any, in n. */ #define MFREE(m, n) ((n) = m_free(m)) /* * Copy mbuf pkthdr from from to to. * from must have M_PKTHDR set, and to must be empty. * aux pointer will be moved to `to'. */ #define M_COPY_PKTHDR(to, from) m_copy_pkthdr(to, from) #define M_COPY_PFTAG(to, from) m_copy_pftag(to, from) #define M_COPY_NECPTAG(to, from) m_copy_necptag(to, from) #define M_COPY_CLASSIFIER(to, from) m_copy_classifier(to, from) /* * Evaluate TRUE if it's safe to write to the mbuf m's data region (this can * be both the local data payload, or an external buffer area, depending on * whether M_EXT is set). */ #define M_WRITABLE(m) (((m)->m_flags & M_EXT) == 0 || !MCLHASREFERENCE(m)) /* * These macros are mapped to the appropriate KPIs, so that private code * can be simply recompiled in order to be forward-compatible with future * changes toward the struture sizes. */ #ifdef XNU_KERNEL_PRIVATE #define MLEN _MLEN #define MHLEN _MHLEN #define MINCLSIZE (MLEN + MHLEN) #else #define MLEN mbuf_get_mlen() /* normal mbuf data len */ #define MHLEN mbuf_get_mhlen() /* data len in an mbuf w/pkthdr */ #define MINCLSIZE mbuf_get_minclsize() /* cluster usage threshold */ #endif /* * Return the address of the start of the buffer associated with an mbuf, * handling external storage, packet-header mbufs, and regular data mbufs. */ #define M_START(m) \ (((m)->m_flags & M_EXT) ? (caddr_t)(m)->m_ext.ext_buf : \ ((m)->m_flags & M_PKTHDR) ? &(m)->m_pktdat[0] : \ &(m)->m_dat[0]) /* * Return the size of the buffer associated with an mbuf, handling external * storage, packet-header mbufs, and regular data mbufs. */ #define M_SIZE(m) \ (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_size : \ ((m)->m_flags & M_PKTHDR) ? MHLEN : \ MLEN) #define M_ALIGN(m, len) m_align(m, len) #define MH_ALIGN(m, len) m_align(m, len) #define MEXT_ALIGN(m, len) m_align(m, len) /* * Compute the amount of space available before the current start of data in * an mbuf. * * The M_WRITABLE() is a temporary, conservative safety measure: the burden * of checking writability of the mbuf data area rests solely with the caller. */ #define M_LEADINGSPACE(m) \ (M_WRITABLE(m) ? ((m)->m_data - (uintptr_t)M_START(m)) : 0) /* * Compute the amount of space available after the end of data in an mbuf. * * The M_WRITABLE() is a temporary, conservative safety measure: the burden * of checking writability of the mbuf data area rests solely with the caller. */ #define M_TRAILINGSPACE(m) \ (M_WRITABLE(m) ? \ ((M_START(m) + M_SIZE(m)) - (mtod(m, caddr_t) + (m)->m_len)) : 0) /* * Arrange to prepend space of size plen to mbuf m. * If a new mbuf must be allocated, how specifies whether to wait. * If how is M_DONTWAIT and allocation fails, the original mbuf chain * is freed and m is set to NULL. */ #define M_PREPEND(m, plen, how, align) \ ((m) = m_prepend_2((m), (plen), (how), (align))) /* change mbuf to new type */ #define MCHTYPE(m, t) m_mchtype(m, t) /* compatiblity with 4.3 */ #define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT) #define MBSHIFT 20 /* 1MB */ #define MBSIZE (1 << MBSHIFT) #define GBSHIFT 30 /* 1GB */ #define GBSIZE (1 << GBSHIFT) /* * M_STRUCT_GET ensures that intermediate protocol header (from "off" to * "off+len") is located in single mbuf, on contiguous memory region. * The pointer to the region will be returned to pointer variable "val", * with type "typ". * * M_STRUCT_GET0 does the same, except that it aligns the structure at * very top of mbuf. GET0 is likely to make memory copy than GET. */ #define M_STRUCT_GET(val, typ, m, off, len) \ do { \ struct mbuf *t; \ int tmp; \ \ if ((m)->m_len >= (off) + (len)) { \ (val) = (typ)(mtod((m), caddr_t) + (off)); \ } else { \ t = m_pulldown((m), (off), (len), &tmp); \ if (t != NULL) { \ if (t->m_len < tmp + (len)) \ panic("m_pulldown malfunction"); \ (val) = (typ)(mtod(t, caddr_t) + tmp); \ } else { \ (val) = (typ)NULL; \ (m) = NULL; \ } \ } \ } while (0) #define M_STRUCT_GET0(val, typ, m, off, len) \ do { \ struct mbuf *t; \ \ if ((off) == 0 && ((m)->m_len >= (len))) { \ (val) = (typ)(void *)mtod(m, caddr_t); \ } else { \ t = m_pulldown((m), (off), (len), NULL); \ if (t != NULL) { \ if (t->m_len < (len)) \ panic("m_pulldown malfunction"); \ (val) = (typ)(void *)mtod(t, caddr_t); \ } else { \ (val) = (typ)NULL; \ (m) = NULL; \ } \ } \ } while (0) #define MBUF_INPUT_CHECK(m, rcvif) \ do { \ if (!(m->m_flags & MBUF_PKTHDR) || \ m->m_len < 0 || \ m->m_len > njclbytes || \ m->m_type == MT_FREE || \ ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) { \ panic_plain("Failed mbuf validity check: mbuf %p len %d " \ "type %d flags 0x%x data %p rcvif %s ifflags 0x%x", \ m, m->m_len, m->m_type, m->m_flags, \ ((m->m_flags & M_EXT) \ ? m->m_ext.ext_buf \ : (caddr_t __unsafe_indexable)m->m_data), \ if_name(rcvif), \ (rcvif->if_flags & 0xffff)); \ } \ } while (0) /* * Simple mbuf queueing system * * This is basically a SIMPLEQ adapted to mbuf use (i.e. using * m_nextpkt instead of field.sqe_next). * * m_next is ignored, so queueing chains of mbufs is possible */ #define MBUFQ_HEAD(name) \ struct name { \ struct mbuf *mq_first; /* first packet */ \ struct mbuf **mq_last; /* addr of last next packet */ \ } #define MBUFQ_INIT(q) do { \ MBUFQ_FIRST(q) = NULL; \ (q)->mq_last = &MBUFQ_FIRST(q); \ } while (0) #define MBUFQ_PREPEND(q, m) do { \ if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL) \ (q)->mq_last = &MBUFQ_NEXT(m); \ MBUFQ_FIRST(q) = (m); \ } while (0) #define MBUFQ_ENQUEUE(q, m) do { \ MBUFQ_NEXT(m) = NULL; \ *(q)->mq_last = (m); \ (q)->mq_last = &MBUFQ_NEXT(m); \ } while (0) #define MBUFQ_ENQUEUE_MULTI(q, m, n) do { \ MBUFQ_NEXT(n) = NULL; \ *(q)->mq_last = (m); \ (q)->mq_last = &MBUFQ_NEXT(n); \ } while (0) #define MBUFQ_DEQUEUE(q, m) do { \ if (((m) = MBUFQ_FIRST(q)) != NULL) { \ if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL) \ (q)->mq_last = &MBUFQ_FIRST(q); \ else \ MBUFQ_NEXT(m) = NULL; \ } \ } while (0) #define MBUFQ_REMOVE(q, m) do { \ if (MBUFQ_FIRST(q) == (m)) { \ MBUFQ_DEQUEUE(q, m); \ } else { \ struct mbuf *_m = MBUFQ_FIRST(q); \ while (MBUFQ_NEXT(_m) != (m)) \ _m = MBUFQ_NEXT(_m); \ if ((MBUFQ_NEXT(_m) = \ MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL) \ (q)->mq_last = &MBUFQ_NEXT(_m); \ } \ } while (0) #define MBUFQ_DRAIN(q) do { \ struct mbuf *__m0; \ while ((__m0 = MBUFQ_FIRST(q)) != NULL) { \ MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0); \ MBUFQ_NEXT(__m0) = NULL; \ m_freem(__m0); \ } \ (q)->mq_last = &MBUFQ_FIRST(q); \ } while (0) #define MBUFQ_DROP_AND_DRAIN(q, d, r) do { \ struct mbuf *__m0; \ while ((__m0 = MBUFQ_FIRST(q)) != NULL) { \ MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0); \ MBUFQ_NEXT(__m0) = NULL; \ m_drop(__m0, (d) | DROPTAP_FLAG_L2_MISSING, (r), NULL, 0); \ } \ (q)->mq_last = &MBUFQ_FIRST(q); \ } while (0) #define MBUFQ_FOREACH(m, q) \ for ((m) = MBUFQ_FIRST(q); \ (m); \ (m) = MBUFQ_NEXT(m)) #define MBUFQ_FOREACH_SAFE(m, q, tvar) \ for ((m) = MBUFQ_FIRST(q); \ (m) && ((tvar) = MBUFQ_NEXT(m), 1); \ (m) = (tvar)) #define MBUFQ_EMPTY(q) ((q)->mq_first == NULL) #define MBUFQ_FIRST(q) ((q)->mq_first) #define MBUFQ_NEXT(m) ((m)->m_nextpkt) /* * mq_last is initialized to point to mq_first, so check if they're * equal and return NULL when the list is empty. Otherwise, we need * to subtract the offset of MBUQ_NEXT (i.e. m_nextpkt field) to get * to the base mbuf address to return to caller. */ #define MBUFQ_LAST(head) \ (((head)->mq_last == &MBUFQ_FIRST(head)) ? NULL : \ __container_of((head)->mq_last, struct mbuf, m_nextpkt)) #if (DEBUG || DEVELOPMENT) #define MBUFQ_ADD_CRUMB_MULTI(_q, _h, _t, _f) do { \ struct mbuf * _saved = (_t)->m_nextpkt; \ struct mbuf * _m; \ for (_m = (_h); _m != NULL; _m = MBUFQ_NEXT(_m)) { \ m_add_crumb((_m), (_f)); \ } \ (_t)->m_nextpkt = _saved; \ } while (0) #define MBUFQ_ADD_CRUMB(_q, _m, _f) do { \ m_add_crumb((_m), (_f)); \ } while (0) #else #define MBUFQ_ADD_CRUMB_MULTI(_q, _h, _t, _f) #define MBUFQ_ADD_CRUMB(_q, _m, _f) #endif /* (DEBUG || DEVELOPMENT) */ struct mbufq { MBUFQ_HEAD(counted_mbufq) mq; uint32_t count; uint32_t bytes; }; inline void mbufq_init(struct mbufq *q) { MBUFQ_INIT(&q->mq); q->bytes = q->count = 0; } inline void mbufq_enqueue(struct mbufq *q, struct mbuf *head, struct mbuf *tail, uint32_t cnt, uint32_t bytes) { MBUFQ_ENQUEUE_MULTI(&q->mq, head, tail); q->count += cnt; q->bytes += bytes; } inline boolean_t mbufq_empty(struct mbufq *q) { return q->count == 0; } inline struct mbuf* mbufq_first(struct mbufq *q) { return MBUFQ_FIRST(&q->mq); } inline struct mbuf* mbufq_last(struct mbufq *q) { return MBUFQ_LAST(&q->mq); } #endif /* XNU_KERNEL_PRIVATE */ /* * Mbuf statistics (legacy). */ struct mbstat { u_int32_t m_mbufs; /* mbufs obtained from page pool */ u_int32_t m_clusters; /* clusters obtained from page pool */ u_int32_t m_spare; /* spare field */ u_int32_t m_clfree; /* free clusters */ u_int32_t m_drops; /* times failed to find space */ u_int32_t m_wait; /* times waited for space */ u_int32_t m_drain; /* times drained protocols for space */ u_short m_mtypes[256]; /* type specific mbuf allocations */ u_int32_t m_mcfail; /* times m_copym failed */ u_int32_t m_mpfail; /* times m_pullup failed */ u_int32_t m_msize; /* length of an mbuf */ u_int32_t m_mclbytes; /* length of an mbuf cluster */ u_int32_t m_minclsize; /* min length of data to allocate a cluster */ u_int32_t m_mlen; /* length of data in an mbuf */ u_int32_t m_mhlen; /* length of data in a header mbuf */ u_int32_t m_bigclusters; /* clusters obtained from page pool */ u_int32_t m_bigclfree; /* free clusters */ u_int32_t m_bigmclbytes; /* length of an mbuf cluster */ u_int32_t m_forcedefunct; /* times we force defunct'ed an app's sockets */ }; /* Compatibillity with 10.3 */ struct ombstat { u_int32_t m_mbufs; /* mbufs obtained from page pool */ u_int32_t m_clusters; /* clusters obtained from page pool */ u_int32_t m_spare; /* spare field */ u_int32_t m_clfree; /* free clusters */ u_int32_t m_drops; /* times failed to find space */ u_int32_t m_wait; /* times waited for space */ u_int32_t m_drain; /* times drained protocols for space */ u_short m_mtypes[256]; /* type specific mbuf allocations */ u_int32_t m_mcfail; /* times m_copym failed */ u_int32_t m_mpfail; /* times m_pullup failed */ u_int32_t m_msize; /* length of an mbuf */ u_int32_t m_mclbytes; /* length of an mbuf cluster */ u_int32_t m_minclsize; /* min length of data to allocate a cluster */ u_int32_t m_mlen; /* length of data in an mbuf */ u_int32_t m_mhlen; /* length of data in a header mbuf */ }; /* * mbuf class statistics. */ #define MAX_MBUF_CNAME 15 #if defined(XNU_KERNEL_PRIVATE) /* For backwards compatibility with 32-bit userland process */ struct omb_class_stat { char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */ u_int32_t mbcl_size; /* buffer size */ u_int32_t mbcl_total; /* # of buffers created */ u_int32_t mbcl_active; /* # of active buffers */ u_int32_t mbcl_infree; /* # of available buffers */ u_int32_t mbcl_slab_cnt; /* # of available slabs */ u_int32_t mbcl_pad; /* padding */ u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */ u_int64_t mbcl_free_cnt; /* # of times free is called */ u_int64_t mbcl_notified; /* # of notified wakeups */ u_int64_t mbcl_purge_cnt; /* # of purges so far */ u_int64_t mbcl_fail_cnt; /* # of allocation failures */ u_int32_t mbcl_ctotal; /* total only for this class */ u_int32_t mbcl_release_cnt; /* amount of memory returned */ /* * Cache layer statistics */ u_int32_t mbcl_mc_state; /* cache state (see below) */ u_int32_t mbcl_mc_cached; /* # of cached buffers */ u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */ u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */ u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */ u_int32_t mbcl_reserved[7]; /* for future use */ } __attribute__((__packed__)); #endif /* XNU_KERNEL_PRIVATE */ typedef struct mb_class_stat { char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */ u_int32_t mbcl_size; /* buffer size */ u_int32_t mbcl_total; /* # of buffers created */ u_int32_t mbcl_active; /* # of active buffers */ u_int32_t mbcl_infree; /* # of available buffers */ u_int32_t mbcl_slab_cnt; /* # of available slabs */ #if defined(KERNEL) || defined(__LP64__) u_int32_t mbcl_pad; /* padding */ #endif /* KERNEL || __LP64__ */ u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */ u_int64_t mbcl_free_cnt; /* # of times free is called */ u_int64_t mbcl_notified; /* # of notified wakeups */ u_int64_t mbcl_purge_cnt; /* # of purges so far */ u_int64_t mbcl_fail_cnt; /* # of allocation failures */ u_int32_t mbcl_ctotal; /* total only for this class */ u_int32_t mbcl_release_cnt; /* amount of memory returned */ /* * Cache layer statistics */ u_int32_t mbcl_mc_state; /* cache state (see below) */ u_int32_t mbcl_mc_cached; /* # of cached buffers */ u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */ u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */ u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */ u_int32_t mbcl_reserved[7]; /* for future use */ } mb_class_stat_t; #define MCS_DISABLED 0 /* cache is permanently disabled */ #define MCS_ONLINE 1 /* cache is online */ #define MCS_PURGING 2 /* cache is being purged */ #define MCS_OFFLINE 3 /* cache is offline (resizing) */ #if defined(XNU_KERNEL_PRIVATE) #define MB_STAT_MAX_MB_CLASSES 8 /* Max number of distinct Mbuf classes. */ #endif /* XNU_KERNEL_PRIVATE */ #if defined(XNU_KERNEL_PRIVATE) /* For backwards compatibility with 32-bit userland process */ struct omb_stat { u_int32_t mbs_cnt; /* number of classes */ u_int32_t mbs_pad; /* padding */ struct omb_class_stat mbs_class[MB_STAT_MAX_MB_CLASSES]; /* class array */ } __attribute__((__packed__)); #endif /* XNU_KERNEL_PRIVATE */ typedef struct mb_stat { u_int32_t mbs_cnt; /* number of classes */ #if defined(KERNEL) || defined(__LP64__) u_int32_t mbs_pad; /* padding */ #endif /* KERNEL || __LP64__ */ #if defined(XNU_KERNEL_PRIVATE) mb_class_stat_t mbs_class[MB_STAT_MAX_MB_CLASSES]; #else /* XNU_KERNEL_PRIVATE */ mb_class_stat_t mbs_class[1]; /* class array */ #endif /* XNU_KERNEL_PRIVATE */ } mb_stat_t; #ifdef PRIVATE #define MLEAK_STACK_DEPTH 16 /* Max PC stack depth */ typedef struct mleak_trace_stat { u_int64_t mltr_collisions; u_int64_t mltr_hitcount; u_int64_t mltr_allocs; u_int64_t mltr_depth; u_int64_t mltr_addr[MLEAK_STACK_DEPTH]; } mleak_trace_stat_t; typedef struct mleak_stat { u_int32_t ml_isaddr64; /* 64-bit KVA? */ u_int32_t ml_cnt; /* number of traces */ mleak_trace_stat_t ml_trace[1]; /* trace array */ } mleak_stat_t; struct mleak_table { u_int32_t mleak_capture; /* sampling capture counter */ u_int32_t mleak_sample_factor; /* sample factor */ /* Times two active records want to occupy the same spot */ u_int64_t alloc_collisions; u_int64_t trace_collisions; /* Times new record lands on spot previously occupied by freed alloc */ u_int64_t alloc_overwrites; u_int64_t trace_overwrites; /* Times a new alloc or trace is put into the hash table */ u_int64_t alloc_recorded; u_int64_t trace_recorded; /* Total number of outstanding allocs */ u_int64_t outstanding_allocs; /* Times mleak_log returned false because couldn't acquire the lock */ u_int64_t total_conflicts; }; #define HAS_M_TAG_STATS 1 struct m_tag_stats { u_int32_t mts_id; u_int16_t mts_type; u_int16_t mts_len; u_int64_t mts_alloc_count; u_int64_t mts_alloc_failed; u_int64_t mts_free_count; }; #define M_TAG_TYPE_NAMES \ "other,dummynet,ipfilt,encap,inet6,ipsec,cfil_udp,pf_reass,aqm,drvaux" #endif /* PRIVATE */ #ifdef KERNEL_PRIVATE __BEGIN_DECLS /* * Exported (private) */ extern struct mbstat mbstat; /* statistics */ __END_DECLS #endif /* KERNEL_PRIVATE */ #ifdef XNU_KERNEL_PRIVATE __BEGIN_DECLS /* * Not exported (xnu private) */ /* flags to m_get/MGET */ /* Need to include malloc.h to get right options for malloc */ #include <sys/malloc.h> struct mbuf; /* length to m_copy to copy all */ #define M_COPYALL 1000000000 #define M_DONTWAIT M_NOWAIT #define M_WAIT M_WAITOK /* modes for m_copym and variants */ #define M_COPYM_NOOP_HDR 0 /* don't copy/move pkthdr contents */ #define M_COPYM_COPY_HDR 1 /* copy pkthdr from old to new */ #define M_COPYM_MOVE_HDR 2 /* move pkthdr from old to new */ #define M_COPYM_MUST_COPY_HDR 3 /* MUST copy pkthdr from old to new */ #define M_COPYM_MUST_MOVE_HDR 4 /* MUST move pkthdr from old to new */ extern void m_freem(struct mbuf *) __XNU_INTERNAL(m_freem); extern void m_drop(mbuf_t, uint16_t, uint32_t, const char *, uint16_t); extern void m_drop_if(mbuf_t, struct ifnet *, uint16_t, uint32_t, const char *, uint16_t); extern void m_drop_list(mbuf_t, struct ifnet *, uint16_t, uint32_t, const char *, uint16_t); extern void m_drop_extended(mbuf_t, struct ifnet *, char *, uint16_t, uint32_t, const char *, uint16_t); extern u_int64_t mcl_to_paddr(char *); extern void m_adj(struct mbuf *, int); extern void m_cat(struct mbuf *, struct mbuf *); extern void m_copydata(struct mbuf *, int, int len, void * __sized_by(len)); extern struct mbuf *m_copym(struct mbuf *, int, int, int); extern struct mbuf *m_copym_mode(struct mbuf *, int, int, int, struct mbuf **, int *, uint32_t); extern struct mbuf *m_get(int, int); extern struct mbuf *m_gethdr(int, int); extern struct mbuf *m_getpacket(void); extern struct mbuf *m_getpackets(int, int, int); extern struct mbuf *m_mclget(struct mbuf *, int); extern void *__unsafe_indexable m_mtod(struct mbuf *); extern struct mbuf *m_prepend_2(struct mbuf *, int, int, int); extern struct mbuf *m_pullup(struct mbuf *, int); extern struct mbuf *m_split(struct mbuf *, int, int); extern void m_mclfree(caddr_t p); extern bool mbuf_class_under_pressure(struct mbuf *m); extern int m_chain_capacity(const struct mbuf *m); /* * Accessors for the mbuf data range. * The "lower bound" is the start of the memory range that m->m_data is allowed * to point into. The "start" is where m->m_data points to; equivalent to the * late m_mtod. The end is where m->m_data + m->m_len points to. The upper bound * is the end of the memory range that m->m_data + m->m_len is allowed to point * into. * In a well-formed range, lower bound <= start <= end <= upper bound. An * ill-formed range always means a programming error. */ __stateful_pure static inline caddr_t __header_bidi_indexable m_mtod_lower_bound(struct mbuf *m) { return M_START(m); } __stateful_pure static inline caddr_t __header_bidi_indexable m_mtod_current(struct mbuf *m) { caddr_t data = m_mtod_lower_bound(m); return data + (m->m_data - (uintptr_t)data); } __stateful_pure static inline caddr_t __header_bidi_indexable m_mtod_end(struct mbuf *m) { return m_mtod_current(m) + m->m_len; } __stateful_pure static inline caddr_t __header_bidi_indexable m_mtod_upper_bound(struct mbuf *m) { return m_mtod_lower_bound(m) + M_SIZE(m); } static inline bool m_has_mtype(const struct mbuf *m, int mtype_flags) { return (1 << m->m_type) & mtype_flags; } static inline int m_capacity(const struct mbuf *m) { return _MSIZE + ((m->m_flags & M_EXT) ? m->m_ext.ext_size : 0); } /* * On platforms which require strict alignment (currently for anything but * i386 or x86_64 or arm64), this macro checks whether the data pointer of an mbuf * is 32-bit aligned (this is the expected minimum alignment for protocol * headers), and assert otherwise. */ #if defined(__i386__) || defined(__x86_64__) || defined(__arm64__) #define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) #else /* !__i386__ && !__x86_64__ && !__arm64__ */ #define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do { \ if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) { \ if (((_m)->m_flags & M_PKTHDR) && \ (_m)->m_pkthdr.rcvif != NULL) { \ panic_plain("\n%s: mbuf %p data ptr %p is not " \ "32-bit aligned [%s: alignerrs=%lld]\n", \ __func__, (_m), \ (caddr_t __unsafe_indexable)(_m)->m_data, \ if_name((_m)->m_pkthdr.rcvif), \ (_m)->m_pkthdr.rcvif->if_alignerrs); \ } else { \ panic_plain("\n%s: mbuf %p data ptr %p is not " \ "32-bit aligned\n", \ __func__, (_m), \ (caddr_t __unsafe_indexable)(_m)->m_data); \ } \ } \ } while (0) #endif /* !__i386__ && !__x86_64__ && !__arm64__ */ /* Maximum number of MBUF_SC values (excluding MBUF_SC_UNSPEC) */ #define MBUF_SC_MAX_CLASSES 10 /* * These conversion macros rely on the corresponding MBUF_SC and * MBUF_TC values in order to establish the following mapping: * * MBUF_SC_BK_SYS ] ==> MBUF_TC_BK * MBUF_SC_BK ] * * MBUF_SC_BE ] ==> MBUF_TC_BE * MBUF_SC_RD ] * MBUF_SC_OAM ] * * MBUF_SC_AV ] ==> MBUF_TC_VI * MBUF_SC_RV ] * MBUF_SC_VI ] * MBUF_SC_SIG ] * * MBUF_SC_VO ] ==> MBUF_TC_VO * MBUF_SC_CTL ] * * The values assigned to each service class allows for a fast mapping to * the corresponding MBUF_TC traffic class values, as well as to retrieve the * assigned index; therefore care must be taken when comparing against these * values. Use the corresponding class and index macros to retrieve the * corresponding portion, and never assume that a higher class corresponds * to a higher index. */ #define MBUF_SCVAL(x) ((x) & 0xffff) #define MBUF_SCIDX(x) ((((x) >> 16) & 0xff) >> 3) #define MBUF_SC2TC(_sc) (MBUF_SCVAL(_sc) >> 7) #define MBUF_TC2SCVAL(_tc) ((_tc) << 7) #define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \ ((_sc) == MBUF_SC_BK)) #define IS_MBUF_SC_REALTIME(_sc) ((_sc) >= MBUF_SC_AV && (_sc) <= MBUF_SC_VO) #define IS_MBUF_SC_BESTEFFORT(_sc) ((_sc) == MBUF_SC_BE || \ (_sc) == MBUF_SC_RD || (_sc) == MBUF_SC_OAM) #define SCIDX_BK_SYS MBUF_SCIDX(MBUF_SC_BK_SYS) #define SCIDX_BK MBUF_SCIDX(MBUF_SC_BK) #define SCIDX_BE MBUF_SCIDX(MBUF_SC_BE) #define SCIDX_RD MBUF_SCIDX(MBUF_SC_RD) #define SCIDX_OAM MBUF_SCIDX(MBUF_SC_OAM) #define SCIDX_AV MBUF_SCIDX(MBUF_SC_AV) #define SCIDX_RV MBUF_SCIDX(MBUF_SC_RV) #define SCIDX_VI MBUF_SCIDX(MBUF_SC_VI) #define SCIDX_SIG MBUF_SCIDX(MBUF_SC_SIG) #define SCIDX_VO MBUF_SCIDX(MBUF_SC_VO) #define SCIDX_CTL MBUF_SCIDX(MBUF_SC_CTL) #define SCVAL_BK_SYS MBUF_SCVAL(MBUF_SC_BK_SYS) #define SCVAL_BK MBUF_SCVAL(MBUF_SC_BK) #define SCVAL_BE MBUF_SCVAL(MBUF_SC_BE) #define SCVAL_RD MBUF_SCVAL(MBUF_SC_RD) #define SCVAL_OAM MBUF_SCVAL(MBUF_SC_OAM) #define SCVAL_AV MBUF_SCVAL(MBUF_SC_AV) #define SCVAL_RV MBUF_SCVAL(MBUF_SC_RV) #define SCVAL_VI MBUF_SCVAL(MBUF_SC_VI) #define SCVAL_SIG MBUF_SCVAL(MBUF_SC_SIG) #define SCVAL_VO MBUF_SCVAL(MBUF_SC_VO) #define SCVAL_CTL MBUF_SCVAL(MBUF_SC_CTL) #define MBUF_VALID_SC(c) \ (c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE || \ c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV || \ c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_SIG || \ c == MBUF_SC_VO || c == MBUF_SC_CTL) #define MBUF_VALID_SCIDX(c) \ (c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE || \ c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV || \ c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_SIG || \ c == SCIDX_VO || c == SCIDX_CTL) #define MBUF_VALID_SCVAL(c) \ (c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE || \ c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV || \ c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_SIG || \ c == SCVAL_VO || SCVAL_CTL) extern unsigned int nmbclusters; /* number of mapped clusters */ extern int njcl; /* # of jumbo clusters */ extern int njclbytes; /* size of a jumbo cluster */ extern int max_hdr; /* largest link+protocol header */ extern int max_datalen; /* MHLEN - max_hdr */ extern int max_linkhdr; /* largest link-level header */ /* Use max_protohdr instead of _max_protohdr */ extern int max_protohdr; /* largest protocol header */ extern uint32_t high_sb_max; __private_extern__ unsigned int mbuf_default_ncl(uint64_t); __private_extern__ void mbinit(void); __private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t __sized_by(extsize), void (*)(caddr_t, u_int, caddr_t), size_t extsize, caddr_t, int, int); __private_extern__ char * __sized_by_or_null(MBIGCLBYTES) m_bigalloc(int); __private_extern__ void m_bigfree(caddr_t, u_int, caddr_t); __private_extern__ struct mbuf *m_mbigget(struct mbuf *, int); __private_extern__ char * __sized_by_or_null(M16KCLBYTES) m_16kalloc(int); __private_extern__ void m_16kfree(caddr_t, u_int, caddr_t); __private_extern__ struct mbuf *m_m16kget(struct mbuf *, int); __private_extern__ int m_reinit(struct mbuf *, int); __private_extern__ struct mbuf *m_free(struct mbuf *) __XNU_INTERNAL(m_free); __private_extern__ struct mbuf *m_getclr(int, int); __private_extern__ struct mbuf *m_getptr(struct mbuf *, int, int *); __private_extern__ unsigned int m_length(struct mbuf *); __private_extern__ unsigned int m_length2(struct mbuf *, struct mbuf **); __private_extern__ unsigned int m_fixhdr(struct mbuf *); __private_extern__ struct mbuf *m_defrag(struct mbuf *, int); __private_extern__ struct mbuf *m_defrag_offset(struct mbuf *, u_int32_t, int); __private_extern__ struct mbuf *m_prepend(struct mbuf *, int, int); __private_extern__ struct mbuf *m_copyup(struct mbuf *, int, int); __private_extern__ struct mbuf *m_retry(int, int); __private_extern__ struct mbuf *m_retryhdr(int, int); __private_extern__ int m_freem_list(struct mbuf *); __private_extern__ int m_append(struct mbuf *, int len, caddr_t __sized_by(len)); __private_extern__ struct mbuf *m_last(struct mbuf *); __private_extern__ struct mbuf *m_pulldown(struct mbuf *, int, int, int *); __private_extern__ struct mbuf *m_getcl(int, int, int); __private_extern__ char * __sized_by_or_null(MCLBYTES) m_mclalloc(int); __private_extern__ int m_mclhasreference(struct mbuf *); __private_extern__ void m_copy_pkthdr(struct mbuf *, struct mbuf *); __private_extern__ int m_dup_pkthdr(struct mbuf *, struct mbuf *, int); __private_extern__ void m_copy_pftag(struct mbuf *, struct mbuf *); __private_extern__ void m_copy_necptag(struct mbuf *, struct mbuf *); __private_extern__ void m_copy_classifier(struct mbuf *, struct mbuf *); __private_extern__ struct mbuf *m_dtom(void *); __private_extern__ int m_mtocl(void *); __private_extern__ union mcluster *m_cltom(int); __private_extern__ void m_align(struct mbuf *, int); __private_extern__ struct mbuf *m_normalize(struct mbuf *m); __private_extern__ void m_mchtype(struct mbuf *m, int t); __private_extern__ void m_copyback(struct mbuf *, int, int len, const void * __sized_by(len)); __private_extern__ struct mbuf *m_copyback_cow(struct mbuf *, int, int len, const void * __sized_by(len), int); __private_extern__ int m_makewritable(struct mbuf **, int, int, int); __private_extern__ struct mbuf *m_dup(struct mbuf *m, int how); __private_extern__ struct mbuf *m_copym_with_hdrs(struct mbuf *, int, int, int, struct mbuf **, int *, uint32_t); __private_extern__ struct mbuf *m_getpackethdrs(int, int); __private_extern__ struct mbuf *m_getpacket_how(int); __private_extern__ struct mbuf *m_getpackets_internal(unsigned int *, int, int, int, size_t); __private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t, unsigned int *, int, int, size_t); __private_extern__ int m_ext_set_prop(struct mbuf *, uint32_t, uint32_t); __private_extern__ uint32_t m_ext_get_prop(struct mbuf *); __private_extern__ int m_ext_paired_is_active(struct mbuf *); __private_extern__ void m_ext_paired_activate(struct mbuf *); __private_extern__ void m_add_crumb(struct mbuf *, uint16_t); __private_extern__ void m_add_hdr_crumb(struct mbuf *, uint64_t, uint64_t); __private_extern__ void m_add_hdr_crumb_chain(struct mbuf *, uint64_t, uint64_t); static inline void m_add_hdr_crumb_interface_output(mbuf_t m, int index, bool chain) { if (chain) { m_add_hdr_crumb_chain(m, index, CRUMB_INTERFACE_FLAG); } else { m_add_hdr_crumb(m, index, CRUMB_INTERFACE_FLAG); } } static inline void m_add_hdr_crumb_interface_input(mbuf_t m, int index, bool chain) { if (chain) { m_add_hdr_crumb_chain(m, index | CRUMB_INPUT_FLAG, CRUMB_INTERFACE_FLAG); } else { m_add_hdr_crumb(m, index | CRUMB_INPUT_FLAG, CRUMB_INTERFACE_FLAG); } } __private_extern__ void mbuf_drain(boolean_t); /* * Packets may have annotations attached by affixing a list of "packet * tags" to the pkthdr structure. Packet tags are dynamically allocated * semi-opaque data structures that have a fixed header (struct m_tag) * that specifies the size of the memory block and an <id,type> pair that * identifies it. The id identifies the module and the type identifies the * type of data for that module. The id of zero is reserved for the kernel. * * By default packet tags are allocated via kalloc except on Intel that still * uses the legacy implementation of using mbufs for packet tags. * * When kalloc is used for allocation, packet tags returned by m_tag_allocate have * the default memory alignment implemented by kalloc. * * When mbufs are used for allocation packets tag returned by m_tag_allocate has * the default memory alignment implemented by malloc. * * To reference the private data one should use a construct like: * struct m_tag *mtag = m_tag_allocate(...); * struct foo *p = (struct foo *)(mtag->m_tag_data); * * There should be no assumption on the location of the private data relative to the * 'struct m_tag' * * When kalloc is used, packet tags that are internal to xnu use KERNEL_MODULE_TAG_ID and * they are allocated with kalloc_type using a single container data structure that has * the 'struct m_tag' followed by a data structure for the private data * * Packet tags that are allocated by KEXTs are external to xnu and type of the private data * is unknown to xnu, so they are allocated in two chunks: * - one allocation with kalloc_type for the 'struct m_tag' * - one allocation using kheap_alloc as for the private data * * Note that packet tags of type KERNEL_TAG_TYPE_DRVAUX are allocated by KEXTs with * a variable length so they are allocated in two chunks * * In all cases the 'struct m_tag' is allocated using kalloc_type to avoid type * confusion. */ #define KERNEL_MODULE_TAG_ID 0 enum { KERNEL_TAG_TYPE_NONE = 0, KERNEL_TAG_TYPE_DUMMYNET = 1, KERNEL_TAG_TYPE_IPFILT = 2, KERNEL_TAG_TYPE_ENCAP = 3, KERNEL_TAG_TYPE_INET6 = 4, KERNEL_TAG_TYPE_IPSEC = 5, KERNEL_TAG_TYPE_CFIL_UDP = 6, KERNEL_TAG_TYPE_PF_REASS = 7, KERNEL_TAG_TYPE_AQM = 8, KERNEL_TAG_TYPE_DRVAUX = 9, KERNEL_TAG_TYPE_COUNT = 10 }; /* Packet tag routines */ __private_extern__ struct m_tag *m_tag_create(u_int32_t, u_int16_t, int, int, struct mbuf *); __private_extern__ void m_tag_free(struct m_tag *); __private_extern__ void m_tag_prepend(struct mbuf *, struct m_tag *); __private_extern__ void m_tag_unlink(struct mbuf *, struct m_tag *); __private_extern__ void m_tag_delete(struct mbuf *, struct m_tag *); __private_extern__ void m_tag_delete_chain(struct mbuf *); __private_extern__ struct m_tag *m_tag_locate(struct mbuf *, u_int32_t, u_int16_t); __private_extern__ struct m_tag *m_tag_copy(struct m_tag *, int); __private_extern__ int m_tag_copy_chain(struct mbuf *, struct mbuf *, int); __private_extern__ void m_tag_init(struct mbuf *, int); __private_extern__ struct m_tag *m_tag_first(struct mbuf *); __private_extern__ struct m_tag *m_tag_next(struct mbuf *, struct m_tag *); typedef struct m_tag * (*m_tag_kalloc_func_t)(u_int32_t id, u_int16_t type, uint16_t len, int wait); typedef void (*m_tag_kfree_func_t)(struct m_tag *tag); int m_register_internal_tag_type(uint16_t type, uint16_t len, m_tag_kalloc_func_t alloc_func, m_tag_kfree_func_t free_func); void m_tag_create_cookie(struct m_tag *); void mbuf_tag_init(void); __private_extern__ u_int32_t m_scratch_get(struct mbuf *, u_int8_t **); __private_extern__ void m_classifier_init(struct mbuf *, uint32_t); __private_extern__ int m_set_service_class(struct mbuf *, mbuf_svc_class_t); __private_extern__ mbuf_svc_class_t m_get_service_class(struct mbuf *); __private_extern__ mbuf_svc_class_t m_service_class_from_idx(u_int32_t); __private_extern__ mbuf_svc_class_t m_service_class_from_val(u_int32_t); __private_extern__ int m_set_traffic_class(struct mbuf *, mbuf_traffic_class_t); __private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *); __private_extern__ void mbuf_tag_init(void); #define ADDCARRY(_x) do { \ while (((_x) >> 16) != 0) \ (_x) = ((_x) >> 16) + ((_x) & 0xffff); \ } while (0) __private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t, u_int32_t, u_int32_t, u_int32_t); __private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t); __private_extern__ void mbuf_set_tx_time(struct mbuf *m, uint64_t tx_time); __private_extern__ struct ext_ref *m_get_rfa(struct mbuf *); __private_extern__ m_ext_free_func_t m_get_ext_free(struct mbuf *); __private_extern__ void m_do_tx_compl_callback(struct mbuf *, struct ifnet *); __private_extern__ mbuf_tx_compl_func m_get_tx_compl_callback(u_int32_t); __END_DECLS /* START - the following can be moved to uipc_mbuf.c once we got rid of CONFIG_MBUF_MCACHE */ typedef enum { MC_MBUF = 0, /* Regular mbuf */ MC_CL, /* Cluster */ MC_BIGCL, /* Large (4KB) cluster */ MC_16KCL, /* Jumbo (16KB) cluster */ MC_MBUF_CL, /* mbuf + cluster */ MC_MBUF_BIGCL, /* mbuf + large (4KB) cluster */ MC_MBUF_16KCL, /* mbuf + jumbo (16KB) cluster */ MC_MAX } mbuf_class_t; typedef struct { mbuf_class_t mtbl_class; /* class type */ #if CONFIG_MBUF_MCACHE mcache_t *mtbl_cache; /* mcache for this buffer class */ TAILQ_HEAD(mcl_slhead, mcl_slab) mtbl_slablist; /* slab list */ mcache_obj_t *mtbl_cobjlist; /* composite objects freelist */ #endif mb_class_stat_t *mtbl_stats; /* statistics fetchable via sysctl */ u_int32_t mtbl_maxsize; /* maximum buffer size */ int mtbl_minlimit; /* minimum allowed */ int mtbl_maxlimit; /* maximum allowed */ u_int32_t mtbl_wantpurge; /* purge during next reclaim */ uint32_t mtbl_avgtotal; /* average total on iOS */ u_int32_t mtbl_expand; /* worker should expand the class */ } mbuf_table_t; /* * Allocation statistics related to mbuf types (up to MT_MAX-1) are updated * atomically and stored in a per-CPU structure which is lock-free; this is * done in order to avoid writing to the global mbstat data structure which * would cause false sharing. During sysctl request for kern.ipc.mbstat, * the statistics across all CPUs will be converged into the mbstat.m_mtypes * array and returned to the application. Any updates for types greater or * equal than MT_MAX would be done atomically to the mbstat; this slows down * performance but is okay since the kernel uses only up to MT_MAX-1 while * anything beyond that (up to type 255) is considered a corner case. */ typedef struct { unsigned int cpu_mtypes[MT_MAX]; } mbuf_mtypes_t; #define MBUF_CLASS_MIN MC_MBUF #define MBUF_CLASS_MAX MC_MBUF_16KCL #define MBUF_CLASS_LAST MC_16KCL #define MBUF_CLASS_COMPOSITE(c) \ ((int)(c) > MBUF_CLASS_LAST) #define m_class(c) mbuf_table[c].mtbl_class #define m_maxsize(c) mbuf_table[c].mtbl_maxsize #define m_minlimit(c) mbuf_table[c].mtbl_minlimit #define m_maxlimit(c) mbuf_table[c].mtbl_maxlimit #define m_cname(c) mbuf_table[c].mtbl_stats->mbcl_cname #define m_size(c) mbuf_table[c].mtbl_stats->mbcl_size #define m_total(c) mbuf_table[c].mtbl_stats->mbcl_total #define m_infree(c) mbuf_table[c].mtbl_stats->mbcl_infree #define NELEM(a) (sizeof (a) / sizeof ((a)[0])) #define MB_WDT_MAXTIME 10 /* # of secs before watchdog panic */ /* * This flag is set for all mbufs that come out of and into the composite * mbuf + cluster caches, i.e. MC_MBUF_CL and MC_MBUF_BIGCL. mbufs that * are marked with such a flag have clusters attached to them, and will be * treated differently when they are freed; instead of being placed back * into the mbuf and cluster freelists, the composite mbuf + cluster objects * are placed back into the appropriate composite cache's freelist, and the * actual freeing is deferred until the composite objects are purged. At * such a time, this flag will be cleared from the mbufs and the objects * will be freed into their own separate freelists. */ #define EXTF_COMPOSITE 0x1 /* * This flag indicates that the external cluster is read-only, i.e. it is * or was referred to by more than one mbufs. Once set, this flag is never * cleared. */ #define EXTF_READONLY 0x2 /* * This flag indicates that the external cluster is paired with the mbuf. * Pairing implies an external free routine defined which will be invoked * when the reference count drops to the minimum at m_free time. This * flag is never cleared. */ #define EXTF_PAIRED 0x4 #define EXTF_MASK \ (EXTF_COMPOSITE | EXTF_READONLY | EXTF_PAIRED) #define MEXT_MINREF(m) ((m_get_rfa(m))->minref) #define MEXT_REF(m) ((m_get_rfa(m))->refcnt) #define MEXT_PREF(m) ((m_get_rfa(m))->prefcnt) #define MEXT_FLAGS(m) ((m_get_rfa(m))->flags) #define MEXT_PRIV(m) ((m_get_rfa(m))->priv) #define MEXT_PMBUF(m) ((m_get_rfa(m))->paired) #define MBUF_IS_COMPOSITE(m) \ (MEXT_REF(m) == MEXT_MINREF(m) && \ (MEXT_FLAGS(m) & EXTF_MASK) == EXTF_COMPOSITE) /* * This macro can be used to test if the mbuf is paired to an external * cluster. The test for MEXT_PMBUF being equal to the mbuf in subject * is important, as EXTF_PAIRED alone is insufficient since it is immutable, * and thus survives calls to m_free_paired. */ #define MBUF_IS_PAIRED(m) \ (((m)->m_flags & M_EXT) && \ (MEXT_FLAGS(m) & EXTF_MASK) == EXTF_PAIRED && \ MEXT_PMBUF(m) == (m)) #define MBUF_CL_INIT(m, buf, rfa, ref, flag) \ mext_init(m, buf, m_maxsize(MC_CL), NULL, NULL, rfa, 0, \ ref, 0, flag, 0, NULL) #define MBUF_BIGCL_INIT(m, buf, rfa, ref, flag) \ mext_init(m, buf, m_maxsize(MC_BIGCL), m_bigfree, NULL, rfa, 0, \ ref, 0, flag, 0, NULL) #define MBUF_16KCL_INIT(m, buf, rfa, ref, flag) \ mext_init(m, buf, m_maxsize(MC_16KCL), m_16kfree, NULL, rfa, 0, \ ref, 0, flag, 0, NULL) #define MBSTAT_MTYPES_MAX \ (sizeof (mbstat.m_mtypes) / sizeof (mbstat.m_mtypes[0])) #define mtype_stat_add(type, n) { \ if ((unsigned)(type) < MT_MAX) { \ mbuf_mtypes_t *mbs = PERCPU_GET(mbuf_mtypes); \ os_atomic_add(&mbs->cpu_mtypes[type], n, relaxed); \ } else if ((unsigned)(type) < (unsigned)MBSTAT_MTYPES_MAX) { \ os_atomic_add((int16_t *)&mbstat.m_mtypes[type], n, relaxed); \ } \ } #define mtype_stat_sub(t, n) mtype_stat_add(t, -(n)) #define mtype_stat_inc(t) mtype_stat_add(t, 1) #define mtype_stat_dec(t) mtype_stat_sub(t, 1) /* END - the following can be moved to uipc_mbuf.c once we got rid of CONFIG_MBUF_MCACHE */ #if CONFIG_MBUF_MCACHE extern lck_mtx_t *const mbuf_mlock; extern int nclusters; /* # of clusters for non-jumbo (legacy) sizes */ extern unsigned char *mbutl; /* start VA of mbuf pool */ extern unsigned int mb_memory_pressure_percentage; extern struct mb_stat *mb_stat; PERCPU_DECL(mbuf_mtypes_t, mbuf_mtypes); extern mbuf_table_t mbuf_table[]; extern void mbuf_mtypes_sync(void); extern void mbuf_stat_sync(void); extern void mbuf_table_init(void); extern void m_incref(struct mbuf *m); extern uint16_t m_decref(struct mbuf *m); extern struct mbuf *m_get_common(int wait, short type, int hdr); extern int m_free_paired(struct mbuf *m); extern caddr_t m_get_ext_arg(struct mbuf *m); extern int mbuf_watchdog_defunct_iterate(proc_t p, void *arg); extern void m_set_ext(struct mbuf *m, struct ext_ref *rfa, m_ext_free_func_t ext_free, caddr_t ext_arg); extern void mext_init(struct mbuf *m, void *__sized_by(size)buf, u_int size, m_ext_free_func_t free, caddr_t free_arg, struct ext_ref *rfa, u_int16_t min, u_int16_t ref, u_int16_t pref, u_int16_t flag, u_int32_t priv, struct mbuf *pm); extern int mbuf_get_class(struct mbuf *m); extern void mbuf_init(struct mbuf *m, int pkthdr, int type); extern void mbuf_mcheck(struct mbuf *m); #endif /* CONFIG_MBUF_MCACHE */ #endif /* XNU_KERNEL_PRIVATE */ #endif /* !_SYS_MBUF_H_ */ |