Index: conf/NOTES =================================================================== --- conf/NOTES (.../head/sys) (revision 192641) +++ conf/NOTES (.../projects/pnet/sys) (revision 192641) @@ -512,6 +512,11 @@ # NETWORKING OPTIONS # +# Support for parallel netisr threads. +# +options NETISR2 + +# # Protocol families # options INET #Internet communications protocols Index: conf/files =================================================================== --- conf/files (.../head/sys) (revision 192641) +++ conf/files (.../projects/pnet/sys) (revision 192641) @@ -2184,6 +2184,7 @@ net/mppcc.c optional netgraph_mppc_compression net/mppcd.c optional netgraph_mppc_compression net/netisr.c standard +net/netisr2.c optional netisr2 net/pfil.c optional ether | inet net/radix.c standard net/radix_mpath.c standard Index: conf/options =================================================================== --- conf/options (.../head/sys) (revision 192641) +++ conf/options (.../projects/pnet/sys) (revision 192641) @@ -406,6 +406,7 @@ MROUTING opt_mrouting.h NCP NETATALK opt_atalk.h +NETISR2 opt_netisr.h NFSLOCKD RADIX_MPATH opt_mpath.h ROUTETABLES opt_route.h Index: kern/kern_poll.c =================================================================== --- kern/kern_poll.c (.../head/sys) (revision 192641) +++ kern/kern_poll.c (.../projects/pnet/sys) (revision 192641) @@ -28,6 +28,7 @@ #include __FBSDID("$FreeBSD$"); +#include "opt_netisr.h" #include "opt_route.h" #include "opt_device_polling.h" @@ -45,11 +46,10 @@ #include /* for IFF_* flags */ #include /* for NETISR_POLL */ +#include #include #include -static void netisr_poll(void); /* the two netisr handlers */ -static void netisr_pollmore(void); static int poll_switch(SYSCTL_HANDLER_ARGS); void hardclock_device_poll(void); /* hook from hardclock */ @@ -110,6 +110,11 @@ SYSCTL_UINT(_kern_polling, OID_AUTO, burst, CTLFLAG_RD, &poll_burst, 0, "Current polling burst size"); +#ifdef NETISR2 +static int netisr_poll_scheduled; +static int netisr_pollmore_scheduled; +#endif + static int poll_burst_max_sysctl(SYSCTL_HANDLER_ARGS) { uint32_t val = poll_burst_max; @@ -264,8 +269,10 @@ { mtx_init(&poll_mtx, "polling", NULL, MTX_DEF); +#ifndef NETISR2 netisr_register(NETISR_POLL, (netisr_t *)netisr_poll, NULL, 0); netisr_register(NETISR_POLLMORE, (netisr_t *)netisr_pollmore, NULL, 0); +#endif } SYSINIT(device_poll, SI_SUB_CLOCKS, SI_ORDER_MIDDLE, init_device_poll, NULL); @@ -314,7 +321,13 @@ if (phase != 0) suspect++; phase = 1; +#ifdef NETISR2 + netisr_poll_scheduled = 1; + netisr_pollmore_scheduled = 1; + netisr2_sched_poll(); +#else schednetisrbits(1 << NETISR_POLL | 1 << NETISR_POLLMORE); +#endif phase = 2; } if (pending_polls++ > 0) @@ -365,9 +378,22 @@ int kern_load; mtx_lock(&poll_mtx); +#ifdef NETISR2 + if (!netisr_pollmore_scheduled) { + mtx_unlock(&poll_mtx); + return; + } + netisr_pollmore_scheduled = 0; +#endif phase = 5; if (residual_burst > 0) { +#ifdef NETISR2 + netisr_poll_scheduled = 1; + netisr_pollmore_scheduled = 1; + netisr2_sched_poll(); +#else schednetisrbits(1 << NETISR_POLL | 1 << NETISR_POLLMORE); +#endif mtx_unlock(&poll_mtx); /* will run immediately on return, followed by netisrs */ return; @@ -397,7 +423,13 @@ poll_burst -= (poll_burst / 8); if (poll_burst < 1) poll_burst = 1; +#ifdef NETISR2 + netisr_poll_scheduled = 1; + netisr_pollmore_scheduled = 1; + netisr2_sched_poll(); +#else schednetisrbits(1 << NETISR_POLL | 1 << NETISR_POLLMORE); +#endif phase = 6; } mtx_unlock(&poll_mtx); @@ -407,13 +439,20 @@ * netisr_poll is scheduled by schednetisr when appropriate, typically once * per tick. */ -static void +void netisr_poll(void) { int i, cycles; enum poll_cmd arg = POLL_ONLY; mtx_lock(&poll_mtx); +#ifdef NETISR2 + if (!netisr_poll_scheduled) { + mtx_unlock(&poll_mtx); + return; + } + netisr_poll_scheduled = 0; +#endif phase = 3; if (residual_burst == 0) { /* first call in this tick */ microuptime(&poll_start_t); Index: opencrypto/cryptodev.c =================================================================== --- opencrypto/cryptodev.c (.../head/sys) (revision 192641) +++ opencrypto/cryptodev.c (.../projects/pnet/sys) (revision 192641) @@ -409,10 +409,8 @@ cse->uio.uio_rw = UIO_WRITE; cse->uio.uio_td = td; cse->uio.uio_iov[0].iov_len = cop->len; - if (cse->thash) { + if (cse->thash) cse->uio.uio_iov[0].iov_len += cse->thash->hashsize; - cse->uio.uio_resid += cse->thash->hashsize; - } cse->uio.uio_iov[0].iov_base = malloc(cse->uio.uio_iov[0].iov_len, M_XDATA, M_WAITOK); Index: netatalk/ddp_usrreq.c =================================================================== --- netatalk/ddp_usrreq.c (.../head/sys) (revision 192641) +++ netatalk/ddp_usrreq.c (.../projects/pnet/sys) (revision 192641) @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2004-2005 Robert N. M. Watson + * Copyright (c) 2004-2009 Robert N. M. Watson * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -50,6 +50,8 @@ * $FreeBSD$ */ +#include "opt_netisr.h" + #include #include #include @@ -60,6 +62,7 @@ #include #include #include +#include #include #include @@ -70,8 +73,35 @@ static u_long ddp_sendspace = DDP_MAXSZ; /* Max ddp size + 1 (ddp_type) */ static u_long ddp_recvspace = 10 * (587 + sizeof(struct sockaddr_at)); +#ifdef NETISR2 +static const struct netisr_handler atalk1_nh = { + .nh_name = "atalk1", + .nh_handler = at1intr, + .nh_proto = NETISR_ATALK1, + .nh_qlimit = IFQ_MAXLEN, + .nh_policy = NETISR_POLICY_SOURCE, +}; + +static const struct netisr_handler atalk2_nh = { + .nh_name = "atalk2", + .nh_handler = at2intr, + .nh_proto = NETISR_ATALK2, + .nh_qlimit = IFQ_MAXLEN, + .nh_policy = NETISR_POLICY_SOURCE, +}; + +static const struct netisr_handler aarp_nh = { + .nh_name = "aarp", + .nh_handler = aarpintr, + .nh_proto = NETISR_AARP, + .nh_qlimit = IFQ_MAXLEN, + .nh_policy = NETISR_POLICY_SOURCE, +}; +#else static struct ifqueue atintrq1, atintrq2, aarpintrq; +#endif + static int ddp_attach(struct socket *so, int proto, struct thread *td) { @@ -256,16 +286,22 @@ ddp_init(void) { + DDP_LIST_LOCK_INIT(); +#ifdef NETISR2 + netisr2_register(&atalk1_nh); + netisr2_register(&atalk2_nh); + netisr2_register(&aarp_nh); +#else atintrq1.ifq_maxlen = IFQ_MAXLEN; atintrq2.ifq_maxlen = IFQ_MAXLEN; aarpintrq.ifq_maxlen = IFQ_MAXLEN; mtx_init(&atintrq1.ifq_mtx, "at1_inq", NULL, MTX_DEF); mtx_init(&atintrq2.ifq_mtx, "at2_inq", NULL, MTX_DEF); mtx_init(&aarpintrq.ifq_mtx, "aarp_inq", NULL, MTX_DEF); - DDP_LIST_LOCK_INIT(); netisr_register(NETISR_ATALK1, at1intr, &atintrq1, 0); netisr_register(NETISR_ATALK2, at2intr, &atintrq2, 0); netisr_register(NETISR_AARP, aarpintr, &aarpintrq, 0); +#endif } #if 0 Index: netnatm/natm_proto.c =================================================================== --- netnatm/natm_proto.c (.../head/sys) (revision 192641) +++ netnatm/natm_proto.c (.../projects/pnet/sys) (revision 192641) @@ -35,6 +35,8 @@ * protocol layer for access to native mode ATM */ +#include "opt_netisr.h" + #include __FBSDID("$FreeBSD$"); @@ -47,6 +49,7 @@ #include #include +#include #include @@ -88,8 +91,19 @@ .dom_protoswNPROTOSW = &natmsw[sizeof(natmsw)/sizeof(natmsw[0])], }; +#ifdef NETISR2 +static struct netisr_handler natm_nh = { + .nh_name = "natm", + .nh_handler = natmintr, + .nh_proto = NETISR_NATM, + .nh_qlimit = 1000 /* IFQ_MAXLEN */, + .nh_policy = NETISR_POLICY_SOURCE, +}; +#else static int natmqmaxlen = 1000 /* IFQ_MAXLEN */; /* max # of packets on queue */ static struct ifqueue natmintrq; +#endif + #ifdef NATM_STAT u_int natm_sodropcnt; /* # mbufs dropped due to full sb */ u_int natm_sodropbytes; /* # of bytes dropped */ @@ -101,11 +115,15 @@ natm_init(void) { LIST_INIT(&natm_pcbs); + NATM_LOCK_INIT(); +#ifdef NETISR2 + netisr2_register(&natm_nh); +#else bzero(&natmintrq, sizeof(natmintrq)); natmintrq.ifq_maxlen = natmqmaxlen; - NATM_LOCK_INIT(); mtx_init(&natmintrq.ifq_mtx, "natm_inq", NULL, MTX_DEF); netisr_register(NETISR_NATM, natmintr, &natmintrq, 0); +#endif } DOMAIN_SET(natm); Index: netinet/ip_divert.c =================================================================== --- netinet/ip_divert.c (.../head/sys) (revision 192641) +++ netinet/ip_divert.c (.../projects/pnet/sys) (revision 192641) @@ -42,6 +42,7 @@ #error "IPDIVERT requires IPFIREWALL" #endif #endif +#include "opt_netisr.h" #include #include @@ -66,6 +67,7 @@ #include #include +#include #include #include @@ -472,7 +474,11 @@ SOCK_UNLOCK(so); #endif /* Send packet to input processing via netisr */ +#ifdef NETISR2 + netisr2_queue_src(NETISR_IP, (uintptr_t)so, m); +#else netisr_queue(NETISR_IP, m); +#endif } return error; Index: netinet/ip_input.c =================================================================== --- netinet/ip_input.c (.../head/sys) (revision 192641) +++ netinet/ip_input.c (.../projects/pnet/sys) (revision 192641) @@ -38,6 +38,7 @@ #include "opt_ipsec.h" #include "opt_route.h" #include "opt_mac.h" +#include "opt_netisr.h" #include "opt_carp.h" #include @@ -63,6 +64,7 @@ #include #include #include +#include #include #include @@ -164,19 +166,32 @@ struct pfil_head inet_pfil_hook; /* Packet filter hooks */ +#ifdef NETISR2 +static struct mbuf *ip_input_m2flow(struct mbuf *m, uintptr_t source); +static struct netisr_handler ip_nh = { + .nh_name = "ip", + .nh_handler = ip_input, + .nh_m2flow = ip_input_m2flow, + .nh_proto = NETISR_IP, + .nh_qlimit = IFQ_MAXLEN, + .nh_policy = NETISR_POLICY_FLOW, +}; +#else static struct ifqueue ipintrq; static int ipqmaxlen = IFQ_MAXLEN; -extern struct domain inetdomain; -extern struct protosw inetsw[]; -u_char ip_protox[IPPROTO_MAX]; - SYSCTL_INT(_net_inet_ip, IPCTL_INTRQMAXLEN, intr_queue_maxlen, CTLFLAG_RW, &ipintrq.ifq_maxlen, 0, "Maximum size of the IP input queue"); SYSCTL_INT(_net_inet_ip, IPCTL_INTRQDROPS, intr_queue_drops, CTLFLAG_RD, &ipintrq.ifq_drops, 0, "Number of packets dropped from the IP input queue"); +#endif +extern struct domain inetdomain; +extern struct protosw inetsw[]; +u_char ip_protox[IPPROTO_MAX]; + + SYSCTL_V_STRUCT(V_NET, vnet_inet, _net_inet_ip, IPCTL_STATS, stats, CTLFLAG_RW, ipstat, ipstat, "IP statistics (struct ipstat, netinet/ip_var.h)"); @@ -249,6 +264,51 @@ SYSINIT(inet, SI_SUB_PROTO_BEGIN, SI_ORDER_FIRST, vnet_inet_register, 0); #endif +#ifdef NETISR2 +static int +sysctl_netinet_intr_queue_maxlen(SYSCTL_HANDLER_ARGS) +{ + int error, qlimit; + + netisr2_getqlimit(&ip_nh, &qlimit); + error = sysctl_handle_int(oidp, &qlimit, 0, req); + if (error || !req->newptr) + return (error); + if (qlimit < 1) + return (EINVAL); + return (netisr2_setqlimit(&ip_nh, qlimit)); +} +SYSCTL_PROC(_net_inet_ip, IPCTL_INTRQMAXLEN, intr_queue_maxlen, + CTLTYPE_INT|CTLFLAG_RW, 0, 0, sysctl_netinet_intr_queue_maxlen, "I", + "Maximum size of the IP input queue"); + +static int +sysctl_netinet_intr_queue_drops(SYSCTL_HANDLER_ARGS) +{ + u_int64_t qdrops_long; + int error, qdrops; + + netisr2_getqdrops(&ip_nh, &qdrops_long); + qdrops = qdrops_long; + error = sysctl_handle_int(oidp, &qdrops, 0, req); + if (error || !req->newptr) + return (error); + if (qdrops != 0) + return (EINVAL); + netisr2_clearqdrops(&ip_nh); + return (0); + +} +SYSCTL_PROC(_net_inet_ip, IPCTL_INTRQDROPS, intr_queue_drops, + CTLTYPE_INT|CTLFLAG_RD, 0, 0, sysctl_netinet_intr_queue_drops, "I", + "Number of packets dropped from the IP input queue"); + +static int ip_m2flow_enable = 1; +SYSCTL_INT(_net_inet_ip, OID_AUTO, m2flow_enable, CTLFLAG_RW, + &ip_m2flow_enable, 0, + "Enable software flow ID calculation for parallel netisr distribution"); +#endif + /* * IP initialization: fill in IP protocol switch table. * All protocols not implemented in kernel go to raw IP protocol handler. @@ -347,10 +407,13 @@ /* Initialize various other remaining things. */ IPQ_LOCK_INIT(); +#ifdef NETISR2 + netisr2_register(&ip_nh); +#else ipintrq.ifq_maxlen = ipqmaxlen; mtx_init(&ipintrq.ifq_mtx, "ip_inq", NULL, MTX_DEF); netisr_register(NETISR_IP, ip_input, &ipintrq, 0); - +#endif ip_ft = flowtable_alloc(ip_output_flowtable_size, FL_PCPU); } @@ -361,7 +424,58 @@ callout_stop(&ipport_tick_callout); } +#ifdef NETISR2 /* + * Calculate a flow ID for an IP packet if one isn't already present; this is + * a subset of the work done by ip_input() necessary to validate and read the + * IP header. We only do stats on the packet if we drop it -- otherwise, the + * normal input routine manages its statistics. + */ +static struct mbuf * +ip_input_m2flow(struct mbuf *m, uintptr_t source) +{ + struct ip *ip; + int hlen; + + M_ASSERTPKTHDR(m); + KASSERT(!(m->m_flags & M_FLOWID), + ("ip_input_m2flow: M_FLOWID already set")); + + if (!ip_m2flow_enable) + return (m); + + if (m->m_pkthdr.len < sizeof(struct ip)) { + IPSTAT_INC(ips_tooshort); + goto bad; + } + if (m->m_len < sizeof (struct ip) && + (m = m_pullup(m, sizeof(struct ip))) == NULL) { + IPSTAT_INC(ips_total); + IPSTAT_INC(ips_toosmall); + return (NULL); + } + ip = mtod(m, struct ip *); + if (ip->ip_v != IPVERSION) { + IPSTAT_INC(ips_badvers); + goto bad; + } + hlen = ip->ip_hl << 2; + if (hlen < sizeof(struct ip)) { + IPSTAT_INC(ips_badhlen); + goto bad; + } + m->m_flags |= M_FLOWID; + m->m_pkthdr.flowid = ip->ip_src.s_addr ^ ip->ip_dst.s_addr; + return (m); + +bad: + IPSTAT_INC(ips_total); + m_freem(m); + return (NULL); +} +#endif + +/* * Ip input routine. Checksum and byte swap header. If fragmented * try to reassemble. Process options. Pass to next level. */ Index: netinet/if_ether.c =================================================================== --- netinet/if_ether.c (.../head/sys) (revision 192641) +++ netinet/if_ether.c (.../projects/pnet/sys) (revision 192641) @@ -41,6 +41,7 @@ #include "opt_inet.h" #include "opt_route.h" #include "opt_mac.h" +#include "opt_netisr.h" #include "opt_carp.h" #include @@ -60,6 +61,7 @@ #include #include #include +#include #include #include #include @@ -96,8 +98,6 @@ SYSCTL_V_INT(V_NET, vnet_inet, _net_link_ether_inet, OID_AUTO, max_age, CTLFLAG_RW, arpt_keep, 0, "ARP entry lifetime in seconds"); -static struct ifqueue arpintrq; - SYSCTL_V_INT(V_NET, vnet_inet, _net_link_ether_inet, OID_AUTO, maxtries, CTLFLAG_RW, arp_maxtries, 0, "ARP resolution attempts before returning error"); @@ -118,6 +118,18 @@ static void in_arpinput(struct mbuf *); #endif +#ifdef NETISR2 +static const struct netisr_handler arp_nh = { + .nh_name = "arp", + .nh_handler = arpintr, + .nh_proto = NETISR_ARP, + .nh_qlimit = 50, + .nh_policy = NETISR_POLICY_SOURCE, +}; +#else +static struct ifqueue arpintrq; +#endif + #ifndef VIMAGE_GLOBALS static const vnet_modinfo_t vnet_arp_modinfo = { .vmi_id = VNET_MOD_ARP, @@ -823,8 +835,12 @@ arp_iattach(NULL); #endif +#ifdef NETISR2 + netisr2_register(&arp_nh); +#else arpintrq.ifq_maxlen = 50; mtx_init(&arpintrq.ifq_mtx, "arp_inq", NULL, MTX_DEF); netisr_register(NETISR_ARP, arpintr, &arpintrq, 0); +#endif } SYSINIT(arp, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY, arp_init, 0); Index: netinet/igmp.c =================================================================== --- netinet/igmp.c (.../head/sys) (revision 192641) +++ netinet/igmp.c (.../projects/pnet/sys) (revision 192641) @@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$"); #include "opt_mac.h" +#include "opt_netisr.h" #include "opt_route.h" #include @@ -68,6 +69,7 @@ #include #include +#include #include #include @@ -144,7 +146,22 @@ static vnet_attach_fn vnet_igmp_iattach; static vnet_detach_fn vnet_igmp_idetach; +#ifdef NETISR2 +static const struct netisr_handler igmp_nh = { + .nh_name = "igmp", + .nh_handler = igmp_intr, + .nh_proto = NETISR_IGMP, + .nh_qlimit = IFQ_MAXLEN, + .nh_policy = NETISR_POLICY_SOURCE, +}; +#else /* + * Global netisr output queue. + */ +struct ifqueue igmpoq; +#endif + +/* * System-wide globals. * * Unlocked access to these is OK, except for the global IGMP output @@ -190,11 +207,6 @@ MALLOC_DEFINE(M_IGMP, "igmp", "igmp state"); /* - * Global netisr output queue. - */ -struct ifqueue igmpoq; - -/* * VIMAGE-wide globals. * * The IGMPv3 timers themselves need to run per-image, however, @@ -3537,12 +3549,15 @@ IGMP_LOCK_INIT(); + m_raopt = igmp_ra_alloc(); + +#ifdef NETISR2 + netisr2_register(&igmp_nh); +#else mtx_init(&igmpoq.ifq_mtx, "igmpoq_mtx", NULL, MTX_DEF); IFQ_SET_MAXLEN(&igmpoq, IFQ_MAXLEN); - - m_raopt = igmp_ra_alloc(); - netisr_register(NETISR_IGMP, igmp_intr, &igmpoq, 0); +#endif } static void @@ -3551,8 +3566,12 @@ CTR1(KTR_IGMPV3, "%s: tearing down", __func__); +#ifdef NETISR2 + netisr2_unregister(&igmp_nh); +#else netisr_unregister(NETISR_IGMP); mtx_destroy(&igmpoq.ifq_mtx); +#endif m_free(m_raopt); m_raopt = NULL; Index: netipx/ipx_input.c =================================================================== --- netipx/ipx_input.c (.../head/sys) (revision 192641) +++ netipx/ipx_input.c (.../projects/pnet/sys) (revision 192641) @@ -62,6 +62,8 @@ * @(#)ipx_input.c */ +#include "opt_netisr.h" + #include __FBSDID("$FreeBSD$"); @@ -77,6 +79,7 @@ #include #include #include +#include #include #include @@ -119,8 +122,18 @@ struct ipxpcbhead ipxpcb_list; struct ipxpcbhead ipxrawpcb_list; +#ifdef NETISR2 +static const struct netisr_handler ipx_nh = { + .nh_name = "ipx", + .nh_handler = ipxintr, + .nh_proto = NETISR_IPX, + .nh_qlimit = IFQ_MAXLEN, + .nh_policy = NETISR_POLICY_SOURCE, +}; +#else static int ipxqmaxlen = IFQ_MAXLEN; static struct ifqueue ipxintrq; +#endif long ipx_pexseq; /* Locked with ipxpcb_list_mtx. */ @@ -151,9 +164,14 @@ ipx_hostmask.sipx_addr.x_net = ipx_broadnet; ipx_hostmask.sipx_addr.x_host = ipx_broadhost; +#ifdef NETISR2 + ipx_nh.nh_qlimit = ipxqmaxlen; + netisr2_register(&ipx_nh); +#else ipxintrq.ifq_maxlen = ipxqmaxlen; mtx_init(&ipxintrq.ifq_mtx, "ipx_inq", NULL, MTX_DEF); netisr_register(NETISR_IPX, ipxintr, &ipxintrq, 0); +#endif } /* Index: net/netisr.c =================================================================== --- net/netisr.c (.../head/sys) (revision 192641) +++ net/netisr.c (.../projects/pnet/sys) (revision 192641) @@ -28,6 +28,7 @@ */ #include "opt_device_polling.h" +#include "opt_netisr.h" #include #include @@ -152,6 +153,7 @@ /* * Call the netisr directly instead of queueing the packet, if possible. */ +#ifndef NETISR2 void netisr_dispatch(int num, struct mbuf *m) { @@ -208,6 +210,7 @@ schednetisr(num); return (0); } +#endif /* !NETISR2 */ static void swi_net(void *dummy) Index: net/netisr2.c =================================================================== --- net/netisr2.c (.../head/sys) (revision 0) +++ net/netisr2.c (.../projects/pnet/sys) (revision 192641) @@ -0,0 +1,1121 @@ +/*- + * Copyright (c) 2007-2009 Robert N. M. Watson + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +/* + * netisr2 is a packet dispatch service, allowing synchronous (directly + * dispatched) and asynchronous (deferred dispatch) processing of packets by + * registered protocol handlers. Callers pass a protocol identifier and + * packet to netisr2, along with a direct dispatch hint, and work will either + * be immediately processed with the registered handler, or passed to a + * kernel software interrupt (SWI) thread for deferred dispatch. + * + * Maintaining ordering for protocol streams is a critical design concern. + * Enforcing ordering limits the opportunity for concurrency, but maintains + * the strong ordering requirements found in some protocols, such as TCP. Of + * related concern is CPU affinity--it is desirable to process all data + * associated with a particular stream on the same CPU over time in order to + * avoid acquiring locks associated with the connection on different CPUs, + * keep connection data in one cache, and to generally encourage associated + * user threads to live on the same CPU as the stream. It's also desirable + * to avoid lock migration and contention where locks are associated with + * more than one flow. + * + * netisr2 supports several policy variations, represented by the + * NETISR_POLICY_* constants, allowing protocols to play a varying role in + * identifying flows, assigning work to CPUs, etc. These are described in + * detail in netisr2.h. + */ + +#include "opt_ddb.h" +#include "opt_device_polling.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef DDB +#include +#endif + +#include +#include +#include +#include + +/*- + * Synchronize use and modification of the registered netisr data structures; + * acquire a read lock while modifying the set of registered protocols to + * prevent partially registered or unregistered protocols from being run. + * + * The following data structures and fields are protected by this lock: + * + * - The np array, including all fields of struct netisr_proto. + * - The nws array, including all fields of struct netisr_worker. + * - The nws_array array. + * + * XXXRW: rmlocks don't support assertions. + */ +static struct rmlock netisr_rmlock; +#define NETISR_LOCK_INIT() rm_init(&netisr_rmlock, "netisr", 0) +#if 0 +#define NETISR_LOCK_ASSERT() rm_assert(&netisr_rmlock, RW_LOCKED) +#else +#define NETISR_LOCK_ASSERT() +#endif +#define NETISR_RLOCK(tracker) rm_rlock(&netisr_rmlock, (tracker)) +#define NETISR_RUNLOCK(tracker) rm_runlock(&netisr_rmlock, (tracker)) +#define NETISR_WLOCK() rm_wlock(&netisr_rmlock) +#define NETISR_WUNLOCK() rm_wunlock(&netisr_rmlock) + +SYSCTL_NODE(_net, OID_AUTO, isr2, CTLFLAG_RW, 0, "netisr2"); + +/*- + * Four direct dispatch policies are supported: + * + * - Always defer: all work is scheduled for a netisr, regardless of context. + * (direct_enable == 0) + * + * - Always direct: if the executing context allows direct dispatch, always + * direct dispatch. + * (direct_enable != 0 && direct_force != 0) + * + * - Hybrid: if the executing context allows direct dispatch, and we're + * running on the CPU the work would be done on, then direct dispatch if it + * wouldn't violate ordering constraints on the workstream. + * (direct_enable != 0 && direct_force == 0 && hybridxcpu_enable == 0) + * + * - Hybrid with cross-CPU dispatch: if the executing context allows direct + * dispatch, then direct dispatch if it wouldn't violate ordering + * constraints on the workstream. + * (direct_enable != 0 && direct_force == 0 && hybridxcpu_enable != 0) + * + * Notice that changing the global policy could lead to short periods of + * disordered processing, but this is considered acceptable as compared to + * the complexity of enforcing ordering during policy changes. + */ +static int netisr_direct_force = 1; /* Always direct dispatch. */ +SYSCTL_INT(_net_isr2, OID_AUTO, direct_force, CTLFLAG_RW, + &netisr_direct_force, 0, "Force direct dispatch"); + +static int netisr_direct_enable = 1; /* Enable direct dispatch. */ +SYSCTL_INT(_net_isr2, OID_AUTO, direct_enable, CTLFLAG_RW, + &netisr_direct_enable, 0, "Enable direct dispatch"); + +static int netisr_hybridxcpu_enable = 1; /* Enable cross-CPU dispatch. */ +SYSCTL_INT(_net_isr2, OID_AUTO, hybridxcpu_enable, CTLFLAG_RW, + &netisr_hybridxcpu_enable, 0, "Enable cross-CPU hybrid direct dispatch."); + +/* + * Allow the administrator to limit the number of threads (CPUs) to use for + * netisr2. Notice that we don't check netisr_maxthreads before creating the + * thread for CPU 0, so in practice we ignore values <= 1. This must be set + * as a tunable, no run-time reconfiguration yet. We will create at most one + * thread per available CPU. + */ +static int netisr_maxthreads = 1; /* Max number of threads. */ +TUNABLE_INT("net.isr2.maxthreads", &netisr_maxthreads); +SYSCTL_INT(_net_isr2, OID_AUTO, maxthreads, CTLFLAG_RD, + &netisr_maxthreads, 0, + "Use at most this many CPUs for netisr2 processing"); + +static int netisr_bindthreads = 0; /* Bind threads to CPUs. */ +TUNABLE_INT("net.isr2.bindthreads", &netisr_bindthreads); +SYSCTL_INT(_net_isr2, OID_AUTO, bindthreads, CTLFLAG_RD, + &netisr_bindthreads, 0, "Bind netisr2 threads to CPUs."); + +/* + * Limit per-workstream queues to at most net.isr2.maxqlimit, both for + * initial configuration and later modification using netisr2_setqlimit(). + */ +#define NETISR_DEFAULT_MAXQLIMIT 10240 +static int netisr_maxqlimit = NETISR_DEFAULT_MAXQLIMIT; +TUNABLE_INT("net.isr2.maxqlimit", &netisr_maxqlimit); +SYSCTL_INT(_net_isr2, OID_AUTO, maxqlimit, CTLFLAG_RW, + &netisr_maxqlimit, 0, + "Maximum netisr2 per-protocol, per-CPU queue depth."); + +/* + * Each protocol is described by an instance of netisr_proto, which holds all + * global per-protocol information. This data structure is set up by + * netisr_register(), and derived from the public struct netisr_handler. + */ +struct netisr_proto { + const char *np_name; /* Character string protocol name. */ + netisr_t *np_handler; /* Protocol handler. */ + netisr_m2flow_t *np_m2flow; /* Query flow for untagged packet. */ + netisr_m2cpu_t *np_m2cpu; /* Query CPU to process packet on. */ + u_int np_qlimit; /* Maximum per-CPU queue depth. */ + u_int np_policy; /* Work placement policy. */ +}; + +#define NETISR_MAXPROT 32 /* Compile-time limit. */ +#define NETISR_ALLPROT 0xffffffff /* Run all protocols. */ + +/* + * The np array describes all registered protocols, indexed by protocol + * number. + */ +static struct netisr_proto np[NETISR_MAXPROT]; + +/* + * Protocol-specific work for each workstream is described by struct + * netisr_work. Each work descriptor consists of an mbuf queue and + * statistics. + * + * XXXRW: Using a lock-free linked list here might be useful. + */ +struct netisr_work { + /* + * Packet queue, linked by m_nextpkt. + */ + struct mbuf *nw_head; + struct mbuf *nw_tail; + u_int nw_len; + u_int nw_qlimit; + u_int nw_watermark; + + /* + * Statistics -- written unlocked, but mostly from curcpu. + */ + u_int64_t nw_dispatched; /* Number of direct dispatches. */ + u_int64_t nw_hybrid_dispatched; /* "" hybrid dispatches. */ + u_int64_t nw_hybrid_xcpudispatched; /* "" cross-CPU hybrid. */ + u_int64_t nw_qdrops; /* "" drops. */ + u_int64_t nw_queued; /* "" enqueues. */ + u_int64_t nw_handled; /* "" handled in worker. */ +}; + +/* + * Workstreams hold a set of ordered work across each protocol, and are + * described by netisr_workstream. Each workstream is associated with a + * worker thread, which in turn is pinned to a CPU. Work associated with a + * workstream can be processd in other threads during direct dispatch; + * concurrent processing is prevented by the NWS_RUNNING flag, which + * indicates that a thread is already processing the work queue. + * + * #workstreams must be <= #CPUs. + */ +struct netisr_workstream { + struct intr_event *nws_intr_event; /* Handler for stream. */ + void *nws_swi_cookie; /* swi(9) cookie for stream. */ + struct mtx nws_mtx; /* Synchronize work. */ + u_int nws_cpu; /* CPU pinning. */ + u_int nws_flags; /* Wakeup flags. */ + u_int nws_pendingwork; /* Across all protos. */ + + /* + * Each protocol has per-workstream data. + */ + struct netisr_work nws_work[NETISR_MAXPROT]; +} __aligned(CACHE_LINE_SIZE); + +/* + * Per-CPU workstream data, indexed by CPU ID. + */ +static struct netisr_workstream nws[MAXCPU]; + +/* + * Map contiguous values between 0 and nws_count into CPU IDs appropriate for + * indexing the nws[] array. This allows constructions of the form + * nws[nws_array(arbitraryvalue % nws_count)]. + */ +static u_int nws_array[MAXCPU]; + +/* + * Number of registered workstreams. Should be the number of running CPUs + * once fully started. + */ +static u_int nws_count; + +/* + * Per-workstream flags. + */ +#define NWS_RUNNING 0x00000001 /* Currently running in a thread. */ +#define NWS_DISPATCHING 0x00000002 /* Currently being direct-dispatched. */ +#define NWS_SCHEDULED 0x00000004 /* Signal issued. */ + +/* + * Synchronization for each workstream: a mutex protects all mutable fields + * in each stream, including per-protocol state (mbuf queues). The SWI is + * woken up if asynchronous dispatch is required. + */ +#define NWS_LOCK(s) mtx_lock(&(s)->nws_mtx) +#define NWS_LOCK_ASSERT(s) mtx_assert(&(s)->nws_mtx, MA_OWNED) +#define NWS_UNLOCK(s) mtx_unlock(&(s)->nws_mtx) +#define NWS_SIGNAL(s) swi_sched((s)->nws_swi_cookie, 0) + +/* + * Utility routines for protocols that implement their own mapping of flows + * to CPUs. + */ +u_int +netisr2_get_cpucount(void) +{ + + return (nws_count); +} + +u_int +netisr2_get_cpuid(u_int cpunumber) +{ + + KASSERT(cpunumber < nws_count, ("netisr2_get_cpuid: %u > %u", + cpunumber, nws_count)); + + return (nws_array[cpunumber]); +} + +/* + * The default implementation of -> CPU ID mapping. + * + * Non-static so that protocols can use it to map their own work to specific + * CPUs in a manner consistent to netisr2 for affinity purposes. + */ +u_int +netisr2_default_flow2cpu(u_int flowid) +{ + + return (nws_array[flowid % nws_count]); +} + +/* + * Register a new netisr handler, which requires initializing per-protocol + * fields for each workstream. All netisr2 work is briefly suspended while + * the protocol is installed. + */ +void +netisr2_register(const struct netisr_handler *nhp) +{ + struct netisr_work *npwp; + const char *name; + u_int i, proto; + + proto = nhp->nh_proto; + name = nhp->nh_name; + KASSERT(proto < NETISR_MAXPROT, + ("netisr2_register(%d, %s): protocol too big", proto, name)); + NETISR_WLOCK(); + + /* + * Test that no existing registration exists for this protocol. + */ + KASSERT(np[proto].np_name == NULL, + ("netisr2_register(%d, %s): name present", proto, name)); + KASSERT(np[proto].np_handler == NULL, + ("netisr2_register(%d, %s): handler present", proto, name)); + + /* + * Test that the requested registration is valid. + */ + KASSERT(nhp->nh_name != NULL, + ("netisr2_register: nh_name NULL for %d", proto)); + KASSERT(nhp->nh_handler != NULL, + ("netisr2_register: nh_handler NULL for %s", name)); + KASSERT(nhp->nh_policy == NETISR_POLICY_SOURCE || + nhp->nh_policy == NETISR_POLICY_FLOW || + nhp->nh_policy == NETISR_POLICY_CPU, + ("netisr2_register: unsupported nh_policy %u for %s", + nhp->nh_policy, name)); + KASSERT(nhp->nh_policy == NETISR_POLICY_FLOW || + nhp->nh_m2flow == NULL, + ("netisr2_register: nh_policy != FLOW but m2flow defined for %s", + name)); + KASSERT(nhp->nh_policy == NETISR_POLICY_CPU || nhp->nh_m2cpu == NULL, + ("netisr2_register: nh_policy != CPU but m2cpu defined for %s", + name)); + KASSERT(nhp->nh_policy != NETISR_POLICY_CPU || nhp->nh_m2cpu != NULL, + ("netisr2_register: nh_policy == CPU but m2cpu not defined for " + "%s", name)); + KASSERT(nhp->nh_qlimit != 0, + ("netisr2_register: nh_qlimit 0 for %s", name)); + + /* + * Initialize global and per-workstream protocol state. + */ + np[proto].np_name = name; + np[proto].np_handler = nhp->nh_handler; + np[proto].np_m2flow = nhp->nh_m2flow; + np[proto].np_m2cpu = nhp->nh_m2cpu; + if (nhp->nh_qlimit > netisr_maxqlimit) { + printf("netisr2_register: %s requested queue limit %u " + "capped to net.isr2.maxqlimit %u\n", name, + nhp->nh_qlimit, netisr_maxqlimit); + np[proto].np_qlimit = netisr_maxqlimit; + } else + np[proto].np_qlimit = nhp->nh_qlimit; + np[proto].np_policy = nhp->nh_policy; + for (i = 0; i < MAXCPU; i++) { + npwp = &nws[i].nws_work[proto]; + bzero(npwp, sizeof(*npwp)); + npwp->nw_qlimit = nhp->nh_qlimit; + } + NETISR_WUNLOCK(); +} + +/* + * Clear drop counters across all workstreams for a protocol. + */ +void +netisr2_clearqdrops(const struct netisr_handler *nhp) +{ + struct netisr_work *npwp; +#ifdef INVARIANTS + const char *name; +#endif + u_int i, proto; + + proto = nhp->nh_proto; +#ifdef INVARIANTS + name = nhp->nh_name; +#endif + KASSERT(proto < NETISR_MAXPROT, + ("netisr_clearqdrops(%d): protocol too big for %s", proto, name)); + NETISR_WLOCK(); + KASSERT(np[proto].np_handler != NULL, + ("netisr_clearqdrops(%d): protocol not registered for %s", proto, + name)); + + for (i = 0; i < MAXCPU; i++) { + npwp = &nws[i].nws_work[proto]; + npwp->nw_qdrops = 0; + } + NETISR_WUNLOCK(); +} + +/* + * Query the current drop counters across all workstreams for a protocol. + */ +void +netisr2_getqdrops(const struct netisr_handler *nhp, u_int64_t *qdropp) +{ + struct netisr_work *npwp; + struct rm_priotracker tracker; +#ifdef INVARIANTS + const char *name; +#endif + u_int i, proto; + + *qdropp = 0; + proto = nhp->nh_proto; +#ifdef INVARIANTS + name = nhp->nh_name; +#endif + KASSERT(proto < NETISR_MAXPROT, + ("netisr_getqdrops(%d): protocol too big for %s", proto, name)); + NETISR_RLOCK(&tracker); + KASSERT(np[proto].np_handler != NULL, + ("netisr_getqdrops(%d): protocol not registered for %s", proto, + name)); + + for (i = 0; i < MAXCPU; i++) { + npwp = &nws[i].nws_work[proto]; + *qdropp += npwp->nw_qdrops; + } + NETISR_RUNLOCK(&tracker); +} + +/* + * Query the current queue limit for per-workstream queues for a protocol. + */ +void +netisr2_getqlimit(const struct netisr_handler *nhp, u_int *qlimitp) +{ + struct rm_priotracker tracker; +#ifdef INVARIANTS + const char *name; +#endif + u_int proto; + + proto = nhp->nh_proto; +#ifdef INVARIANTS + name = nhp->nh_name; +#endif + KASSERT(proto < NETISR_MAXPROT, + ("netisr_getqlimit(%d): protocol too big for %s", proto, name)); + NETISR_RLOCK(&tracker); + KASSERT(np[proto].np_handler != NULL, + ("netisr_getqlimit(%d): protocol not registered for %s", proto, + name)); + *qlimitp = np[proto].np_qlimit; + NETISR_RUNLOCK(&tracker); +} + +/* + * Update the queue limit across per-workstream queues for a protocol. We + * simply change the limits, and don't drain overflowed packets as they will + * (hopefully) take care of themselves shortly. + */ +int +netisr2_setqlimit(const struct netisr_handler *nhp, u_int qlimit) +{ + struct netisr_work *npwp; +#ifdef INVARIANTS + const char *name; +#endif + u_int i, proto; + + if (qlimit > netisr_maxqlimit) + return (EINVAL); + + proto = nhp->nh_proto; +#ifdef INVARIANTS + name = nhp->nh_name; +#endif + KASSERT(proto < NETISR_MAXPROT, + ("netisr_setqlimit(%d): protocol too big for %s", proto, name)); + NETISR_WLOCK(); + KASSERT(np[proto].np_handler != NULL, + ("netisr_setqlimit(%d): protocol not registered for %s", proto, + name)); + + np[proto].np_qlimit = qlimit; + for (i = 0; i < MAXCPU; i++) { + npwp = &nws[i].nws_work[proto]; + npwp->nw_qlimit = qlimit; + } + NETISR_WUNLOCK(); + return (0); +} + +/* + * Drain all packets currently held in a particular protocol work queue. + */ +static void +netisr2_drain_proto(struct netisr_work *npwp) +{ + struct mbuf *m; + + while ((m = npwp->nw_head) != NULL) { + npwp->nw_head = m->m_nextpkt; + m->m_nextpkt = NULL; + if (npwp->nw_head == NULL) + npwp->nw_tail = NULL; + npwp->nw_len--; + m_freem(m); + } + KASSERT(npwp->nw_tail == NULL, ("netisr_drain_proto: tail")); + KASSERT(npwp->nw_len == 0, ("netisr_drain_proto: len")); +} + +/* + * Remove the registration of a network protocol, which requires clearing + * per-protocol fields across all workstreams, including freeing all mbufs in + * the queues at time of unregister. All work in netisr2 is briefly + * suspended while this takes place. + */ +void +netisr2_unregister(const struct netisr_handler *nhp) +{ + struct netisr_work *npwp; +#ifdef INVARIANTS + const char *name; +#endif + u_int i, proto; + + proto = nhp->nh_proto; +#ifdef INVARIANTS + name = nhp->nh_name; +#endif + KASSERT(proto < NETISR_MAXPROT, + ("netisr_unregister(%d): protocol too big for %s", proto, name)); + NETISR_WLOCK(); + KASSERT(np[proto].np_handler != NULL, + ("netisr_unregister(%d): protocol not registered for %s", proto, + name)); + + np[proto].np_name = NULL; + np[proto].np_handler = NULL; + np[proto].np_m2flow = NULL; + np[proto].np_m2cpu = NULL; + np[proto].np_qlimit = 0; + np[proto].np_policy = 0; + for (i = 0; i < MAXCPU; i++) { + npwp = &nws[i].nws_work[proto]; + netisr2_drain_proto(npwp); + bzero(npwp, sizeof(*npwp)); + } + NETISR_WUNLOCK(); +} + +/* + * Look up the workstream given a packet and source identifier. Do this by + * checking the protocol's policy, and optionally call out to the protocol + * for assistance if required. + */ +static struct mbuf * +netisr2_selectcpu(struct netisr_proto *npp, uintptr_t source, struct mbuf *m, + u_int *cpuidp) +{ + struct ifnet *ifp; + + NETISR_LOCK_ASSERT(); + + /* + * In the event we have only one worker, shortcut and deliver to it + * without further ado. + */ + if (nws_count == 1) { + *cpuidp = nws_array[0]; + return (m); + } + + /* + * What happens next depends on the policy selected by the protocol. + * If we want to support per-interface policies, we should do that + * here first. + */ + switch (npp->np_policy) { + case NETISR_POLICY_CPU: + return (npp->np_m2cpu(m, source, cpuidp)); + + case NETISR_POLICY_FLOW: + if (!(m->m_flags & M_FLOWID) && npp->np_m2flow != NULL) { + m = npp->np_m2flow(m, source); + if (m == NULL) + return (NULL); + } + if (m->m_flags & M_FLOWID) { + *cpuidp = + netisr2_default_flow2cpu(m->m_pkthdr.flowid); + return (m); + } + /* FALLTHROUGH */ + + case NETISR_POLICY_SOURCE: + ifp = m->m_pkthdr.rcvif; + if (ifp != NULL) + *cpuidp = (ifp->if_index + source) % nws_count; + else + *cpuidp = source % nws_count; + return (m); + + default: + panic("netisr2_selectcpu: invalid policy %u for %s", + npp->np_policy, npp->np_name); + } +} + +/* + * Process packets associated with a workstream and protocol. For reasons of + * fairness, we process up to one complete netisr queue at a time, moving the + * queue to a stack-local queue for processing, but do not loop refreshing + * from the global queue. The caller is responsible for deciding whether to + * loop, and for setting the NWS_RUNNING flag. The passed workstream will be + * locked on entry and relocked before return, but will be released while + * processing. + */ +static void +netisr2_process_workstream_proto(struct netisr_workstream *nwsp, int proto) +{ + struct netisr_work local_npw, *npwp; + u_int handled; + struct mbuf *m; + + NWS_LOCK_ASSERT(nwsp); + + KASSERT(nwsp->nws_flags & NWS_RUNNING, + ("netisr_process_workstream_proto(%d): not running", proto)); + KASSERT(proto >= 0 && proto < NETISR_MAXPROT, + ("netisr_process_workstream_proto(%d): invalid proto\n", proto)); + + npwp = &nwsp->nws_work[proto]; + if (npwp->nw_len == 0) + return; + + /* + * Move the global work queue to a thread-local work queue. + * + * Notice that this means the effective maximum length of the queue + * is actually twice that of the maximum queue length specified in + * the protocol registration call. + */ + handled = npwp->nw_len; + local_npw = *npwp; + npwp->nw_head = NULL; + npwp->nw_tail = NULL; + npwp->nw_len = 0; + nwsp->nws_pendingwork -= handled; + NWS_UNLOCK(nwsp); + while ((m = local_npw.nw_head) != NULL) { + local_npw.nw_head = m->m_nextpkt; + m->m_nextpkt = NULL; + if (local_npw.nw_head == NULL) + local_npw.nw_tail = NULL; + local_npw.nw_len--; + np[proto].np_handler(m); + } + KASSERT(local_npw.nw_len == 0, + ("netisr_process_proto(%d): len %d", proto, local_npw.nw_len)); + NWS_LOCK(nwsp); + npwp->nw_handled += handled; +} + +/* + * Process either one or all protocols associated with a specific workstream. + * Handle only existing work for each protocol processed, not new work that + * may arrive while processing. Set the running flag so that other threads + * don't also try to process work in the queue; however, the lock on the + * workstream will be released by netisr_process_workstream_proto() while + * entering the protocol so that producers can continue to queue new work. + * + * The consumer is responsible for making sure that either all available work + * is performed until there is no more work to perform, or that the worker is + * scheduled to pick up where the consumer left off. They are also + * responsible for checking the running flag before entering this function. + */ +static void +netisr2_process_workstream(struct netisr_workstream *nwsp, int proto) +{ + u_int i; + + NETISR_LOCK_ASSERT(); + NWS_LOCK_ASSERT(nwsp); + + KASSERT(nwsp->nws_flags & NWS_RUNNING, + ("netisr2_process_workstream: not running")); + KASSERT(!(nwsp->nws_flags & NWS_DISPATCHING), + ("netisr2_process_workstream: dispatching")); + if (proto == NETISR_ALLPROT) { + for (i = 0; i < NETISR_MAXPROT; i++) + netisr2_process_workstream_proto(nwsp, i); + } else + netisr2_process_workstream_proto(nwsp, proto); +} + +/* + * SWI handler for netisr2 -- processes prackets in a set of workstreams that + * it owns, woken up by calls to NWS_SIGNAL(). If this workstream is already + * being direct dispatched, go back to sleep and wait for the dispatching + * thread to wake us up again. + */ +static void +swi_net(void *arg) +{ + struct rm_priotracker tracker; + struct netisr_workstream *nwsp; + + nwsp = arg; + +#ifdef DEVICE_POLLING + KASSERT(nws_count == 1, + ("swi_net: device_polling but nws_count != 1")); + netisr_poll(); +#endif + + NETISR_RLOCK(&tracker); + NWS_LOCK(nwsp); + KASSERT(!(nwsp->nws_flags & NWS_RUNNING), ("swi_net: running")); + if (nwsp->nws_flags & NWS_DISPATCHING) + goto out; + nwsp->nws_flags |= NWS_RUNNING; + nwsp->nws_flags &= ~NWS_SCHEDULED; + while (nwsp->nws_pendingwork != 0) + netisr2_process_workstream(nwsp, NETISR_ALLPROT); + nwsp->nws_flags &= ~NWS_RUNNING; +out: + NWS_UNLOCK(nwsp); + NETISR_RUNLOCK(&tracker); + +#ifdef DEVICE_POLLING + netisr_pollmore(); +#endif +} + +static int +netisr2_queue_workstream(struct netisr_workstream *nwsp, + struct netisr_work *npwp, struct mbuf *m, int *dosignalp) +{ + + NWS_LOCK_ASSERT(nwsp); + + *dosignalp = 0; + if (npwp->nw_len < npwp->nw_qlimit) { + m->m_nextpkt = NULL; + if (npwp->nw_head == NULL) { + npwp->nw_head = m; + npwp->nw_tail = m; + } else { + npwp->nw_tail->m_nextpkt = m; + npwp->nw_tail = m; + } + npwp->nw_len++; + if (npwp->nw_len > npwp->nw_watermark) + npwp->nw_watermark = npwp->nw_len; + nwsp->nws_pendingwork++; + if (!(nwsp->nws_flags & (NWS_SCHEDULED | NWS_RUNNING))) { + nwsp->nws_flags |= NWS_SCHEDULED; + *dosignalp = 1; /* Defer until unlocked. */ + } + npwp->nw_queued++; + return (0); + } else { + npwp->nw_qdrops++; + return (ENOBUFS); + } +} + +static int +netisr2_queue_internal(u_int proto, struct mbuf *m, u_int cpuid) +{ + struct netisr_workstream *nwsp; + struct netisr_work *npwp; + int dosignal, error; + + NETISR_LOCK_ASSERT(); + + dosignal = 0; + error = 0; + nwsp = &nws[cpuid]; + npwp = &nwsp->nws_work[proto]; + NWS_LOCK(nwsp); + error = netisr2_queue_workstream(nwsp, npwp, m, &dosignal); + NWS_UNLOCK(nwsp); + if (dosignal) + NWS_SIGNAL(nwsp); + return (error); +} + +int +netisr2_queue_src(u_int proto, uintptr_t source, struct mbuf *m) +{ + struct rm_priotracker tracker; + u_int cpuid, error; + + KASSERT(proto < NETISR_MAXPROT, + ("netisr2_queue_src: invalid proto %d", proto)); + + NETISR_RLOCK(&tracker); + KASSERT(np[proto].np_handler != NULL, + ("netisr2_queue_src: invalid proto %d", proto)); + + m = netisr2_selectcpu(&np[proto], source, m, &cpuid); + if (m != NULL) + error = netisr2_queue_internal(proto, m, cpuid); + else + error = ENOBUFS; + NETISR_RUNLOCK(&tracker); + return (error); +} + +int +netisr2_queue(u_int proto, struct mbuf *m) +{ + + return (netisr2_queue_src(proto, 0, m)); +} + +int +netisr_queue(int proto, struct mbuf *m) +{ + + KASSERT(proto >= 0, ("netisr_queue: proto < 0")); + + return (netisr2_queue(proto, m)); +} + +/* + * Dispatch a packet for netisr2 processing, direct dispatch permitted by + * calling context. + */ +int +netisr2_dispatch_src(u_int proto, uintptr_t source, struct mbuf *m) +{ + struct rm_priotracker tracker; + struct netisr_workstream *nwsp; + struct netisr_work *npwp; + int dosignal, error; + u_int cpuid; + + /* + * If direct dispatch is entirely disabled, fall back on queueing. + */ + if (!netisr_direct_enable) + return (netisr2_queue_src(proto, source, m)); + + KASSERT(proto < NETISR_MAXPROT, + ("netisr2_dispatch_src: invalid proto %u", proto)); + NETISR_RLOCK(&tracker); + KASSERT(np[proto].np_handler != NULL, + ("netisr2_dispatch_src: invalid proto %u", proto)); + + /* + * If direct dispatch is forced, then unconditionally dispatch + * without a formal CPU selection. Borrow the current CPU's stats, + * even if there's no worker on it. + */ + if (netisr_direct_force) { + nwsp = &nws[curcpu]; + npwp = &nwsp->nws_work[proto]; + npwp->nw_dispatched++; + npwp->nw_handled++; + np[proto].np_handler(m); + NETISR_RUNLOCK(&tracker); + return (0); + } + + /* + * Otherwise, we execute in a hybrid mode where we will try to direct + * dispatch if we're on the right CPU and the netisr worker isn't + * already running. + */ + m = netisr2_selectcpu(&np[proto], source, m, &cpuid); + if (m == NULL) { + NETISR_RUNLOCK(&tracker); + return (ENOBUFS); + } + sched_pin(); + if (!netisr_hybridxcpu_enable && (cpuid != curcpu)) + goto queue_fallback; + nwsp = &nws[cpuid]; + npwp = &nwsp->nws_work[proto]; + + /*- + * We are willing to direct dispatch only if three conditions hold: + * + * (1) The netisr worker isn't already running, + * (2) Another thread isn't already directly dispatching, and + * (3) The netisr hasn't already been woken up. + */ + NWS_LOCK(nwsp); + if (nwsp->nws_flags & (NWS_RUNNING | NWS_DISPATCHING | NWS_SCHEDULED)) { + error = netisr2_queue_workstream(nwsp, npwp, m, &dosignal); + NWS_UNLOCK(nws); + if (dosignal) + NWS_SIGNAL(nwsp); + sched_unpin(); + NETISR_RUNLOCK(&tracker); + return (error); + } + + /* + * The current thread is now effectively the netisr worker, so set + * the dispatching flag to prevent concurrent processing of the + * stream from another thread (even the netisr worker), which could + * otherwise lead to effective misordering of the stream. + */ + nwsp->nws_flags |= NWS_DISPATCHING; + NWS_UNLOCK(nwsp); + np[proto].np_handler(m); + NWS_LOCK(nwsp); + nwsp->nws_flags &= ~NWS_DISPATCHING; + npwp->nw_handled++; + if (curcpu == cpuid) + npwp->nw_hybrid_dispatched++; + else + npwp->nw_hybrid_xcpudispatched++; + + /* + * If other work was enqueued by another thread while we were direct + * dispatching, we need to signal the netisr worker to do that work. + * In the future, we might want to do some of that work in the + * current thread, rather than trigger further context switches. If + * so, we'll want to establish a reasonable bound on the work done in + * the "borrowed" context. + */ + if (nwsp->nws_pendingwork != 0) { + nwsp->nws_flags |= NWS_SCHEDULED; + dosignal = 1; + } else + dosignal = 0; + NWS_UNLOCK(nwsp); + if (dosignal) + NWS_SIGNAL(nwsp); + NETISR_RUNLOCK(&tracker); + return (0); + +queue_fallback: + error = netisr2_queue_internal(proto, m, cpuid); + sched_unpin(); + NETISR_RUNLOCK(&tracker); + return (error); +} + +int +netisr2_dispatch(u_int proto, struct mbuf *m) +{ + + return (netisr2_dispatch_src(proto, 0, m)); +} + +void +netisr_dispatch(int proto, struct mbuf *m) +{ + + KASSERT(proto >= 0, ("netisr_dispatch: proto < 0")); + + (void)netisr2_dispatch(proto, m); +} + +#ifdef DEVICE_POLLING +/* + * Kernel polling borrows a netisr2 thread to run interface polling in; this + * function allows kernel polling to request that the netisr2 thread be + * scheduled even if no packets are pending for protocols. + */ +void +netisr2_sched_poll(void) +{ + struct netisr_workstream *nwsp; + + nwsp = &nws[nws_array[0]]; + NWS_SIGNAL(nwsp); +} +#endif + +static void +netisr2_start_swi(u_int cpuid, struct pcpu *pc) +{ + char swiname[12]; + struct netisr_workstream *nwsp; + int error; + + nwsp = &nws[cpuid]; + mtx_init(&nwsp->nws_mtx, "netisr2_mtx", NULL, MTX_DEF); + nwsp->nws_cpu = cpuid; + snprintf(swiname, sizeof(swiname), "netisr %d", cpuid); + error = swi_add(&nwsp->nws_intr_event, swiname, swi_net, nwsp, + SWI_NET, INTR_MPSAFE, &nwsp->nws_swi_cookie); + if (error) + panic("netisr2_init: swi_add %d", error); + pc->pc_netisr2 = nwsp->nws_intr_event; + if (netisr_bindthreads) { + error = intr_event_bind(nwsp->nws_intr_event, cpuid); + if (error != 0) + printf("netisr2_start_swi cpu %d: intr_event_bind: %d", + cpuid, error); + } + NETISR_WLOCK(); + nws_array[nws_count] = nwsp->nws_cpu; + nws_count++; + NETISR_WUNLOCK(); +} + +/* + * Initialize the netisr subsystem. We rely on BSS and static initialization + * of most fields in global data structures. + * + * Start a worker thread for the boot CPU so that we can support network + * traffic immediately in case the netowrk stack is used before additional + * CPUs are started (for example, diskless boot). + */ +static void +netisr2_init(void *arg) +{ + + KASSERT(curcpu == 0, ("netisr2_init: not on CPU 0")); + + NETISR_LOCK_INIT(); + if (netisr_maxthreads < 1) + netisr_maxthreads = 1; + if (netisr_maxthreads > MAXCPU) + netisr_maxthreads = MAXCPU; +#ifdef DEVICE_POLLING + /* + * The device polling code is not yet aware of how to deal with + * multiple netisr threads, so for the time being compiling in device + * polling disables parallel netisr workers. + */ + netisr_maxthreads = 1; + netisr_bindthreads = 0; +#endif + + netisr2_start_swi(curcpu, pcpu_find(curcpu)); +} +SYSINIT(netisr2_init, SI_SUB_SOFTINTR, SI_ORDER_FIRST, netisr2_init, NULL); + +/* + * Start worker threads for additional CPUs. No attempt to gracefully handle + * work reassignment, we don't yet support dynamic reconfiguration. + */ +static void +netisr2_start(void *arg) +{ + struct pcpu *pc; + + SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { + if (nws_count >= netisr_maxthreads) + break; + /* XXXRW: Is skipping absent CPUs still required here? */ + if (CPU_ABSENT(pc->pc_cpuid)) + continue; + /* Worker will already be present for boot CPU. */ + if (pc->pc_netisr2 != NULL) + continue; + netisr2_start_swi(pc->pc_cpuid, pc); + } +} +SYSINIT(netisr2_start, SI_SUB_SMP, SI_ORDER_MIDDLE, netisr2_start, NULL); + +#ifdef DDB +DB_SHOW_COMMAND(netisr2, db_show_netisr2) +{ + struct netisr_workstream *nwsp; + struct netisr_work *nwp; + int cpu, first, proto; + + db_printf("%3s %5s %6s %5s %5s %5s %8s %8s %8s %8s %8s\n", "CPU", + "Pend", "Proto", "Len", "WMark", "Max", "Disp", "HDisp", + "XHDisp", "Drop", "Queue"); + for (cpu = 0; cpu < MAXCPU; cpu++) { + nwsp = &nws[cpu]; + if (nwsp->nws_intr_event == NULL) + continue; + first = 1; + for (proto = 0; proto < NETISR_MAXPROT; proto++) { + if (np[proto].np_handler == NULL) + continue; + nwp = &nwsp->nws_work[proto]; + if (first) { + db_printf("%3d %5d ", cpu, + nwsp->nws_pendingwork); + first = 0; + } else + db_printf("%3s %5s ", "", ""); + db_printf("%6s %5d %5d %5d %8ju %8ju %8ju %8ju %8ju\n", + np[proto].np_name, nwp->nw_len, + nwp->nw_watermark, nwp->nw_qlimit, + nwp->nw_dispatched, nwp->nw_hybrid_dispatched, + nwp->nw_hybrid_xcpudispatched, nwp->nw_qdrops, + nwp->nw_queued); + } + } +} +#endif Property changes on: net/netisr2.c ___________________________________________________________________ Added: svn:mime-type + text/plain Added: svn:keywords + FreeBSD=%H Added: svn:eol-style + native Index: net/netisr2.h =================================================================== --- net/netisr2.h (.../head/sys) (revision 0) +++ net/netisr2.h (.../projects/pnet/sys) (revision 192641) @@ -0,0 +1,134 @@ +/*- + * Copyright (c) 2007-2009 Robert N. M. Watson + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _NET_NETISR2_H_ +#define _NET_NETISR2_H_ + +#ifndef _KERNEL +#error "no user-serviceable parts inside" +#endif + +/*- + * Protocols express ordering constraints and affinity preferences by + * implementing one or neither of nh_m2flow and nh_m2cpu, which are used by + * netisr2 to determine which per-CPU workstream to assign mbufs to. + * + * The following policies may be used by protocols: + * + * NETISR_POLICY_SOURCE - netisr2 should maintain source ordering without + * advice from the protocol. netisr2 will ignore any + * flow IDs present on the mbuf for the purposes of + * work placement. + * + * NETISR_POLICY_FLOW - netisr2 should maintain flow ordering as defined by + * the mbuf header flow ID field. If the protocol + * implements nh_m2flow, then netisr2 will query the + * protocol in the event that the mbuf doesn't have a + * flow ID, falling back on source ordering. + * + * NETISR_POLICY_CPU - netisr2 will delegate all work placement decisions to + * the protocol, querying nh_m2cpu for each packet. + * + * Protocols might make decisions about work placement based on an existing + * calculated flow ID on the mbuf, such as one provided in hardware, the + * receive interface pointed to by the mbuf (if any), the optional source + * identifier passed at some dispatch points, or even parse packet headers to + * calculate a flow. Both protocol handlers may return a new mbuf pointer + * for the chain, or NULL if the packet proves invalid or m_pullup() fails. + * + * XXXRW: If we eventually support dynamic reconfiguration, there should be + * protocol handlers to notify them of CPU configuration changes so that they + * can rebalance work. + */ +typedef struct mbuf *netisr_m2cpu_t(struct mbuf *m, uintptr_t source, + u_int *cpuid); +typedef struct mbuf *netisr_m2flow_t(struct mbuf *m, uintptr_t source); + +#define NETISR_POLICY_SOURCE 1 /* Maintain source ordering. */ +#define NETISR_POLICY_FLOW 2 /* Maintain flow ordering. */ +#define NETISR_POLICY_CPU 3 /* Protocol determines CPU placement. */ + +/* + * Data structure describing a protocol handler. + */ +struct netisr_handler { + const char *nh_name; /* Character string protocol name. */ + netisr_t *nh_handler; /* Protocol handler. */ + netisr_m2flow_t *nh_m2flow; /* Query flow for untagged packet. */ + netisr_m2cpu_t *nh_m2cpu; /* Query CPU to process packet on. */ + u_int nh_proto; /* Integer protocol ID. */ + u_int nh_qlimit; /* Maximum per-CPU queue depth. */ + u_int nh_policy; /* Work placement policy. */ + u_int nh_ispare[5]; /* For future use. */ + void *nh_pspare[4]; /* For future use. */ +}; + +/* + * Register, unregister, and other netisr2 handler management functions. + */ +void netisr2_clearqdrops(const struct netisr_handler *nhp); +void netisr2_getqdrops(const struct netisr_handler *nhp, + u_int64_t *qdropsp); +void netisr2_getqlimit(const struct netisr_handler *nhp, u_int *qlimitp); +void netisr2_register(const struct netisr_handler *nhp); +int netisr2_setqlimit(const struct netisr_handler *nhp, u_int qlimit); +void netisr2_unregister(const struct netisr_handler *nhp); + +/* + * Process a packet destined for a protocol, and attempt direct dispatch. + * Supplemental source ordering information can be passed using the _src + * variant. + */ +//int netisr_dispatch(u_int proto, struct mbuf *m); +//int netisr_queue(u_int proto, struct mbuf *m); +int netisr2_dispatch(u_int proto, struct mbuf *m); +int netisr2_dispatch_src(u_int proto, uintptr_t source, struct mbuf *m); +int netisr2_queue(u_int proto, struct mbuf *m); +int netisr2_queue_src(u_int proto, uintptr_t source, struct mbuf *m); + +/* + * Provide a default implementation of "map a ID to a cpu ID". + */ +u_int netisr2_default_flow2cpu(u_int flowid); + +/* + * Utility routines to return the number of CPUs participting in netisr2, and + * to return a mapping from a number to a CPU ID that can be used with the + * scheduler. + */ +u_int netisr2_get_cpucount(void); +u_int netisr2_get_cpuid(u_int cpunumber); + +/* + * Interfaces between DEVICE_POLLING and netisr2. + */ +void netisr2_sched_poll(void); +void netisr_poll(void); +void netisr_pollmore(void); + +#endif /* !_NET_NETISR2_H_ */ Property changes on: net/netisr2.h ___________________________________________________________________ Added: svn:mime-type + text/plain Added: svn:keywords + FreeBSD=%H Added: svn:eol-style + native Index: net/rtsock.c =================================================================== --- net/rtsock.c (.../head/sys) (revision 192641) +++ net/rtsock.c (.../projects/pnet/sys) (revision 192641) @@ -34,6 +34,7 @@ #include "opt_route.h" #include "opt_inet.h" #include "opt_inet6.h" +#include "opt_netisr.h" #include #include @@ -57,6 +58,7 @@ #include #include #include +#include #include #include #include @@ -90,11 +92,13 @@ #define RTSOCK_UNLOCK() mtx_unlock(&rtsock_mtx) #define RTSOCK_LOCK_ASSERT() mtx_assert(&rtsock_mtx, MA_OWNED) +SYSCTL_NODE(_net, OID_AUTO, route, CTLFLAG_RD, 0, ""); + +#ifndef NETISR2 static struct ifqueue rtsintrq; - -SYSCTL_NODE(_net, OID_AUTO, route, CTLFLAG_RD, 0, ""); SYSCTL_INT(_net_route, OID_AUTO, netisr_maxqlen, CTLFLAG_RW, &rtsintrq.ifq_maxlen, 0, "maximum routing socket dispatch queue length"); +#endif struct walkarg { int w_tmemsize; @@ -119,16 +123,49 @@ struct rt_metrics *out); static void rt_dispatch(struct mbuf *, const struct sockaddr *); +#ifdef NETISR2 +static struct netisr_handler rtsock_nh = { + .nh_name = "rtsock", + .nh_handler = rts_input, + .nh_proto = NETISR_ROUTE, + .nh_qlimit = 256, + .nh_policy = NETISR_POLICY_SOURCE, +}; + +static int +sysctl_route_netisr_maxqlen(SYSCTL_HANDLER_ARGS) +{ + int error, qlimit; + + netisr2_getqlimit(&rtsock_nh, &qlimit); + error = sysctl_handle_int(oidp, &qlimit, 0, req); + if (error || !req->newptr) + return (error); + if (qlimit < 1) + return (EINVAL); + return (netisr2_setqlimit(&rtsock_nh, qlimit)); +} +SYSCTL_PROC(_net_route, OID_AUTO, netisr_maxqlen, CTLTYPE_INT|CTLFLAG_RW, + 0, 0, sysctl_route_netisr_maxqlen, "I", + "maximum routing socket dispatch queue length"); +#endif + static void rts_init(void) { int tmp; +#ifdef NETISR2 + if (TUNABLE_INT_FETCH("net.route.netisr_maxqlen", &tmp)) + rtsock_nh.nh_qlimit = tmp; + netisr2_register(&rtsock_nh); +#else + mtx_init(&rtsintrq.ifq_mtx, "rts_inq", NULL, MTX_DEF); rtsintrq.ifq_maxlen = 256; if (TUNABLE_INT_FETCH("net.route.netisr_maxqlen", &tmp)) rtsintrq.ifq_maxlen = tmp; - mtx_init(&rtsintrq.ifq_mtx, "rts_inq", NULL, MTX_DEF); netisr_register(NETISR_ROUTE, rts_input, &rtsintrq, 0); +#endif } SYSINIT(rtsock, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, rts_init, 0); Index: netinet6/ip6_input.c =================================================================== --- netinet6/ip6_input.c (.../head/sys) (revision 192641) +++ netinet6/ip6_input.c (.../projects/pnet/sys) (revision 192641) @@ -66,6 +66,7 @@ #include "opt_inet.h" #include "opt_inet6.h" #include "opt_ipsec.h" +#include "opt_netisr.h" #include "opt_route.h" #include @@ -88,6 +89,7 @@ #include #include #include +#include #include #include @@ -120,7 +122,18 @@ extern struct domain inet6domain; u_char ip6_protox[IPPROTO_MAX]; + +#ifdef NETISR2 +static struct netisr_handler ip6_nh = { + .nh_name = "ip6", + .nh_handler = ip6_input, + .nh_proto = NETISR_IPV6, + .nh_qlimit = IFQ_MAXLEN, + .nh_policy = NETISR_POLICY_FLOW, +}; +#else static struct ifqueue ip6intrq; +#endif #ifndef VIMAGE #ifndef VIMAGE_GLOBALS @@ -296,9 +309,14 @@ printf("%s: WARNING: unable to register pfil hook, " "error %d\n", __func__, i); +#ifdef NETISR2 + ip6_nh.nh_qlimit = V_ip6qmaxlen; + netisr2_register(&ip6_nh); +#else ip6intrq.ifq_maxlen = V_ip6qmaxlen; /* XXX */ mtx_init(&ip6intrq.ifq_mtx, "ip6_inq", NULL, MTX_DEF); netisr_register(NETISR_IPV6, ip6_input, &ip6intrq, 0); +#endif } static int Property changes on: cddl/boot/zfs/zfssubr.c ___________________________________________________________________ Deleted: fbsd:nokeywords - yes Added: svn:keywords + FreeBSD=%H Index: netipsec/ipsec_input.c =================================================================== --- netipsec/ipsec_input.c (.../head/sys) (revision 192641) +++ netipsec/ipsec_input.c (.../projects/pnet/sys) (revision 192641) @@ -44,6 +44,7 @@ #include "opt_inet6.h" #include "opt_ipsec.h" #include "opt_enc.h" +#include "opt_netisr.h" #include #include @@ -60,6 +61,7 @@ #include #include #include +#include #include #include @@ -481,7 +483,15 @@ /* * Re-dispatch via software interrupt. */ +#ifdef NETISR2 + /* + * XXXRW: Is this ordering sufficient? Perhaps should be + * m->m_pkthdr.rcvif? + */ + if ((error = netisr2_queue(NETISR_IP, (uintptr_t)sav, m))) { +#else if ((error = netisr_queue(NETISR_IP, m))) { +#endif IPSEC_ISTAT(sproto, V_espstat.esps_qfull, V_ahstat.ahs_qfull, V_ipcompstat.ipcomps_qfull); Index: sys/pcpu.h =================================================================== --- sys/pcpu.h (.../head/sys) (revision 192641) +++ sys/pcpu.h (.../projects/pnet/sys) (revision 192641) @@ -86,6 +86,7 @@ struct vmmeter pc_cnt; /* VM stats counters */ long pc_cp_time[CPUSTATES]; /* statclock ticks */ struct device *pc_device; + void *pc_netisr2; /* netisr2 SWI cookie. */ /* * Stuff for read mostly lock Property changes on: . ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys:r186685-192635