Rework global locks for interface list and index management, correcting
several critical bugs, including race conditions and lock order issues:

(1) Add a new 'reserved' ifnet pointer value for use in the ifindex
    array, IFNET_HOLD, allowing an index to be reserved without
    connecting the interface to the array prematurely.  This allows the
    global ifnet lock to be released in if_alloc() between reserving an
    index and making the new ifnet available the index.

(2) Replace the single rwlock, ifnet_lock, with an rwlock and an sxlock;
    either can be held to stabilize the lists and indexes, but both are
    required to write.  This allows the list to be held stable in both
    network interrupt contexts, and also across potentially sleeping
    memory allocations or device driver interactions.  As before, writes
    to the interface list must occur from sleepable contexts.

This allows us to introduce missing locking, and fix existing locking.
Unless otherwise indicated through the use of modified lock macros,
IFNET_RLOCK() now acquires a sleepable lock.

This also corrects the problem in which if_ef panicked on load.

Index: nfsclient/bootp_subr.c
===================================================================
--- nfsclient/bootp_subr.c	(revision 196425)
+++ nfsclient/bootp_subr.c	(working copy)
@@ -389,7 +389,7 @@
 	struct ifaddr *ifa;
 
 	printf("Interface list:\n");
-	IFNET_RLOCK(); /* could sleep, but okay for debugging XXX */
+	IFNET_RLOCK();
 	for (ifp = TAILQ_FIRST(&V_ifnet);
 	     ifp != NULL;
 	     ifp = TAILQ_NEXT(ifp, if_link)) {
Index: kern/kern_uuid.c
===================================================================
--- kern/kern_uuid.c	(revision 196425)
+++ kern/kern_uuid.c	(working copy)
@@ -95,7 +95,7 @@
 	int i;
 
 	CURVNET_SET(TD_TO_VNET(curthread));
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		/* Walk the address list */
 		IF_ADDR_LOCK(ifp);
@@ -106,14 +106,14 @@
 				/* Got a MAC address. */
 				bcopy(LLADDR(sdl), node, UUID_NODE_LEN);
 				IF_ADDR_UNLOCK(ifp);
-				IFNET_RUNLOCK();
+				IFNET_RUNLOCK_NOSLEEP();
 				CURVNET_RESTORE();
 				return;
 			}
 		}
 		IF_ADDR_UNLOCK(ifp);
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	for (i = 0; i < (UUID_NODE_LEN>>1); i++)
 		node[i] = (uint16_t)arc4random();
Index: netinet/in.c
===================================================================
--- netinet/in.c	(revision 196425)
+++ netinet/in.c	(working copy)
@@ -1407,12 +1407,7 @@
 	} arpc;
 	int error, i;
 
-	/* XXXXX
-	 * current IFNET_RLOCK() is mapped to IFNET_WLOCK()
-	 * so it is okay to use this ASSERT, change it when
-	 * IFNET lock is finalized
-	 */
-	IFNET_WLOCK_ASSERT();
+	IFNET_RLOCK_ASSERT();
 
 	error = 0;
 	for (i = 0; i < LLTBL_HASHTBL_SIZE; i++) {
Index: netgraph/ng_gif.c
===================================================================
--- netgraph/ng_gif.c	(revision 196425)
+++ netgraph/ng_gif.c	(working copy)
@@ -560,7 +560,7 @@
 		ng_gif_input_orphan_p = ng_gif_input_orphan;
 
 		/* Create nodes for any already-existing gif interfaces */
-		VNET_LIST_RLOCK_NOSLEEP();
+		VNET_LIST_RLOCK();
 		IFNET_RLOCK();
 		VNET_FOREACH(vnet_iter) {
 			CURVNET_SET_QUIET(vnet_iter); /* XXX revisit quiet */
@@ -571,7 +571,7 @@
 			CURVNET_RESTORE();
 		}
 		IFNET_RUNLOCK();
-		VNET_LIST_RUNLOCK_NOSLEEP();
+		VNET_LIST_RUNLOCK();
 		break;
 
 	case MOD_UNLOAD:
Index: compat/linux/linux_ioctl.c
===================================================================
--- compat/linux/linux_ioctl.c	(revision 196425)
+++ compat/linux/linux_ioctl.c	(working copy)
@@ -2061,22 +2061,20 @@
 	struct ifnet *ifscan;
 	int ethno;
 
+	IFNET_RLOCK_ASSERT();
+
 	/* Short-circuit non ethernet interfaces */
 	if (!IFP_IS_ETH(ifp))
 		return (strlcpy(buffer, ifp->if_xname, buflen));
 
 	/* Determine the (relative) unit number for ethernet interfaces */
 	ethno = 0;
-	IFNET_RLOCK();
 	TAILQ_FOREACH(ifscan, &V_ifnet, if_link) {
-		if (ifscan == ifp) {
-			IFNET_RUNLOCK();
+		if (ifscan == ifp)
 			return (snprintf(buffer, buflen, "eth%d", ethno));
-		}
 		if (IFP_IS_ETH(ifscan))
 			ethno++;
 	}
-	IFNET_RUNLOCK();
 
 	return (0);
 }
@@ -2177,7 +2175,7 @@
 	valid_len = 0;
 
 	/* Return all AF_INET addresses of all interfaces */
-	IFNET_RLOCK();		/* could sleep XXX */
+	IFNET_RLOCK();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		int addrs = 0;
 
Index: net/if.c
===================================================================
--- net/if.c	(revision 196425)
+++ net/if.c	(working copy)
@@ -154,14 +154,26 @@
 VNET_DEFINE(int, if_index);
 static VNET_DEFINE(int, if_indexlim) = 8;
 
-/* Table of ifnet by index.  Locked with ifnet_lock. */
+/* Table of ifnet by index. */
 static VNET_DEFINE(struct ifindex_entry *, ifindex_table);
 
 #define	V_if_indexlim		VNET(if_indexlim)
 #define	V_ifindex_table		VNET(ifindex_table)
 
 int	ifqmaxlen = IFQ_MAXLEN;
-struct rwlock ifnet_lock;
+
+/*
+ * The global network interface list (V_ifnet) and related state (such as
+ * if_index, if_indexlim, and ifindex_table) are protected by an sxlock and
+ * an rwlock.  Either may be acquired shared to stablize the list, but both
+ * must be acquired writable to modify the list.  This model allows us to
+ * both stablize the interface list during interrupt thread processing, but
+ * also to stablize it over long-running ioctls, without introducing priority
+ * inversions and deadlocks.
+ */
+struct rwlock ifnet_rwlock;
+struct sx ifnet_sxlock;
+
 static	if_com_alloc_t *if_com_alloc[256];
 static	if_com_free_t *if_com_free[256];
 
@@ -174,12 +186,21 @@
 MALLOC_DEFINE(M_IFADDR, "ifaddr", "interface address");
 MALLOC_DEFINE(M_IFMADDR, "ether_multi", "link-level multicast address");
 
+/*
+ * The allocation of network interfaces is a rather non-atomic affair; we
+ * need to select an index before we are ready to expose the interface for
+ * use, so will use this pointer value to indicate reservation.
+ */
+#define	IFNET_HOLD	(void *)(uintptr_t)(-1)
+
 struct ifnet *
 ifnet_byindex_locked(u_short idx)
 {
 
 	if (idx > V_if_index)
 		return (NULL);
+	if (V_ifindex_table[idx].ife_ifnet == IFNET_HOLD)
+		return (NULL);
 	return (V_ifindex_table[idx].ife_ifnet);
 }
 
@@ -188,9 +209,9 @@
 {
 	struct ifnet *ifp;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	ifp = ifnet_byindex_locked(idx);
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifp);
 }
 
@@ -199,19 +220,19 @@
 {
 	struct ifnet *ifp;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	ifp = ifnet_byindex_locked(idx);
 	if (ifp == NULL || (ifp->if_flags & IFF_DYING)) {
-		IFNET_RUNLOCK();
+		IFNET_RUNLOCK_NOSLEEP();
 		return (NULL);
 	}
 	if_ref(ifp);
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifp);
 }
 
 static void
-ifnet_setbyindex(u_short idx, struct ifnet *ifp)
+ifnet_setbyindex_locked(u_short idx, struct ifnet *ifp)
 {
 
 	IFNET_WLOCK_ASSERT();
@@ -219,16 +240,25 @@
 	V_ifindex_table[idx].ife_ifnet = ifp;
 }
 
+static void
+ifnet_setbyindex(u_short idx, struct ifnet *ifp)
+{
+
+	IFNET_WLOCK();
+	ifnet_setbyindex_locked(idx, ifp);
+	IFNET_WUNLOCK();
+}
+
 struct ifaddr *
 ifaddr_byindex(u_short idx)
 {
 	struct ifaddr *ifa;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	ifa = ifnet_byindex_locked(idx)->if_addr;
 	if (ifa != NULL)
 		ifa_ref(ifa);
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifa);
 }
 
@@ -317,17 +347,17 @@
 	ifp = malloc(sizeof(struct ifnet), M_IFNET, M_WAITOK|M_ZERO);
 
 	/*
-	 * Try to find an empty slot below if_index.  If we fail, take
-	 * the next slot.
-	 *
-	 * XXX: should be locked!
+	 * Try to find an empty slot below if_index.  If we fail, take the
+	 * next slot.
 	 */
+	IFNET_WLOCK();
 	for (ifp->if_index = 1; ifp->if_index <= V_if_index; ifp->if_index++) {
-		if (ifnet_byindex(ifp->if_index) == NULL)
+		if (ifnet_byindex_locked(ifp->if_index) == NULL)
 			break;
 	}
 	/* Catch if_index overflow. */
 	if (ifp->if_index < 1) {
+		IFNET_WUNLOCK();
 		free(ifp, M_IFNET);
 		return (NULL);
 	}
@@ -335,6 +365,8 @@
 		V_if_index = ifp->if_index;
 	if (V_if_index >= V_if_indexlim)
 		if_grow();
+	ifnet_setbyindex_locked(ifp->if_index, IFNET_HOLD);
+	IFNET_WUNLOCK();
 
 	ifp->if_type = type;
 	ifp->if_alloctype = type;
@@ -342,6 +374,7 @@
 	if (if_com_alloc[type] != NULL) {
 		ifp->if_l2com = if_com_alloc[type](type, ifp);
 		if (ifp->if_l2com == NULL) {
+			ifnet_setbyindex(ifp->if_index, NULL);
 			free(ifp, M_IFNET);
 			return (NULL);
 		}
@@ -361,9 +394,7 @@
 	ifq_init(&ifp->if_snd, ifp);
 
 	refcount_init(&ifp->if_refcount, 1);	/* Index reference. */
-	IFNET_WLOCK();
 	ifnet_setbyindex(ifp->if_index, ifp);
-	IFNET_WUNLOCK();
 	return (ifp);
 }
 
@@ -383,7 +414,7 @@
 	KASSERT(ifp == ifnet_byindex_locked(ifp->if_index),
 	    ("%s: freeing unallocated ifnet", ifp->if_xname));
 
-	ifnet_setbyindex(ifp->if_index, NULL);
+	ifnet_setbyindex_locked(ifp->if_index, NULL);
 	while (V_if_index > 0 && ifnet_byindex_locked(V_if_index) == NULL)
 		V_if_index--;
 	IFNET_WUNLOCK();
@@ -855,14 +886,16 @@
 	if_detach_internal(ifp, 1);
 
 	/*
-	 * Unlink the ifnet from ifindex_table[] in current vnet,
-	 * and shrink the if_index for that vnet if possible.
+	 * Unlink the ifnet from ifindex_table[] in current vnet, and shrink
+	 * the if_index for that vnet if possible.
+	 *
+	 * NOTE: IFNET_WLOCK/IFNET_WUNLOCK() are assumed to be unvirtualized,
+	 * or we'd lock on one vnet and unlock on another.
 	 */
 	IFNET_WLOCK();
-	ifnet_setbyindex(ifp->if_index, NULL);
+	ifnet_setbyindex_locked(ifp->if_index, NULL);
 	while (V_if_index > 0 && ifnet_byindex_locked(V_if_index) == NULL)
 		V_if_index--;
-	IFNET_WUNLOCK();
 
 	/*
 	 * Switch to the context of the target vnet.
@@ -873,7 +906,6 @@
 	 * Try to find an empty slot below if_index.  If we fail, take 
 	 * the next slot.
 	 */
-	IFNET_WLOCK();
 	for (ifp->if_index = 1; ifp->if_index <= V_if_index; ifp->if_index++) {
 		if (ifnet_byindex_locked(ifp->if_index) == NULL)
 			break;
@@ -886,7 +918,7 @@
 		V_if_index = ifp->if_index;
 	if (V_if_index >= V_if_indexlim)
 		if_grow();
-	ifnet_setbyindex(ifp->if_index, ifp);
+	ifnet_setbyindex_locked(ifp->if_index, ifp);
 	IFNET_WUNLOCK();
 
 	if_attach_internal(ifp, 1);
@@ -1368,7 +1400,7 @@
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		IF_ADDR_LOCK(ifp);
 		TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
@@ -1395,7 +1427,7 @@
 	}
 	ifa = NULL;
 done:
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifa);
 }
 
@@ -1423,7 +1455,7 @@
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		IF_ADDR_LOCK(ifp);
 		TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
@@ -1442,7 +1474,7 @@
 	}
 	ifa = NULL;
 done:
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifa);
 }
 
@@ -1456,7 +1488,7 @@
 	struct ifnet *ifp;
 	struct ifaddr *ifa;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		if ((ifp->if_flags & IFF_POINTOPOINT) == 0)
 			continue;
@@ -1475,7 +1507,7 @@
 	}
 	ifa = NULL;
 done:
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifa);
 }
 
@@ -1508,7 +1540,7 @@
 	 * we find one, as we release the IF_ADDR_LOCK() that kept it stable
 	 * when we move onto the next interface.
 	 */
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		IF_ADDR_LOCK(ifp);
 		TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
@@ -1584,7 +1616,7 @@
 	ifa = ifa_maybe;
 	ifa_maybe = NULL;
 done:
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	if (ifa_maybe != NULL)
 		ifa_free(ifa_maybe);
 	return (ifa);
@@ -1863,7 +1895,7 @@
 	int s = splimp();
 
 	VNET_LIST_RLOCK_NOSLEEP();
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
 		TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
@@ -1874,7 +1906,7 @@
 		}
 		CURVNET_RESTORE();
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	VNET_LIST_RUNLOCK_NOSLEEP();
 	splx(s);
 	timeout(if_slowtimo, (void *)0, hz / IFNET_SLOWHZ);
@@ -1889,7 +1921,7 @@
 {
 	struct ifnet *ifp;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		if (strncmp(name, ifp->if_xname, IFNAMSIZ) == 0 &&
 		    !(ifp->if_flags & IFF_DYING))
@@ -1897,7 +1929,7 @@
 	}
 	if (ifp != NULL)
 		if_ref(ifp);
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifp);
 }
 
@@ -1906,12 +1938,12 @@
 {
 	struct ifnet *ifp;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		if (strncmp(name, ifp->if_xname, IFNAMSIZ) == 0)
 			break;
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	return (ifp);
 }
 
@@ -2521,7 +2553,7 @@
 	max_len = 0;
 	valid_len = 0;
 
-	IFNET_RLOCK();		/* could sleep XXX */
+	IFNET_RLOCK();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		int addrs;
 
@@ -2846,13 +2878,13 @@
 #ifdef INVARIANTS
 	struct ifnet *oifp;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(oifp, &V_ifnet, if_link)
 		if (ifp == oifp)
 			break;
 	if (ifp != oifp)
 		ifp = NULL;
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	KASSERT(ifp != NULL, ("%s: ifnet went away", __func__));
 #endif
@@ -2895,7 +2927,7 @@
 	} else {
 		struct ifnet *oifp;
 
-		IFNET_RLOCK();
+		IFNET_RLOCK_NOSLEEP();
 		TAILQ_FOREACH(oifp, &V_ifnet, if_link)
 			if (ifp == oifp)
 				break;
@@ -2903,7 +2935,7 @@
 			printf("%s: ifnet %p disappeared\n", __func__, ifp);
 			ifp = NULL;
 		}
-		IFNET_RUNLOCK();
+		IFNET_RUNLOCK_NOSLEEP();
 	}
 #endif
 	/*
Index: net/if_ef.c
===================================================================
--- net/if_ef.c	(revision 196425)
+++ net/if_ef.c	(working copy)
@@ -492,7 +492,8 @@
 	VNET_LIST_RLOCK();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
-		IFNET_RLOCK();
+
+		IFNET_RLOCK_MAYWRITE();
 		TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 			if (ifp->if_type != IFT_ETHER) continue;
 			EFDEBUG("Found interface %s\n", ifp->if_xname);
@@ -523,7 +524,7 @@
 			efcount++;
 			SLIST_INSERT_HEAD(&efdev, efl, el_next);
 		}
-		IFNET_RUNLOCK();
+		IFNET_RUNLOCK_MAYWRITE();
 		CURVNET_RESTORE();
 	}
 	VNET_LIST_RUNLOCK();
Index: net/bridgestp.c
===================================================================
--- net/bridgestp.c	(revision 196425)
+++ net/bridgestp.c	(working copy)
@@ -2019,7 +2019,7 @@
 	 * not need to be part of the bridge, it just needs to be a unique
 	 * value.
 	 */
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		if (ifp->if_type != IFT_ETHER)
 			continue;
@@ -2036,7 +2036,7 @@
 			continue;
 		}
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	if (LIST_EMPTY(&bs->bs_bplist) || mif == NULL) {
 		/* Set the bridge and root id (lower bits) to zero */
Index: net/if_vlan.c
===================================================================
--- net/if_vlan.c	(revision 196425)
+++ net/if_vlan.c	(working copy)
@@ -580,7 +580,7 @@
 	int t = 0;
 
 	/* Check for <etherif>.<vlan> style interface names. */
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 		if (ifp->if_type != IFT_ETHER)
 			continue;
@@ -598,7 +598,7 @@
 			*tag = t;
 		break;
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	return (ifp);
 }
Index: net/if_llatbl.c
===================================================================
--- net/if_llatbl.c	(revision 196425)
+++ net/if_llatbl.c	(working copy)
@@ -200,14 +200,14 @@
 {
 	struct lltable *llt;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	SLIST_FOREACH(llt, &lltables, llt_link) {
 		if (llt->llt_af != af)
 			continue;
 
 		llt->llt_prefix_free(llt, prefix, mask);
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 }
 
 
@@ -300,13 +300,13 @@
 	}
 
 	/* XXX linked list may be too expensive */
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	SLIST_FOREACH(llt, &lltables, llt_link) {
 		if (llt->llt_af == dst->sa_family &&
 		    llt->llt_ifp == ifp)
 			break;
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	KASSERT(llt != NULL, ("Yep, ugly hacks are bad\n"));
 
 	if (flags && LLE_CREATE)
Index: net/if_var.h
===================================================================
--- net/if_var.h	(revision 196425)
+++ net/if_var.h	(working copy)
@@ -85,6 +85,7 @@
 #include <sys/lock.h>		/* XXX */
 #include <sys/mutex.h>		/* XXX */
 #include <sys/rwlock.h>		/* XXX */
+#include <sys/sx.h>		/* XXX */
 #include <sys/event.h>		/* XXX */
 #include <sys/_task.h>
 
@@ -754,16 +755,49 @@
 
 #ifdef _KERNEL
 
-extern	struct rwlock ifnet_lock;
-#define	IFNET_LOCK_INIT() \
-   rw_init_flags(&ifnet_lock, "ifnet",  RW_RECURSE)
-#define	IFNET_WLOCK()		rw_wlock(&ifnet_lock)
-#define	IFNET_WUNLOCK()		rw_wunlock(&ifnet_lock)
-#define	IFNET_WLOCK_ASSERT()	rw_assert(&ifnet_lock, RA_LOCKED)
-#define	IFNET_RLOCK()		rw_rlock(&ifnet_lock)
-#define	IFNET_RUNLOCK()		rw_runlock(&ifnet_lock)	
+extern	struct rwlock ifnet_rwlock;
+extern	struct sx ifnet_sxlock;
 
+#define	IFNET_LOCK_INIT() do {						\
+	rw_init_flags(&ifnet_rwlock, "ifnet_rw",  RW_RECURSE);		\
+	sx_init_flags(&ifnet_sxlock, "ifnet_sx",  SX_RECURSE);		\
+} while(0)
+
+#define	IFNET_WLOCK() do {						\
+	sx_xlock(&ifnet_sxlock);					\
+	rw_wlock(&ifnet_rwlock);					\
+} while (0)
+
+#define	IFNET_WUNLOCK() do {						\
+	rw_wunlock(&ifnet_rwlock);					\
+	sx_xunlock(&ifnet_sxlock);					\
+} while (0)
+
 /*
+ * To assert the ifnet lock, you must know not only whether it's for read or
+ * write, but also whether it was acquired with sleep support or not.
+ */
+#define	IFNET_RLOCK_ASSERT()		sx_assert(&ifnet_sxlock, SA_SLOCKED)
+#define	IFNET_RLOCK_NOSLEEP_ASSERT()	rw_assert(&ifnet_rwlock, RA_RLOCKED)
+#define	IFNET_WLOCK_ASSERT() do {					\
+	sx_assert(&ifnet_sxlock, SA_XLOCKED);				\
+	rw_assert(&ifnet_rwlock, RA_WLOCKED);				\
+} while (0)
+
+#define	IFNET_RLOCK()		sx_slock(&ifnet_sxlock)
+#define	IFNET_RLOCK_NOSLEEP()	rw_rlock(&ifnet_rwlock)
+#define	IFNET_RUNLOCK()		sx_sunlock(&ifnet_sxlock)
+#define	IFNET_RUNLOCK_NOSLEEP()	rw_runlock(&ifnet_rwlock)	
+
+/*
+ * In one kernel module, we need to walk the list of ifnets while potentially
+ * modifying it.  Use these variations on the read-locking theme to allow
+ * this to be safe (since recursion is alllowed).
+ */
+#define	IFNET_RLOCK_MAYWRITE()		sx_xlock(&ifnet_sxlock)
+#define	IFNET_RUNLOCK_MAYWRITE()	sx_xunlock(&ifnet_sxlock)
+
+/*
  * Look up an ifnet given its index; the _ref variant also acquires a
  * reference that must be freed using if_rele().  It is almost always a bug
  * to call ifnet_byindex() instead if ifnet_byindex_ref().
Index: netinet6/in6_ifattach.c
===================================================================
--- netinet6/in6_ifattach.c	(revision 196425)
+++ netinet6/in6_ifattach.c	(working copy)
@@ -398,7 +398,7 @@
 	}
 
 	/* next, try to get it from some other hardware interface */
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	for (ifp = V_ifnet.tqh_first; ifp; ifp = ifp->if_list.tqe_next) {
 		if (ifp == ifp0)
 			continue;
@@ -413,11 +413,11 @@
 			nd6log((LOG_DEBUG,
 			    "%s: borrow interface identifier from %s\n",
 			    if_name(ifp0), if_name(ifp)));
-			IFNET_RUNLOCK();
+			IFNET_RUNLOCK_NOSLEEP();
 			goto success;
 		}
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	/* last resort: get from random number source */
 	if (get_rand_ifid(ifp, in6) == 0) {
Index: netinet6/icmp6.c
===================================================================
--- netinet6/icmp6.c	(revision 196425)
+++ netinet6/icmp6.c	(working copy)
@@ -1705,7 +1705,7 @@
 		}
 	}
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	for (ifp = TAILQ_FIRST(&V_ifnet); ifp; ifp = TAILQ_NEXT(ifp, if_list)) {
 		addrsofif = 0;
 		IF_ADDR_LOCK(ifp);
@@ -1762,13 +1762,13 @@
 		IF_ADDR_UNLOCK(ifp);
 		if (iffound) {
 			*ifpp = ifp;
-			IFNET_RUNLOCK();
+			IFNET_RUNLOCK_NOSLEEP();
 			return (addrsofif);
 		}
 
 		addrs += addrsofif;
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	return (addrs);
 }
@@ -1789,7 +1789,7 @@
 	if (ifp0 == NULL && !(niflags & NI_NODEADDR_FLAG_ALL))
 		return (0);	/* needless to copy */
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
   again:
 
 	for (; ifp; ifp = TAILQ_NEXT(ifp, if_list)) {
@@ -1854,7 +1854,7 @@
 				 * Set the truncate flag and return.
 				 */
 				nni6->ni_flags |= NI_NODEADDR_FLAG_TRUNCATE;
-				IFNET_RUNLOCK();
+				IFNET_RUNLOCK_NOSLEEP();
 				return (copied);
 			}
 
@@ -1907,7 +1907,7 @@
 		goto again;
 	}
 
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 
 	return (copied);
 }
Index: netinet6/nd6.c
===================================================================
--- netinet6/nd6.c	(revision 196425)
+++ netinet6/nd6.c	(working copy)
@@ -1662,7 +1662,7 @@
 
 	callout_reset(&V_nd6_slowtimo_ch, ND6_SLOWTIMER_INTERVAL * hz,
 	    nd6_slowtimo, curvnet);
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	for (ifp = TAILQ_FIRST(&V_ifnet); ifp;
 	    ifp = TAILQ_NEXT(ifp, if_list)) {
 		nd6if = ND_IFINFO(ifp);
@@ -1678,7 +1678,7 @@
 			nd6if->reachable = ND_COMPUTE_RTIME(nd6if->basereachable);
 		}
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	CURVNET_RESTORE();
 }
 
Index: netinet6/in6.c
===================================================================
--- netinet6/in6.c	(revision 196425)
+++ netinet6/in6.c	(working copy)
@@ -2234,7 +2234,7 @@
 	unsigned long maxmtu = 0;
 	struct ifnet *ifp;
 
-	IFNET_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
 	for (ifp = TAILQ_FIRST(&V_ifnet); ifp;
 	    ifp = TAILQ_NEXT(ifp, if_list)) {
 		/* this function can be called during ifnet initialization */
@@ -2244,7 +2244,7 @@
 		    IN6_LINKMTU(ifp) > maxmtu)
 			maxmtu = IN6_LINKMTU(ifp);
 	}
-	IFNET_RUNLOCK();
+	IFNET_RUNLOCK_NOSLEEP();
 	if (maxmtu)	     /* update only when maxmtu is positive */
 		V_in6_maxmtu = maxmtu;
 }
@@ -2495,12 +2495,7 @@
 	} ndpc;
 	int i, error;
 
-	/* XXXXX
-	 * current IFNET_RLOCK() is mapped to IFNET_WLOCK()
-	 * so it is okay to use this ASSERT, change it when
-	 * IFNET lock is finalized
-	 */
-	IFNET_WLOCK_ASSERT();
+	IFNET_RLOCK_ASSERT();
 
 	error = 0;
 	for (i = 0; i < LLTBL_HASHTBL_SIZE; i++) {
Index: contrib/altq/altq/altq_subr.c
===================================================================
--- contrib/altq/altq/altq_subr.c	(revision 196425)
+++ contrib/altq/altq/altq_subr.c	(working copy)
@@ -462,8 +462,8 @@
 	s = splimp();
 #endif
 #if defined(__FreeBSD__) && (__FreeBSD_version >= 500000)
-	IFNET_RLOCK();
-	VNET_LIST_RLOCK();
+	IFNET_RLOCK_NOSLEEP();
+	VNET_LIST_RLOCK_NOSLEEP();
 	VNET_FOREACH(vnet_iter) {
 		CURVNET_SET(vnet_iter);
 #endif
@@ -480,8 +480,8 @@
 #if defined(__FreeBSD__) && (__FreeBSD_version >= 500000)
 		CURVNET_RESTORE();
 	}
-	VNET_LIST_RUNLOCK();
-	IFNET_RUNLOCK();
+	VNET_LIST_RUNLOCK_NOSLEEP();
+	IFNET_RUNLOCK_NOSLEEP();
 #endif
 	splx(s);
 	if (active > 0)
Index: netipsec/xform_ipip.c
===================================================================
--- netipsec/xform_ipip.c	(revision 196425)
+++ netipsec/xform_ipip.c	(working copy)
@@ -303,7 +303,7 @@
 	if ((m->m_pkthdr.rcvif == NULL ||
 	    !(m->m_pkthdr.rcvif->if_flags & IFF_LOOPBACK)) &&
 	    V_ipip_allow != 2) {
-	    	IFNET_RLOCK();
+	    	IFNET_RLOCK_NOSLEEP();
 		TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
 			TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
 #ifdef INET
@@ -318,7 +318,7 @@
 					    ipo->ip_src.s_addr)	{
 						V_ipipstat.ipips_spoof++;
 						m_freem(m);
-						IFNET_RUNLOCK();
+						IFNET_RUNLOCK_NOSLEEP();
 						return;
 					}
 				}
@@ -335,7 +335,7 @@
 					if (IN6_ARE_ADDR_EQUAL(&sin6->sin6_addr, &ip6->ip6_src)) {
 						V_ipipstat.ipips_spoof++;
 						m_freem(m);
-						IFNET_RUNLOCK();
+						IFNET_RUNLOCK_NOSLEEP();
 						return;
 					}
 
@@ -343,7 +343,7 @@
 #endif /* INET6 */
 			}
 		}
-		IFNET_RUNLOCK();
+		IFNET_RUNLOCK_NOSLEEP();
 	}
 
 	/* Statistics */