--- //depot/projects/netperf_socket/sys/kern/kern_descrip.c	2004/02/28 09:51:12
+++ //depot/user/rwatson/netperf/sys/kern/kern_descrip.c	2004/03/03 20:58:29
@@ -2041,6 +2041,8 @@
 void
 fputsock(struct socket *so)
 {
+
+	NET_ASSERT_GIANT();
 	SOCK_LOCK(so);
 	sorele(so);
 }
@@ -2067,6 +2069,10 @@
 	}
 	/* We have the last ref so we can proceed without the file lock. */
 	FILE_UNLOCK(fp);
+	/*
+	 * XXXRW: With sockets locked, can we push this Giant grab into
+	 * the purely VFS case?
+	 */
 	mtx_lock(&Giant);
 	if (fp->f_count < 0)
 		panic("fdrop: count < 0");
--- //depot/projects/netperf_socket/sys/kern/kern_synch.c	2004/03/02 16:12:45
+++ //depot/user/rwatson/netperf/sys/kern/kern_synch.c	2004/03/06 21:44:42
@@ -448,6 +448,8 @@
 	int i, nrun;
 	struct loadavg *avg;
 
+	GIANT_REQUIRED;
+
 	nrun = sched_load();
 	avg = &averunnable;
 
--- //depot/projects/netperf_socket/sys/kern/kern_timeout.c	2004/01/18 09:46:57
+++ //depot/user/rwatson/netperf/sys/kern/kern_timeout.c	2004/03/08 08:37:54
@@ -47,6 +47,7 @@
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
+#include <sys/sbuf.h>
 #include <sys/sysctl.h>
 
 static int avg_depth;
@@ -58,6 +59,89 @@
 static int avg_mpcalls;
 SYSCTL_INT(_debug, OID_AUTO, to_avg_mpcalls, CTLFLAG_RD, &avg_mpcalls, 0,
     "Average number of MP callouts made per softclock call. Units = 1/1000");
+
+/*-
+ * Sampling buffer of function pointers executed by timeouts and callouts.
+ * This circular buffer wraps when it fills, and uses an inefficient
+ * sbuf-based sysctl to dump sample data to userspace.  Sysctls can select
+ * to monitor mpsafe and !mpsafe callouts/timeouts as desired.  Suggested
+ * use is: (1) set sample of interest (mpsafe/notmpsafe), (2) reset the
+ * buffer, (3) do some benchmark/test, (5) disable sampling, (6) dump
+ * buffer.
+ *
+ * XXX: ifdef TIMEOUT_SAMPLING?
+ */
+
+#define	MAXFUNC	200000
+static void * func_array[MAXFUNC];
+static int array_off;
+
+static void
+push_cfunc(void *ptr)
+{
+
+	/* XXX */
+	func_array[array_off % MAXFUNC] = ptr;
+	array_off++;
+}
+
+static int
+sysctl_cfunc(SYSCTL_HANDLER_ARGS)
+{
+	struct sbuf sb;
+	int error, i;
+
+	if (req->newptr != NULL)
+		return (EINVAL);
+
+	sbuf_new(&sb, NULL, 0, SBUF_AUTOEXTEND);
+
+	for (i = 0; i < MAXFUNC; i++) {
+		if (func_array[i] == NULL)
+			break;
+		sbuf_printf(&sb, "%p ", func_array[i]);
+	}
+	sbuf_finish(&sb);
+
+	error = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb) + 1, req);
+
+	sbuf_delete(&sb);
+
+	return (error);
+}
+
+SYSCTL_PROC(_debug, OID_AUTO, to_cfunc, CTLTYPE_STRING|CTLFLAG_RD, 0, 0,
+    sysctl_cfunc, "A", "callout/timeout sample");
+
+static int
+sysctl_cfunc_reset(SYSCTL_HANDLER_ARGS)
+{
+	int dummy, error;
+
+	dummy = 0;
+	error = sysctl_handle_int(oidp, &dummy, 0, req);
+	if (error)
+		return (error);
+
+	if (dummy != 0) {
+		bzero(func_array, sizeof(void *) * MAXFUNC);
+		array_off = 0;
+	}
+
+	return (0);
+}
+
+SYSCTL_PROC(_debug, OID_AUTO, to_cfunc_reset, CTLTYPE_INT|CTLFLAG_RW, 0, 0,
+    sysctl_cfunc_reset, "I", "Reset sample");
+
+static int cfunc_sample_mpsafe;
+static int cfunc_sample_notmpsafe;
+
+SYSCTL_INT(_debug, OID_AUTO, to_cfunc_mpsafe, CTLFLAG_RW,
+    &cfunc_sample_mpsafe, 0, "Sample mpsafe callouts");
+SYSCTL_INT(_debug, OID_AUTO, to_cfunc_notmpsafe, CTLFLAG_RW,
+    &cfunc_sample_notmpsafe, 0, "Sample !mpsafe callouts");
+
 /*
  * TODO:
  *	allocate more timeout table slots when table overflows.
@@ -216,8 +300,12 @@
 				if (!(c_flags & CALLOUT_MPSAFE)) {
 					mtx_lock(&Giant);
 					gcalls++;
+					if (cfunc_sample_mpsafe)
+						push_cfunc(c_func);
 				} else {
 					mpcalls++;
+					if (cfunc_sample_notmpsafe)
+						push_cfunc(c_func);
 				}
 #ifdef DIAGNOSTIC
 				binuptime(&bt1);
--- //depot/projects/netperf_socket/sys/kern/subr_log.c	2004/02/28 09:51:12
+++ //depot/user/rwatson/netperf/sys/kern/subr_log.c	2004/03/06 09:01:33
@@ -87,7 +87,12 @@
 	struct	callout sc_callout;	/* callout to wakeup syslog  */
 } logsoftc;
 
-int	log_open;			/* also used in log() */
+/*
+ * log_mtx protects logsoftc, log_open.  Note that log_mtx does *not*
+ * protect the structures associated with msgbuf, which require Giant.
+ */
+struct mtx	log_mtx;
+int		log_open;		/* also used in log() */
 
 /* Times per second to check for a pending syslog wakeup. */
 static int	log_wakeups_per_second = 5;
@@ -98,17 +103,24 @@
 static	int
 logopen(dev_t dev, int flags, int mode, struct thread *td)
 {
-	if (log_open)
+
+	mtx_lock(&log_mtx);
+	if (log_open) {
+		mtx_unlock(&log_mtx);
 		return (EBUSY);
+	}
 	log_open = 1;
-	callout_init(&logsoftc.sc_callout, 0);
+	callout_init(&logsoftc.sc_callout, CALLOUT_MPSAFE);
+	mtx_unlock(&log_mtx);
 	fsetown(td->td_proc->p_pid, &logsoftc.sc_sigio);	/* signal process only */
+	mtx_lock(&log_mtx);
 	if (log_wakeups_per_second < 1) {
 		printf("syslog wakeup is less than one.  Adjusting to 1.\n");
 		log_wakeups_per_second = 1;
 	}
 	callout_reset(&logsoftc.sc_callout, hz / log_wakeups_per_second,
 	    logtimeout, NULL);
+	mtx_unlock(&log_mtx);
 	return (0);
 }
 
@@ -117,9 +129,11 @@
 logclose(dev_t dev, int flag, int mode, struct thread *td)
 {
 
+	mtx_lock(&log_mtx);
 	log_open = 0;
 	callout_stop(&logsoftc.sc_callout);
 	logsoftc.sc_state = 0;
+	mtx_unlock(&log_mtx);
 	funsetown(&logsoftc.sc_sigio);
 	return (0);
 }
@@ -138,14 +152,18 @@
 			splx(s);
 			return (EWOULDBLOCK);
 		}
+		mtx_lock(&log_mtx);
 		logsoftc.sc_state |= LOG_RDWAIT;
+		mtx_unlock(&log_mtx);
 		if ((error = tsleep(mbp, LOG_RDPRI | PCATCH, "klog", 0))) {
 			splx(s);
 			return (error);
 		}
 	}
 	splx(s);
+	mtx_lock(&log_mtx);
 	logsoftc.sc_state &= ~LOG_RDWAIT;
+	mtx_unlock(&log_mtx);
 
 	while (uio->uio_resid > 0) {
 		l = imin(sizeof(buf), uio->uio_resid);
@@ -182,8 +200,11 @@
 logtimeout(void *arg)
 {
 
-	if (!log_open)
+	mtx_lock(&log_mtx);
+	if (!log_open) {
+		mtx_unlock(&log_mtx);
 		return;
+	}
 	if (log_wakeups_per_second < 1) {
 		printf("syslog wakeup is less than one.  Adjusting to 1.\n");
 		log_wakeups_per_second = 1;
@@ -191,6 +212,7 @@
 	if (msgbuftrigger == 0) {
 		callout_reset(&logsoftc.sc_callout,
 		    hz / log_wakeups_per_second, logtimeout, NULL);
+		mtx_unlock(&log_mtx);
 		return;
 	}
 	msgbuftrigger = 0;
@@ -203,6 +225,7 @@
 	}
 	callout_reset(&logsoftc.sc_callout, hz / log_wakeups_per_second,
 	    logtimeout, NULL);
+	mtx_unlock(&log_mtx);
 }
 
 /*ARGSUSED*/
@@ -221,10 +244,12 @@
 		break;
 
 	case FIOASYNC:
+		mtx_lock(&log_mtx);
 		if (*(int *)data)
 			logsoftc.sc_state |= LOG_ASYNC;
 		else
 			logsoftc.sc_state &= ~LOG_ASYNC;
+		mtx_unlock(&log_mtx);
 		break;
 
 	case FIOSETOWN:
@@ -253,6 +278,7 @@
 log_drvinit(void *unused)
 {
 
+	mtx_init(&log_mtx, "log_mtx", NULL, MTX_DEF);
 	make_dev(&log_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600, "klog");
 }
 
--- //depot/projects/netperf_socket/sys/kern/subr_prf.c	2004/02/28 09:51:12
+++ //depot/user/rwatson/netperf/sys/kern/subr_prf.c	2004/03/06 09:01:33
@@ -87,6 +87,9 @@
 	size_t	remain;
 };
 
+/*
+ * XXXRW: We access subr_log.c's log_open variable unlocked.
+ */
 extern	int log_open;
 
 static void  msglogchar(int c, int pri);
--- //depot/projects/netperf_socket/sys/kern/sys_socket.c	2004/02/28 10:48:04
+++ //depot/user/rwatson/netperf/sys/kern/sys_socket.c	2004/03/01 16:13:52
@@ -80,11 +80,14 @@
 	int error;
 
 #ifdef MAC
+	/* XXX: Socket lock needed here? */
 	error = mac_check_socket_receive(active_cred, so);
 	if (error)
 		return (error);
 #endif
+	NET_LOCK_GIANT();
 	error = so->so_proto->pr_usrreqs->pru_soreceive(so, 0, uio, 0, 0, 0);
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
@@ -101,12 +104,15 @@
 	int error;
 
 #ifdef MAC
+	/* XXX: Socket lock needed here? */
 	error = mac_check_socket_send(active_cred, so);
 	if (error)
 		return (error);
 #endif
+	NET_LOCK_GIANT();
 	error = so->so_proto->pr_usrreqs->pru_sosend(so, 0, uio, 0, 0, 0,
 						    uio->uio_td);
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
--- //depot/projects/netperf_socket/sys/kern/uipc_domain.c	2004/01/18 18:40:56
+++ //depot/user/rwatson/netperf/sys/kern/uipc_domain.c	2004/02/29 20:52:43
@@ -134,8 +134,13 @@
 	if (max_linkhdr < 16)		/* XXX */
 		max_linkhdr = 16;
 
-	callout_init(&pffast_callout, CALLOUT_MPSAFE);
-	callout_init(&pfslow_callout, CALLOUT_MPSAFE);
+	if (debug_mpsafenet) {
+		callout_init(&pffast_callout, CALLOUT_MPSAFE);
+		callout_init(&pfslow_callout, CALLOUT_MPSAFE);
+	} else {
+		callout_init(&pffast_callout, 0);
+		callout_init(&pfslow_callout, 0);
+	}
 
 	callout_reset(&pffast_callout, 1, pffasttimo, NULL);
 	callout_reset(&pfslow_callout, 1, pfslowtimo, NULL);
--- //depot/projects/netperf_socket/sys/kern/uipc_syscalls.c	2004/03/03 19:53:17
+++ //depot/user/rwatson/netperf/sys/kern/uipc_syscalls.c	2004/03/03 21:04:46
@@ -117,8 +117,10 @@
 	if (error)
 		return (error);
 	/* An extra reference on `fp' has been held for us by falloc(). */
+	NET_LOCK_GIANT();
 	error = socreate(uap->domain, &so, uap->type, uap->protocol,
 	    td->td_ucred, td);
+	NET_UNLOCK_GIANT();
 	FILEDESC_LOCK(fdp);
 	if (error) {
 		if (fdp->fd_ofiles[fd] == fp) {
@@ -172,6 +174,7 @@
 	struct socket *so;
 	int error;
 
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, fd, &so, NULL)) != 0)
 		goto done2;
 #ifdef MAC
@@ -186,6 +189,7 @@
 #endif
 	fputsock(so);
 done2:
+	NET_UNLOCK_GIANT();
 	FREE(sa, M_SONAME);
 	return (error);
 }
@@ -205,6 +209,7 @@
 	struct socket *so;
 	int error;
 
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, uap->s, &so, NULL)) == 0) {
 #ifdef MAC
 		/* XXXRW: MAC requires socket lock? */
@@ -218,6 +223,7 @@
 #endif
 		fputsock(so);
 	}
+	NET_UNLOCK_GIANT();
 	return(error);
 }
 
@@ -256,6 +262,7 @@
 			goto done3;
 		}
 	}
+	NET_LOCK_GIANT();
 	error = fgetsock(td, uap->s, &head, &fflag);
 	if (error)
 		goto done2;
@@ -297,7 +304,8 @@
 	 * ready to be accepted. Remove it from the queue prior to
 	 * allocating the file descriptor for it since falloc() may
 	 * block allowing another process to accept the connection
-	 * instead.
+	 * instead.  The reference previously owned by the socket queue
+	 * is now thread-local, letting us release the lock on the head.
 	 */
 	so = TAILQ_FIRST(&head->so_comp);
 	TAILQ_REMOVE(&head->so_comp, so, so_list);
@@ -310,7 +318,8 @@
 		 * Probably ran out of file descriptors. Put the
 		 * unaccepted connection back onto the queue and
 		 * do another wakeup so some other process might
-		 * have a chance at it.
+		 * have a chance at it.  Note that strict ordering
+		 * is lost.
 		 */
 		SOCK_LOCK(head);
 		TAILQ_INSERT_HEAD(&head->so_comp, so, so_list);
@@ -327,6 +336,9 @@
 	/* connection has been removed from the listen queue */
 	KNOTE(&head->so_rcv.sb_sel.si_note, 0);
 
+	/*
+	 * XXXRW: so should be locked to modify so_state here?
+	 */
 	so->so_state &= ~SS_COMP;
 	so->so_head = NULL;
 	pgid = fgetown(&head->so_sigio);
@@ -411,6 +423,7 @@
 		fdrop(nfp, td);
 	fputsock(head);
 done2:
+	NET_UNLOCK_GIANT();
 done3:
 	return (error);
 }
@@ -475,6 +488,7 @@
 	int error, s;
 	int interrupted = 0;
 
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, fd, &so, NULL)) != 0)
 		goto done2;
 	if (so->so_state & SS_ISCONNECTING) {
@@ -519,6 +533,7 @@
 done1:
 	fputsock(so);
 done2:
+	NET_UNLOCK_GIANT();
 	FREE(sa, M_SONAME);
 	return (error);
 }
@@ -541,6 +556,7 @@
 	struct socket *so1, *so2;
 	int fd, error, sv[2];
 
+	NET_LOCK_GIANT();
 	error = socreate(uap->domain, &so1, uap->type, uap->protocol,
 	    td->td_ucred, td);
 	if (error)
@@ -612,6 +628,7 @@
 free1:
 	(void)soclose(so1);
 done2:
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
@@ -697,6 +714,7 @@
 	int iovlen;
 #endif
 
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, s, &so, NULL)) != 0)
 		goto bad2;
 
@@ -758,6 +776,7 @@
 bad:
 	fputsock(so);
 bad2:
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
@@ -937,7 +956,9 @@
 	int iovlen;
 #endif
 
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, s, &so, NULL)) != 0) {
+		NET_UNLOCK_GIANT();
 		return (error);
 	}
 
@@ -1069,6 +1090,7 @@
 	}
 out:
 	fputsock(so);
+	NET_UNLOCK_GIANT();
 	if (fromsa)
 		FREE(fromsa, M_SONAME);
 	if (control)
@@ -1283,10 +1305,12 @@
 	struct socket *so;
 	int error;
 
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, uap->s, &so, NULL)) == 0) {
 		error = soshutdown(so, uap->how);
 		fputsock(so);
 	}
+	NET_UNLOCK_GIANT();
 	return(error);
 }
 
@@ -1314,6 +1338,7 @@
 	if (uap->valsize < 0)
 		return (EINVAL);
 
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, uap->s, &so, NULL)) == 0) {
 		sopt.sopt_dir = SOPT_SET;
 		sopt.sopt_level = uap->level;
@@ -1324,6 +1349,7 @@
 		error = sosetopt(so, &sopt);
 		fputsock(so);
 	}
+	NET_UNLOCK_GIANT();
 	return(error);
 }
 
@@ -1347,6 +1373,7 @@
 	struct  socket *so;
 	struct	sockopt sopt;
 
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, uap->s, &so, NULL)) != 0)
 		goto done2;
 	if (uap->val) {
@@ -1376,6 +1403,7 @@
 done1:
 	fputsock(so);
 done2:
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
@@ -1400,6 +1428,7 @@
 	socklen_t len;
 	int error;
 
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, uap->fdes, &so, NULL)) != 0)
 		goto done2;
 	error = copyin(uap->alen, &len, sizeof (len));
@@ -1433,6 +1462,7 @@
 done1:
 	fputsock(so);
 done2:
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
@@ -1483,6 +1513,7 @@
 	socklen_t len;
 	int error;
 
+	NET_LOCK_GIANT();
 	if ((error = fgetsock(td, uap->fdes, &so, NULL)) != 0)
 		goto done2;
 	if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) {
@@ -1521,6 +1552,7 @@
 done1:
 	fputsock(so);
 done2:
+	NET_UNLOCK_GIANT();
 	return (error);
 }
 
@@ -1674,6 +1706,8 @@
 	int error, s, headersize = 0, headersent = 0;
 	struct iovec *hdr_iov = NULL;
 
+	NET_LOCK_GIANT();
+
 	hdtr_size = 0;
 
 	/*
@@ -2051,6 +2085,8 @@
 	if (m_header)
 		m_freem(m_header);
 
+	NET_UNLOCK_GIANT();
+
 	if (error == ERESTART)
 		error = EINTR;
 
--- //depot/projects/netperf_socket/sys/net/bpf.c	2004/02/29 12:24:51
+++ //depot/user/rwatson/netperf/sys/net/bpf.c	2004/03/01 16:13:52
@@ -585,7 +585,10 @@
 	BPFD_UNLOCK(d);
 #endif
 	/* NB: the driver frees the mbuf */
-	return (*ifp->if_output)(ifp, m, &dst, (struct rtentry *)0);
+	NET_LOCK_GIANT();
+	error = (*ifp->if_output)(ifp, m, &dst, (struct rtentry *)0);
+	NET_UNLOCK_GIANT();
+	return (error);
 }
 
 /*
--- //depot/projects/netperf_socket/sys/net/bpfdesc.h	2004/02/29 12:24:51
+++ //depot/user/rwatson/netperf/sys/net/bpfdesc.h	2004/02/29 22:51:24
@@ -102,7 +102,10 @@
 
 #define BPFD_LOCK(bd)		mtx_lock(&(bd)->bd_mtx)
 #define BPFD_UNLOCK(bd)		mtx_unlock(&(bd)->bd_mtx)
-#define BPFD_LOCK_ASSERT(bd)	mtx_assert(&(bd)->bd_mtx, MA_OWNED)
+#define BPFD_LOCK_ASSERT(bd)	do {					\
+	mtx_assert(&(bd)->bd_mtx, MA_OWNED);				\
+	NET_ASSERT_GIANT();						\
+} while (0)
 
 /* Test whether a BPF is ready for read(). */
 #define	bpf_ready(bd)						 \
--- //depot/projects/netperf_socket/sys/net/if.c	2004/02/28 09:51:12
+++ //depot/user/rwatson/netperf/sys/net/if.c	2004/03/07 19:10:47
@@ -659,6 +659,7 @@
 
 /*
  * Create a clone network interface.
+ * XXXRW: Locking?
  */
 int
 if_clone_create(char *name, int len)
@@ -731,6 +732,7 @@
 
 /*
  * Destroy a clone network interface.
+ * XXXRW: Locking?
  */
 int
 if_clone_destroy(const char *name)
@@ -768,6 +770,7 @@
 
 /*
  * Look up a network interface cloner.
+ * XXXRW: Locking?
  */
 static struct if_clone *
 if_clone_lookup(const char *name, int *unitp)
@@ -809,6 +812,7 @@
 
 /*
  * Register a network interface cloner.
+ * XXXRW: Locking?
  */
 void
 if_clone_attach(struct if_clone *ifc)
@@ -851,6 +855,7 @@
 
 /*
  * Unregister a network interface cloner.
+ * XXXRW: Locking?
  */
 void
 if_clone_detach(struct if_clone *ifc)
@@ -863,6 +868,7 @@
 
 /*
  * Provide list of interface cloners to userspace.
+ * XXXRW: Locking?
  */
 static int
 if_clone_list(struct if_clonereq *ifcr)
--- //depot/projects/netperf_socket/sys/net/if_disc.c	2004/01/18 09:46:57
+++ //depot/user/rwatson/netperf/sys/net/if_disc.c	2004/03/07 17:51:42
@@ -76,6 +76,7 @@
 static int	disc_clone_create(struct if_clone *, int);
 static void	disc_clone_destroy(struct ifnet *);
 
+static struct mtx disc_mtx;
 static MALLOC_DEFINE(M_DISC, DISCNAME, "Discard interface");
 static LIST_HEAD(, disc_softc) disc_softc_list;
 static struct if_clone disc_cloner = IF_CLONE_INITIALIZER(DISCNAME,
@@ -104,39 +105,59 @@
 	ifp->if_snd.ifq_maxlen = 20;
 	if_attach(ifp);
 	bpfattach(ifp, DLT_NULL, sizeof(u_int));
+	mtx_lock(&disc_mtx);
 	LIST_INSERT_HEAD(&disc_softc_list, sc, sc_list);
+	mtx_unlock(&disc_mtx);
 
 	return (0);
 }
 
 static void
+disc_destroy(struct disc_softc *sc)
+{
+
+	bpfdetach(&sc->sc_if);
+	if_detach(&sc->sc_if);
+
+	free(sc, M_DISC);
+}
+
+static void
 disc_clone_destroy(struct ifnet *ifp)
 {
 	struct disc_softc	*sc;
 
 	sc = ifp->if_softc;
-
+	mtx_lock(&disc_mtx);
 	LIST_REMOVE(sc, sc_list);
-	bpfdetach(ifp);
-	if_detach(ifp);
+	mtx_unlock(&disc_mtx);
 
-	free(sc, M_DISC);
+	disc_destroy(sc);
 }
 
 static int
 disc_modevent(module_t mod, int type, void *data) 
-{ 
+{
+	struct disc_softc *sc;
+
 	switch (type) { 
 	case MOD_LOAD: 
+		mtx_init(&disc_mtx, "disc_mtx", NULL, MTX_DEF);
 		LIST_INIT(&disc_softc_list);
 		if_clone_attach(&disc_cloner);
 		break; 
 	case MOD_UNLOAD: 
 		if_clone_detach(&disc_cloner);
 
-		while (!LIST_EMPTY(&disc_softc_list))
-			disc_clone_destroy(
-			    &LIST_FIRST(&disc_softc_list)->sc_if);
+		mtx_lock(&disc_mtx);
+		while ((sc = LIST_FIRST(&disc_softc_list)) != NULL) {
+			LIST_REMOVE(sc, sc_list);
+			mtx_unlock(&disc_mtx);
+			disc_destroy(sc);
+			mtx_lock(&disc_mtx);
+		}
+		mtx_unlock(&disc_mtx);
+		mtx_destroy(&disc_mtx);
 		break;
 	} 
 	return 0; 
--- //depot/projects/netperf_socket/sys/net/if_faith.c	2004/01/18 09:46:57
+++ //depot/user/rwatson/netperf/sys/net/if_faith.c	2004/03/07 17:56:46
@@ -99,11 +99,13 @@
 
 static int faithmodevent(module_t, int, void *);
 
+static struct mtx faith_mtx;
 static MALLOC_DEFINE(M_FAITH, FAITHNAME, "Firewall Assisted Tunnel Interface");
 static LIST_HEAD(, faith_softc) faith_softc_list;
 
 int	faith_clone_create(struct if_clone *, int);
 void	faith_clone_destroy(struct ifnet *);
+static void	faith_destroy(struct faith_softc *);
 
 struct if_clone faith_cloner = IF_CLONE_INITIALIZER(FAITHNAME,
     faith_clone_create, faith_clone_destroy, 0, IF_MAXUNIT);
@@ -116,9 +118,11 @@
 	int type;
 	void *data;
 {
+	struct faith_softc *sc;
 
 	switch (type) {
 	case MOD_LOAD:
+		mtx_init(&faith_mtx, "faith_mtx", NULL, MTX_DEF);
 		LIST_INIT(&faith_softc_list);
 		if_clone_attach(&faith_cloner);
 
@@ -134,10 +138,15 @@
 
 		if_clone_detach(&faith_cloner);
 
-		while (!LIST_EMPTY(&faith_softc_list))
-			faith_clone_destroy(
-			    &LIST_FIRST(&faith_softc_list)->sc_if);
-
+		mtx_lock(&faith_mtx);
+		while ((sc = LIST_FIRST(&faith_softc_list)) != NULL) {
+			LIST_REMOVE(sc, sc_list);
+			mtx_unlock(&faith_mtx);
+			faith_destroy(sc);
+			mtx_lock(&faith_mtx);
+		}
+		mtx_unlock(&faith_mtx);
+		mtx_destroy(&faith_mtx);
 		break;
 	}
 	return 0;
@@ -176,21 +185,32 @@
 	sc->sc_if.if_snd.ifq_maxlen = ifqmaxlen;
 	if_attach(&sc->sc_if);
 	bpfattach(&sc->sc_if, DLT_NULL, sizeof(u_int));
+	mtx_lock(&faith_mtx);
 	LIST_INSERT_HEAD(&faith_softc_list, sc, sc_list);
+	mtx_unlock(&faith_mtx);
 	return (0);
 }
 
+static void
+faith_destroy(struct faith_softc *sc)
+{
+
+	bpfdetach(&sc->sc_if);
+	if_detach(&sc->sc_if);
+	free(sc, M_FAITH);
+}
+
 void
 faith_clone_destroy(ifp)
 	struct ifnet *ifp;
 {
 	struct faith_softc *sc = (void *) ifp;
 
+	mtx_lock(&faith_mtx);
 	LIST_REMOVE(sc, sc_list);
-	bpfdetach(ifp);
-	if_detach(ifp);
+	mtx_unlock(&faith_mtx);
 
-	free(sc, M_FAITH);
+	faith_destroy(sc);
 }
 
 int
--- //depot/projects/netperf_socket/sys/net/if_loop.c	2004/01/18 09:46:57
+++ //depot/user/rwatson/netperf/sys/net/if_loop.c	2004/03/07 18:16:39
@@ -113,6 +113,7 @@
 
 static MALLOC_DEFINE(M_LO, LONAME, "Loopback Interface");
 
+static struct mtx lo_mtx;
 static LIST_HEAD(lo_list, lo_softc) lo_list;
 
 struct if_clone lo_cloner = IF_CLONE_INITIALIZER(LONAME,
@@ -129,9 +130,13 @@
 	/* XXX: destroying lo0 will lead to panics. */
 	KASSERT(loif != ifp, ("%s: destroying lo0", __func__));
 
+	mtx_lock(&lo_mtx);
+	LIST_REMOVE(sc, sc_next);
+	mtx_unlock(&lo_mtx);
+
 	bpfdetach(ifp);
 	if_detach(ifp);
-	LIST_REMOVE(sc, sc_next);
+
 	free(sc, M_LO);
 }
 
@@ -154,7 +159,10 @@
 	sc->sc_if.if_softc = sc;
 	if_attach(&sc->sc_if);
 	bpfattach(&sc->sc_if, DLT_NULL, sizeof(u_int));
+
+	mtx_lock(&lo_mtx);
 	LIST_INSERT_HEAD(&lo_list, sc, sc_next);
+	mtx_unlock(&lo_mtx);
 	if (loif == NULL)
 		loif = &sc->sc_if;
 
@@ -166,6 +174,7 @@
 { 
 	switch (type) { 
 	case MOD_LOAD: 
+		mtx_init(&lo_mtx, "lo_mtx", NULL, MTX_DEF);
 		LIST_INIT(&lo_list);
 		if_clone_attach(&lo_cloner);
 		break; 
--- //depot/projects/netperf_socket/sys/net/if_stf.c	2004/03/07 09:36:30
+++ //depot/user/rwatson/netperf/sys/net/if_stf.c	2004/03/07 18:21:11
@@ -138,6 +138,10 @@
 	LIST_ENTRY(stf_softc) sc_list;	/* all stf's are linked */
 };
 
+/*
+ * All mutable global variables in if_stf.c are protected by stf_mtx.
+ */
+static struct mtx stf_mtx;
 static LIST_HEAD(, stf_softc) stf_softc_list;
 
 static MALLOC_DEFINE(M_STF, STFNAME, "6to4 Tunnel Interface");
@@ -197,24 +201,36 @@
 	sc->sc_if.if_snd.ifq_maxlen = IFQ_MAXLEN;
 	if_attach(&sc->sc_if);
 	bpfattach(&sc->sc_if, DLT_NULL, sizeof(u_int));
+	mtx_lock(&stf_mtx);
 	LIST_INSERT_HEAD(&stf_softc_list, sc, sc_list);
+	mtx_unlock(&stf_mtx);
 	return (0);
 }
 
+static void
+stf_destroy(struct stf_softc *sc)
+{
+	int err;
+
+	err = encap_detach(sc->encap_cookie);
+	KASSERT(err == 0, ("Unexpected error detaching encap_cookie"));
+	bpfdetach(&sc->sc_if);
+	if_detach(&sc->sc_if);
+
+	free(sc, M_STF);
+}
+
 void
 stf_clone_destroy(ifp)
 	struct ifnet *ifp;
 {
-	int err;
 	struct stf_softc *sc = (void *) ifp;
 
+	mtx_lock(&stf_mtx);
 	LIST_REMOVE(sc, sc_list);
-	err = encap_detach(sc->encap_cookie);
-	KASSERT(err == 0, ("Unexpected error detaching encap_cookie"));
-	bpfdetach(ifp);
-	if_detach(ifp);
+	mtx_unlock(&stf_mtx);
 
-	free(sc, M_STF);
+	stf_destroy(sc);
 }
 
 static int
@@ -223,9 +239,11 @@
 	int type;
 	void *data;
 {
+	struct stf_softc *sc;
 
 	switch (type) {
 	case MOD_LOAD:
+		mtx_init(&stf_mtx, "stf_mtx", NULL, MTX_DEF);
 		LIST_INIT(&stf_softc_list);
 		if_clone_attach(&stf_cloner);
 
@@ -233,8 +251,15 @@
 	case MOD_UNLOAD:
 		if_clone_detach(&stf_cloner);
 
-		while (!LIST_EMPTY(&stf_softc_list))
-			stf_clone_destroy(&LIST_FIRST(&stf_softc_list)->sc_if);
+		mtx_lock(&stf_mtx);
+		while ((sc = LIST_FIRST(&stf_softc_list)) != NULL) {
+			LIST_REMOVE(sc, sc_list);
+			mtx_unlock(&stf_mtx);
+			stf_destroy(sc);
+			mtx_lock(&stf_mtx);
+		}
+		mtx_unlock(&stf_mtx);
+		mtx_destroy(&stf_mtx);
 		break;
 	}
 
--- //depot/projects/netperf_socket/sys/net/if_tap.c	2004/02/28 09:51:12
+++ //depot/user/rwatson/netperf/sys/net/if_tap.c	2004/03/07 16:34:34
@@ -108,6 +108,12 @@
 	.d_name =	CDEV_NAME,
 };
 
+/*
+ * All global variables in if_tap.c are locked with tapmtx, with the
+ * exception of tapdebug, which is accessed unlocked, and tapclones,
+ * which currently suffers from a bad API (XXX).
+ */
+static struct mtx		tapmtx;
 static int			tapdebug = 0;        /* debug flag   */
 static SLIST_HEAD(, tap_softc)	taphead;             /* first device */
 static struct clonedevs 	*tapclones;
@@ -138,22 +144,40 @@
 
 		/* intitialize device */
 
+		mtx_init(&tapmtx, "tapmtx", NULL, MTX_DEF);
 		SLIST_INIT(&taphead);
 
 		eh_tag = EVENTHANDLER_REGISTER(dev_clone, tapclone, 0, 1000);
-		if (eh_tag == NULL)
+		if (eh_tag == NULL) {
+			mtx_destroy(&tapmtx);
 			return (ENOMEM);
+		}
 		return (0);
 
 	case MOD_UNLOAD:
-		SLIST_FOREACH(tp, &taphead, tap_next)
-			if (tp->tap_flags & TAP_OPEN)
+		/*
+		 * The EBUSY algorithm here can't quite atomically
+		 * guarantee that this is race-free since we have to
+		 * release the tap mtx to deregister the clone handler.
+		 */
+		mtx_lock(&tapmtx);
+		SLIST_FOREACH(tp, &taphead, tap_next) {
+			mtx_lock(&tp->tap_mtx);
+			if (tp->tap_flags & TAP_OPEN) {
+				mtx_unlock(&tp->tap_mtx);
+				mtx_unlock(&tapmtx);
 				return (EBUSY);
+			}
+			mtx_unlock(&tp->tap_mtx);
+		}
+		mtx_unlock(&tapmtx);
 
 		EVENTHANDLER_DEREGISTER(dev_clone, eh_tag);
 
+		mtx_lock(&tapmtx);
 		while ((tp = SLIST_FIRST(&taphead)) != NULL) {
 			SLIST_REMOVE_HEAD(&taphead, tap_next);
+			mtx_unlock(&tapmtx);
 
 			ifp = &tp->tap_if;
 
@@ -167,8 +191,11 @@
 			ether_ifdetach(ifp);
 			splx(s);
 
+			mtx_destroy(&tp->tap_mtx);
 			free(tp, M_TAP);
+			mtx_lock(&tapmtx);
 		}
+		mtx_unlock(&tapmtx);
 		clone_cleanup(&tapclones);
 
 		break;
@@ -245,7 +272,10 @@
 
 	/* allocate driver storage and create device */
 	MALLOC(tp, struct tap_softc *, sizeof(*tp), M_TAP, M_WAITOK | M_ZERO);
+	mtx_init(&tp->tap_mtx, "tap_mtx", NULL, MTX_DEF);
+	mtx_lock(&tapmtx);
 	SLIST_INSERT_HEAD(&taphead, tp, tap_next);
+	mtx_unlock(&tapmtx);
 
 	unit = dev2unit(dev) & TAPMAXUNIT;
 
@@ -320,13 +350,16 @@
 		tp = dev->si_drv1;
 	}
 
+	/* Unlocked read. */
 	KASSERT(!(tp->tap_flags & TAP_OPEN), 
 		("%s flags is out of sync", tp->tap_if.if_xname));
 
 	bcopy(tp->arpcom.ac_enaddr, tp->ether_addr, sizeof(tp->ether_addr));
 
 	tp->tap_pid = td->td_proc->p_pid;
+	mtx_lock(&tp->tap_mtx);
 	tp->tap_flags |= TAP_OPEN;
+	mtx_unlock(&tp->tap_mtx);
 
 	TAPDEBUG("%s is open. minor = %#x\n", 
 		tp->tap_if.if_xname, minor(dev));
@@ -359,7 +392,9 @@
 	 * interface, if we are in VMnet mode. just close the device.
 	 */
 
+	mtx_lock(&tp->tap_mtx);
 	if (((tp->tap_flags & TAP_VMNET) == 0) && (ifp->if_flags & IFF_UP)) {
+		mtx_unlock(&tp->tap_mtx);
 		s = splimp();
 		if_down(ifp);
 		if (ifp->if_flags & IFF_RUNNING) {
@@ -383,12 +418,15 @@
 			ifp->if_flags &= ~IFF_RUNNING;
 		}
 		splx(s);
-	}
+	} else
+		mtx_unlock(&tp->tap_mtx);
 
 	funsetown(&tp->tap_sigio);
 	selwakeuppri(&tp->tap_rsel, PZERO+1);
 
+	mtx_lock(&tp->tap_mtx);
 	tp->tap_flags &= ~TAP_OPEN;
+	mtx_unlock(&tp->tap_mtx);
 	tp->tap_pid = 0;
 
 	TAPDEBUG("%s is closed. minor = %#x\n", 
@@ -482,10 +520,14 @@
 	 * XXX: can this do any harm because of queue overflow?
 	 */
 
+	mtx_lock(&tp->tap_mtx);
 	if (((tp->tap_flags & TAP_VMNET) == 0) &&
 	    ((tp->tap_flags & TAP_READY) != TAP_READY)) {
 		struct mbuf	*m = NULL;
 
+		mtx_unlock(&tp->tap_mtx);
+
+		/* Unlocked read. */
 		TAPDEBUG("%s not ready, tap_flags = 0x%x\n", ifp->if_xname, 
 		    tp->tap_flags);
 
@@ -505,13 +547,17 @@
 	ifp->if_flags |= IFF_OACTIVE;
 
 	if (ifp->if_snd.ifq_len != 0) {
+		mtx_lock(&tp->tap_mtx);
 		if (tp->tap_flags & TAP_RWAIT) {
 			tp->tap_flags &= ~TAP_RWAIT;
 			wakeup(tp);
 		}
 
-		if ((tp->tap_flags & TAP_ASYNC) && (tp->tap_sigio != NULL))
+		if ((tp->tap_flags & TAP_ASYNC) && (tp->tap_sigio != NULL)) {
+			mtx_unlock(&tp->tap_mtx);
 			pgsigio(&tp->tap_sigio, SIGIO, 0);
+		} else
+			mtx_unlock(&tp->tap_mtx);
 
 		selwakeuppri(&tp->tap_rsel, PZERO+1);
 		ifp->if_opackets ++; /* obytes are counted in ether_output */
@@ -571,10 +617,12 @@
 
 		case FIOASYNC:
 			s = splimp();
+			mtx_lock(&tp->tap_mtx);
 			if (*(int *)data)
 				tp->tap_flags |= TAP_ASYNC;
 			else
 				tp->tap_flags &= ~TAP_ASYNC;
+			mtx_unlock(&tp->tap_mtx);
 			splx(s);
 			break;
 
@@ -658,7 +706,11 @@
 
 	TAPDEBUG("%s reading, minor = %#x\n", ifp->if_xname, minor(dev));
 
+	mtx_lock(&tp->tap_mtx);
 	if ((tp->tap_flags & TAP_READY) != TAP_READY) {
+		mtx_unlock(&tp->tap_mtx);
+
+		/* Unlocked read. */
 		TAPDEBUG("%s not ready. minor = %#x, tap_flags = 0x%x\n",
 			ifp->if_xname, minor(dev), tp->tap_flags);
 
@@ -666,6 +718,7 @@
 	}
 
 	tp->tap_flags &= ~TAP_RWAIT;
+	mtx_unlock(&tp->tap_mtx);
 
 	/* sleep until we get a packet */
 	do {
@@ -677,7 +730,10 @@
 			if (flag & IO_NDELAY)
 				return (EWOULDBLOCK);
 
+			/* This looks like a wanna-be condition variable. */
+			mtx_lock(&tp->tap_mtx);
 			tp->tap_flags |= TAP_RWAIT;
+			mtx_unlock(&tp->tap_mtx);
 			error = tsleep(tp,PCATCH|(PZERO+1),"taprd",0);
 			if (error)
 				return (error);
--- //depot/projects/netperf_socket/sys/net/if_tapvar.h	2004/02/28 09:51:12
+++ //depot/user/rwatson/netperf/sys/net/if_tapvar.h	2004/03/07 16:34:34
@@ -41,6 +41,10 @@
 #ifndef _NET_IF_TAPVAR_H_
 #define _NET_IF_TAPVAR_H_
 
+/*
+ * tap_mtx locks tap_flags.  tap_next locked with global tapmtx.  Other
+ * fields locked by owning subsystems, or constant (tap_pid) after open.
+ */
 struct tap_softc {
 	struct arpcom	arpcom;			/* ethernet common data      */
 #define tap_if		arpcom.ac_if
@@ -62,6 +66,7 @@
 
 	SLIST_ENTRY(tap_softc)	tap_next;	/* next device in chain      */
 	dev_t		 tap_dev;
+	struct mtx	 tap_mtx;		/* per-softc mutex */
 };
 
 #endif /* !_NET_IF_TAPVAR_H_ */
--- //depot/projects/netperf_socket/sys/net/if_tun.c	2004/02/28 09:51:12
+++ //depot/user/rwatson/netperf/sys/net/if_tun.c	2004/03/06 20:55:10
@@ -81,6 +81,15 @@
 #define TUNDEBUG	if (tundebug) if_printf
 #define	TUNNAME		"tun"
 
+/*
+ * All mutable global variables in if_tun are locked using tunmtx, with
+ * the exception of tundebug, which is used unlocked.  Per-softc data
+ * not yet locked.
+ *
+ * XXXRW: tunclones can't be properly locked with the current API.  Bug
+ * phk about this.
+ */
+static struct mtx tunmtx;
 static MALLOC_DEFINE(M_TUN, TUNNAME, "Tunnel Interface");
 static int tundebug = 0;
 static struct clonedevs *tunclones;
@@ -150,6 +159,7 @@
 
 	switch (type) {
 	case MOD_LOAD:
+		mtx_init(&tunmtx, "tunmtx", NULL, MTX_DEF);
 		tag = EVENTHANDLER_REGISTER(dev_clone, tunclone, 0, 1000);
 		if (tag == NULL)
 			return (ENOMEM);
@@ -157,19 +167,25 @@
 	case MOD_UNLOAD:
 		EVENTHANDLER_DEREGISTER(dev_clone, tag);
 
-		while (!TAILQ_EMPTY(&tunhead)) {
-			tp = TAILQ_FIRST(&tunhead);
+		mtx_lock(&tunmtx);
+		while ((tp = TAILQ_FIRST(&tunhead)) != NULL) {
+			TAILQ_REMOVE(&tunhead, tp, tun_list);
+			mtx_unlock(&tunmtx);
+
 			KASSERT((tp->tun_flags & TUN_OPEN) == 0,
 			    ("tununits is out of sync - unit %d",
 			    tp->tun_if.if_dunit));
-			TAILQ_REMOVE(&tunhead, tp, tun_list);
+
 			dev = tp->tun_dev;
 			bpfdetach(&tp->tun_if);
 			if_detach(&tp->tun_if);
 			destroy_dev(dev);
 			free(tp, M_TUN);
+			mtx_lock(&tunmtx);
 		}
+		mtx_unlock(&tunmtx);
 		clone_cleanup(&tunclones);
+		mtx_destroy(&tunmtx);
 		break;
 	}
 	return 0;
@@ -208,7 +224,9 @@
 	MALLOC(sc, struct tun_softc *, sizeof(*sc), M_TUN, M_WAITOK | M_ZERO);
 	sc->tun_flags = TUN_INITED;
 	sc->tun_dev = dev;
+	mtx_lock(&tunmtx);
 	TAILQ_INSERT_TAIL(&tunhead, sc, tun_list);
+	mtx_unlock(&tunmtx);
 
 	ifp = &sc->tun_if;
 	if_initname(ifp, TUNNAME, dev2unit(dev));
--- //depot/projects/netperf_socket/sys/netinet/in_pcb.h	2004/01/18 09:46:57
+++ //depot/user/rwatson/netperf/sys/netinet/in_pcb.h	2004/02/29 22:51:24
@@ -248,9 +248,14 @@
 #define INP_LOCK(inp)		mtx_lock(&(inp)->inp_mtx)
 #define INP_UNLOCK(inp)		mtx_unlock(&(inp)->inp_mtx)
 #ifndef INET6
-#define INP_LOCK_ASSERT(inp)	mtx_assert(&(inp)->inp_mtx, MA_OWNED)
+#define INP_LOCK_ASSERT(inp)	do {					\
+	mtx_assert(&(inp)->inp_mtx, MA_OWNED);				\
+	NET_ASSERT_GIANT();						\
+} while (0)
 #else
-#define INP_LOCK_ASSERT(inp)
+#define INP_LOCK_ASSERT(inp)	do {					\
+	NET_ASSERT_GIANT();						\
+} while (0)
 #endif
 
 #define INP_INFO_LOCK_INIT(ipi, d) \
@@ -260,11 +265,21 @@
 #define INP_INFO_RUNLOCK(ipi)	mtx_unlock(&(ipi)->ipi_mtx)
 #define INP_INFO_WUNLOCK(ipi)	mtx_unlock(&(ipi)->ipi_mtx)
 #ifndef INET6
-#define INP_INFO_RLOCK_ASSERT(ipi)	mtx_assert(&(ipi)->ipi_mtx, MA_OWNED)
-#define INP_INFO_WLOCK_ASSERT(ipi)	mtx_assert(&(ipi)->ipi_mtx, MA_OWNED)
+#define INP_INFO_RLOCK_ASSERT(ipi)	do {				\
+	mtx_assert(&(ipi)->ipi_mtx, MA_OWNED);				\
+	NET_ASSERT_GIANT();						\
+} while (0)
+#define INP_INFO_WLOCK_ASSERT(ipi)	do {				\
+	mtx_assert(&(ipi)->ipi_mtx, MA_OWNED);				\
+	NET_ASSERT_GIANT();						\
+} while (0)
 #else
-#define INP_INFO_RLOCK_ASSERT(ipi)
-#define INP_INFO_WLOCK_ASSERT(ipi)
+#define INP_INFO_RLOCK_ASSERT(ipi)	do {				\
+	NET_ASSERT_GIANT();						\
+} while (0)
+#define INP_INFO_WLOCK_ASSERT(ipi)	do {				\
+	NET_ASSERT_GIANT();						\
+} while (0)
 #endif
 
 #define INP_PCBHASH(faddr, lport, fport, mask) \
--- //depot/projects/netperf_socket/sys/netinet/ip_dummynet.c	2004/03/03 18:07:13
+++ //depot/user/rwatson/netperf/sys/netinet/ip_dummynet.c	2004/03/03 20:00:06
@@ -171,7 +171,10 @@
 #define	DUMMYNET_LOCK_DESTROY()	mtx_destroy(&dummynet_mtx)
 #define	DUMMYNET_LOCK()		mtx_lock(&dummynet_mtx)
 #define	DUMMYNET_UNLOCK()	mtx_unlock(&dummynet_mtx)
-#define	DUMMYNET_LOCK_ASSERT()	mtx_assert(&dummynet_mtx, MA_OWNED)
+#define	DUMMYNET_LOCK_ASSERT()	do {				\
+	mtx_assert(&dummynet_mtx, MA_OWNED);			\
+	NET_ASSERT_GIANT();					\
+} while (0)
 
 static int config_pipe(struct dn_pipe *p);
 static int ip_dn_ctl(struct sockopt *sopt);
--- //depot/projects/netperf_socket/sys/netinet/ip_fw2.c	2004/02/28 09:51:12
+++ //depot/user/rwatson/netperf/sys/netinet/ip_fw2.c	2004/02/29 22:51:24
@@ -122,7 +122,10 @@
 #define	IPFW_LOCK_DESTROY(_chain)	mtx_destroy(&(_chain)->mtx)
 #define	IPFW_LOCK(_chain)	mtx_lock(&(_chain)->mtx)
 #define	IPFW_UNLOCK(_chain)	mtx_unlock(&(_chain)->mtx)
-#define	IPFW_LOCK_ASSERT(_chain)	mtx_assert(&(_chain)->mtx, MA_OWNED)
+#define	IPFW_LOCK_ASSERT(_chain)	do {				\
+	mtx_assert(&(_chain)->mtx, MA_OWNED);				\
+	NET_ASSERT_GIANT();						\
+} while (0)
 
 /*
  * list of rules for layer 3
--- //depot/projects/netperf_socket/sys/netinet/ip_mroute.c	2004/01/19 09:56:28
+++ //depot/user/rwatson/netperf/sys/netinet/ip_mroute.c	2004/03/05 07:48:03
@@ -103,7 +103,10 @@
 static struct mtx mfc_mtx;
 #define	MFC_LOCK()	mtx_lock(&mfc_mtx)
 #define	MFC_UNLOCK()	mtx_unlock(&mfc_mtx)
-#define	MFC_LOCK_ASSERT()	mtx_assert(&mfc_mtx, MA_OWNED)
+#define	MFC_LOCK_ASSERT()	do {					\
+	mtx_assert(&mfc_mtx, MA_OWNED);					\
+	NET_ASSERT_GIANT();						\
+} while (0)
 #define	MFC_LOCK_INIT()	mtx_init(&mfc_mtx, "mroute mfc table", NULL, MTX_DEF)
 #define	MFC_LOCK_DESTROY()	mtx_destroy(&mfc_mtx)
 
--- //depot/projects/netperf_socket/sys/netinet/tcp_subr.c	2004/02/28 09:51:12
+++ //depot/user/rwatson/netperf/sys/netinet/tcp_subr.c	2004/03/05 23:20:26
@@ -560,6 +560,7 @@
 #ifdef INET6
 	int isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
 #endif /* INET6 */
+	int callout_flag;
 
 	tm = uma_zalloc(tcpcb_zone, M_NOWAIT | M_ZERO);
 	if (tm == NULL)
@@ -573,11 +574,17 @@
 		tcp_mssdflt;
 
 	/* Set up our timeouts. */
-	callout_init(tp->tt_rexmt = &tm->tcpcb_mem_rexmt, 0);
-	callout_init(tp->tt_persist = &tm->tcpcb_mem_persist, 0);
-	callout_init(tp->tt_keep = &tm->tcpcb_mem_keep, 0);
-	callout_init(tp->tt_2msl = &tm->tcpcb_mem_2msl, 0);
-	callout_init(tp->tt_delack = &tm->tcpcb_mem_delack, 0);
+	/*
+	 * XXXRW: Are these actually MPSAFE?  I think so, but need to
+	 * review the timed wait code, as it has some list variables,
+	 * etc, that are global.
+	 */
+	callout_flag = debug_mpsafenet ? CALLOUT_MPSAFE : 0;
+	callout_init(tp->tt_rexmt = &tm->tcpcb_mem_rexmt, callout_flag);
+	callout_init(tp->tt_persist = &tm->tcpcb_mem_persist, callout_flag);
+	callout_init(tp->tt_keep = &tm->tcpcb_mem_keep, callout_flag);
+	callout_init(tp->tt_2msl = &tm->tcpcb_mem_2msl, callout_flag);
+	callout_init(tp->tt_delack = &tm->tcpcb_mem_delack, callout_flag);
 
 	if (tcp_do_rfc1323)
 		tp->t_flags = (TF_REQ_SCALE|TF_REQ_TSTMP);
--- //depot/projects/netperf_socket/sys/netinet/tcp_syncache.c	2004/02/28 09:51:12
+++ //depot/user/rwatson/netperf/sys/netinet/tcp_syncache.c	2004/03/01 18:05:01
@@ -540,6 +540,7 @@
 	struct socket *so;
 	struct tcpcb *tp;
 
+	NET_ASSERT_GIANT();
 	INP_INFO_WLOCK_ASSERT(&tcbinfo);
 
 	/*
--- //depot/projects/netperf_socket/sys/netinet/tcp_timer.c	2004/01/18 09:46:57
+++ //depot/user/rwatson/netperf/sys/netinet/tcp_timer.c	2004/03/06 21:55:09
@@ -273,6 +273,9 @@
 	}
 }
 
+/*
+ * XXXRW: This doesn't look MPSAFE.
+ */
 void
 tcp_timer_2msl_reset(struct tcptw *tw, int timeo)
 {
@@ -287,6 +290,9 @@
 	LIST_INSERT_BEFORE(tw_tail, tw, tw_2msl);
 }
 
+/*
+ * XXXRW: This doesn't look MPSAFE.
+ */
 void
 tcp_timer_2msl_stop(struct tcptw *tw)
 {
@@ -295,6 +301,9 @@
 		LIST_REMOVE(tw, tw_2msl);
 }
 
+/*
+ * XXXRW: This doesn't look MPSAFE.
+ */
 struct tcptw *
 tcp_timer_2msl_tw(int reuse)
 {
--- //depot/projects/netperf_socket/sys/netinet6/ah_core.c	2004/01/18 09:46:57
+++ //depot/user/rwatson/netperf/sys/netinet6/ah_core.c	2004/03/07 10:15:41
@@ -153,47 +153,39 @@
 static void ah_update_mbuf __P((struct mbuf *, int, int,
 	const struct ah_algorithm *, struct ah_algorithm_state *));
 
+static const struct ah_algorithm ah_algorithms[] = {
+	{ ah_sumsiz_1216, ah_common_mature, 128, 128, "hmac-md5",
+		ah_hmac_md5_init, ah_hmac_md5_loop, ah_hmac_md5_result, },
+	{ ah_sumsiz_1216, ah_common_mature, 160, 160, "hmac-sha1",
+		ah_hmac_sha1_init, ah_hmac_sha1_loop, ah_hmac_sha1_result, },
+	{ ah_sumsiz_1216, ah_keyed_md5_mature, 128, 128, "keyed-md5",
+		ah_keyed_md5_init, ah_keyed_md5_loop, ah_keyed_md5_result, },
+	{ ah_sumsiz_1216, ah_common_mature, 160, 160, "keyed-sha1",
+		ah_keyed_sha1_init, ah_keyed_sha1_loop,
+		ah_keyed_sha1_result, },
+	{ ah_sumsiz_zero, ah_none_mature, 0, 2048, "none",
+		ah_none_init, ah_none_loop, ah_none_result, },
+	{ ah_sumsiz_1216, ah_common_mature, 256, 256,
+		"hmac-sha2-256", ah_hmac_sha2_256_init, ah_hmac_sha2_256_loop,
+		ah_hmac_sha2_256_result, },
+	{ ah_sumsiz_1216, ah_common_mature, 384, 384,
+		"hmac-sha2-384", ah_hmac_sha2_384_init, ah_hmac_sha2_384_loop,
+		ah_hmac_sha2_384_result, },
+	{ ah_sumsiz_1216, ah_common_mature, 512, 512,
+		"hmac-sha2-512", ah_hmac_sha2_512_init, ah_hmac_sha2_512_loop,
+		ah_hmac_sha2_512_result, },
+	{ ah_sumsiz_1216, ah_common_mature, 160, 160,
+		"hmac-ripemd160", ah_hmac_ripemd160_init,
+		ah_hmac_ripemd160_loop, ah_hmac_ripemd160_result, },
+	{ ah_sumsiz_1216, ah_common_mature, 128, 128,
+		"aes-xcbc-mac", ah_aes_xcbc_mac_init, ah_aes_xcbc_mac_loop,
+		ah_aes_xcbc_mac_result, },
+};
+
 const struct ah_algorithm *
 ah_algorithm_lookup(idx)
 	int idx;
 {
-	/* checksum algorithms */
-	static struct ah_algorithm ah_algorithms[] = {
-		{ ah_sumsiz_1216, ah_common_mature, 128, 128, "hmac-md5",
-			ah_hmac_md5_init, ah_hmac_md5_loop,
-			ah_hmac_md5_result, },
-		{ ah_sumsiz_1216, ah_common_mature, 160, 160, "hmac-sha1",
-			ah_hmac_sha1_init, ah_hmac_sha1_loop,
-			ah_hmac_sha1_result, },
-		{ ah_sumsiz_1216, ah_keyed_md5_mature, 128, 128, "keyed-md5",
-			ah_keyed_md5_init, ah_keyed_md5_loop,
-			ah_keyed_md5_result, },
-		{ ah_sumsiz_1216, ah_common_mature, 160, 160, "keyed-sha1",
-			ah_keyed_sha1_init, ah_keyed_sha1_loop,
-			ah_keyed_sha1_result, },
-		{ ah_sumsiz_zero, ah_none_mature, 0, 2048, "none",
-			ah_none_init, ah_none_loop, ah_none_result, },
-		{ ah_sumsiz_1216, ah_common_mature, 256, 256,
-			"hmac-sha2-256",
-			ah_hmac_sha2_256_init, ah_hmac_sha2_256_loop,
-			ah_hmac_sha2_256_result, },
-		{ ah_sumsiz_1216, ah_common_mature, 384, 384,
-			"hmac-sha2-384",
-			ah_hmac_sha2_384_init, ah_hmac_sha2_384_loop,
-			ah_hmac_sha2_384_result, },
-		{ ah_sumsiz_1216, ah_common_mature, 512, 512,
-			"hmac-sha2-512",
-			ah_hmac_sha2_512_init, ah_hmac_sha2_512_loop,
-			ah_hmac_sha2_512_result, },
-		{ ah_sumsiz_1216, ah_common_mature, 160, 160,
-			"hmac-ripemd160",
-			ah_hmac_ripemd160_init, ah_hmac_ripemd160_loop,
-			ah_hmac_ripemd160_result, },
-		{ ah_sumsiz_1216, ah_common_mature, 128, 128,
-			"aes-xcbc-mac",
-			ah_aes_xcbc_mac_init, ah_aes_xcbc_mac_loop,
-			ah_aes_xcbc_mac_result, },
-	};
 
 	switch (idx) {
 	case SADB_AALG_MD5HMAC:
--- //depot/projects/netperf_socket/sys/netinet6/in6_ifattach.c	2004/02/28 09:51:12
+++ //depot/user/rwatson/netperf/sys/netinet6/in6_ifattach.c	2004/03/07 10:23:16
@@ -226,8 +226,8 @@
 	struct sockaddr_dl *sdl;
 	u_int8_t *addr;
 	size_t addrlen;
-	static u_int8_t allzero[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
-	static u_int8_t allone[8] =
+	static const u_int8_t allzero[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
+	static const u_int8_t allone[8] =
 		{ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 
 	for (ifa = ifp->if_addrlist.tqh_first;
--- //depot/projects/netperf_socket/sys/netinet6/in6_prefix.c	2004/01/18 09:46:57
+++ //depot/user/rwatson/netperf/sys/netinet6/in6_prefix.c	2004/03/07 11:31:09
@@ -100,6 +100,8 @@
 static int link_stray_ia6s __P((struct rr_prefix *rpp));
 static void	rp_remove __P((struct rr_prefix *rpp));
 
+static int	delete_each_prefix __P((struct rr_prefix *rpp, u_char origin));
+
 /*
  * Copy bits from src to tgt, from off bit for len bits.
  * Caller must specify collect tgtsize and srcsize.
@@ -955,7 +957,7 @@
 	}
 }
 
-int
+static int
 delete_each_prefix(struct rr_prefix *rpp, u_char origin)
 {
 	int error = 0;
--- //depot/projects/netperf_socket/sys/netinet6/in6_prefix.h	2004/01/18 09:46:57
+++ //depot/user/rwatson/netperf/sys/netinet6/in6_prefix.h	2004/03/07 11:31:09
@@ -88,4 +88,3 @@
 
 void in6_rr_timer __P((void *));
 extern struct callout in6_rr_timer_ch;
-int delete_each_prefix  __P((struct rr_prefix *rpp, u_char origin));
--- //depot/projects/netperf_socket/sys/nfsclient/bootp_subr.c	2004/01/18 16:31:40
+++ //depot/user/rwatson/netperf/sys/nfsclient/bootp_subr.c	2004/03/01 17:07:22
@@ -589,6 +589,8 @@
 	int retry;
 	const char *s;
 
+	NET_ASSERT_GIANT();
+
 	/*
 	 * Create socket and set its recieve timeout.
 	 */
@@ -979,6 +981,8 @@
 	struct ifaddr *ifa;
 	struct sockaddr_dl *sdl;
 
+	NET_ASSERT_GIANT();
+
 	error = socreate(AF_INET, &ifctx->so, SOCK_DGRAM, 0, td->td_ucred, td);
 	if (error != 0)
 		panic("nfs_boot: socreate, error=%d", error);
--- //depot/projects/netperf_socket/sys/nfsclient/nfs_socket.c	2004/01/18 16:31:40
+++ //depot/user/rwatson/netperf/sys/nfsclient/nfs_socket.c	2004/03/01 17:07:22
@@ -165,6 +165,8 @@
 	struct sockaddr *saddr;
 	struct thread *td = &thread0; /* only used for socreate and sobind */
 
+	NET_ASSERT_GIANT();
+
 	nmp->nm_so = NULL;
 	saddr = nmp->nm_nam;
 	error = socreate(saddr->sa_family, &nmp->nm_so, nmp->nm_sotype,
@@ -381,6 +383,8 @@
 {
 	struct socket *so;
 
+	NET_ASSERT_GIANT();
+
 	if (nmp->nm_so) {
 		so = nmp->nm_so;
 		nmp->nm_so = NULL;
@@ -415,6 +419,8 @@
 	struct sockaddr *sendnam;
 	int error, soflags, flags;
 
+	NET_ASSERT_GIANT();
+
 	KASSERT(rep, ("nfs_send: called with rep == NULL"));
 
 	if (rep->r_flags & R_SOFTTERM) {
@@ -488,6 +494,8 @@
 	int error, sotype, rcvflg;
 	struct thread *td = curthread;	/* XXX */
 
+	NET_ASSERT_GIANT();
+
 	/*
 	 * Set up arguments for soreceive()
 	 */
--- //depot/projects/netperf_socket/sys/nfsclient/nfs_vfsops.c	2004/02/29 20:38:43
+++ //depot/user/rwatson/netperf/sys/nfsclient/nfs_vfsops.c	2004/03/01 17:07:22
@@ -388,6 +388,8 @@
 	u_long l;
 	char buf[128];
 
+	NET_ASSERT_GIANT();
+
 #if defined(BOOTP_NFSROOT) && defined(BOOTP)
 	bootpc_init();		/* use bootp to get nfs_diskless filled in */
 #elif defined(NFS_ROOT)
--- //depot/projects/netperf_socket/sys/nfsserver/nfs_srvsock.c	2004/03/07 09:36:30
+++ //depot/user/rwatson/netperf/sys/nfsserver/nfs_srvsock.c	2004/03/07 09:40:54
@@ -425,6 +425,8 @@
 		goto dorecs;
 	}
 #endif
+	NET_ASSERT_GIANT();
+
 	auio.uio_td = NULL;
 	if (so->so_type == SOCK_STREAM) {
 		/*
@@ -725,6 +727,8 @@
 	struct sockaddr *sendnam;
 	int error, soflags, flags;
 
+	NET_ASSERT_GIANT();
+
 	soflags = so->so_proto->pr_flags;
 	if ((soflags & PR_CONNREQUIRED) || (so->so_state & SS_ISCONNECTED))
 		sendnam = NULL;
--- //depot/projects/netperf_socket/sys/nfsserver/nfs_syscalls.c	2004/01/18 16:31:40
+++ //depot/user/rwatson/netperf/sys/nfsserver/nfs_syscalls.c	2004/03/01 17:07:22
@@ -199,6 +199,8 @@
 	struct socket *so;
 	int error, s;
 
+	NET_ASSERT_GIANT();
+
 	so = fp->f_data;
 #if 0
 	tslp = NULL;
--- //depot/projects/netperf_socket/sys/rpc/rpcclnt.c	2004/03/07 09:36:30
+++ //depot/user/rwatson/netperf/sys/rpc/rpcclnt.c	2004/03/07 09:40:54
@@ -362,6 +362,8 @@
 		RPC_RETURN(EFAULT);
 	}
 
+	NET_ASSERT_GIANT();
+
 	/* create the socket */
 	rpc->rc_so = NULL;
 
@@ -618,6 +620,8 @@
 {
 	struct socket  *so;
 
+	NET_ASSERT_GIANT();
+
 	if (rpc->rc_so) {
 		so = rpc->rc_so;
 		rpc->rc_so = NULL;
@@ -667,6 +671,8 @@
 #endif
 	int error, soflags, flags;
 
+	NET_ASSERT_GIANT();
+
 	if (rep) {
 		if (rep->r_flags & R_SOFTTERM) {
 			m_freem(top);
@@ -750,6 +756,8 @@
 #endif
 	int error, sotype, rcvflg;
 
+	NET_ASSERT_GIANT();
+
 	/*
 	 * Set up arguments for soreceive()
 	 */
--- //depot/projects/netperf_socket/sys/sys/mutex.h	2004/03/04 16:13:19
+++ //depot/user/rwatson/netperf/sys/sys/mutex.h	2004/03/05 07:52:54
@@ -357,15 +357,15 @@
  */
 extern	int debug_mpsafenet;		/* defined in net/netisr.c */
 #define	NET_LOCK_GIANT() do {						\
-	if (debug_mpsafenet)						\
+	if (!debug_mpsafenet)						\
 		mtx_lock(&Giant);					\
 } while (0)
 #define	NET_UNLOCK_GIANT() do {						\
-	if (debug_mpsafenet)						\
+	if (!debug_mpsafenet)						\
 		mtx_unlock(&Giant);					\
 } while (0)
 #define	NET_ASSERT_GIANT() do {						\
-	if (debug_mpsafenet)						\
+	if (!debug_mpsafenet)						\
 		mtx_assert(&Giant, MA_OWNED);				\
 } while (0)