Re: connect problems on SMP

[Joerg Sonnenberger <joerg@xxxxxxxxxxxxxxxxx>]

Attached is Jeffrey's work-in-progress patch with same additions to get
a working kernel. This works fine for me.

Joerg

Index: kern/uipc_msg.c
===================================================================
RCS file: /extern/Daten/Sources/repository/dragonflybsd/src/sys/kern/uipc_msg.c,v
retrieving revision 1.5
diff -u -r1.5 uipc_msg.c
--- kern/uipc_msg.c	24 Mar 2004 21:58:44 -0000	1.5
+++ kern/uipc_msg.c	5 Apr 2004 13:44:45 -0000
@@ -139,7 +139,7 @@
 	if (!so->so_proto->pr_mport)
 		return ((*so->so_proto->pr_usrreqs->pru_connect)(so, nam, td));
 
-	port = so->so_proto->pr_mport(so, NULL);
+	port = so->so_proto->pr_mport(so, nam);
 	lwkt_initmsg(&msg.nm_lmsg, CMD_NETMSG_PRU_CONNECT);
 	msg.nm_handler = netmsg_pru_dispatcher;
 	msg.nm_prufn = so->so_proto->pr_usrreqs->pru_connect;
Index: netinet/in_pcb.c
===================================================================
RCS file: /extern/Daten/Sources/repository/dragonflybsd/src/sys/netinet/in_pcb.c,v
retrieving revision 1.16
diff -u -r1.16 in_pcb.c
--- netinet/in_pcb.c	31 Mar 2004 00:43:09 -0000	1.16
+++ netinet/in_pcb.c	5 Apr 2004 13:44:45 -0000
@@ -538,7 +538,7 @@
 	}
 	inp->inp_faddr = sin->sin_addr;
 	inp->inp_fport = sin->sin_port;
-	in_pcbinsconnhash(inp);
+	in_pcbinsconnhash(inp, inp->inp_pcbinfo);
 	return (0);
 }
 
@@ -559,12 +559,12 @@
 	struct inpcb *inp;
 {
 	struct socket *so = inp->inp_socket;
-	struct inpcbinfo *ipi = inp->inp_pcbinfo;
+	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 
 #ifdef IPSEC
 	ipsec4_delete_pcbpolicy(inp);
 #endif /*IPSEC*/
-	inp->inp_gencnt = ++ipi->ipi_gencnt;
+	inp->inp_gencnt = ++pcbinfo->ipi_gencnt;
 	in_pcbremlists(inp);
 	so->so_pcb = 0;
 	sofree(so);
@@ -574,7 +574,7 @@
 		rtfree(inp->inp_route.ro_rt);
 	ip_freemoptions(inp->inp_moptions);
 	inp->inp_vflag = 0;
-	zfree(ipi->ipi_zone, inp);
+	zfree(pcbinfo->ipi_zone, inp);
 }
 
 /*
@@ -908,9 +908,8 @@
  * Insert PCB into connection hash table.
  */
 void
-in_pcbinsconnhash(struct inpcb *inp)
+in_pcbinsconnhash(struct inpcb *inp, struct inpcbinfo *pcbinfo)
 {
-	struct inpcbinfo *pcbinfo = inp->inp_pcbinfo;
 	struct inpcbhead *bucket;
 	u_int32_t hashkey_faddr, hashkey_laddr;
 
Index: netinet/in_pcb.h
===================================================================
RCS file: /extern/Daten/Sources/repository/dragonflybsd/src/sys/netinet/in_pcb.h,v
retrieving revision 1.9
diff -u -r1.9 in_pcb.h
--- netinet/in_pcb.h	31 Mar 2004 00:43:09 -0000	1.9
+++ netinet/in_pcb.h	5 Apr 2004 13:44:45 -0000
@@ -329,7 +329,7 @@
 void	in_pcbdetach (struct inpcb *);
 void	in_pcbdisconnect (struct inpcb *);
 void	in_pcbinswildcardhash(struct inpcb *inp);
-void	in_pcbinsconnhash(struct inpcb *inp);
+void	in_pcbinsconnhash(struct inpcb *inp, struct inpcbinfo *);
 int	in_pcbinsporthash (struct inpcb *);
 int	in_pcbladdr (struct inpcb *, struct sockaddr *,
 	    struct sockaddr_in **);
Index: netinet/in_proto.c
===================================================================
RCS file: /extern/Daten/Sources/repository/dragonflybsd/src/sys/netinet/in_proto.c,v
retrieving revision 1.6
diff -u -r1.6 in_proto.c
--- netinet/in_proto.c	6 Mar 2004 01:58:55 -0000	1.6
+++ netinet/in_proto.c	5 Apr 2004 14:30:31 -0000
@@ -48,6 +48,7 @@
 #include <sys/protosw.h>
 #include <sys/queue.h>
 #include <sys/sysctl.h>
+#include <sys/thread.h>
 
 #include <net/if.h>
 #include <net/route.h>
Index: netinet/ip_demux.c
===================================================================
RCS file: /extern/Daten/Sources/repository/dragonflybsd/src/sys/netinet/ip_demux.c,v
retrieving revision 1.14
diff -u -r1.14 ip_demux.c
--- netinet/ip_demux.c	5 Apr 2004 09:17:48 -0000	1.14
+++ netinet/ip_demux.c	5 Apr 2004 14:36:59 -0000
@@ -166,7 +166,7 @@
 	 *   attach() has a null socket parameter
 	 *   Fast and slow timeouts pass in two NULLs
 	 */
-	if (nam != NULL || so == NULL)
+	if (so == NULL || nam != NULL)
 		return (&tcp_thread[0].td_msgport);
 
 	/*
@@ -189,6 +189,13 @@
 	    inp->inp_laddr.s_addr, inp->inp_fport, inp->inp_lport)].td_msgport);
 }
 
+lwkt_port_t
+tcp_addrport(in_addr_t faddr, in_port_t fport, in_addr_t laddr, in_port_t lport)
+{
+	return (&tcp_thread[tcp_addrcpu(faddr, fport,
+					laddr, lport)].td_msgport);
+}
+
 /*
  * Map a UDP socket to a protocol processing thread.
  */
Index: netinet/tcp_output.c
===================================================================
RCS file: /extern/Daten/Sources/repository/dragonflybsd/src/sys/netinet/tcp_output.c,v
retrieving revision 1.10
diff -u -r1.10 tcp_output.c
--- netinet/tcp_output.c	8 Mar 2004 00:36:30 -0000	1.10
+++ netinet/tcp_output.c	5 Apr 2004 14:30:54 -0000
@@ -49,6 +49,7 @@
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/in_cksum.h>
+#include <sys/thread.h>
 
 #include <net/route.h>
 
Index: netinet/tcp_timer.c
===================================================================
RCS file: /extern/Daten/Sources/repository/dragonflybsd/src/sys/netinet/tcp_timer.c,v
retrieving revision 1.6
diff -u -r1.6 tcp_timer.c
--- netinet/tcp_timer.c	8 Mar 2004 00:39:00 -0000	1.6
+++ netinet/tcp_timer.c	5 Apr 2004 14:31:14 -0000
@@ -48,7 +48,7 @@
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/protosw.h>
-
+#include <sys/thread.h>
 #include <machine/cpu.h>	/* before tcp_seq.h, for tcp_random18() */
 
 #include <net/route.h>
Index: netinet/tcp_usrreq.c
===================================================================
RCS file: /extern/Daten/Sources/repository/dragonflybsd/src/sys/netinet/tcp_usrreq.c,v
retrieving revision 1.10
diff -u -r1.10 tcp_usrreq.c
--- netinet/tcp_usrreq.c	31 Mar 2004 10:23:10 -0000	1.10
+++ netinet/tcp_usrreq.c	5 Apr 2004 14:32:41 -0000
@@ -42,6 +42,7 @@
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
+#include <sys/malloc.h>
 #include <sys/sysctl.h>
 #include <sys/globaldata.h>
 #include <sys/thread.h>
@@ -54,7 +55,10 @@
 #include <sys/socketvar.h>
 #include <sys/protosw.h>
 
+#include <sys/msgport2.h>
+
 #include <net/if.h>
+#include <net/netisr.h>
 #include <net/route.h>
 
 #include <netinet/in.h>
@@ -694,47 +698,29 @@
 };
 #endif /* INET6 */
 
-/*
- * Common subroutine to open a TCP connection to remote host specified
- * by struct sockaddr_in in mbuf *nam.  Call in_pcbbind to assign a local
- * port number if needed.  Call in_pcbladdr to do the routing and to choose
- * a local host address (interface).  If there is an existing incarnation
- * of the same connection in TIME-WAIT state and if the remote host was
- * sending CC options and if the connection duration was < MSL, then
- * truncate the previous TIME-WAIT state and proceed.
- * Initialize connection parameters and enter SYN-SENT state.
- */
+struct netmsg_tcp_connect {
+	struct lwkt_msg		nm_lmsg;
+	netisr_fn_t		nm_handler;
+	struct tcpcb		*nm_tp;
+	struct sockaddr_in	*nm_sin;
+	struct sockaddr_in	*nm_ifsin;
+};
+
 static int
-tcp_connect(struct tcpcb *tp, struct sockaddr *nam, struct thread *td)
+tcp_connect_oncpu(struct tcpcb *tp, struct sockaddr_in *sin,
+    struct sockaddr_in *if_sin)
 {
 	struct inpcb *inp = tp->t_inpcb, *oinp;
 	struct socket *so = inp->inp_socket;
 	struct tcpcb *otp;
-	struct sockaddr_in *sin = (struct sockaddr_in *)nam;
-	struct sockaddr_in *if_sin;
 	struct rmxp_tao *taop;
 	struct rmxp_tao tao_noncached;
-	int error;
-
-	if (inp->inp_lport == 0) {
-		error = in_pcbbind(inp, (struct sockaddr *)0, td);
-		if (error)
-			return (error);
-	}
 
-	/*
-	 * Cannot simply call in_pcbconnect, because there might be an
-	 * earlier incarnation of this same connection still in
-	 * TIME_WAIT state, creating an ADDRINUSE error.
-	 */
-	error = in_pcbladdr(inp, nam, &if_sin);
-	if (error)
-		return (error);
-	oinp = in_pcblookup_hash(inp->inp_pcbinfo,
+	oinp = in_pcblookup_hash(&tcbinfo[mycpu->gd_cpuid],
 	    sin->sin_addr, sin->sin_port,
 	    inp->inp_laddr.s_addr != INADDR_ANY ?
 	        inp->inp_laddr : if_sin->sin_addr,
-	    inp->inp_lport,  0, NULL);
+	    inp->inp_lport, 0, NULL);
 	if (oinp != NULL) {
 		if (oinp != inp && (otp = intotcpcb(oinp)) != NULL &&
 		    otp->t_state == TCPS_TIME_WAIT &&
@@ -748,7 +734,7 @@
 		inp->inp_laddr = if_sin->sin_addr;
 	inp->inp_faddr = sin->sin_addr;
 	inp->inp_fport = sin->sin_port;
-	in_pcbinsconnhash(inp);
+	in_pcbinsconnhash(inp, &tcbinfo[mycpu->gd_cpuid]);
 
 	/* Compute window scaling to request.  */
 	while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
@@ -784,6 +770,82 @@
 	return (0);
 }
 
+static int
+tcp_connect_handler(struct netmsg *msg0)
+{
+	struct netmsg_tcp_connect *msg = (struct netmsg_tcp_connect *)msg0;
+	int error;
+
+	error = tcp_connect_oncpu(msg->nm_tp, msg->nm_sin, msg->nm_ifsin);
+	return (error);
+}
+
+/*
+ * Common subroutine to open a TCP connection to remote host specified
+ * by struct sockaddr_in in mbuf *nam.  Call in_pcbbind to assign a local
+ * port number if needed.  Call in_pcbladdr to do the routing and to choose
+ * a local host address (interface).  If there is an existing incarnation
+ * of the same connection in TIME-WAIT state and if the remote host was
+ * sending CC options and if the connection duration was < MSL, then
+ * truncate the previous TIME-WAIT state and proceed.
+ * Initialize connection parameters and enter SYN-SENT state.
+ */
+static int
+tcp_connect(struct tcpcb *tp, struct sockaddr *nam, struct thread *td)
+{
+	struct inpcb *inp = tp->t_inpcb;
+	struct sockaddr_in *sin = (struct sockaddr_in *)nam;
+	struct sockaddr_in *if_sin;
+	int error;
+	boolean_t didbind = FALSE;
+	lwkt_port_t port;
+
+	if (inp->inp_lport == 0) {
+		error = in_pcbbind(inp, (struct sockaddr *)NULL, td);
+		if (error)
+			return (error);
+		didbind = TRUE;
+	}
+
+	/*
+	 * Cannot simply call in_pcbconnect, because there might be an
+	 * earlier incarnation of this same connection still in
+	 * TIME_WAIT state, creating an ADDRINUSE error.
+	 */
+	error = in_pcbladdr(inp, nam, &if_sin);
+	if (error)
+		return (error);
+
+	port = tcp_addrport(sin->sin_addr.s_addr, sin->sin_port,
+	    inp->inp_laddr.s_addr ?
+		inp->inp_laddr.s_addr : if_sin->sin_addr.s_addr,
+	    inp->inp_lport);
+
+	if (port->mp_td == curthread) {
+		error = tcp_connect_oncpu(tp, sin, if_sin);
+	} else {
+		struct netmsg_tcp_connect *msg;
+
+		msg = malloc(sizeof(struct netmsg_tcp_connect), M_LWKTMSG,
+		    M_NOWAIT);
+		if (msg == NULL) {
+			if (didbind) {	/* need to unwind bind */
+				inp->inp_lport = 0;
+				inp->inp_laddr.s_addr = INADDR_ANY;
+			}
+			return (ENOMEM);
+		}
+		lwkt_initmsg(&msg->nm_lmsg, CMD_NETMSG_ONCPU);
+		msg->nm_handler = tcp_connect_handler;
+		msg->nm_tp = tp;
+		msg->nm_sin = sin;
+		msg->nm_ifsin = if_sin;
+		error = lwkt_domsg(port, &msg->nm_lmsg);
+	}
+
+	return (error);
+}
+
 #ifdef INET6
 static int
 tcp6_connect(struct tcpcb *tp, struct sockaddr *nam, struct thread *td)
@@ -831,7 +893,7 @@
 	inp->inp_fport = sin6->sin6_port;
 	if ((sin6->sin6_flowinfo & IPV6_FLOWINFO_MASK) != NULL)
 		inp->in6p_flowinfo = sin6->sin6_flowinfo;
-	in_pcbinsconnhash(inp);
+	in_pcbinsconnhash(inp, &tcbinfo[mycpu->gd_cpuid]);
 
 	/* Compute window scaling to request.  */
 	while (tp->request_r_scale < TCP_MAX_WINSHIFT &&
Index: netinet/tcp_var.h
===================================================================
RCS file: /extern/Daten/Sources/repository/dragonflybsd/src/sys/netinet/tcp_var.h,v
retrieving revision 1.15
diff -u -r1.15 tcp_var.h
--- netinet/tcp_var.h	14 Mar 2004 08:21:53 -0000	1.15
+++ netinet/tcp_var.h	5 Apr 2004 13:44:45 -0000
@@ -467,6 +467,9 @@
 
 int	 tcp_addrcpu(in_addr_t faddr, in_port_t fport,
 	    in_addr_t laddr, in_port_t lport);
+lwkt_port_t
+	tcp_addrport(in_addr_t faddr, in_port_t fport,
+	    in_addr_t laddr, in_port_t lport);
 void	 tcp_canceltimers (struct tcpcb *);
 struct tcpcb *
 	 tcp_close (struct tcpcb *);
Index: netinet6/in6_pcb.c
===================================================================
RCS file: /extern/Daten/Sources/repository/dragonflybsd/src/sys/netinet6/in6_pcb.c,v
retrieving revision 1.12
diff -u -r1.12 in6_pcb.c
--- netinet6/in6_pcb.c	31 Mar 2004 00:43:09 -0000	1.12
+++ netinet6/in6_pcb.c	5 Apr 2004 14:33:51 -0000
@@ -372,7 +372,7 @@
 		inp->in6p_flowinfo |=
 		    (htonl(ip6_flow_seq++) & IPV6_FLOWLABEL_MASK);
 
-	in_pcbinsconnhash(inp);
+	in_pcbinsconnhash(inp, inp->inp_pcbinfo);
 	return (0);
 }
 
Index: netinet6/in6_proto.c
===================================================================
RCS file: /extern/Daten/Sources/repository/dragonflybsd/src/sys/netinet6/in6_proto.c,v
retrieving revision 1.5
diff -u -r1.5 in6_proto.c
--- netinet6/in6_proto.c	6 Mar 2004 01:58:56 -0000	1.5
+++ netinet6/in6_proto.c	5 Apr 2004 14:34:15 -0000
@@ -79,6 +79,7 @@
 #include <sys/mbuf.h>
 #include <sys/systm.h>
 #include <sys/sysctl.h>
+#include <sys/thread.h>
 
 #include <net/if.h>
 #include <net/radix.h>
Index: netinet6/in6_rmx.c
===================================================================
RCS file: /extern/Daten/Sources/repository/dragonflybsd/src/sys/netinet6/in6_rmx.c,v
retrieving revision 1.4
diff -u -r1.4 in6_rmx.c
--- netinet6/in6_rmx.c	7 Jan 2004 11:07:04 -0000	1.4
+++ netinet6/in6_rmx.c	5 Apr 2004 14:34:32 -0000
@@ -83,6 +83,7 @@
 #include <sys/socketvar.h>
 #include <sys/mbuf.h>
 #include <sys/syslog.h>
+#include <sys/thread.h>
 
 #include <net/if.h>
 #include <net/route.h>