From 28f8ba80cd733e14e0540c414a18134b3c3fcc94 Mon Sep 17 00:00:00 2001
From: FanBin <fanbin12@huawei.com>
Date: Wed, 15 Feb 2023 10:09:39 +0800
Subject: [PATCH] lwip reuse ip port

---
 src/core/tcp.c                   | 40 +++++++++++++++++++++++++++++---
 src/core/tcp_in.c                | 32 +++++++++++++++++++++++++
 src/include/lwip/api.h           |  4 ++++
 src/include/lwip/priv/tcp_priv.h | 19 +++++++++++++++
 src/include/lwip/tcp.h           |  8 +++++++
 src/include/lwipopts.h           |  4 ++++
 6 files changed, 104 insertions(+), 3 deletions(-)

diff --git a/src/core/tcp.c b/src/core/tcp.c
index f75d214..3171c5e 100644
--- a/src/core/tcp.c
+++ b/src/core/tcp.c
@@ -111,6 +111,7 @@
 #include "lwip/ip6.h"
 #include "lwip/ip6_addr.h"
 #include "lwip/nd6.h"
+#include "lwip/api.h"
 
 #include <string.h>
 #include <pthread.h>
@@ -772,6 +773,9 @@ tcp_bind(struct tcp_pcb *pcb, const ip_addr_t *ipaddr, u16_t port)
     /* Check if the address already is in use (on all lists) */
     for (i = 0; i < max_pcb_list; i++) {
       for (cpcb = *tcp_pcb_lists[i]; cpcb != NULL; cpcb = cpcb->next) {
+#if REUSE_IPPORT
+              continue;
+#else
         if (cpcb->local_port == port) {
 #if SO_REUSE
           /* Omit checking for the same port if both pcbs have REUSEADDR set.
@@ -790,6 +794,7 @@ tcp_bind(struct tcp_pcb *pcb, const ip_addr_t *ipaddr, u16_t port)
             }
           }
         }
+#endif /* REUSE_IPORT */
       }
     }
   }
@@ -921,7 +926,18 @@ tcp_listen_with_backlog_and_err(struct tcp_pcb *pcb, u8_t backlog, err_t *err)
     res = ERR_ALREADY;
     goto done;
   }
-#if SO_REUSE
+
+#if REUSE_IPPORT
+  struct tcp_pcb_listen *first_same_port_pcb = NULL;
+  for (lpcb = tcp_listen_pcbs.listen_pcbs; lpcb != NULL; lpcb = lpcb->next) {
+    if ((lpcb->local_port == pcb->local_port) &&
+         ip_addr_cmp(&lpcb->local_ip, &pcb->local_ip)) {
+      /* this address/port is already used */
+      first_same_port_pcb = lpcb;
+      break;
+    }
+  }
+#else
   if (ip_get_option(pcb, SOF_REUSEADDR)) {
     /* Since SOF_REUSEADDR allows reusing a local address before the pcb's usage
        is declared (listen-/connection-pcb), we have to make sure now that
@@ -936,7 +952,7 @@ tcp_listen_with_backlog_and_err(struct tcp_pcb *pcb, u8_t backlog, err_t *err)
       }
     }
   }
-#endif /* SO_REUSE */
+#endif /* REUSE_IPPORT */
 
 #if USE_LIBOS
   vdev_reg_done(REG_RING_TCP_LISTEN, pcb);
@@ -955,6 +971,16 @@ tcp_listen_with_backlog_and_err(struct tcp_pcb *pcb, u8_t backlog, err_t *err)
   lpcb->netif_idx = pcb->netif_idx;
   lpcb->ttl = pcb->ttl;
   lpcb->tos = pcb->tos;
+
+#if REUSE_IPPORT
+  lpcb->connect_num = 0;
+  lpcb->next_same_port_pcb = NULL;
+
+  struct netconn* conn = pcb->callback_arg;
+  lpcb->socket_fd = conn->socket;
+  lpcb->master_lpcb = conn->is_master_fd;
+#endif
+
 #if LWIP_IPV4 && LWIP_IPV6
   IP_SET_TYPE_VAL(lpcb->remote_ip, pcb->local_ip.type);
 #endif /* LWIP_IPV4 && LWIP_IPV6 */
@@ -979,7 +1005,15 @@ tcp_listen_with_backlog_and_err(struct tcp_pcb *pcb, u8_t backlog, err_t *err)
   lpcb->accepts_pending = 0;
   tcp_backlog_set(lpcb, backlog);
 #endif /* TCP_LISTEN_BACKLOG */
-  TCP_REG(&tcp_listen_pcbs.pcbs, (struct tcp_pcb *)lpcb);
+
+#if REUSE_IPPORT
+  if (first_same_port_pcb != NULL) {
+    TCP_REG_SAMEPORT((struct tcp_pcb_listen *)first_same_port_pcb, (struct tcp_pcb_listen *)lpcb);
+  } else
+#endif
+  {
+    TCP_REG(&tcp_listen_pcbs.pcbs, (struct tcp_pcb *)lpcb);
+  }
   res = ERR_OK;
 done:
   if (err != NULL) {
diff --git a/src/core/tcp_in.c b/src/core/tcp_in.c
index 35ec6d9..9f5c34a 100644
--- a/src/core/tcp_in.c
+++ b/src/core/tcp_in.c
@@ -356,6 +356,9 @@ tcp_input(struct pbuf *p, struct netif *inp)
       }
     }
 
+#if REUSE_IPPORT
+    struct tcp_pcb_listen *min_cnts_lpcb = NULL;
+#endif
     /* Finally, if we still did not get a match, we check all PCBs that
        are LISTENing for incoming connections. */
     prev = NULL;
@@ -379,6 +382,30 @@ tcp_input(struct pbuf *p, struct netif *inp)
         } else if (IP_ADDR_PCB_VERSION_MATCH_EXACT(lpcb, ip_current_dest_addr())) {
           if (ip_addr_cmp(&lpcb->local_ip, ip_current_dest_addr())) {
             /* found an exact match */
+#if REUSE_IPPORT
+            // check master fd
+            struct tcp_pcb_listen *tmp_lpcb = lpcb;
+            u8_t have_master_fd = 0;
+            while (tmp_lpcb != NULL) {
+              if (tmp_lpcb->master_lpcb) {
+	        have_master_fd = 1;
+	      }
+              tmp_lpcb = tmp_lpcb->next_same_port_pcb;
+	    }
+
+            tmp_lpcb = lpcb;
+            min_cnts_lpcb = lpcb;
+            u16_t min_conn_num = MAX_CONN_NUM_PER_THREAD;
+            while (tmp_lpcb != NULL) {
+              if (!have_master_fd || tmp_lpcb->master_lpcb) {
+                if (tmp_lpcb->connect_num < min_conn_num) {
+                  min_cnts_lpcb = tmp_lpcb;
+		  min_conn_num = tmp_lpcb->connect_num;
+	        }
+	      }
+              tmp_lpcb = tmp_lpcb->next_same_port_pcb;
+	    }
+#endif
             break;
           } else if (ip_addr_isany(&lpcb->local_ip)) {
             /* found an ANY-match */
@@ -428,7 +455,12 @@ tcp_input(struct pbuf *p, struct netif *inp)
                                      tcphdr_opt1len, tcphdr_opt2, p) == ERR_OK)
 #endif
       {
+#if REUSE_IPPORT
+        tcp_listen_input(min_cnts_lpcb);
+	min_cnts_lpcb->connect_num++;
+#else
         tcp_listen_input(lpcb);
+#endif
       }
       pbuf_free(p);
       return;
diff --git a/src/include/lwip/api.h b/src/include/lwip/api.h
index 6dec8c0..430a7a0 100644
--- a/src/include/lwip/api.h
+++ b/src/include/lwip/api.h
@@ -318,6 +318,10 @@ struct netconn {
 #endif /* LWIP_TCP */
   /** A callback function that is informed about events for this netconn */
   netconn_callback callback;
+
+#if REUSE_IPPORT
+  u8_t is_master_fd;
+#endif
 };
 
 /** This vector type is passed to @ref netconn_write_vectors_partly to send
diff --git a/src/include/lwip/priv/tcp_priv.h b/src/include/lwip/priv/tcp_priv.h
index b242428..97f799e 100644
--- a/src/include/lwip/priv/tcp_priv.h
+++ b/src/include/lwip/priv/tcp_priv.h
@@ -353,6 +353,15 @@ static inline int vdev_reg_done(enum reg_ring_type reg_type, const struct tcp_pc
   qtuple.dst_ip     = pcb->remote_ip.addr;
   qtuple.dst_port   = lwip_htons(pcb->remote_port);
 
+#if REUSE_IPPORT
+  if (reg_type == REG_RING_TCP_CONNECT_CLOSE) {
+    struct tcp_pcb_listen* lpcb = pcb->listener;
+    if (lpcb != NULL) {
+      lpcb->connect_num--;
+    }
+  }
+#endif
+
   return vdev_reg_xmit(reg_type, &qtuple);
 }
 static inline void vdev_unreg_done(const struct tcp_pcb *pcb)
@@ -473,6 +482,16 @@ static inline void vdev_unreg_done(const struct tcp_pcb *pcb)
     tcp_timer_needed();                            \
   } while (0)
 
+#define TCP_REG_SAMEPORT(first_pcb, lpcb)          \
+  do {                                             \
+    struct tcp_pcb_listen *tmp_pcb = first_pcb;    \
+    while (tmp_pcb->next_same_port_pcb != NULL) {  \
+      tmp_pcb = tmp_pcb->next_same_port_pcb;       \
+    };                                             \
+    tmp_pcb->next_same_port_pcb = lpcb;            \
+    tcp_timer_needed();                            \
+  } while (0)
+
 #define TCP_RMV_HASH(pcbs, npcb)                   \
   do {                                             \
     hlist_del_init(&(npcb)->tcp_node);             \
diff --git a/src/include/lwip/tcp.h b/src/include/lwip/tcp.h
index 0b65b01..312320b 100644
--- a/src/include/lwip/tcp.h
+++ b/src/include/lwip/tcp.h
@@ -252,6 +252,14 @@ struct tcp_pcb_listen {
   u8_t backlog;
   u8_t accepts_pending;
 #endif /* TCP_LISTEN_BACKLOG */
+
+#if REUSE_IPPORT
+  struct tcp_pcb_listen* next_same_port_pcb;
+  u16_t connect_num;
+  int socket_fd;
+  u8_t master_lpcb;
+#endif
+
 };
 
 
diff --git a/src/include/lwipopts.h b/src/include/lwipopts.h
index fedded9..be58ec3 100644
--- a/src/include/lwipopts.h
+++ b/src/include/lwipopts.h
@@ -143,6 +143,10 @@
 
 #define USE_LIBOS_ZC_RING 0
 
+#define REUSE_IPPORT 1
+
+#define MAX_CONN_NUM_PER_THREAD 65535
+
 #define SO_REUSE 1
 
 #define SIOCSHIWAT 1
-- 
2.33.0