From 0f126e8723824823cddc608c2a6493ea9fc99e89 Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng12@huawei.com>
Date: Tue, 8 Mar 2022 10:59:59 +0800
Subject: [PATCH 04/34] recvmsg/sendmsg should use
 recvmsg_from_stack/sendmsg_to_stack instead of
 rpc_call_recvmsg/rpc_call_sendmsg

---
 src/lstack/api/lstack_wrap.c     | 33 +--------------
 src/lstack/core/lstack_lwip.c    | 88 ++++++++++++++++++++++++++++++++++++++++
 src/lstack/include/lstack_lwip.h |  3 ++
 3 files changed, 93 insertions(+), 31 deletions(-)

diff --git a/src/lstack/api/lstack_wrap.c b/src/lstack/api/lstack_wrap.c
index 6488c62..7804ce9 100644
--- a/src/lstack/api/lstack_wrap.c
+++ b/src/lstack/api/lstack_wrap.c
@@ -103,10 +103,6 @@ static inline int32_t do_epoll_wait(int32_t epfd, struct epoll_event* events, in
 
 static inline int32_t do_accept(int32_t s, struct sockaddr *addr, socklen_t *addrlen)
 {
-    if (addr == NULL || addrlen == NULL) {
-        GAZELLE_RETURN(EINVAL);
-    }
-
     if (select_path(s) == PATH_KERNEL) {
         return posix_api->accept_fn(s, addr, addrlen);
     }
@@ -292,31 +288,6 @@ static inline ssize_t do_read(int32_t s, void *mem, size_t len)
     return posix_api->read_fn(s, mem, len);
 }
 
-static inline ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags)
-{
-    if (buf == NULL) {
-        GAZELLE_RETURN(EINVAL);
-    }
-
-    if (len == 0) {
-        return 0;
-    }
-
-    struct lwip_sock *sock = get_socket(fd);
-    if (sock == NULL) {
-        GAZELLE_RETURN(EINVAL);
-    }
-
-    ssize_t send = write_stack_data(sock, buf, len);
-    if (send < 0 || sock->have_rpc_send) {
-        return send;
-    }
-
-    sock->have_rpc_send = true;
-    ssize_t ret = rpc_call_send(fd, buf, len, flags);
-    return (ret < 0) ? ret : send;
-}
-
 static inline ssize_t do_send(int32_t sockfd, const void *buf, size_t len, int32_t flags)
 {
     if (select_path(sockfd) != PATH_LWIP) {
@@ -342,7 +313,7 @@ static inline ssize_t do_recvmsg(int32_t s, struct msghdr *message, int32_t flag
     }
 
     if (select_path(s) == PATH_LWIP) {
-        return rpc_call_recvmsg(s, message, flags);
+        return recvmsg_from_stack(s, message, flags);
     }
 
     return posix_api->recv_msg(s, message, flags);
@@ -355,7 +326,7 @@ static inline ssize_t do_sendmsg(int32_t s, const struct msghdr *message, int32_
     }
 
     if (select_path(s) == PATH_LWIP) {
-        return rpc_call_sendmsg(s, message, flags);
+        return sendmsg_to_stack(s, message, flags);
     }
 
     return posix_api->send_msg(s, message, flags);
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index fbb4d62..0a71aae 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -386,6 +386,94 @@ ssize_t read_lwip_data(struct lwip_sock *sock, int32_t flags, u8_t apiflags)
     return recv_len;
 }
 
+ssize_t recvmsg_from_stack(int32_t s, struct msghdr *message, int32_t flags)
+{
+    ssize_t buflen = 0;
+    int32_t i;
+
+    if (message == NULL || message->msg_iovlen <= 0 || message->msg_iovlen > IOV_MAX) {
+        GAZELLE_RETURN(EINVAL);
+    }
+    for (i = 0; i < message->msg_iovlen; i++) {
+        if ((message->msg_iov[i].iov_base == NULL) || ((ssize_t)message->msg_iov[i].iov_len <= 0) ||
+            ((size_t)(ssize_t)message->msg_iov[i].iov_len != message->msg_iov[i].iov_len) ||
+            ((ssize_t)(buflen + (ssize_t)message->msg_iov[i].iov_len) <= 0)) {
+          GAZELLE_RETURN(EINVAL);
+        }
+        buflen = (ssize_t)(buflen + (ssize_t)message->msg_iov[i].iov_len);
+    }
+    buflen = 0;
+    for (i = 0; i < message->msg_iovlen; i++) {
+        ssize_t recvd_local = read_stack_data(s, message->msg_iov[i].iov_base, message->msg_iov[i].iov_len, flags);
+        if (recvd_local > 0) {
+            buflen += recvd_local;
+        }
+        if (recvd_local < 0 || (recvd_local < (int)message->msg_iov[i].iov_len) || (flags & MSG_PEEK)) {
+            if (buflen <= 0) {
+                buflen = recvd_local;
+            }
+            break;
+        }
+        flags |= MSG_DONTWAIT;
+    }
+
+    return buflen;
+}
+
+ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags)
+{
+    if (buf == NULL) {
+        GAZELLE_RETURN(EINVAL);
+    }
+
+    if (len == 0) {
+        return 0;
+    }
+
+    struct lwip_sock *sock = get_socket(fd);
+    if (sock == NULL) {
+        GAZELLE_RETURN(EINVAL);
+    }
+
+    ssize_t send = write_stack_data(sock, buf, len);
+    if (send < 0 || sock->have_rpc_send) {
+        return send;
+    }
+
+    sock->have_rpc_send = true;
+    ssize_t ret = rpc_call_send(fd, buf, len, flags);
+    return (ret < 0) ? ret : send;
+}
+
+ssize_t sendmsg_to_stack(int32_t s, const struct msghdr *message, int32_t flags)
+{
+    int32_t ret;
+    int32_t i;
+    ssize_t buflen = 0;
+
+    if (message == NULL || message->msg_iovlen <= 0 || message->msg_iovlen > IOV_MAX) {
+        GAZELLE_RETURN(EINVAL);
+    }
+    for (i = 0; i < message->msg_iovlen; i++) {
+        if ((message->msg_iov[i].iov_base == NULL) || ((ssize_t)message->msg_iov[i].iov_len <= 0) ||
+            ((size_t)(ssize_t)message->msg_iov[i].iov_len != message->msg_iov[i].iov_len) ||
+            ((ssize_t)(buflen + (ssize_t)message->msg_iov[i].iov_len) <= 0)) {
+           GAZELLE_RETURN(EINVAL);
+        }
+        buflen = (ssize_t)(buflen + (ssize_t)message->msg_iov[i].iov_len);
+    }
+
+    for (i = 0; i < message->msg_iovlen; i++) {
+        ret = gazelle_send(s, message->msg_iov[i].iov_base, message->msg_iov[i].iov_len, flags);
+        if (ret < 0) {
+           return  buflen == 0 ? ret : buflen;
+        }
+        buflen += ret;
+    }
+
+    return buflen;
+}
+
 ssize_t read_stack_data(int32_t fd, void *buf, size_t len, int32_t flags)
 {
     size_t recv_left = len;
diff --git a/src/lstack/include/lstack_lwip.h b/src/lstack/include/lstack_lwip.h
index 8bf0f29..581b9fe 100644
--- a/src/lstack/include/lstack_lwip.h
+++ b/src/lstack/include/lstack_lwip.h
@@ -39,5 +39,8 @@ void stack_recvlist_count(struct rpc_msg *msg);
 void stack_replenish_send_idlembuf(struct protocol_stack *stack);
 int32_t gazelle_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, uint32_t num);
 void gazelle_free_pbuf(struct pbuf *pbuf);
+ssize_t sendmsg_to_stack(int32_t s, const struct msghdr *message, int32_t flags);
+ssize_t recvmsg_from_stack(int32_t s, struct msghdr *message, int32_t flags);
+ssize_t gazelle_send(int32_t fd, const void *buf, size_t len, int32_t flags);
 
 #endif
-- 
1.8.3.1