From 835185f7c4739993c2ca26d737bb0a45277ad932 Mon Sep 17 00:00:00 2001
From: jikai <jikai11@huawei.com>
Date: Wed, 20 Mar 2024 15:48:42 +0800
Subject: [PATCH 28/43] use supervisor to notify sandbox exit event

Signed-off-by: jikai <jikai11@huawei.com>
---
 src/daemon/modules/api/container_api.h        |  2 +-
 .../modules/container/restore/restore.c       |  6 +++-
 .../modules/container/supervisor/supervisor.c | 15 +++++++-
 .../modules/service/service_container.c       |  9 +++--
 .../controller/shim/shim_controller.cc        | 35 ++-----------------
 src/daemon/sandbox/sandbox_ops.cc             | 23 ++++++++++++
 src/daemon/sandbox/sandbox_ops.h              |  2 ++
 7 files changed, 53 insertions(+), 39 deletions(-)

diff --git a/src/daemon/modules/api/container_api.h b/src/daemon/modules/api/container_api.h
index 4602d244..43d66d64 100644
--- a/src/daemon/modules/api/container_api.h
+++ b/src/daemon/modules/api/container_api.h
@@ -270,7 +270,7 @@ bool container_is_valid_state_string(const char *state);
 void container_update_health_monitor(const char *container_id);
 
 extern int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name,
-                                                 const char *runtime);
+                                                 const char *runtime, bool sandbox_container);
 
 extern char *container_exit_fifo_create(const char *cont_state_path);
 
diff --git a/src/daemon/modules/container/restore/restore.c b/src/daemon/modules/container/restore/restore.c
index 2669ea22..76868e28 100644
--- a/src/daemon/modules/container/restore/restore.c
+++ b/src/daemon/modules/container/restore/restore.c
@@ -57,6 +57,7 @@ static int restore_supervisor(const container_t *cont)
     char *statepath = cont->state_path;
     char *runtime = cont->runtime;
     pid_ppid_info_t pid_info = { 0 };
+    bool sandbox_container = false;
 
     nret = snprintf(container_state, sizeof(container_state), "%s/%s", statepath, id);
     if (nret < 0 || (size_t)nret >= sizeof(container_state)) {
@@ -90,8 +91,11 @@ static int restore_supervisor(const container_t *cont)
     pid_info.ppid = cont->state->state->p_pid;
     pid_info.start_time = cont->state->state->start_time;
     pid_info.pstart_time = cont->state->state->p_start_time;
+#ifdef ENABLE_CRI_API_V1
+    sandbox_container = is_sandbox_container(cont->common_config->sandbox_info);
+#endif
 
-    if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime)) {
+    if (container_supervisor_add_exit_monitor(exit_fifo_fd, &pid_info, id, runtime, sandbox_container)) {
         ERROR("Failed to add exit monitor to supervisor");
         ret = -1;
         goto out;
diff --git a/src/daemon/modules/container/supervisor/supervisor.c b/src/daemon/modules/container/supervisor/supervisor.c
index 1f9a043c..63289283 100644
--- a/src/daemon/modules/container/supervisor/supervisor.c
+++ b/src/daemon/modules/container/supervisor/supervisor.c
@@ -38,6 +38,9 @@
 #include "container_api.h"
 #include "event_type.h"
 #include "utils_file.h"
+#ifdef ENABLE_CRI_API_V1
+#include "sandbox_ops.h"
+#endif
 
 pthread_mutex_t g_supervisor_lock = PTHREAD_MUTEX_INITIALIZER;
 struct epoll_descr g_supervisor_descr;
@@ -47,6 +50,7 @@ struct supervisor_handler_data {
     int exit_code;
     char *name;
     char *runtime;
+    bool is_sandbox_container;
     pid_ppid_info_t pid_info;
 };
 
@@ -211,6 +215,14 @@ retry:
 
     (void)isulad_monitor_send_container_event(name, STOPPED, (int)pid, data->exit_code, NULL, NULL);
 
+#ifdef ENABLE_CRI_API_V1
+    if (data->is_sandbox_container) {
+        if (sandbox_on_sandbox_exit(name, data->exit_code) < 0) {
+            ERROR("Failed to handle sandbox %s exit", name);
+        }
+    }
+#endif
+
     supervisor_handler_data_free(data);
 
     DAEMON_CLEAR_ERRMSG();
@@ -259,7 +271,7 @@ static int supervisor_exit_cb(int fd, uint32_t events, void *cbdata, struct epol
 
 /* supervisor add exit monitor */
 int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_info, const char *name,
-                                          const char *runtime)
+                                          const char *runtime, bool sandbox_container)
 {
     int ret = 0;
     struct supervisor_handler_data *data = NULL;
@@ -285,6 +297,7 @@ int container_supervisor_add_exit_monitor(int fd, const pid_ppid_info_t *pid_inf
     data->fd = fd;
     data->name = util_strdup_s(name);
     data->runtime = util_strdup_s(runtime);
+    data->is_sandbox_container = sandbox_container;
     data->pid_info.pid = pid_info->pid;
     data->pid_info.start_time = pid_info->start_time;
     data->pid_info.ppid = pid_info->ppid;
diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
index a3606a82..7b34cc7f 100644
--- a/src/daemon/modules/service/service_container.c
+++ b/src/daemon/modules/service/service_container.c
@@ -275,13 +275,14 @@ static void clean_resources_on_failure(const container_t *cont, const char *engi
     return;
 }
 
-static int do_post_start_on_success(const char *id, const char *runtime, const char *pidfile, int exit_fifo_fd,
+static int do_post_start_on_success(const char *id, const char *runtime, bool sandbox_container,
+                                    const char *pidfile, int exit_fifo_fd,
                                     const pid_ppid_info_t *pid_info)
 {
     int ret = 0;
 
     // exit_fifo_fd was closed in container_supervisor_add_exit_monitor
-    if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime)) {
+    if (container_supervisor_add_exit_monitor(exit_fifo_fd, pid_info, id, runtime, sandbox_container)) {
         ERROR("Failed to add exit monitor to supervisor");
         ret = -1;
     }
@@ -749,6 +750,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo
     oci_runtime_spec *oci_spec = NULL;
     rt_create_params_t create_params = { 0 };
     rt_start_params_t start_params = { 0 };
+    bool sandbox_container;
 
     nret = snprintf(bundle, sizeof(bundle), "%s/%s", cont->root_path, id);
     if (nret < 0 || (size_t)nret >= sizeof(bundle)) {
@@ -897,6 +899,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo
     if (cont->common_config->sandbox_info != NULL) {
         create_params.task_addr = cont->common_config->sandbox_info->task_address;
     }
+    sandbox_container = is_sandbox_container(cont->common_config->sandbox_info);
 #endif
 
     if (runtime_create(id, runtime, &create_params) != 0) {
@@ -921,7 +924,7 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo
 
     ret = runtime_start(id, runtime, &start_params, pid_info);
     if (ret == 0) {
-        if (do_post_start_on_success(id, runtime, pidfile, exit_fifo_fd, pid_info) != 0) {
+        if (do_post_start_on_success(id, runtime, sandbox_container, pidfile, exit_fifo_fd, pid_info) != 0) {
             ERROR("Failed to do post start on runtime start success");
             ret = -1;
             goto clean_resources;
diff --git a/src/daemon/sandbox/controller/shim/shim_controller.cc b/src/daemon/sandbox/controller/shim/shim_controller.cc
index 39fcf8ea..593fade9 100644
--- a/src/daemon/sandbox/controller/shim/shim_controller.cc
+++ b/src/daemon/sandbox/controller/shim/shim_controller.cc
@@ -397,39 +397,8 @@ bool ShimController::Stop(const std::string &sandboxId, uint32_t timeoutSecs, Er
 
 bool ShimController::Wait(std::shared_ptr<SandboxStatusCallback> cb, const std::string &sandboxId, Errors &error)
 {
-    std::thread([this, cb, sandboxId]() {
-        if (m_cb == nullptr || m_cb->container.wait == nullptr) {
-            ERROR("Unimplemented callback");
-            return;
-        }
-
-        auto requestWrapper = makeUniquePtrCStructWrapper<container_wait_request>(free_container_wait_request);
-        if (requestWrapper == nullptr) {
-            ERROR("Out of memory");
-            return;
-        }
-        auto request = requestWrapper->get();
-        request->id = isula_strdup_s(sandboxId.c_str());
-        request->condition = WAIT_CONDITION_STOPPED;
-        container_wait_response *response { nullptr };
-
-        int ret = m_cb->container.wait(request, &response);
-        auto responseWrapper = makeUniquePtrCStructWrapper<container_wait_response>(response, free_container_wait_response);
-
-        if (ret != 0) {
-            std::string msg = (response != nullptr && response->errmsg != nullptr) ? response->errmsg : "internal";
-            ERROR("Failed to wait sandbox %s: %s", sandboxId.c_str(), msg.c_str());
-            return;
-        }
-
-        ControllerExitInfo info;
-        auto currentTime = std::chrono::high_resolution_clock::now();
-        auto duration = currentTime.time_since_epoch();
-        info.exitedAt = std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count();
-        info.exitStatus = response->exit_code;
-        cb->OnSandboxExit(info);
-    }).detach();
-
+    // ShimController will use sandbox_on_exit callback of supervisor in lower container level
+    // to notify the sandbox exit event
     return true;
 }
 
diff --git a/src/daemon/sandbox/sandbox_ops.cc b/src/daemon/sandbox/sandbox_ops.cc
index 005063c0..b7fb40bf 100644
--- a/src/daemon/sandbox/sandbox_ops.cc
+++ b/src/daemon/sandbox/sandbox_ops.cc
@@ -18,6 +18,7 @@
 #include <isula_libutils/log.h>
 
 #include "controller_manager.h"
+#include "sandbox_manager.h"
 #include "namespace.h"
 #include "utils.h"
 
@@ -175,3 +176,25 @@ int sandbox_purge_exec(const container_config_v2_common_config *config, const ch
 {
     return do_sandbox_purge(config, exec_id);
 }
+
+int sandbox_on_sandbox_exit(const char *sandbox_id, int exit_code)
+{
+    if (nullptr == sandbox_id) {
+        ERROR("Invalid parameter: sandbox_id");
+        return -1;
+    }
+
+    auto sandbox = sandbox::SandboxManager::GetInstance()->GetSandbox(sandbox_id);
+    if (nullptr == sandbox) {
+        ERROR("Sandbox %s not found", sandbox_id);
+        return -1;
+    }
+
+    sandbox::ControllerExitInfo info;
+    auto currentTime = std::chrono::high_resolution_clock::now();
+    auto duration = currentTime.time_since_epoch();
+    info.exitedAt = std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count();
+    info.exitStatus = exit_code;
+    sandbox->OnSandboxExit(info);
+    return 0;
+}
diff --git a/src/daemon/sandbox/sandbox_ops.h b/src/daemon/sandbox/sandbox_ops.h
index bef884fb..8189efd6 100644
--- a/src/daemon/sandbox/sandbox_ops.h
+++ b/src/daemon/sandbox/sandbox_ops.h
@@ -36,6 +36,8 @@ int sandbox_purge_container(const container_config_v2_common_config *config);
 
 int sandbox_purge_exec(const container_config_v2_common_config *config, const char *exec_id);
 
+int sandbox_on_sandbox_exit(const char *sandbox_id, int exit_code);
+
 #ifdef __cplusplus
 }
 #endif
-- 
2.34.1