From 27a99ca97da9540200068d75152896492ecd0064 Mon Sep 17 00:00:00 2001
From: jikui <jikui2@huawei.com>
Date: Tue, 14 Dec 2021 16:33:02 +0800
Subject: [PATCH 18/24] implement cmd hooks

---
 cmd/checker.go                         |  52 ++++++++++
 cmd/cleanup.go                         |  10 +-
 cmd/configs.go                         | 125 ++++++++++++++++++++++++-
 cmd/configs_test.go                    |   2 +-
 cmd/delete.go                          |  12 ++-
 cmd/deploy.go                          |   9 +-
 cmd/join.go                            |  14 ++-
 cmd/opts.go                            |  12 +++
 docs/hooks_of_eggo.md                  |   3 +-
 pkg/api/types.go                       |  18 ++--
 pkg/clusterdeployment/binary/binary.go |  25 +++++
 pkg/constants/constants.go             |   5 +
 pkg/utils/dependency/cmdhooks.go       | 115 +++++++++++++++++++++++
 pkg/utils/dependency/cmdhooks_test.go  |  43 +++++++++
 pkg/utils/runner/runner.go             |   4 +-
 pkg/utils/utils.go                     |  15 +++
 16 files changed, 444 insertions(+), 20 deletions(-)
 create mode 100644 pkg/utils/dependency/cmdhooks.go
 create mode 100644 pkg/utils/dependency/cmdhooks_test.go

diff --git a/cmd/checker.go b/cmd/checker.go
index 9d1fda6..07068e9 100644
--- a/cmd/checker.go
+++ b/cmd/checker.go
@@ -19,11 +19,15 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"os"
+	"path"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	"isula.org/eggo/pkg/api"
+	"isula.org/eggo/pkg/constants"
 	"isula.org/eggo/pkg/utils"
 	"isula.org/eggo/pkg/utils/endpoint"
 	chain "isula.org/eggo/pkg/utils/responsibilitychain"
@@ -383,6 +387,54 @@ func checkPackageConfig(pc *PackageConfig) error {
 	return nil
 }
 
+func checkCmdHooksParameter(pa ...string) error {
+	for _, v := range pa {
+		if v == "" {
+			continue
+		}
+		res := strings.Split(v, ",")
+		if len(res) < 1 || len(res) > 2 {
+			return fmt.Errorf("invalid hook parameter with:%s\n", v)
+		}
+	}
+
+	return nil
+}
+
+func checkHookFile(fileName string) error {
+	file, err := os.Stat(fileName)
+	if err != nil {
+		return err
+
+	}
+
+	if !path.IsAbs(fileName) {
+		return fmt.Errorf("%s is not Abs path", fileName)
+	}
+	if !file.Mode().IsRegular() {
+		return fmt.Errorf("%s is not regular file", file.Name())
+	}
+	if file.Mode().Perm() != os.FileMode(constants.HookFileMode) {
+		return fmt.Errorf("file mode of %s is incorrect", file.Name())
+	}
+	if file.Size() > constants.MaxHookFileSize || file.Size() == 0 {
+		return fmt.Errorf("%s is too large or small", file.Name())
+	}
+	if !(strings.HasSuffix(fileName, ".sh") || strings.HasSuffix(fileName, ".bash")) {
+		return fmt.Errorf("%s is not shell file", file.Name())
+	}
+
+	user, group, err := utils.GetUserIDAndGroupID(fileName)
+	if err != nil {
+		return fmt.Errorf("get user ID and group ID with file %s failed", file.Name())
+	}
+	if user != os.Getuid() && group != os.Getgid() {
+		return fmt.Errorf("user id and group id of %s mismatch with process", file.Name())
+	}
+
+	return nil
+}
+
 func (ccr *InstallConfigResponsibility) Execute() error {
 	if ccr.conf.PackageSrc != nil {
 		if ccr.conf.PackageSrc.DstPath != "" {
diff --git a/cmd/cleanup.go b/cmd/cleanup.go
index 37bb87f..7a78b15 100644
--- a/cmd/cleanup.go
+++ b/cmd/cleanup.go
@@ -54,17 +54,25 @@ func cleanupCluster(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("load deploy config file %v failed: %v", confPath, err)
 	}
 
+	if err = checkCmdHooksParameter(opts.clusterPrehook, opts.clusterPosthook); err != nil {
+		return err
+	}
 	if err = RunChecker(conf); err != nil {
 		return err
 	}
 
+	hooksConf, err := getClusterHookConf(api.HookOpCleanup)
+	if err != nil {
+		return fmt.Errorf("get cmd hooks config failed:%v", err)
+	}
+
 	holder, err := NewProcessPlaceHolder(eggoPlaceHolderPath(conf.ClusterID))
 	if err != nil {
 		return fmt.Errorf("create process holder failed: %v, mayebe other eggo is running with cluster: %s", err, conf.ClusterID)
 	}
 	defer holder.Remove()
 
-	if err = cleanup(toClusterdeploymentConfig(conf)); err != nil {
+	if err = cleanup(toClusterdeploymentConfig(conf, hooksConf)); err != nil {
 		return err
 	}
 
diff --git a/cmd/configs.go b/cmd/configs.go
index 326e889..4d7a4b9 100644
--- a/cmd/configs.go
+++ b/cmd/configs.go
@@ -20,6 +20,7 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -559,7 +560,7 @@ func fillExtrArgs(ccfg *api.ClusterConfig, eargs []*ConfigExtraArgs) {
 	}
 }
 
-func toClusterdeploymentConfig(conf *DeployConfig) *api.ClusterConfig {
+func toClusterdeploymentConfig(conf *DeployConfig, hooks []*api.ClusterHookConf) *api.ClusterConfig {
 	ccfg := getDefaultClusterdeploymentConfig()
 
 	setIfStrConfigNotEmpty(&ccfg.Name, conf.ClusterID)
@@ -601,10 +602,132 @@ func toClusterdeploymentConfig(conf *DeployConfig) *api.ClusterConfig {
 	ccfg.WorkerConfig.KubeletConf.EnableServer = conf.EnableKubeletServing
 
 	fillExtrArgs(ccfg, conf.ConfigExtraArgs)
+	ccfg.HooksConf = hooks
 
 	return ccfg
 }
 
+func getClusterHookConf(op api.HookOperator) ([]*api.ClusterHookConf, error) {
+	var hooks []*api.ClusterHookConf
+
+	if opts.clusterPrehook != "" {
+		hook, err := getCmdHooks(opts.clusterPrehook, api.ClusterPrehookType, op)
+		if err != nil {
+			return nil, err
+		}
+		hooks = append(hooks, hook)
+	}
+
+	if opts.clusterPosthook != "" {
+		hook, err := getCmdHooks(opts.clusterPosthook, api.ClusterPosthookType, op)
+		if err != nil {
+			return nil, err
+		}
+		hooks = append(hooks, hook)
+	}
+
+	if opts.prehook != "" {
+		hook, err := getCmdHooks(opts.prehook, api.PreHookType, op)
+		if err != nil {
+			return nil, err
+		}
+		hooks = append(hooks, hook)
+	}
+
+	if opts.posthook != "" {
+		hook, err := getCmdHooks(opts.posthook, api.PostHookType, op)
+		if err != nil {
+			return nil, err
+		}
+		hooks = append(hooks, hook)
+	}
+	return hooks, nil
+}
+
+func getCmdHooks(hopts string, ty api.HookType, op api.HookOperator) (*api.ClusterHookConf, error) {
+	path, target, err := getHookPathAndTarget(hopts)
+	if err != nil {
+		return nil, err
+	}
+	hook, err := getResolvedHook(path, ty, op, target)
+	if err != nil {
+		return nil, err
+	}
+	return hook, nil
+}
+
+func getHookPathAndTarget(hook string) (string, uint16, error) {
+	pathAndTarget := strings.Split(hook, ",")
+	if len(pathAndTarget) == 1 {
+		pathAndTarget = append(pathAndTarget, "master")
+	}
+	target, ok := toTypeInt[pathAndTarget[1]]
+	if !ok {
+		return "", 0x0, fmt.Errorf("invalid role:%s", pathAndTarget[1])
+	}
+
+	return pathAndTarget[0], target, nil
+}
+
+func getResolvedHook(path string, ty api.HookType, op api.HookOperator, target uint16) (*api.ClusterHookConf, error) {
+
+	dir, shells, err := getDirAndShells(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return &api.ClusterHookConf{
+		Type:       ty,
+		Operator:   op,
+		Target:     target,
+		HookSrcDir: dir,
+		HookFiles:  shells,
+	}, nil
+}
+
+func getDirAndShells(path string) (string, []string, error) {
+	file, err := os.Stat(path)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if !file.IsDir() {
+		return resolveFile(path)
+	}
+
+	return resolvePath(path)
+}
+
+func resolveFile(p string) (string, []string, error) {
+	dir := path.Dir(p)
+	fileName := path.Base(p)
+	if err := checkHookFile(p); err != nil {
+		return "", nil, err
+	}
+
+	return dir, []string{fileName}, nil
+}
+
+func resolvePath(p string) (string, []string, error) {
+	var files []string
+	rd, err := ioutil.ReadDir(p)
+	if err != nil {
+		return "", nil, err
+	}
+
+	for _, fi := range rd {
+		if err := checkHookFile(path.Join(p, fi.Name())); err == nil {
+			files = append(files, fi.Name())
+		} else {
+			logrus.Debugf("check hook file failed:%v", err)
+		}
+	}
+	if len(files) == 0 {
+		return "", nil, fmt.Errorf("empty folder:%s", p)
+	}
+	return p, files, nil
+}
+
 func getHostconfigs(format string, ips []string) []*HostConfig {
 	var confs []*HostConfig
 	for i, ip := range ips {
diff --git a/cmd/configs_test.go b/cmd/configs_test.go
index 46cb163..04afc51 100644
--- a/cmd/configs_test.go
+++ b/cmd/configs_test.go
@@ -44,7 +44,7 @@ func TestCmdConfigs(t *testing.T) {
 		t.Fatalf("load deploy config file failed: %v", err)
 	}
 
-	ccfg := toClusterdeploymentConfig(conf)
+	ccfg := toClusterdeploymentConfig(conf, nil)
 	d, err := yaml.Marshal(ccfg)
 	if err != nil {
 		t.Fatalf("marshal cluster config failed: %v", err)
diff --git a/cmd/delete.go b/cmd/delete.go
index 9d911a9..5990a42 100644
--- a/cmd/delete.go
+++ b/cmd/delete.go
@@ -63,7 +63,7 @@ func getDeletedAndDiffConfigs(conf *DeployConfig, delNames []string) (*DeployCon
 		return nil, nil, fmt.Errorf("forbidden to delete first master")
 	}
 
-	clusterConfig := toClusterdeploymentConfig(&diffConfig)
+	clusterConfig := toClusterdeploymentConfig(&diffConfig, nil)
 	if len(clusterConfig.Nodes) == 0 {
 		return nil, nil, fmt.Errorf("no valid ip or name found")
 	}
@@ -89,11 +89,19 @@ func deleteCluster(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("load saved deploy config failed: %v", err)
 	}
 
+	if err := checkCmdHooksParameter(opts.prehook, opts.posthook); err != nil {
+		return err
+	}
 	// check saved deploy config
 	if err = RunChecker(conf); err != nil {
 		return err
 	}
 
+	hooksConf, err := getClusterHookConf(api.HookOpDelete)
+	if err != nil {
+		return fmt.Errorf("get cmd hooks config failed:%v", err)
+	}
+
 	holder, err := NewProcessPlaceHolder(eggoPlaceHolderPath(conf.ClusterID))
 	if err != nil {
 		return fmt.Errorf("create process holder failed: %v, mayebe other eggo is running with cluster: %s", err, conf.ClusterID)
@@ -110,7 +118,7 @@ func deleteCluster(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if err = clusterdeployment.DeleteNodes(toClusterdeploymentConfig(conf), diffHostconfigs); err != nil {
+	if err = clusterdeployment.DeleteNodes(toClusterdeploymentConfig(conf, hooksConf), diffHostconfigs); err != nil {
 		return err
 	}
 
diff --git a/cmd/deploy.go b/cmd/deploy.go
index e21bcc5..2d7c441 100644
--- a/cmd/deploy.go
+++ b/cmd/deploy.go
@@ -71,7 +71,11 @@ func deploy(conf *DeployConfig) error {
 		return fmt.Errorf("save deploy config failed: %v", err)
 	}
 
-	ccfg := toClusterdeploymentConfig(conf)
+	hooksConf, err := getClusterHookConf(api.HookOpDeploy)
+	if err != nil {
+		return fmt.Errorf("get cmd hooks config failed:%v", err)
+	}
+	ccfg := toClusterdeploymentConfig(conf, hooksConf)
 
 	cstatus, err := clusterdeployment.CreateCluster(ccfg, opts.deployEnableRollback)
 	if err != nil {
@@ -116,6 +120,9 @@ func deployCluster(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("load deploy config file failed: %v", err)
 	}
 
+	if err = checkCmdHooksParameter(opts.clusterPrehook, opts.clusterPosthook); err != nil {
+		return err
+	}
 	if err = RunChecker(conf); err != nil {
 		return err
 	}
diff --git a/cmd/join.go b/cmd/join.go
index 79d68fc..d035bfe 100644
--- a/cmd/join.go
+++ b/cmd/join.go
@@ -128,7 +128,7 @@ func getMergedAndDiffConfigs(conf *DeployConfig, joinConf *DeployConfig) (*Deplo
 		diffConfig.Workers = append(diffConfig.Workers, h)
 	}
 
-	return &mergedConfig, toClusterdeploymentConfig(&diffConfig).Nodes, nil
+	return &mergedConfig, toClusterdeploymentConfig(&diffConfig, nil).Nodes, nil
 }
 
 func getFailedConfigs(diffConfigs []*api.HostConfig, cstatus api.ClusterStatus) []*api.HostConfig {
@@ -206,6 +206,9 @@ func joinCluster(cmd *cobra.Command, args []string) error {
 	}
 	var err error
 
+	if err = checkCmdHooksParameter(opts.prehook, opts.posthook); err != nil {
+		return err
+	}
 	joinConf, err := parseJoinInput(opts.joinYaml, &opts.joinHost, opts.joinType, opts.joinClusterID)
 	if err != nil {
 		return err
@@ -237,11 +240,16 @@ func joinCluster(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	cstatus, err := clusterdeployment.JoinNodes(toClusterdeploymentConfig(conf), diffConfigs)
+	hooksConf, err := getClusterHookConf(api.HookOpJoin)
+	if err != nil {
+		return fmt.Errorf("get cmd hooks config failed:%v", err)
+	}
+
+	cstatus, err := clusterdeployment.JoinNodes(toClusterdeploymentConfig(conf, hooksConf), diffConfigs)
 	if err != nil {
 		failedConfigs := getFailedConfigs(diffConfigs, cstatus)
 		// rollback
-		if err1 := clusterdeployment.DeleteNodes(toClusterdeploymentConfig(mergedConf), failedConfigs); err1 != nil {
+		if err1 := clusterdeployment.DeleteNodes(toClusterdeploymentConfig(mergedConf, nil), failedConfigs); err1 != nil {
 			logrus.Errorf("delete nodes failed when join failed: %v", err1)
 		}
 
diff --git a/cmd/opts.go b/cmd/opts.go
index f5204f2..7bb8297 100644
--- a/cmd/opts.go
+++ b/cmd/opts.go
@@ -43,6 +43,10 @@ type eggoOptions struct {
 	joinYaml             string
 	joinHost             HostConfig
 	delClusterID         string
+	clusterPrehook       string
+	clusterPosthook      string
+	prehook              string
+	posthook             string
 }
 
 var opts eggoOptions
@@ -66,12 +70,16 @@ func setupDeployCmdOpts(deployCmd *cobra.Command) {
 	flags := deployCmd.Flags()
 	flags.StringVarP(&opts.deployConfig, "file", "f", defaultDeployConfigPath(), "location of cluster deploy config file, default $HOME/.eggo/deploy.yaml")
 	flags.BoolVarP(&opts.deployEnableRollback, "rollback", "", true, "rollback failed node to cleanup")
+	flags.StringVarP(&opts.clusterPrehook, "cluster-prehook", "", "", "cluser prehooks when deploy cluser")
+	flags.StringVarP(&opts.clusterPosthook, "cluster-posthook", "", "", "cluster posthook when deploy cluster")
 }
 
 func setupCleanupCmdOpts(cleanupCmd *cobra.Command) {
 	flags := cleanupCmd.Flags()
 	flags.StringVarP(&opts.cleanupConfig, "file", "f", "", "location of cluster deploy config file")
 	flags.StringVarP(&opts.cleanupClusterID, "id", "", "", "cluster id")
+	flags.StringVarP(&opts.clusterPrehook, "cluster-prehook", "", "", "cluser prehooks when clenaup cluser")
+	flags.StringVarP(&opts.clusterPosthook, "cluster-posthook", "", "", "cluster posthook when cleaup cluster")
 }
 
 func setupJoinCmdOpts(joinCmd *cobra.Command) {
@@ -82,11 +90,15 @@ func setupJoinCmdOpts(joinCmd *cobra.Command) {
 	flags.IntVarP(&opts.joinHost.Port, "port", "p", 0, "host's ssh port")
 	flags.StringVarP(&opts.joinClusterID, "id", "", "", "cluster id")
 	flags.StringVarP(&opts.joinYaml, "file", "f", "", "yaml file contain nodes infomation")
+	flags.StringVarP(&opts.prehook, "prehook", "", "", "prehook when join cluster")
+	flags.StringVarP(&opts.posthook, "posthook", "", "", "posthook when join cluster")
 }
 
 func setupDeleteCmdOpts(deleteCmd *cobra.Command) {
 	flags := deleteCmd.Flags()
 	flags.StringVarP(&opts.delClusterID, "id", "", "", "cluster id")
+	flags.StringVarP(&opts.prehook, "prehook", "", "", "prehook when delete cluster")
+	flags.StringVarP(&opts.posthook, "posthook", "", "", "posthook when delete cluster")
 }
 
 func setupTemplateCmdOpts(templateCmd *cobra.Command) {
diff --git a/docs/hooks_of_eggo.md b/docs/hooks_of_eggo.md
index b1f09cb..fd9ce35 100644
--- a/docs/hooks_of_eggo.md
+++ b/docs/hooks_of_eggo.md
@@ -21,8 +21,9 @@
 说明：
 
 - 脚本目录下的所有脚本都会被执行，而子目录中的脚本不会被执行；
-- 每个脚本的超时时间为60s；
+- 每个脚本的超时时间为120s；
 - role可以为master,worker,etcd或者loadbalance；
+- 命令行参数指定的hooks脚本默认拷贝到目标机器的/root/.eggo/package/file/cmdhooks目录下,脚本大小限制1M字节;
 
 ### 配置文件参数方式
 
diff --git a/pkg/api/types.go b/pkg/api/types.go
index e5e1958..5cb7121 100644
--- a/pkg/api/types.go
+++ b/pkg/api/types.go
@@ -47,8 +47,10 @@ const (
 type HookType string
 
 const (
-	PreHookType  HookType = "prehook"
-	PostHookType HookType = "posthook"
+	ClusterPrehookType  HookType = "cluster-prehook"
+	ClusterPosthookType HookType = "cluster-posthook"
+	PreHookType         HookType = "prehook"
+	PostHookType        HookType = "posthook"
 )
 
 type HookRunConfig struct {
@@ -233,11 +235,11 @@ type AddonConfig struct {
 }
 
 type ClusterHookConf struct {
-	Type      HookType
-	Operator  HookOperator
-	Target    uint16
-	HookDir   string
-	HookFiles []string
+	Type       HookType
+	Operator   HookOperator
+	Target     uint16
+	HookSrcDir string
+	HookFiles  []string
 }
 
 type ClusterConfig struct {
@@ -258,7 +260,7 @@ type ClusterConfig struct {
 	RoleInfra       map[uint16]*RoleInfra   `json:"role-infra"`
 
 	// do not encode hooks, just set before use it
-	HooksConf *ClusterHookConf `json:"-"`
+	HooksConf []*ClusterHookConf `json:"-"`
 
 	// TODO: add other configurations at here
 }
diff --git a/pkg/clusterdeployment/binary/binary.go b/pkg/clusterdeployment/binary/binary.go
index 363de0e..478e081 100644
--- a/pkg/clusterdeployment/binary/binary.go
+++ b/pkg/clusterdeployment/binary/binary.go
@@ -419,6 +419,10 @@ func (bcp *BinaryClusterDeployment) Finish() {
 
 func (bcp *BinaryClusterDeployment) PreCreateClusterHooks() error {
 	role := []uint16{api.LoadBalance, api.ETCD, api.Master, api.Worker}
+	if err := dependency.ExecuteCmdHooks(bcp.config, bcp.config.Nodes, api.HookOpDeploy, api.ClusterPrehookType); err != nil {
+		return err
+	}
+
 	if err := dependency.HookSchedule(bcp.config, bcp.config.Nodes, role, api.SchedulePreJoin); err != nil {
 		return err
 	}
@@ -434,11 +438,17 @@ func (bcp *BinaryClusterDeployment) PostCreateClusterHooks(nodes []*api.HostConf
 	if err := checkK8sServices(nodes); err != nil {
 		return err
 	}
+	if err := dependency.ExecuteCmdHooks(bcp.config, bcp.config.Nodes, api.HookOpDeploy, api.ClusterPosthookType); err != nil {
+		return err
+	}
 	return nil
 }
 
 func (bcp *BinaryClusterDeployment) PreDeleteClusterHooks() {
 	role := []uint16{api.Worker, api.Master, api.ETCD, api.LoadBalance}
+	if err := dependency.ExecuteCmdHooks(bcp.config, bcp.config.Nodes, api.HookOpCleanup, api.ClusterPrehookType); err != nil {
+		logrus.Warnf("Ignore: Delete cluster prehook failed:%v", err)
+	}
 	if err := dependency.HookSchedule(bcp.config, bcp.config.Nodes, role, api.SchedulePreCleanup); err != nil {
 		logrus.Warnf("Ignore: Delete cluster PreHook failed: %v", err)
 	}
@@ -449,10 +459,16 @@ func (bcp *BinaryClusterDeployment) PostDeleteClusterHooks() {
 	if err := dependency.HookSchedule(bcp.config, bcp.config.Nodes, role, api.SchedulePostCleanup); err != nil {
 		logrus.Warnf("Ignore: Delete cluster PostHook failed: %v", err)
 	}
+	if err := dependency.ExecuteCmdHooks(bcp.config, bcp.config.Nodes, api.HookOpCleanup, api.ClusterPosthookType); err != nil {
+		logrus.Warnf("Ignore: Delete cluster posthook failed:%v", err)
+	}
 }
 
 func (bcp *BinaryClusterDeployment) PreNodeJoinHooks(node *api.HostConfig) error {
 	role := []uint16{api.Master, api.Worker, api.ETCD}
+	if err := dependency.ExecuteCmdHooks(bcp.config, []*api.HostConfig{node}, api.HookOpJoin, api.PreHookType); err != nil {
+		return err
+	}
 	if err := dependency.HookSchedule(bcp.config, []*api.HostConfig{node}, role, api.SchedulePreJoin); err != nil {
 		return err
 	}
@@ -525,6 +541,9 @@ func (bcp *BinaryClusterDeployment) PostNodeJoinHooks(node *api.HostConfig) erro
 	if err := dependency.HookSchedule(bcp.config, []*api.HostConfig{node}, role, api.SchedulePostJoin); err != nil {
 		return err
 	}
+	if err := dependency.ExecuteCmdHooks(bcp.config, []*api.HostConfig{node}, api.HookOpJoin, api.PostHookType); err != nil {
+		return err
+	}
 
 	// taint and label for master node
 	roles := node.Type
@@ -552,6 +571,9 @@ func (bcp *BinaryClusterDeployment) PostNodeJoinHooks(node *api.HostConfig) erro
 
 func (bcp *BinaryClusterDeployment) PreNodeCleanupHooks(node *api.HostConfig) {
 	role := []uint16{api.Worker, api.Master, api.ETCD}
+	if err := dependency.ExecuteCmdHooks(bcp.config, []*api.HostConfig{node}, api.HookOpDelete, api.PreHookType); err != nil {
+		logrus.Warnf("Ignore: Delete Node Cmd Prehook failed: %v", err)
+	}
 	if err := dependency.HookSchedule(bcp.config, []*api.HostConfig{node}, role, api.SchedulePreCleanup); err != nil {
 		logrus.Warnf("Ignore: Delete Node PreHook failed: %v", err)
 	}
@@ -562,6 +584,9 @@ func (bcp *BinaryClusterDeployment) PostNodeCleanupHooks(node *api.HostConfig) {
 	if err := dependency.HookSchedule(bcp.config, []*api.HostConfig{node}, role, api.SchedulePostCleanup); err != nil {
 		logrus.Warnf("Ignore: Delete Node PostHook failed: %v", err)
 	}
+	if err := dependency.ExecuteCmdHooks(bcp.config, []*api.HostConfig{node}, api.HookOpDelete, api.PostHookType); err != nil {
+		logrus.Warnf("Ignore: Delete Node Cmd Posthook failed: %v", err)
+	}
 }
 
 func (bcp *BinaryClusterDeployment) CleanupLastStep(nodeName string) error {
diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go
index ee02e24..c60d061 100644
--- a/pkg/constants/constants.go
+++ b/pkg/constants/constants.go
@@ -17,6 +17,7 @@ const (
 	DefaultPkgPath     = "/pkg"
 	DefaultBinPath     = "/bin"
 	DefaultFilePath    = "/file"
+	DefaultHookPath    = "/file/cmdhook"
 	DefaultDirPath     = "/dir"
 	DefaultImagePath   = "/image"
 
@@ -27,4 +28,8 @@ const (
 
 	// network plugin arguments key
 	NetworkPluginArgKeyYamlPath = "NetworkYamlPath"
+
+	MaxHookFileSize = int64(1 << 20)
+	// 750: rwxr-x---
+	HookFileMode = uint32(0750)
 )
diff --git a/pkg/utils/dependency/cmdhooks.go b/pkg/utils/dependency/cmdhooks.go
new file mode 100644
index 0000000..e6fd9af
--- /dev/null
+++ b/pkg/utils/dependency/cmdhooks.go
@@ -0,0 +1,115 @@
+/******************************************************************************
+ * Copyright (c) Huawei Technologies Co., Ltd. 2021. All rights reserved.
+ * eggo licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ * Author: jikui
+ * Create: 2021-12-11
+ * Description: eggo cmd hooks implement
+ ******************************************************************************/
+
+package dependency
+
+import (
+	"fmt"
+	"path"
+
+	"github.com/sirupsen/logrus"
+	"isula.org/eggo/pkg/api"
+	"isula.org/eggo/pkg/constants"
+	"isula.org/eggo/pkg/utils"
+	"isula.org/eggo/pkg/utils/nodemanager"
+	"isula.org/eggo/pkg/utils/runner"
+	"isula.org/eggo/pkg/utils/task"
+)
+
+type CopyHooksTask struct {
+	hooks *api.ClusterHookConf
+}
+
+func (ch *CopyHooksTask) Name() string {
+	return "CopyHooksTask"
+}
+
+func (ch *CopyHooksTask) Run(r runner.Runner, hcg *api.HostConfig) error {
+	dstDir := path.Join(constants.DefaultPackagePath, constants.DefaultHookPath)
+
+	if _, err := r.RunCommand(fmt.Sprintf("sudo -E /bin/sh -c \"test -d %s || mkdir -p %s\"", dstDir, dstDir)); err != nil {
+		return err
+	}
+
+	if err := r.Copy(ch.hooks.HookSrcDir, dstDir); err != nil {
+		return fmt.Errorf("copy from %s to %s for %s failed:%v", ch.hooks.HookSrcDir, dstDir, hcg.Address, err)
+	}
+
+	return nil
+}
+
+func ExecuteCmdHooks(ccfg *api.ClusterConfig, nodes []*api.HostConfig, op api.HookOperator, ty api.HookType) error {
+	for _, hooks := range ccfg.HooksConf {
+		for _, node := range nodes {
+			if !utils.IsType(node.Type, hooks.Target) {
+				continue
+			}
+
+			shell := getCmdShell(hooks, hooks.Target, op, ty)
+			if shell == nil {
+				return nil
+			}
+			if err := doCopyHooks(hooks, node); err != nil {
+				return err
+			}
+			if err := executeCmdHooks(ccfg, hooks, node, shell); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func executeCmdHooks(ccfg *api.ClusterConfig, hooks *api.ClusterHookConf, hcf *api.HostConfig, shell []*api.PackageConfig) error {
+	hookConf := &api.HookRunConfig{
+		ClusterID:          ccfg.Name,
+		ClusterApiEndpoint: ccfg.APIEndpoint.GetUrl(),
+		ClusterConfigDir:   ccfg.ConfigDir,
+		HookType:           hooks.Type,
+		Operator:           hooks.Operator,
+		Node:               hcf,
+		HookDir:            path.Join(ccfg.PackageSrc.GetPkgDstPath(), constants.DefaultHookPath),
+		Hooks:              shell,
+	}
+
+	return ExecuteHooks(hookConf)
+}
+
+func getCmdShell(hooks *api.ClusterHookConf, target uint16, op api.HookOperator, ty api.HookType) []*api.PackageConfig {
+	res := make([]*api.PackageConfig, len(hooks.HookFiles))
+
+	if hooks.Target != target || hooks.Operator != op || hooks.Type != ty {
+		return nil
+	}
+	for i, v := range hooks.HookFiles {
+		res[i] = &api.PackageConfig{
+			Name:    v,
+			TimeOut: "120s",
+		}
+	}
+	return res
+}
+
+func doCopyHooks(hcc *api.ClusterHookConf, node *api.HostConfig) error {
+	copyHooksTask := task.NewTaskInstance(&CopyHooksTask{
+		hooks: hcc,
+	})
+
+	if err := nodemanager.RunTaskOnNodes(copyHooksTask, []string{node.Address}); err != nil {
+		logrus.Errorf("Copy hooks failed with:%v", err)
+		return err
+	}
+	return nil
+}
diff --git a/pkg/utils/dependency/cmdhooks_test.go b/pkg/utils/dependency/cmdhooks_test.go
new file mode 100644
index 0000000..106518a
--- /dev/null
+++ b/pkg/utils/dependency/cmdhooks_test.go
@@ -0,0 +1,43 @@
+package dependency
+
+import (
+	"testing"
+
+	"isula.org/eggo/pkg/api"
+)
+
+func TestCopyHooks(t *testing.T) {
+	var mr MockRunner
+
+	hs := &api.ClusterHookConf{
+		Type:       api.PreHookType,
+		Operator:   api.HookOpDeploy,
+		Target:     api.Master,
+		HookSrcDir: "/tmp",
+		HookFiles:  []string{"test.sh", "test2.bash"},
+	}
+
+	node := &api.HostConfig{}
+
+	ct := &CopyHooksTask{hooks: hs}
+	if err := ct.Run(&mr, node); err != nil {
+		t.Fatalf("run test failed: %v", err)
+	}
+}
+
+func TestExecuteCmdHooks(t *testing.T) {
+	hooks := &api.ClusterHookConf{
+		Target:   api.Master,
+		Operator: api.HookOpDeploy,
+		Type:     api.PreHookType,
+	}
+	host := &api.HostConfig{
+		Type: api.Master,
+	}
+	ccfg := &api.ClusterConfig{
+		HooksConf: []*api.ClusterHookConf{hooks},
+	}
+	if err := ExecuteCmdHooks(ccfg, []*api.HostConfig{host}, api.HookOpJoin, api.PostHookType); err != nil {
+		t.Fatalf("run test failed: %v", err)
+	}
+}
diff --git a/pkg/utils/runner/runner.go b/pkg/utils/runner/runner.go
index 9a739ca..09c9e1d 100644
--- a/pkg/utils/runner/runner.go
+++ b/pkg/utils/runner/runner.go
@@ -227,7 +227,7 @@ func (ssh *SSHRunner) copyDir(srcDir, dstDir string) error {
 		return err
 	}
 	tmpCpyDir := api.GetUserTempDir(ssh.Host.User)
-	tmpPkiFile := filepath.Join(tmpCpyDir, "pkg.tar")
+	tmpPkiFile := filepath.Join(tmpCpyDir, "remote-pkg.tar")
 	// scp to user home directory
 	err = ssh.Copy(tmpPkgFile, tmpPkiFile)
 	if err != nil {
@@ -235,7 +235,7 @@ func (ssh *SSHRunner) copyDir(srcDir, dstDir string) error {
 		return err
 	}
 	// untar tmp file
-	_, err = ssh.RunCommand(fmt.Sprintf("sudo -E /bin/sh -c \"cd %s && mv %s . && tar -xf %s && rm -rf %s\"", dstDir, tmpPkiFile, "pki.tar", tmpPkiFile))
+	_, err = ssh.RunCommand(fmt.Sprintf("sudo -E /bin/sh -c \"cd %s && mv %s . && tar -xf %s && rm -rf %s\"", dstDir, tmpPkiFile, "remote-pkg.tar", "remote-pkg.tar"))
 	if err != nil {
 		logrus.Errorf("[%s] untar tmp tar failed: %v", ssh.Host.Name, err)
 		return err
diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
index 8272439..059516c 100644
--- a/pkg/utils/utils.go
+++ b/pkg/utils/utils.go
@@ -16,10 +16,12 @@
 package utils
 
 import (
+	"fmt"
 	"os"
 	"os/user"
 	"path/filepath"
 	"strings"
+	"syscall"
 
 	"isula.org/eggo/pkg/api"
 )
@@ -107,3 +109,16 @@ func IsDocker(engine string) bool {
 func IsContainerd(engine string) bool {
 	return strings.ToLower(engine) == "containerd"
 }
+
+func GetUserIDAndGroupID(file string) (int, int, error) {
+	fileInfo, err := os.Stat(file)
+	if err != nil {
+		return 0, 0, err
+	}
+	statInfo, ok := fileInfo.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, 0, fmt.Errorf("Assert failed when stat %s", file)
+	}
+
+	return int(statInfo.Uid), int(statInfo.Gid), nil
+}
-- 
2.25.1