diff --git a/Makefile b/Makefile index 08428072aa08ce8302100798d3b18c883854d7d8..224828be52a79aa9a98f9c4e2e3c17d61d7e4ae8 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ IMAGE_REPOSITORY = ccr.ccs.tencentyun.com/koderover-rc VERSION ?= $(shell date +'%Y%m%d%H%M%S') -TARGETS = aslan cron hub-agent hub-server jenkins-plugin podexec predator-plugin resource-server ua warpdrive policy user picket config init +TARGETS = aslan cron hub-agent hub-server jenkins-plugin podexec predator-plugin packager-plugin resource-server ua warpdrive policy user picket config init REAPER_OS= focal xenial bionic ALL_IMAGES=$(TARGETS:=.image) diff --git a/README-zh-CN.md b/README-zh-CN.md index 3bd629bace746e12316ebde00f4e36948333ad85..cbfb90056f9b1f5e788c2cf376052a65dc462d06 100644 --- a/README-zh-CN.md +++ b/README-zh-CN.md @@ -75,7 +75,7 @@ Zadig 是一款面向开发者设计的云原生持续交付(Continuous Delivery ### 快速使用 -请参阅 [快速入门](https://docs.koderover.com/zadig/quick-start/try-out-install) +请参阅 [快速入门](https://docs.koderover.com/zadig/quick-start/try-out-install/) ### 训练营 diff --git a/README.md b/README.md index 7ba402d46c964a0559f558ab2c6ba89ce529a86a..b004e68e76c800a72f214f1606f5aeb39af10b26 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ The Highlighted Features: ### How to use? -Please follow [Quick Start](https://docs.koderover.com/zadig/quick-start/try-out-install) +Please follow [Quick Start](https://docs.koderover.com/zadig/quick-start/try-out-install/) ### Bootcamps diff --git a/System-Architecture-Overview-zh-CN.md b/System-Architecture-Overview-zh-CN.md index 4f10c078aea0e1280cc4481a1592aca3ce853eb1..dcef018dff66cc5e84a1740d54990954730177f1 100644 --- a/System-Architecture-Overview-zh-CN.md +++ b/System-Architecture-Overview-zh-CN.md @@ -12,10 +12,10 @@ - Zadig Toolkit：vscode 开发者插件 API 网关： -- Gloo Edge：Zadig 的 API 网关组件 -- OPA：认证和授权组件 +- [Gloo Edge](https://github.com/solo-io/gloo)：Zadig 的 API 网关组件 +- [OPA](https://github.com/open-policy-agent/opa)：认证和授权组件 +- [Dex](https://github.com/dexidp/dex)：Zadig 的身份认证服务，用于连接其他第三方认证系统，比如 AD / LDAP / OAuth2 / GitHub / .. - User：用户管理，Token 生成 -- Dex：Zadig 的身份认证服务，用于连接其他第三方认证系统，比如 AD / LDAP / OAuth2 / GitHub / .. Zadig 核心业务： - Picket：数据聚合服务 diff --git a/System-Architecture-Overview.md b/System-Architecture-Overview.md index 61e0597798d97af0fe6f6a06d78f990004b1e9af..9c9414992911e9a950c1b7af560cf2d3b4c5779d 100644 --- a/System-Architecture-Overview.md +++ b/System-Architecture-Overview.md @@ -12,10 +12,10 @@ User Interface： - Zadig Toolkit：vscode plugin API Gateway： -- Gloo Edge：Zadig API gateway -- OPA：Authentication and authorization +- [Gloo Edge](https://github.com/solo-io/gloo)：Zadig API gateway +- [OPA](https://github.com/open-policy-agent/opa)：Authentication and authorization +- [Dex](https://github.com/dexidp/dex)：Identity service for Zadig, acts as a portal to other identity providers like AD / LDAP / OAuth2 / GitHub / .. - User：User management, token generation -- Dex：Identity service for Zadig, acts as a portal to other identity providers like AD / LDAP / OAuth2 / GitHub / .. Zadig Core： - Picket：backend for frontend service. diff --git a/cmd/packager/main.go b/cmd/packager/main.go new file mode 100644 index 0000000000000000000000000000000000000000..bd2fa8bc1bbdca5db6e0112706415a8c2985d95a --- /dev/null +++ b/cmd/packager/main.go @@ -0,0 +1,29 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "log" + + "github.com/koderover/zadig/pkg/microservice/packager/executor" +) + +func main() { + if err := executor.Execute(); err != nil { + log.Fatalf("Failed to run packager, the error is: %+v", err) + } +} diff --git a/docker/service/packager-plugin.Dockerfile b/docker/service/packager-plugin.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..5ef3c5d5ec8f8d6652ff733c4d1eed8f7de32c4a --- /dev/null +++ b/docker/service/packager-plugin.Dockerfile @@ -0,0 +1,11 @@ +#golang.Dockerfile + +RUN go build -v -o /packager-plugin ./cmd/packager/main.go + +#alpine.Dockerfile + +WORKDIR /app + +COPY --from=build /packager-plugin . + +ENTRYPOINT ["/app/packager-plugin"] diff --git a/go.mod b/go.mod index c00c919065e760f78fd345e008787e8240aba5e2..b62c82d82ba71b38a82a6e8f8ad542bb23489556 100644 --- a/go.mod +++ b/go.mod @@ -14,11 +14,12 @@ require ( github.com/bugsnag/bugsnag-go v2.1.0+incompatible // indirect github.com/bugsnag/panicwrap v1.3.1 // indirect github.com/cenkalti/backoff/v4 v4.1.1 + github.com/chartmuseum/helm-push v0.10.1 github.com/coocood/freecache v1.1.0 github.com/coreos/go-oidc/v3 v3.0.0 github.com/dexidp/dex v0.0.0-20210802203454-3fac2ab6bc3b github.com/docker/distribution v2.7.1+incompatible - github.com/docker/docker v20.10.7+incompatible + github.com/docker/docker v20.10.11+incompatible github.com/docker/go-connections v0.4.0 github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/dsnet/compress v0.0.1 // indirect @@ -40,7 +41,7 @@ require ( github.com/jasonlvhit/gocron v0.0.0-20171226191223-3c914c8681c3 github.com/jinzhu/now v1.1.2 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect - github.com/mittwald/go-helm-client v0.8.2 + github.com/mittwald/go-helm-client v0.8.3 github.com/moby/buildkit v0.9.1 github.com/nsqio/go-nsq v1.0.7 github.com/nwaples/rardecode v1.0.0 // indirect @@ -66,8 +67,8 @@ require ( github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 // indirect go.mongodb.org/mongo-driver v1.5.0 go.uber.org/zap v1.19.0 - golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a - golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 + golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect @@ -78,25 +79,27 @@ require ( gorm.io/driver/mysql v1.1.2 gorm.io/gorm v1.21.16 helm.sh/helm/v3 v3.7.1 - k8s.io/api v0.22.1 - k8s.io/apimachinery v0.22.1 - k8s.io/apiserver v0.22.1 // indirect - k8s.io/client-go v0.22.1 - k8s.io/kubectl v0.22.1 - k8s.io/utils v0.0.0-20210802155522-efc7438f0176 + k8s.io/api v0.22.4 + k8s.io/apimachinery v0.22.4 + k8s.io/client-go v0.22.4 + k8s.io/kubectl v0.22.4 + k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a sigs.k8s.io/controller-runtime v0.10.0 - sigs.k8s.io/yaml v1.2.0 + sigs.k8s.io/yaml v1.3.0 ) replace ( github.com/docker/distribution => github.com/docker/distribution v2.6.0-rc.1.0.20170726174610-edc3ab29cdff+incompatible github.com/docker/docker => github.com/docker/docker v0.0.0-20180502112750-51a9119f6b81 github.com/docker/go-connections => github.com/docker/go-connections v0.3.1-0.20180212134524-7beb39f0b969 + github.com/go-logr/logr => github.com/go-logr/logr v0.4.0 k8s.io/api => k8s.io/api v0.21.0 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.21.0 k8s.io/apimachinery => k8s.io/apimachinery v0.21.0 k8s.io/cli-runtime => k8s.io/cli-runtime v0.21.0 k8s.io/client-go => k8s.io/client-go v0.21.0 + k8s.io/klog/v2 => k8s.io/klog/v2 v2.9.0 k8s.io/kubectl => k8s.io/kubectl v0.21.0 + oras.land/oras-go => oras.land/oras-go v0.4.0 ) diff --git a/go.sum b/go.sum index 79c316c2d976c79014db0e97198000d073f790fa..80995ae8713551d1c08e16008e41b066d9c241d6 100644 --- a/go.sum +++ b/go.sum @@ -111,8 +111,9 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28= github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= +github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -135,8 +136,9 @@ github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZC github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= -github.com/Masterminds/squirrel v1.5.0 h1:JukIZisrUXadA9pl3rMkjhiamxiB0cXiu+HGp/Y8cY8= github.com/Masterminds/squirrel v1.5.0/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/Masterminds/squirrel v1.5.2 h1:UiOEi2ZX4RCSkpiNDQN5kro/XIBpSRk9iTqdIRPzUXE= +github.com/Masterminds/squirrel v1.5.2/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Masterminds/vcs v1.13.1/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= @@ -158,8 +160,9 @@ github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2 github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= github.com/Microsoft/hcsshim v0.8.18/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.8.21 h1:btRfUDThBE5IKcvI8O8jOiIkujUsAMBSRsYDYmEi6oM= github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.23 h1:47MSwtKGXet80aIn+7h4YI6fwPmwIghAnsx2aOUrG2M= +github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= github.com/Microsoft/hcsshim/test v0.0.0-20200826032352-301c83a30e7c/go.mod h1:30A5igQ91GEmhYJF8TaRP79pMBOYynRsyOByfVV0dU4= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= @@ -286,6 +289,8 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= +github.com/chartmuseum/helm-push v0.10.1 h1:NqStAmarEy0GnrDCk0zubGKeRmkhOm/rRi5au8h76BA= +github.com/chartmuseum/helm-push v0.10.1/go.mod h1:s6xTICU31jKdLkOXS+GgaR61E+oU4h8TWb1yZcHq8OE= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -337,6 +342,7 @@ github.com/containerd/containerd v1.4.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX github.com/containerd/containerd v1.4.1-0.20201117152358-0edc412565dc/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= @@ -345,8 +351,9 @@ github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTV github.com/containerd/containerd v1.5.2/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.3/go.mod h1:sx18RgvW6ABJ4iYUw7Q5x7bgFOAB9B6G7+yO0XBc4zw= github.com/containerd/containerd v1.5.4/go.mod h1:sx18RgvW6ABJ4iYUw7Q5x7bgFOAB9B6G7+yO0XBc4zw= -github.com/containerd/containerd v1.5.7 h1:rQyoYtj4KddB3bxG6SAqd4+08gePNyJjRqvOIfV3rkM= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= +github.com/containerd/containerd v1.5.8 h1:NmkCC1/QxyZFBny8JogwLpOy2f+VEbO/f6bV2Mqtwuw= +github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -384,6 +391,7 @@ github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDG github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= @@ -433,12 +441,14 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= +github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= +github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -463,14 +473,16 @@ github.com/docker/cli v0.0.0-20190925022749-754388324470/go.mod h1:JLrzqnKDaYBop github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.0-beta1.0.20201029214301-1d20b15adc38+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/cli v20.10.7+incompatible h1:pv/3NqibQKphWZiAskMzdz8w0PRbtTaEB+f6NwdU7Is= github.com/docker/cli v20.10.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v20.10.11+incompatible h1:tXU1ezXcruZQRrMP8RN2z9N91h+6egZTS1gsPsKantc= +github.com/docker/cli v20.10.11+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.6.0-rc.1.0.20170726174610-edc3ab29cdff+incompatible h1:357nGVUC8gSpeSc2Axup8HfrfTLLUfWfCsCUhiQSKIg= github.com/docker/distribution v2.6.0-rc.1.0.20170726174610-edc3ab29cdff+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.0.0-20180502112750-51a9119f6b81 h1:VnJAjP/7FbY46wddaiKakrc1ogXW+3MlP7cg+S4j4Og= github.com/docker/docker v0.0.0-20180502112750-51a9119f6b81/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= +github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o= +github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= github.com/docker/go-connections v0.3.1-0.20180212134524-7beb39f0b969 h1:Z8LR4yJgA3vaXpBiUWcWdQZpB9h+dzj4NdLjP9Sfdb0= github.com/docker/go-connections v0.3.1-0.20180212134524-7beb39f0b969/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= @@ -497,7 +509,6 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -537,6 +548,7 @@ github.com/garyburd/redigo v1.6.2 h1:yE/pwKCrbLpLpQICzYTeZ7JsTA/C53wFTJHaEtRqniM github.com/garyburd/redigo v1.6.2/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/gzip v0.0.1 h1:ezvKOL6jH+jlzdHNE4h9h8q8uMpDQjyl0NN0Jd7jozc= github.com/gin-contrib/gzip v0.0.1/go.mod h1:fGBJBCdt6qCZuCAOwWuFhBB4OOq9EFqlo5dEaFhhu5w= @@ -571,8 +583,6 @@ github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTD github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= @@ -991,8 +1001,9 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= -github.com/jmoiron/sqlx v1.3.1 h1:aLN7YINNZ7cYOPK3QC83dbM6KT0NMqVMw961TqrejlE= github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= +github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -1067,8 +1078,9 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= @@ -1141,8 +1153,9 @@ github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.2/go.mod h1:6iaV0fGdElS6dPBx0EApTxHrcWvmJphyh2n8YBLPPZ4= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= -github.com/mitchellh/copystructure v1.1.1 h1:Bp6x9R1Wn16SIz3OfeDr0b7RnCG2OB66Y7PQyC/cvq4= github.com/mitchellh/copystructure v1.1.1/go.mod h1:EBArHfARyrSWO/+Wyr9zwEkc6XMFB9XyNgFNmRkZZU4= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk= @@ -1159,10 +1172,11 @@ github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxd github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/mittwald/go-helm-client v0.8.2 h1:tWPe+e4Bx2bAC0lY55nxuOxOvNEP4aBfCdizvGpM24U= -github.com/mittwald/go-helm-client v0.8.2/go.mod h1:DVtyXr7KDPIT44X320DkxF39nfbZn8KdjvibnN7iHnQ= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mittwald/go-helm-client v0.8.3 h1:4l8W1PBZiNyapjJjoyQUU3qI8yWhOhpdD6wWdG3Z33g= +github.com/mittwald/go-helm-client v0.8.3/go.mod h1:ZdBFetgIkFiJivTy+P/Ruz6y5YtQMk5BnMSja4+wMdA= github.com/moby/buildkit v0.8.1/go.mod h1:/kyU1hKy/aYCuP39GZA9MaKioovHku57N6cqlKZIaiQ= github.com/moby/buildkit v0.9.1 h1:6noq8jvkaRs3OQGTIDf5U5Etkgq6zsPmQHGWi5yhh88= github.com/moby/buildkit v0.9.1/go.mod h1:oVZKk3TMm0MlDx7XxnlF0wKmcpyrzOs9GEp0VXKWFPk= @@ -1252,8 +1266,9 @@ github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQ github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc10/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -1692,8 +1707,9 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871 h1:/pEO3GD/ABYAjuakUS6xSEmmlyVS4kxBNkeA9tLJiTI= +golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1801,8 +1817,9 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1945,6 +1962,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo= @@ -2324,8 +2342,9 @@ k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= k8s.io/apiserver v0.21.0/go.mod h1:w2YSn4/WIwYuxG5zJmcqtRdtqgW/J2JRgFAqps3bBpg= -k8s.io/apiserver v0.22.1 h1:Ul9Iv8OMB2s45h2tl5XWPpAZo1VPIJ/6N+MESeed7L8= k8s.io/apiserver v0.22.1/go.mod h1:2mcM6dzSt+XndzVQJX21Gx0/Klo7Aen7i0Ai6tIa400= +k8s.io/apiserver v0.22.4 h1:L+220cy+94UWmyBl1kiVTklBXrBtKsbjlPV60eL2u6s= +k8s.io/apiserver v0.22.4/go.mod h1:38WmcUZiiy41A7Aty8/VorWRa8vDGqoUzDf2XYlku0E= k8s.io/cli-runtime v0.21.0 h1:/V2Kkxtf6x5NI2z+Sd/mIrq4FQyQ8jzZAUD6N5RnN7Y= k8s.io/cli-runtime v0.21.0/go.mod h1:XoaHP93mGPF37MkLbjGVYqg3S1MnsFdKtiA/RZzzxOo= k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag= @@ -2338,8 +2357,9 @@ k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeY k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= k8s.io/component-base v0.21.0/go.mod h1:qvtjz6X0USWXbgmbfXR+Agik4RZ3jv2Bgr5QnZzdPYw= -k8s.io/component-base v0.22.1 h1:SFqIXsEN3v3Kkr1bS6rstrs1wd45StJqbtgbQ4nRQdo= k8s.io/component-base v0.22.1/go.mod h1:0D+Bl8rrnsPN9v0dyYvkqFfBeAd4u7n77ze+p8CMiPo= +k8s.io/component-base v0.22.4 h1:7qwLJnua2ppGNZrRGDQ0vhsFebI39VGbZ4zdR5ArViI= +k8s.io/component-base v0.22.4/go.mod h1:MrSaQy4a3tFVViff8TZL6JHYSewNCLshZCwHYM58v5A= k8s.io/component-helpers v0.21.0/go.mod h1:tezqefP7lxfvJyR+0a+6QtVrkZ/wIkyMLK4WcQ3Cj8U= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= @@ -2350,22 +2370,21 @@ k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/helm v2.17.0+incompatible h1:Bpn6o1wKLYqKM3+Osh8e+1/K2g/GsQJ4F4yNF2+deao= +k8s.io/helm v2.17.0+incompatible/go.mod h1:LZzlS4LQBHfciFOurYBFkCMTaZ0D1l+p0teMg7TSULI= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c h1:jvamsI1tn9V0S8jicyX82qaFC0H/NKxv2e5mbqsgR80= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kubectl v0.21.0 h1:WZXlnG/yjcE4LWO2g6ULjFxtzK6H1TKzsfaBFuVIhNg= k8s.io/kubectl v0.21.0/go.mod h1:EU37NukZRXn1TpAkMUoy8Z/B2u6wjHDS4aInsDzVvks= k8s.io/kubernetes v1.11.10/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= @@ -2375,8 +2394,9 @@ k8s.io/metrics v0.21.0/go.mod h1:L3Ji9EGPP1YBbfm9sPfEXSpnj8i24bfQbAFAsW0NueQ= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176 h1:Mx0aa+SUAcNRQbs5jUzV8lkDlGFU8laZsY9jrcVX5SY= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= @@ -2416,7 +2436,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4= diff --git a/pkg/cli/initconfig/cmd/healthz.go b/pkg/cli/initconfig/cmd/healthz.go index 7e270b8cdba8bf4abaed96da8fef232af2ee4e60..bdc4d036e6c8fb56a34088aac8f1025a8dbf58b3 100644 --- a/pkg/cli/initconfig/cmd/healthz.go +++ b/pkg/cli/initconfig/cmd/healthz.go @@ -19,6 +19,8 @@ package cmd import ( "fmt" + "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/shared/client/aslan" "github.com/koderover/zadig/pkg/shared/client/policy" "github.com/koderover/zadig/pkg/shared/client/user" ) @@ -30,6 +32,9 @@ func Healthz() error { if err := checkPolicyServiceHealth(); err != nil { return fmt.Errorf("checkPolicyServiceHealth error:%s", err) } + if err := checkAslanServiceHealth(); err != nil { + return fmt.Errorf("checkPolicyServiceHealth error:%s", err) + } return nil } @@ -40,3 +45,7 @@ func checkUserServiceHealth() error { func checkPolicyServiceHealth() error { return policy.NewDefault().Healthz() } + +func checkAslanServiceHealth() error { + return aslan.New(config.AslanServiceAddress()).Healthz() +} diff --git a/pkg/cli/initconfig/cmd/init.go b/pkg/cli/initconfig/cmd/init.go index 5e5ea0f265ca6a8eadb61a725b5be74b1a9256a4..b27f0ed816d87c0bbb67a62a24166a36336f26eb 100644 --- a/pkg/cli/initconfig/cmd/init.go +++ b/pkg/cli/initconfig/cmd/init.go @@ -29,6 +29,7 @@ import ( "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/client/aslan" "github.com/koderover/zadig/pkg/shared/client/policy" "github.com/koderover/zadig/pkg/shared/client/user" "github.com/koderover/zadig/pkg/tool/httpclient" @@ -97,9 +98,15 @@ func initSystemConfig() error { } if err := presetRoleBinding(uid); err != nil { - log.Errorf("presetRoleBinding :%s", err) + log.Errorf("presetRoleBinding err:%s", err) return err } + + if err := createLocalCluster(); err != nil { + log.Errorf("createLocalCluster err:%s", err) + return err + } + return nil } @@ -195,3 +202,14 @@ func presetRole() error { } return nil } + +func createLocalCluster() error { + cluster, err := aslan.New(config.AslanServiceAddress()).GetLocalCluster() + if err != nil { + return err + } + if cluster != nil { + return nil + } + return aslan.New(config.AslanServiceAddress()).AddLocalCluster() +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 78ff453365f3a39ea16a15def51f5b88f3f63c72..4f2ef7d943dc6452224b2e1871a79f1ebc0d50cd 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -210,6 +210,10 @@ func ObjectStorageTemplatePath(name, kind string) string { return filepath.Join("templates", kind, name) } +func ObjectStorageDeliveryVersionPath(project string) string { + return filepath.Join("delivery-distributes", "files", project) +} + func ObjectStorageChartTemplatePath(name string) string { return ObjectStorageTemplatePath(name, setting.ChartTemplatesPath) } @@ -257,3 +261,7 @@ func AdminEmail() string { func AdminPassword() string { return viper.GetString(setting.ENVAdminPassword) } + +func Namespace() string { + return viper.GetString(setting.ENVNamespace) +} diff --git a/pkg/microservice/aslan/config/config.go b/pkg/microservice/aslan/config/config.go index c9c115c8d42f6e41e4ac0239c2eb21baac321f2c..121add2cc15ef20c762eada221bd54a9ac120672 100644 --- a/pkg/microservice/aslan/config/config.go +++ b/pkg/microservice/aslan/config/config.go @@ -190,6 +190,10 @@ func PredatorImage() string { return viper.GetString(setting.ENVPredatorImage) } +func PackagerImage() string { + return viper.GetString(setting.EnvPackagerImage) +} + func DockerHosts() []string { return strings.Split(viper.GetString(setting.ENVDockerHosts), ",") } @@ -230,6 +234,10 @@ func LocalServicePathWithRevision(project, service string, revision int64) strin return configbase.LocalServicePathWithRevision(project, service, fmt.Sprintf("%d", revision)) } +func LocalDeliveryChartPathWithRevision(project, service string, revision int64) string { + return configbase.LocalServicePathWithRevision(project, service, fmt.Sprintf("delivery/%d", revision)) +} + func ServiceNameWithRevision(serviceName string, revision int64) string { return fmt.Sprintf("%s-%d", serviceName, revision) } diff --git a/pkg/microservice/aslan/config/consts.go b/pkg/microservice/aslan/config/consts.go index a028737fd8ae4149db702d6d0a9cb552722735a2..9b91eb4c0a0cce8584a1d0e8932a4313b5bb71e1 100644 --- a/pkg/microservice/aslan/config/consts.go +++ b/pkg/microservice/aslan/config/consts.go @@ -83,6 +83,8 @@ const ( TestType PipelineType = "test" // ServiceType 服务 ServiceType PipelineType = "service" + // ArtifactPackageType package artifact + ArtifactType PipelineType = "artifact" ) type Status string @@ -117,20 +119,21 @@ const ( type TaskType string const ( - TaskPipeline TaskType = "pipeline" - TaskBuild TaskType = "buildv2" - TaskJenkinsBuild TaskType = "jenkins_build" - TaskArtifact TaskType = "artifact" - TaskArtifactDeploy TaskType = "artifact_deploy" - TaskDeploy TaskType = "deploy" - TaskTestingV2 TaskType = "testingv2" - TaskDistributeToS3 TaskType = "distribute2kodo" - TaskReleaseImage TaskType = "release_image" - TaskJira TaskType = "jira" - TaskDockerBuild TaskType = "docker_build" - TaskSecurity TaskType = "security" - TaskResetImage TaskType = "reset_image" - TaskDistribute TaskType = "distribute" + TaskPipeline TaskType = "pipeline" + TaskBuild TaskType = "buildv2" + TaskJenkinsBuild TaskType = "jenkins_build" + TaskArtifact TaskType = "artifact" + TaskArtifactDeploy TaskType = "artifact_deploy" + TaskDeploy TaskType = "deploy" + TaskTestingV2 TaskType = "testingv2" + TaskDistributeToS3 TaskType = "distribute2kodo" + TaskReleaseImage TaskType = "release_image" + TaskJira TaskType = "jira" + TaskDockerBuild TaskType = "docker_build" + TaskSecurity TaskType = "security" + TaskResetImage TaskType = "reset_image" + TaskDistribute TaskType = "distribute" + TaskArtifactPackage TaskType = "artifact_package" ) type DistributeType string @@ -138,6 +141,7 @@ type DistributeType string const ( File DistributeType = "file" Image DistributeType = "image" + Chart DistributeType = "chart" ) type NotifyType int diff --git a/pkg/microservice/aslan/core/common/repository/models/build.go b/pkg/microservice/aslan/core/common/repository/models/build.go index 265507624517963cc793b730bb3c1978a001ca21..1cf8392deff8d62c578b7b53f4b25c54e64322a1 100644 --- a/pkg/microservice/aslan/core/common/repository/models/build.go +++ b/pkg/microservice/aslan/core/common/repository/models/build.go @@ -66,7 +66,9 @@ type PreBuild struct { // Parameters Parameters []*Parameter `bson:"parameters,omitempty" json:"parameters"` // UploadPkg uploads package to s3 - UploadPkg bool `bson:"upload_pkg" json:"upload_pkg"` + UploadPkg bool `bson:"upload_pkg" json:"upload_pkg"` + ClusterID string `bson:"cluster_id" json:"cluster_id"` + Namespace string `bson:"namespace" json:"namespace"` } type BuildObj struct { diff --git a/pkg/microservice/aslan/core/common/repository/models/delivery_distribute.go b/pkg/microservice/aslan/core/common/repository/models/delivery_distribute.go index db24294d6654dad48afde50f71c224c24ec35cb1..732e24fc0f7005bbc2707ea83cea7e2c825adb6d 100644 --- a/pkg/microservice/aslan/core/common/repository/models/delivery_distribute.go +++ b/pkg/microservice/aslan/core/common/repository/models/delivery_distribute.go @@ -23,18 +23,25 @@ import ( ) type DeliveryDistribute struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` ReleaseID primitive.ObjectID `bson:"release_id" json:"releaseId"` - ServiceName string `bson:"service_name" json:"serviceName"` + ServiceName string `bson:"service_name" json:"serviceName,omitempty"` DistributeType config.DistributeType `bson:"distribute_type" json:"distributeType"` RegistryName string `bson:"registry_name" json:"registryName"` - Namespace string `bson:"namespace" json:"namespace"` - PackageFile string `bson:"package_file" json:"packageFile"` - RemoteFileKey string `bson:"remote_file_key" json:"remoteFileKey"` - DestStorageURL string `bson:"dest_storage_url" json:"destStorageUrl"` - SrcStorageURL string `bson:"src_storage_url" json:"srcStorageUrl"` - StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty"` - EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty"` + ChartVersion string `bson:"chart_version" json:"chartVersion,omitempty"` + ChartName string `bson:"chart_name" json:"chartName,omitempty"` + ChartRepoName string `bson:"chart_repo_name" json:"chartRepoName,omitempty"` + SubDistributes []*DeliveryDistribute `bson:"-" json:"subDistributes,omitempty"` + Namespace string `bson:"namespace" json:"namespace,omitempty"` + PackageFile string `bson:"package_file" json:"packageFile,omitempty"` + RemoteFileKey string `bson:"remote_file_key" json:"remoteFileKey,omitempty"` + DestStorageURL string `bson:"dest_storage_url" json:"destStorageUrl,omitempty"` + S3StorageID string `bson:"s3_storage_id" json:"s3StorageID"` + StorageURL string `bson:"-" json:"storageUrl"` + StorageBucket string `bson:"-" json:"storageBucket"` + SrcStorageURL string `bson:"src_storage_url" json:"srcStorageUrl,omitempty"` + StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty,omitempty"` + EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty,omitempty"` CreatedAt int64 `bson:"created_at" json:"created_at"` DeletedAt int64 `bson:"deleted_at" json:"deleted_at"` } diff --git a/pkg/microservice/aslan/core/common/repository/models/delivery_version.go b/pkg/microservice/aslan/core/common/repository/models/delivery_version.go index 459d7f96f70c9e62e184a5fc380fc318c9aa0d94..ed851379dcb5b451b4e5379add4777f64ffa5aa9 100644 --- a/pkg/microservice/aslan/core/common/repository/models/delivery_version.go +++ b/pkg/microservice/aslan/core/common/repository/models/delivery_version.go @@ -20,18 +20,30 @@ import ( "go.mongodb.org/mongo-driver/bson/primitive" ) +type DeliveryVersionProgress struct { + SuccessChartCount int `json:"successChartCount"` + TotalChartCount int `json:"totalChartCount"` + PackageUploadStatus string `json:"packageStatus"` + Error string `json:"error"` +} + type DeliveryVersion struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` - Version string `bson:"version" json:"version"` - ProductName string `bson:"product_name" json:"productName"` - WorkflowName string `bson:"workflow_name" json:"workflowName"` - TaskID int `bson:"task_id" json:"taskId"` - Desc string `bson:"desc" json:"desc"` - Labels []string `bson:"labels" json:"labels"` - ProductEnvInfo *Product `bson:"product_env_info" json:"productEnvInfo"` - CreatedBy string `bson:"created_by" json:"createdBy"` - CreatedAt int64 `bson:"created_at" json:"created_at"` - DeletedAt int64 `bson:"deleted_at" json:"deleted_at"` + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + Version string `bson:"version" json:"version"` + ProductName string `bson:"product_name" json:"productName"` + WorkflowName string `bson:"workflow_name" json:"workflowName"` + Type string `bson:"type" json:"type"` + TaskID int `bson:"task_id" json:"taskId"` + Desc string `bson:"desc" json:"desc"` + Labels []string `bson:"labels" json:"labels"` + ProductEnvInfo *Product `bson:"product_env_info" json:"productEnvInfo"` + Status string `bson:"status" json:"status"` + Error string `bson:"error" json:"-"` + Progress *DeliveryVersionProgress `bson:"-" json:"progress"` + CreateArgument interface{} `bson:"createArgument" json:"-"` + CreatedBy string `bson:"created_by" json:"createdBy"` + CreatedAt int64 `bson:"created_at" json:"created_at"` + DeletedAt int64 `bson:"deleted_at" json:"deleted_at"` } func (DeliveryVersion) TableName() string { diff --git a/pkg/microservice/aslan/core/common/repository/models/k8s_cluster.go b/pkg/microservice/aslan/core/common/repository/models/k8s_cluster.go index 777a2a466835f4f19fc69684874f65cd5f15ea7c..816689683f8336d7da84ac883bfd2c04cb7cdd1f 100644 --- a/pkg/microservice/aslan/core/common/repository/models/k8s_cluster.go +++ b/pkg/microservice/aslan/core/common/repository/models/k8s_cluster.go @@ -17,33 +17,36 @@ limitations under the License. package models import ( - "fmt" - "regexp" - "strings" - "go.mongodb.org/mongo-driver/bson/primitive" + corev1 "k8s.io/api/core/v1" "github.com/koderover/zadig/pkg/setting" ) -var namePattern = regexp.MustCompile(`^[0-9a-zA-Z_.-]{1,32}$`) - type K8SCluster struct { - ID primitive.ObjectID `json:"id,omitempty" bson:"_id,omitempty"` - Name string `json:"name" bson:"name"` - Tags []string `json:"tags" bson:"tags"` - Description string `json:"description" bson:"description"` - Namespace string `json:"namespace" bson:"namespace"` - Info *K8SClusterInfo `json:"info,omitempty" bson:"info,omitempty"` - Status setting.K8SClusterStatus `json:"status" bson:"status"` - Error string `json:"error" bson:"error"` - Yaml string `json:"yaml" bson:"yaml"` - Production bool `json:"production" bson:"production"` - CreatedAt int64 `json:"createdAt" bson:"createdAt"` - CreatedBy string `json:"createdBy" bson:"createdBy"` - Disconnected bool `json:"-" bson:"disconnected"` - Token string `json:"token" bson:"-"` - Provider int8 `json:"provider" bson:"provider"` + ID primitive.ObjectID `json:"id,omitempty" bson:"_id,omitempty"` + Name string `json:"name" bson:"name"` + Tags []string `json:"tags" bson:"tags"` + Description string `json:"description" bson:"description"` + Namespace string `json:"namespace" bson:"namespace"` + Info *K8SClusterInfo `json:"info,omitempty" bson:"info,omitempty"` + AdvancedConfig *AdvancedConfig `json:"advanced_config,omitempty" bson:"advanced_config,omitempty"` + Status setting.K8SClusterStatus `json:"status" bson:"status"` + Error string `json:"error" bson:"error"` + Yaml string `json:"yaml" bson:"yaml"` + Production bool `json:"production" bson:"production"` + CreatedAt int64 `json:"createdAt" bson:"createdAt"` + CreatedBy string `json:"createdBy" bson:"createdBy"` + Disconnected bool `json:"-" bson:"disconnected"` + Token string `json:"token" bson:"-"` + Provider int8 `json:"provider" bson:"provider"` + Local bool `json:"local" bson:"local"` +} + +type K8SClusterResp struct { + ID string `json:"id" bson:"_id,omitempty"` + Name string `json:"name" bson:"name"` + AdvancedConfig *AdvancedConfig `json:"config,omitempty" bson:"config,omitempty"` } type K8SClusterInfo struct { @@ -53,26 +56,17 @@ type K8SClusterInfo struct { Memory string `json:"memory" bson:"memory"` } -func (K8SCluster) TableName() string { - return "k8s_cluster" +type AdvancedConfig struct { + Strategy string `json:"strategy,omitempty" bson:"strategy,omitempty"` + NodeLabels []*NodeSelectorRequirement `json:"node_labels,omitempty" bson:"node_labels,omitempty"` } -func (k *K8SCluster) Clean() error { - k.Name = strings.TrimSpace(k.Name) - tags := k.Tags - k.Tags = []string{} - for _, tag := range tags { - tag = strings.TrimSpace(tag) - if tag != "" { - k.Tags = append(k.Tags, tag) - } - } - - k.Description = strings.TrimSpace(k.Description) - - if !namePattern.MatchString(k.Name) { - return fmt.Errorf("集群名称不符合规则") - } +type NodeSelectorRequirement struct { + Key string `json:"key" bson:"key"` + Value []string `json:"value" bson:"value"` + Operator corev1.NodeSelectorOperator `json:"operator" bson:"operator"` +} - return nil +func (K8SCluster) TableName() string { + return "k8s_cluster" } diff --git a/pkg/microservice/aslan/core/common/repository/models/product.go b/pkg/microservice/aslan/core/common/repository/models/product.go index 5bfa65c97f744ecfd1e18ba65234bfd2ea81ed7a..af7ecc6b076a162e3fc9afdcbb484486a72efe34 100644 --- a/pkg/microservice/aslan/core/common/repository/models/product.go +++ b/pkg/microservice/aslan/core/common/repository/models/product.go @@ -50,6 +50,7 @@ type Product struct { RecycleDay int `bson:"recycle_day" json:"recycle_day"` Source string `bson:"source" json:"source"` IsOpenSource bool `bson:"is_opensource" json:"is_opensource"` + RegistryID string `bson:"registry_id" json:"registry_id"` // TODO: temp flag IsForkedProduct bool `bson:"-" json:"-"` } diff --git a/pkg/microservice/aslan/core/common/repository/models/queue.go b/pkg/microservice/aslan/core/common/repository/models/queue.go index 7801b2c461d2aa333e225199734147e45ef10b10..8aee0760d125a9e88809ca9c36886c57889f1c4e 100644 --- a/pkg/microservice/aslan/core/common/repository/models/queue.go +++ b/pkg/microservice/aslan/core/common/repository/models/queue.go @@ -25,63 +25,50 @@ import ( ) type Queue struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` - TaskID int64 `bson:"task_id" json:"task_id"` - ProductName string `bson:"product_name" json:"product_name"` - PipelineName string `bson:"pipeline_name" json:"pipeline_name"` - Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` - Type config.PipelineType `bson:"type" json:"type"` - Status config.Status `bson:"status" json:"status,omitempty"` - Description string `bson:"description,omitempty" json:"description,omitempty"` - TaskCreator string `bson:"task_creator" json:"task_creator,omitempty"` - TaskRevoker string `bson:"task_revoker,omitempty" json:"task_revoker,omitempty"` - CreateTime int64 `bson:"create_time" json:"create_time,omitempty"` - StartTime int64 `bson:"start_time" json:"start_time,omitempty"` - EndTime int64 `bson:"end_time" json:"end_time,omitempty"` - SubTasks []map[string]interface{} `bson:"sub_tasks" json:"sub_tasks"` - Stages []*Stage `bson:"stages" json:"stages"` - ReqID string `bson:"req_id,omitempty" json:"req_id,omitempty"` - AgentHost string `bson:"agent_host,omitempty" json:"agent_host,omitempty"` - DockerHost string `bson:"-" json:"docker_host,omitempty"` - TeamID int `bson:"team_id,omitempty" json:"team_id,omitempty"` - TeamName string `bson:"team,omitempty" json:"team,omitempty"` - IsDeleted bool `bson:"is_deleted" json:"is_deleted"` - IsArchived bool `bson:"is_archived" json:"is_archived"` - AgentID string `bson:"agent_id" json:"agent_id"` - // 是否允许同时运行多次 - MultiRun bool `bson:"multi_run" json:"multi_run"` - // target 服务名称, k8s为容器名称, 物理机为服务名 - Target string `bson:"target,omitempty" json:"target"` - // 使用预定义编译管理模块中的内容生成SubTasks, - // 查询条件为 服务模板名称: ServiceTmpl, 版本: BuildModuleVer - // 如果为空，则使用pipeline自定义SubTasks - BuildModuleVer string `bson:"build_module_ver,omitempty" json:"build_module_ver"` - ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"` - // TaskArgs 单服务工作流任务参数 - TaskArgs *TaskArgs `bson:"task_args,omitempty" json:"task_args,omitempty"` - // WorkflowArgs 多服务工作流任务参数 - WorkflowArgs *WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args,omitempty"` - // TestArgs 测试任务参数 - TestArgs *TestTaskArgs `bson:"test_args,omitempty" json:"test_args,omitempty"` - // ServiceTaskArgs 脚本部署工作流任务参数 - ServiceTaskArgs *ServiceTaskArgs `bson:"service_args,omitempty" json:"service_args,omitempty"` - ConfigPayload *ConfigPayload `json:"config_payload,omitempty"` - Error string `bson:"error,omitempty" json:"error,omitempty"` - Services [][]*ProductService `bson:"services" json:"services"` - Render *RenderInfo `bson:"render" json:"render"` - StorageURI string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"` - // interface{} 为types.TestReport - TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"` - - RwLock sync.Mutex `bson:"-" json:"-"` - - ResetImage bool `json:"resetImage" bson:"resetImage"` - - TriggerBy *TriggerBy `json:"trigger_by,omitempty" bson:"trigger_by,omitempty"` - - Features []string `bson:"features" json:"features"` - IsRestart bool `bson:"is_restart" json:"is_restart"` - StorageEndpoint string `bson:"storage_endpoint" json:"storage_endpoint"` + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + TaskID int64 `bson:"task_id" json:"task_id"` + ProductName string `bson:"product_name" json:"product_name"` + PipelineName string `bson:"pipeline_name" json:"pipeline_name"` + Namespace string `bson:"namespace,omitempty" json:"namespace,omitempty"` + Type config.PipelineType `bson:"type" json:"type"` + Status config.Status `bson:"status" json:"status,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + TaskCreator string `bson:"task_creator" json:"task_creator,omitempty"` + TaskRevoker string `bson:"task_revoker,omitempty" json:"task_revoker,omitempty"` + CreateTime int64 `bson:"create_time" json:"create_time,omitempty"` + StartTime int64 `bson:"start_time" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time" json:"end_time,omitempty"` + SubTasks []map[string]interface{} `bson:"sub_tasks" json:"sub_tasks"` + Stages []*Stage `bson:"stages" json:"stages"` + ReqID string `bson:"req_id,omitempty" json:"req_id,omitempty"` + AgentHost string `bson:"agent_host,omitempty" json:"agent_host,omitempty"` + DockerHost string `bson:"-" json:"docker_host,omitempty"` + TeamID int `bson:"team_id,omitempty" json:"team_id,omitempty"` + TeamName string `bson:"team,omitempty" json:"team,omitempty"` + IsDeleted bool `bson:"is_deleted" json:"is_deleted"` + IsArchived bool `bson:"is_archived" json:"is_archived"` + AgentID string `bson:"agent_id" json:"agent_id"` + MultiRun bool `bson:"multi_run" json:"multi_run"` + Target string `bson:"target,omitempty" json:"target"` // target service name, for k8s: containerName, for pm: serviceName + BuildModuleVer string `bson:"build_module_ver,omitempty" json:"build_module_ver"` + ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"` + TaskArgs *TaskArgs `bson:"task_args,omitempty" json:"task_args,omitempty"` // TaskArgs job parameters for single-service workflow + WorkflowArgs *WorkflowTaskArgs `bson:"workflow_args" json:"workflow_args,omitempty"` // WorkflowArgs job parameters for multi-service workflow + TestArgs *TestTaskArgs `bson:"test_args,omitempty" json:"test_args,omitempty"` // TestArgs parameters for testing + ServiceTaskArgs *ServiceTaskArgs `bson:"service_args,omitempty" json:"service_args,omitempty"` // ServiceTaskArgs parameters for script-deployed workflows + ArtifactPackageTaskArgs *ArtifactPackageTaskArgs `bson:"artifact_package_args,omitempty" json:"artifact_package_args,omitempty"` + ConfigPayload *ConfigPayload `bson:"configpayload,omitempty" json:"config_payload"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + Services [][]*ProductService `bson:"services" json:"services"` + Render *RenderInfo `bson:"render" json:"render"` + StorageURI string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"` + TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"` + RwLock sync.Mutex `bson:"-" json:"-"` + ResetImage bool `bson:"resetImage" json:"resetImage"` + TriggerBy *TriggerBy `bson:"trigger_by,omitempty" json:"trigger_by,omitempty"` + Features []string `bson:"features" json:"features"` + IsRestart bool `bson:"is_restart" json:"is_restart"` + StorageEndpoint string `bson:"storage_endpoint" json:"storage_endpoint"` } type TriggerBy struct { @@ -108,6 +95,26 @@ type ServiceTaskArgs struct { Updatable bool `bson:"updatable" json:"updatable"` } +type ImageData struct { + ImageUrl string `bson:"image_url" json:"image_url"` + ImageName string `bson:"image_name" json:"image_name"` + ImageTag string `bson:"image_tag" json:"image_tag"` + RegistryID string `bson:"registry_id" json:"registry_id"` +} + +type ImagesByService struct { + ServiceName string `bson:"service_name" json:"service_name"` + Images []*ImageData `bson:"images" json:"images"` +} + +type ArtifactPackageTaskArgs struct { + ProjectName string `bson:"project_name" json:"project_name"` + EnvName string `bson:"env_name" json:"env_name"` + Images []*ImagesByService `bson:"images" json:"images"` + SourceRegistries []string `json:"source_registries" json:"source_registries"` + TargetRegistries []string `json:"target_registries" json:"target_registries"` +} + type ConfigPayload struct { Proxy Proxy `json:"proxy"` S3Storage S3Config `json:"s3_storage"` @@ -126,6 +133,7 @@ type ConfigPayload struct { CustomDNSSupported bool `json:"custom_dns_supported"` HubServerAddr string DeployClusterID string + AesKey string `json:"aes_key"` RepoConfigs map[string]*RegistryNamespace @@ -136,6 +144,10 @@ type ConfigPayload struct { ResetCache bool `json:"reset_cache"` JenkinsBuildConfig JenkinsBuildConfig `json:"jenkins_build_config"` PrivateKeys []*PrivateKey `json:"private_keys"` + K8SClusters []*K8SClusterResp `json:"k8s_clusters"` + + // RegistryID is the id of product registry + RegistryID string `json:"registry_id"` } type AslanConfig struct { @@ -208,6 +220,9 @@ type ReleaseConfig struct { // PredatorImage sets docker build image // e.g. xxx.com/resources/predator-plugin:v0.1.0 PredatorImage string + // PackagerImage sets docker build image + // e.g. xxx.com/resources/predator-plugin:v0.1.0 + PackagerImage string } type ImageReleaseConfig struct { diff --git a/pkg/microservice/aslan/core/common/repository/models/task/artifact_package.go b/pkg/microservice/aslan/core/common/repository/models/task/artifact_package.go new file mode 100644 index 0000000000000000000000000000000000000000..89f89de42a98c629063640d9a351b938fd3bf2db --- /dev/null +++ b/pkg/microservice/aslan/core/common/repository/models/task/artifact_package.go @@ -0,0 +1,75 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "encoding/json" + "fmt" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" +) + +type ImageData struct { + ImageUrl string `bson:"image_url" json:"image_url"` + ImageName string `bson:"image_name" json:"image_name"` + ImageNamespace string `bson:"image_namespace" json:"image_namespace"` + ImageTag string `bson:"image_tag" json:"image_tag"` +} + +type ServicePackageResult struct { + ServiceName string `json:"service_name"` + Result string `json:"result"` + ErrorMsg string `json:"error_msg"` + ImageData []*ImageData `json:"image_data"` +} + +type ArtifactPackage struct { + TaskType config.TaskType `bson:"type" json:"type"` + TaskStatus config.Status `bson:"status" json:"status"` + Enabled bool `bson:"enabled" json:"enabled"` + Timeout int `bson:"timeout,omitempty" json:"timeout,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty"` + LogFile string `bson:"log_file" json:"log_file"` + Progress string `bson:"progress" json:"progress"` + + // source images + Images []*models.ImagesByService `bson:"images" json:"images"` + // source registries need auth to pull images + SourceRegistries []string `bson:"source_registries" json:"source_registries"` + // target registries to push images + TargetRegistries []string `bson:"target_registries" json:"target_registries"` +} + +func (ri *ArtifactPackage) ToSubTask() (map[string]interface{}, error) { + var task map[string]interface{} + if err := IToi(ri, &task); err != nil { + return nil, fmt.Errorf("convert ReleaseImageTask to interface error: %v", err) + } + return task, nil +} + +func (ri *ArtifactPackage) GetProgress() ([]*ServicePackageResult, error) { + if len(ri.Progress) == 0 { + return nil, nil + } + ret := make([]*ServicePackageResult, 0) + err := json.Unmarshal([]byte(ri.Progress), &ret) + return ret, err +} diff --git a/pkg/microservice/aslan/core/common/repository/models/task/build.go b/pkg/microservice/aslan/core/common/repository/models/task/build.go index f4d41e2fe5d44cc5f29923aa27b9080e912d1a6a..97bbbef45ea196ecd355a3bca79d30ea3ab89c37 100644 --- a/pkg/microservice/aslan/core/common/repository/models/task/build.go +++ b/pkg/microservice/aslan/core/common/repository/models/task/build.go @@ -60,6 +60,7 @@ type Build struct { // Get the host bound to the environment of the cloud host service configuration EnvHostInfo map[string][]string `bson:"env_host_info,omitempty" json:"env_host_info,omitempty"` ArtifactInfo *ArtifactInfo `bson:"artifact_info,omitempty" json:"artifact_info,omitempty"` + ClusterID string `bson:"cluster_id,omitempty" json:"cluster_id,omitempty"` } type ArtifactInfo struct { diff --git a/pkg/microservice/aslan/core/common/repository/models/task/model.go b/pkg/microservice/aslan/core/common/repository/models/task/model.go index c503d9c6f71f74c47fae1387887d51e1656018c3..d4783d2e6ea1dee6a46802012e7bbe5be5536029 100644 --- a/pkg/microservice/aslan/core/common/repository/models/task/model.go +++ b/pkg/microservice/aslan/core/common/repository/models/task/model.go @@ -28,7 +28,7 @@ import ( ) type Task struct { - ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` + ID primitive.ObjectID `bson:"_id,omitempty" json:"id,omitempty"` TaskID int64 `bson:"task_id" json:"task_id"` ProductName string `bson:"product_name" json:"product_name"` PipelineName string `bson:"pipeline_name" json:"pipeline_name"` @@ -48,7 +48,7 @@ type Task struct { TeamName string `bson:"team,omitempty" json:"team,omitempty"` IsDeleted bool `bson:"is_deleted" json:"is_deleted"` IsArchived bool `bson:"is_archived" json:"is_archived"` - AgentID string `bson:"agent_id" json:"agent_id"` + AgentID string `bson:"agent_id" json:"agent_id"` // 是否允许同时运行多次 MultiRun bool `bson:"multi_run" json:"multi_run"` // target 服务名称, k8s为容器名称, 物理机为服务名 @@ -57,8 +57,7 @@ type Task struct { // 查询条件为 服务模板名称: ServiceTmpl, 版本: BuildModuleVer // 如果为空，则使用pipeline自定义SubTasks BuildModuleVer string `bson:"build_module_ver,omitempty" json:"build_module_ver"` - - ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"` + ServiceName string `bson:"service_name,omitempty" json:"service_name,omitempty"` // TaskArgs 单服务工作流任务参数 TaskArgs *models.TaskArgs `bson:"task_args,omitempty" json:"task_args,omitempty"` // WorkflowArgs 多服务工作流任务参数 @@ -67,24 +66,22 @@ type Task struct { TestArgs *models.TestTaskArgs `bson:"test_args,omitempty" json:"test_args,omitempty"` // ServiceTaskArgs 脚本部署工作流任务参数 ServiceTaskArgs *models.ServiceTaskArgs `bson:"service_args,omitempty" json:"service_args,omitempty"` + // ArtifactPackageTaskArgs arguments for artifact-package type tasks + ArtifactPackageTaskArgs *models.ArtifactPackageTaskArgs `bson:"artifact_package_args,omitempty" json:"artifact_package_args,omitempty"` // ConfigPayload 系统配置信息 - ConfigPayload *models.ConfigPayload `json:"config_payload,omitempty"` - Error string `bson:"error,omitempty" json:"error,omitempty"` - Services [][]*models.ProductService `bson:"services" json:"services"` - Render *models.RenderInfo `bson:"render" json:"render"` - StorageURI string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"` + ConfigPayload *models.ConfigPayload `bson:"configpayload" json:"config_payload,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + Services [][]*models.ProductService `bson:"services" json:"services"` + Render *models.RenderInfo `bson:"render" json:"render"` + StorageURI string `bson:"storage_uri,omitempty" json:"storage_uri,omitempty"` // interface{} 为types.TestReport - TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"` - - RwLock sync.Mutex `bson:"-" json:"-"` - - ResetImage bool `json:"resetImage" bson:"resetImage"` - - TriggerBy *models.TriggerBy `json:"trigger_by,omitempty" bson:"trigger_by,omitempty"` - - Features []string `bson:"features" json:"features"` - IsRestart bool `bson:"is_restart" json:"is_restart"` - StorageEndpoint string `bson:"storage_endpoint" json:"storage_endpoint"` + TestReports map[string]interface{} `bson:"test_reports,omitempty" json:"test_reports,omitempty"` + RwLock sync.Mutex `bson:"-" json:"-"` + ResetImage bool `bson:"resetImage" json:"resetImage"` + TriggerBy *models.TriggerBy `bson:"trigger_by,omitempty" json:"trigger_by,omitempty"` + Features []string `bson:"features" json:"features"` + IsRestart bool `bson:"is_restart" json:"is_restart"` + StorageEndpoint string `bson:"storage_endpoint" json:"storage_endpoint"` } //type RenderInfo struct { diff --git a/pkg/microservice/aslan/core/common/repository/models/task/testing.go b/pkg/microservice/aslan/core/common/repository/models/task/testing.go index 460a7ca07a6b947aeec06082473a9710d32d0f22..0105494f6ea5715cd2eccf91f3ccda0d24391506 100644 --- a/pkg/microservice/aslan/core/common/repository/models/task/testing.go +++ b/pkg/microservice/aslan/core/common/repository/models/task/testing.go @@ -47,6 +47,8 @@ type Testing struct { ReportReady bool `bson:"report_ready" json:"report_ready"` IsRestart bool `bson:"is_restart" json:"is_restart"` Registries []*models.RegistryNamespace `bson:"-" json:"registries"` + ClusterID string `bson:"cluster_id,omitempty" json:"cluster_id,omitempty"` + Namespace string `bson:"namespace" json:"namespace"` } func (t *Testing) ToSubTask() (map[string]interface{}, error) { diff --git a/pkg/microservice/aslan/core/common/repository/models/testing.go b/pkg/microservice/aslan/core/common/repository/models/testing.go index fa223496138d5d3796b016c8a94d431b8eff148f..005f1eb835ac33dfb107be3ae44c7dcea049e564 100644 --- a/pkg/microservice/aslan/core/common/repository/models/testing.go +++ b/pkg/microservice/aslan/core/common/repository/models/testing.go @@ -79,7 +79,9 @@ type PreTest struct { // Envs stores user defined env key val for build Envs []*KeyVal `bson:"envs,omitempty" json:"envs"` // EnableProxy - EnableProxy bool `bson:"enable_proxy" json:"enable_proxy"` + EnableProxy bool `bson:"enable_proxy" json:"enable_proxy"` + ClusterID string `bson:"cluster_id" json:"cluster_id"` + Namespace string `bson:"namespace" json:"namespace"` } func (Testing) TableName() string { diff --git a/pkg/microservice/aslan/core/common/repository/models/workflow.go b/pkg/microservice/aslan/core/common/repository/models/workflow.go index 3cd5e8368a8d19b9d3ebc31a7fcee5cff886f393..61fcbb4e847349f1fde63edf081bfab452873310 100644 --- a/pkg/microservice/aslan/core/common/repository/models/workflow.go +++ b/pkg/microservice/aslan/core/common/repository/models/workflow.go @@ -168,6 +168,7 @@ type WorkflowTaskArgs struct { // 请求模式，openAPI表示外部客户调用 RequestMode string `json:"request_mode,omitempty"` IsParallel bool `json:"is_parallel" bson:"is_parallel"` + EnvName string `json:"env_name" bson:"-"` } type TestTaskArgs struct { @@ -308,6 +309,7 @@ type HookPayload struct { type TargetArgs struct { Name string `bson:"name" json:"name"` ServiceName string `bson:"service_name" json:"service_name"` + ServiceType string `bson:"service_type,omitempty" json:"service_type,omitempty"` ProductName string `bson:"product_name" json:"product_name"` Build *BuildArgs `bson:"build" json:"build"` Deploy []DeployEnv `bson:"deloy" json:"deploy"` diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/delivery_artifact.go b/pkg/microservice/aslan/core/common/repository/mongodb/delivery_artifact.go index e0e0cc75bf96705655b734a363445b840a9903de..a7fb5ebc58ca96a9656d133ca423a839080014a1 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/delivery_artifact.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/delivery_artifact.go @@ -140,13 +140,20 @@ func (c *DeliveryArtifactColl) Get(args *DeliveryArtifactArgs) (*models.Delivery return nil, errors.New("nil delivery_artifact args") } resp := new(models.DeliveryArtifact) - id, err := primitive.ObjectIDFromHex(args.ID) - if err != nil { - return nil, err + query := bson.M{} + if args.ID != "" { + id, err := primitive.ObjectIDFromHex(args.ID) + if err != nil { + return nil, err + } + query["_id"] = id + } + + if args.Image != "" { + query["image"] = args.Image } - query := bson.M{"_id": id} - err = c.FindOne(context.TODO(), query).Decode(&resp) + err := c.FindOne(context.TODO(), query).Decode(&resp) if err != nil { return nil, err } diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/delivery_distribute.go b/pkg/microservice/aslan/core/common/repository/mongodb/delivery_distribute.go index 0b3ce01a77f8ce7d851896e64ed66dc6818d7155..98a9c39ede04379452247b336046399c646ca913 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/delivery_distribute.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/delivery_distribute.go @@ -35,6 +35,7 @@ type DeliveryDistributeArgs struct { ID string `json:"id"` ReleaseID string `json:"releaseId"` ServiceName string `json:"serviceName"` + ChartName string `json:"chartName"` DistributeType config.DistributeType `json:"distributeType"` } @@ -62,6 +63,7 @@ func (c *DeliveryDistributeColl) EnsureIndex(ctx context.Context) error { Keys: bson.D{ bson.E{Key: "release_id", Value: 1}, bson.E{Key: "service_name", Value: 1}, + bson.E{Key: "chart_name", Value: 1}, bson.E{Key: "deleted_at", Value: 1}, }, Options: options.Index().SetUnique(false), @@ -115,8 +117,11 @@ func (c *DeliveryDistributeColl) Find(args *DeliveryDistributeArgs) ([]*models.D } query := bson.M{"release_id": releaseID, "deleted_at": 0} - if args.DistributeType != "" { - query["distribute_type"] = config.File + if len(args.DistributeType) > 0 { + query["distribute_type"] = string(args.DistributeType) + } + if len(args.ChartName) > 0 { + query["chart_name"] = args.ChartName } cursor, err := c.Collection.Find(context.TODO(), query) if err != nil { diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/delivery_version.go b/pkg/microservice/aslan/core/common/repository/mongodb/delivery_version.go index bae5fc668297d8d018f4da5b3bec632bd2137955..e0adcfc8ec95d190284f8a4451882c88fd75767b 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/delivery_version.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/delivery_version.go @@ -34,6 +34,7 @@ import ( type DeliveryVersionArgs struct { ID string `json:"id"` ProductName string `json:"productName"` + Version string `json:"version"` WorkflowName string `json:"workflowName"` TaskID int `json:"taskId"` PerPage int `json:"perPage"` @@ -89,7 +90,6 @@ func (c *DeliveryVersionColl) Find(args *DeliveryVersionArgs) ([]*models.Deliver return nil, errors.New("nil delivery_version args") } - var resp []*models.DeliveryVersion query := bson.M{"deleted_at": 0} if args.ProductName != "" { query["product_name"] = args.ProductName @@ -113,6 +113,7 @@ func (c *DeliveryVersionColl) Find(args *DeliveryVersionArgs) ([]*models.Deliver return nil, err } + resp := make([]*models.DeliveryVersion, 0) err = cursor.All(ctx, &resp) if err != nil { return nil, err @@ -164,7 +165,13 @@ func (c *DeliveryVersionColl) Get(args *DeliveryVersionArgs) (*models.DeliveryVe resp := new(models.DeliveryVersion) var query map[string]interface{} if args.ID != "" { - query = bson.M{"_id": args.ID, "deleted_at": 0} + oid, err := primitive.ObjectIDFromHex(args.ID) + if err != nil { + return nil, err + } + query = bson.M{"_id": oid, "deleted_at": 0} + } else if len(args.Version) > 0 { + query = bson.M{"product_name": args.ProductName, "version": args.Version, "deleted_at": 0} } else { query = bson.M{"product_name": args.ProductName, "workflow_name": args.WorkflowName, "task_id": args.TaskID, "deleted_at": 0} } @@ -190,6 +197,16 @@ func (c *DeliveryVersionColl) Insert(args *models.DeliveryVersion) error { return nil } +func (c *DeliveryVersionColl) UpdateStatusByName(versionName, status, errorStr string) error { + query := bson.M{"version": versionName, "deleted_at": 0} + change := bson.M{"$set": bson.M{ + "status": status, + "error": errorStr, + }} + _, err := c.UpdateOne(context.TODO(), query, change) + return err +} + func (c *DeliveryVersionColl) Update(args *models.DeliveryVersion) error { if args == nil { return errors.New("nil delivery_version args") diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/helm_repo.go b/pkg/microservice/aslan/core/common/repository/mongodb/helm_repo.go index 6d7731676f098c7856ba584d058b0e291bbe1745..86fff875ecd553c15ebee5683c494086ea901a18 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/helm_repo.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/helm_repo.go @@ -37,6 +37,11 @@ type HelmRepoColl struct { coll string } +type HelmRepoFindOption struct { + Id string + RepoName string +} + func NewHelmRepoColl() *HelmRepoColl { name := models.HelmRepo{}.TableName() coll := &HelmRepoColl{Collection: mongotool.Database(config.MongoDatabase()).Collection(name), coll: name} @@ -49,7 +54,12 @@ func (c *HelmRepoColl) GetCollectionName() string { } func (c *HelmRepoColl) EnsureIndex(ctx context.Context) error { - return nil + mod := mongo.IndexModel{ + Keys: bson.M{"repo_name": 1}, + Options: options.Index().SetUnique(false), + } + _, err := c.Indexes().CreateOne(ctx, mod) + return err } func (c *HelmRepoColl) Create(args *models.HelmRepo) error { @@ -64,6 +74,26 @@ func (c *HelmRepoColl) Create(args *models.HelmRepo) error { return err } +func (c *HelmRepoColl) Find(opt *HelmRepoFindOption) (*models.HelmRepo, error) { + query := bson.M{} + if len(opt.Id) > 0 { + oid, err := primitive.ObjectIDFromHex(opt.Id) + if err != nil { + return nil, err + } + query["_id"] = oid + } + if len(opt.RepoName) > 0 { + query["repo_name"] = opt.RepoName + } + ret := new(models.HelmRepo) + err := c.FindOne(context.TODO(), query).Decode(ret) + if err != nil { + return nil, err + } + return ret, nil +} + func (c *HelmRepoColl) Update(id string, args *models.HelmRepo) error { oid, err := primitive.ObjectIDFromHex(id) if err != nil { diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/k8s_cluster.go b/pkg/microservice/aslan/core/common/repository/mongodb/k8s_cluster.go index f49abb0cd1ce61f3d8c2c41b81f3e10435a71d28..a9464a013edee7b2064913b63c2b7d618e4fdf1a 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/k8s_cluster.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/k8s_cluster.go @@ -67,7 +67,10 @@ func (c *K8SClusterColl) Delete(id string) error { return err } -func (c *K8SClusterColl) Create(cluster *models.K8SCluster) error { +func (c *K8SClusterColl) Create(cluster *models.K8SCluster, id string) error { + if id != "" { + cluster.ID, _ = primitive.ObjectIDFromHex(id) + } res, err := c.InsertOne(context.TODO(), cluster) if oid, ok := res.InsertedID.(primitive.ObjectID); ok { cluster.ID = oid @@ -177,14 +180,20 @@ func (c *K8SClusterColl) FindByName(name string) (*models.K8SCluster, error) { return res, err } -func (c *K8SClusterColl) UpdateMutableFields(cluster *models.K8SCluster) error { - _, err := c.UpdateOne(context.TODO(), +func (c *K8SClusterColl) UpdateMutableFields(cluster *models.K8SCluster, id string) error { + var err error + cluster.ID, err = primitive.ObjectIDFromHex(id) + if err != nil { + return err + } + _, err = c.UpdateOne(context.TODO(), bson.M{"_id": cluster.ID}, bson.M{"$set": bson.M{ - "name": cluster.Name, - "description": cluster.Description, - "tags": cluster.Tags, - "namespace": cluster.Namespace, - "production": cluster.Production, + "name": cluster.Name, + "description": cluster.Description, + "tags": cluster.Tags, + "namespace": cluster.Namespace, + "production": cluster.Production, + "advanced_config": cluster.AdvancedConfig, }}, ) diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/product.go b/pkg/microservice/aslan/core/common/repository/mongodb/product.go index ca490b5bae493c4153a6988e86851f9170a108ad..023ba43f4dfc8a1d7bb27d528aa77590eeaa546e 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/product.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/product.go @@ -33,8 +33,9 @@ import ( ) type ProductFindOptions struct { - Name string - EnvName string + Name string + EnvName string + Namespace string } // ClusterId is a primitive.ObjectID{}.Hex() @@ -130,6 +131,9 @@ func (c *ProductColl) Find(opt *ProductFindOptions) (*models.Product, error) { if opt.EnvName != "" { query["env_name"] = opt.EnvName } + if opt.Namespace != "" { + query["namespace"] = opt.Namespace + } err := c.FindOne(context.TODO(), query).Decode(res) return res, err @@ -241,6 +245,16 @@ func (c *ProductColl) UpdateErrors(owner, productName, errorMsg string) error { return err } +func (c *ProductColl) UpdateRegistry(namespace, registryId string) error { + query := bson.M{"namespace": namespace} + change := bson.M{"$set": bson.M{ + "registry_id": registryId, + }} + _, err := c.UpdateOne(context.TODO(), query, change) + + return err +} + func (c *ProductColl) UpdateRender(envName, productName string, render *models.RenderInfo) error { query := bson.M{"env_name": envName, "product_name": productName} change := bson.M{"$set": bson.M{ diff --git a/pkg/microservice/aslan/core/common/repository/mongodb/template/product.go b/pkg/microservice/aslan/core/common/repository/mongodb/template/product.go index 4cf141c2d551bc18045e14545c3da2f68bcfe616..474d2e960aad2053630e6cff75054d0ed75f73ad 100644 --- a/pkg/microservice/aslan/core/common/repository/mongodb/template/product.go +++ b/pkg/microservice/aslan/core/common/repository/mongodb/template/product.go @@ -39,6 +39,7 @@ type ProjectInfo struct { UpdatedBy string `bson:"update_by"` OnboardStatus int `bson:"onboarding_status"` Public bool `bson:"public"` + DeployType string `bson:"deploy_type"` } type ProductColl struct { @@ -82,8 +83,8 @@ func (c *ProductColl) FindProjectName(project string) (*template.Product, error) } func (c *ProductColl) ListNames(inNames []string) ([]string, error) { - res, err := c.listProjects(inNames, bson.D{ - {"product_name", 1}, + res, err := c.listProjects(inNames, bson.M{ + "product_name": "$product_name", }) if err != nil { return nil, err @@ -98,29 +99,34 @@ func (c *ProductColl) ListNames(inNames []string) ([]string, error) { } func (c *ProductColl) ListProjectBriefs(inNames []string) ([]*ProjectInfo, error) { - return c.listProjects(inNames, bson.D{ - {"product_name", 1}, - {"project_name", 1}, - {"description", 1}, - {"update_time", 1}, - {"update_by", 1}, - {"onboarding_status", 1}, - {"public", 1}, + return c.listProjects(inNames, bson.M{ + "product_name": "$product_name", + "project_name": "$project_name", + "description": "$description", + "update_time": "$update_time", + "update_by": "$update_by", + "onboarding_status": "$onboarding_status", + "public": "$public", + "deploy_type": "$product_feature.deploy_type", }) } -func (c *ProductColl) listProjects(inNames []string, projection bson.D) ([]*ProjectInfo, error) { - opts := options.Find() +func (c *ProductColl) listProjects(inNames []string, projection bson.M) ([]*ProjectInfo, error) { filter := bson.M{} if len(inNames) > 0 { filter["product_name"] = bson.M{"$in": inNames} } - if len(projection) > 0 { - opts.SetProjection(projection) + pipeline := []bson.M{ + { + "$match": filter, + }, + { + "$project": projection, + }, } - cursor, err := c.Collection.Find(context.TODO(), filter, opts) + cursor, err := c.Collection.Aggregate(context.TODO(), pipeline) if err != nil { return nil, err } @@ -130,7 +136,6 @@ func (c *ProductColl) listProjects(inNames []string, projection bson.D) ([]*Proj if err != nil { return nil, err } - return res, nil } diff --git a/pkg/microservice/aslan/core/common/service/base/convert.go b/pkg/microservice/aslan/core/common/service/base/convert.go index b691354808ebed368ce70555c71ae619c29d397c..95ae174088517cbfe5b5bdfaa6d2765aae3be3d0 100644 --- a/pkg/microservice/aslan/core/common/service/base/convert.go +++ b/pkg/microservice/aslan/core/common/service/base/convert.go @@ -92,6 +92,14 @@ func ToReleaseImageTask(sb map[string]interface{}) (*task.ReleaseImage, error) { return t, nil } +func ToArtifactPackageImageTask(sb map[string]interface{}) (*task.ArtifactPackage, error) { + var t *task.ArtifactPackage + if err := task.IToi(sb, &t); err != nil { + return nil, fmt.Errorf("convert interface to ReleaseImageTask error: %v", err) + } + return t, nil +} + func ToJiraTask(sb map[string]interface{}) (*task.Jira, error) { var t *task.Jira if err := task.IToi(sb, &t); err != nil { diff --git a/pkg/microservice/aslan/core/common/service/config_payload.go b/pkg/microservice/aslan/core/common/service/config_payload.go index ac244fe381d5de048b400dfc6f43d2966894ff6e..87905f8220b334743a04e2aeccf29792939cd238 100644 --- a/pkg/microservice/aslan/core/common/service/config_payload.go +++ b/pkg/microservice/aslan/core/common/service/config_payload.go @@ -22,6 +22,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/shared/client/systemconfig" + "github.com/koderover/zadig/pkg/tool/crypto" ) func GetConfigPayload(codeHostID int) *models.ConfigPayload { @@ -54,6 +55,7 @@ func GetConfigPayload(codeHostID int) *models.ConfigPayload { ReaperImage: config.ReaperImage(), ReaperBinaryFile: config.ReaperBinaryFile(), PredatorImage: config.PredatorImage(), + PackagerImage: config.PackagerImage(), }, Docker: models.DockerConfig{ HostList: config.DockerHosts(), @@ -64,6 +66,7 @@ func GetConfigPayload(codeHostID int) *models.ConfigPayload { JenkinsBuildConfig: models.JenkinsBuildConfig{ JenkinsBuildImage: config.JenkinsImage(), }, + AesKey: crypto.GetAesKey(), } githubApps, _ := mongodb.NewGithubAppColl().Find() @@ -89,5 +92,17 @@ func GetConfigPayload(codeHostID int) *models.ConfigPayload { payload.PrivateKeys = privateKeys } + k8sClusters, _ := mongodb.NewK8SClusterColl().List(nil) + if len(k8sClusters) != 0 { + var K8SClusterResp []*models.K8SClusterResp + for _, k8sCluster := range k8sClusters { + K8SClusterResp = append(K8SClusterResp, &models.K8SClusterResp{ + ID: k8sCluster.ID.Hex(), + Name: k8sCluster.Name, + AdvancedConfig: k8sCluster.AdvancedConfig, + }) + } + payload.K8SClusters = K8SClusterResp + } return payload } diff --git a/pkg/microservice/aslan/core/common/service/environment.go b/pkg/microservice/aslan/core/common/service/environment.go index 97a1f15eebbb15184e5dfec7dd69012f139c30fe..3d4a41eac43566f329e030cdf1468d861c61c665 100644 --- a/pkg/microservice/aslan/core/common/service/environment.go +++ b/pkg/microservice/aslan/core/common/service/environment.go @@ -30,12 +30,13 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" @@ -212,7 +213,7 @@ func ListWorkloads(envName, clusterID, namespace, productName string, perPage, p log.Infof("Start to list workloads in namespace %s", namespace) var resp = make([]*ServiceResp, 0) - kubeClient, err := kube.GetKubeClient(clusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) if err != nil { log.Errorf("[%s][%s] error: %v", envName, namespace, err) return 0, resp, e.ErrListGroups.AddDesc(err.Error()) diff --git a/pkg/microservice/aslan/core/common/service/fs/s3.go b/pkg/microservice/aslan/core/common/service/fs/s3.go index 67c5db651baa1fc7032c20e9bd06db8529655d9d..dfe02ea2ce115c95132fd0d6892313f9de750648 100644 --- a/pkg/microservice/aslan/core/common/service/fs/s3.go +++ b/pkg/microservice/aslan/core/common/service/fs/s3.go @@ -24,15 +24,35 @@ import ( "go.uber.org/zap" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" s3service "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" "github.com/koderover/zadig/pkg/setting" s3tool "github.com/koderover/zadig/pkg/tool/s3" fsutil "github.com/koderover/zadig/pkg/util/fs" ) -// ArchiveAndUploadFilesToS3 archive local files and upload to default s3 storage -// if multiple names appointed, s3storage.copy will be used to handle extra names +func ArchiveAndUploadFilesToSpecifiedS3(fileTree fs.FS, names []string, s3Base, s3Id string, logger *zap.SugaredLogger) error { + s3Storage, err := s3service.FindS3ById(s3Id) + if err != nil { + logger.Errorf("Failed to find default s3, err:%v", err) + return err + } + return archiveAndUploadFiles(fileTree, names, s3Base, s3Storage, logger) +} + func ArchiveAndUploadFilesToS3(fileTree fs.FS, names []string, s3Base string, logger *zap.SugaredLogger) error { + s3Storage, err := s3service.FindDefaultS3() + if err != nil { + logger.Errorf("Failed to find default s3, err:%v", err) + return err + } + return archiveAndUploadFiles(fileTree, names, s3Base, s3Storage, logger) +} + +// archiveAndUploadFiles archive local files and upload to default s3 storage +// if multiple names appointed, s3storage.copy will be used to handle extra names +func archiveAndUploadFiles(fileTree fs.FS, names []string, s3Base string, s3Storage *s3.S3, logger *zap.SugaredLogger) error { + if len(names) == 0 { return fmt.Errorf("names not appointed") } @@ -53,11 +73,7 @@ func ArchiveAndUploadFilesToS3(fileTree fs.FS, names []string, s3Base string, lo logger.Errorf("Failed to archive tarball %s, err: %s", localPath, err) return err } - s3Storage, err := s3service.FindDefaultS3() - if err != nil { - logger.Errorf("Failed to find default s3, err:%v", err) - return err - } + forcedPathStyle := true if s3Storage.Provider == setting.ProviderSourceAli { forcedPathStyle = false diff --git a/pkg/microservice/aslan/core/common/service/kube/service.go b/pkg/microservice/aslan/core/common/service/kube/service.go index 65791f9f48172367adc8c03c33fcb314e38288d4..d31ee8770c8dd740d5e1b9ed19874256940a44b3 100644 --- a/pkg/microservice/aslan/core/common/service/kube/service.go +++ b/pkg/microservice/aslan/core/common/service/kube/service.go @@ -32,26 +32,23 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/tool/crypto" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/kube/multicluster" ) -func GetKubeClient(clusterID string) (client.Client, error) { - return multicluster.GetKubeClient(config.HubServerAddress(), clusterID) -} - func GetKubeAPIReader(clusterID string) (client.Reader, error) { - return multicluster.GetKubeAPIReader(config.HubServerAddress(), clusterID) + return kubeclient.GetKubeAPIReader(config.HubServerAddress(), clusterID) } func GetRESTConfig(clusterID string) (*rest.Config, error) { - return multicluster.GetRESTConfig(config.HubServerAddress(), clusterID) + return kubeclient.GetRESTConfig(config.HubServerAddress(), clusterID) } // GetClientset returns a client to interact with APIServer which implements kubernetes.Interface func GetClientset(clusterID string) (kubernetes.Interface, error) { - return multicluster.GetClientset(config.HubServerAddress(), clusterID) + return kubeclient.GetClientset(config.HubServerAddress(), clusterID) } type Service struct { @@ -97,18 +94,18 @@ func (s *Service) ListClusters(clusterType string, logger *zap.SugaredLogger) ([ return clusters, nil } -func (s *Service) CreateCluster(cluster *models.K8SCluster, logger *zap.SugaredLogger) (*models.K8SCluster, error) { +func (s *Service) CreateCluster(cluster *models.K8SCluster, id string, logger *zap.SugaredLogger) (*models.K8SCluster, error) { _, err := s.coll.FindByName(cluster.Name) - if err == nil { logger.Errorf("failed to create cluster %s %v", cluster.Name, err) return nil, e.ErrCreateCluster.AddDesc(e.DuplicateClusterNameFound) } - cluster.Status = setting.Pending - - err = s.coll.Create(cluster) - + if id == setting.LocalClusterID { + cluster.Status = setting.Normal + cluster.Local = true + } + err = s.coll.Create(cluster, id) if err != nil { return nil, e.ErrCreateCluster.AddErr(err) } @@ -136,7 +133,7 @@ func (s *Service) UpdateCluster(id string, cluster *models.K8SCluster, logger *z return nil, e.ErrUpdateCluster.AddDesc(e.DuplicateClusterNameFound) } - err = s.coll.UpdateMutableFields(cluster) + err = s.coll.UpdateMutableFields(cluster, id) if err != nil { logger.Errorf("failed to update mutable fields %v", err) return nil, e.ErrUpdateCluster.AddErr(err) diff --git a/pkg/microservice/aslan/core/common/service/product.go b/pkg/microservice/aslan/core/common/service/product.go index 9986f3575f95d8ab55a029697fc8049b3592f718..4e4249b94b69f96dba47e517544cc4ddb131e329 100644 --- a/pkg/microservice/aslan/core/common/service/product.go +++ b/pkg/microservice/aslan/core/common/service/product.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" @@ -34,6 +35,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" helmtool "github.com/koderover/zadig/pkg/tool/helmclient" "github.com/koderover/zadig/pkg/tool/kube/updater" @@ -51,7 +53,7 @@ func DeleteProduct(username, envName, productName, requestID string, log *zap.Su return e.ErrDeleteEnv.AddDesc("not found") } - kubeClient, err := kube.GetKubeClient(productInfo.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), productInfo.ClusterID) if err != nil { return e.ErrDeleteEnv.AddErr(err) } diff --git a/pkg/microservice/aslan/core/common/service/registry.go b/pkg/microservice/aslan/core/common/service/registry.go index ecf5a1c37500577c5da506c5071f7e482062ded2..62c35be8e3f49a764a6d54f5dadbbb66de549b6c 100644 --- a/pkg/microservice/aslan/core/common/service/registry.go +++ b/pkg/microservice/aslan/core/common/service/registry.go @@ -30,14 +30,16 @@ import ( "github.com/koderover/zadig/pkg/util" ) -func FindDefaultRegistry(log *zap.SugaredLogger) (*models.RegistryNamespace, error) { +func FindRegistryById(registryId string, log *zap.SugaredLogger) (*models.RegistryNamespace, error) { + return findRegisty(&mongodb.FindRegOps{ID: registryId}, log) +} + +func findRegisty(regOps *mongodb.FindRegOps, log *zap.SugaredLogger) (*models.RegistryNamespace, error) { // TODO: 多租户适配 - resp, err := mongodb.NewRegistryNamespaceColl().Find(&mongodb.FindRegOps{ - IsDefault: true, - }) + resp, err := mongodb.NewRegistryNamespaceColl().Find(regOps) if err != nil { - log.Warnf("RegistryNamespace.Find error: %v", err) + log.Warnf("RegistryNamespace.Find error: %s", err) resp = &models.RegistryNamespace{ RegAddr: config.RegistryAddress(), AccessKey: config.RegistryAccessKey(), @@ -58,11 +60,15 @@ func FindDefaultRegistry(log *zap.SugaredLogger) (*models.RegistryNamespace, err return resp, nil } +func FindDefaultRegistry(log *zap.SugaredLogger) (*models.RegistryNamespace, error) { + return findRegisty(&mongodb.FindRegOps{IsDefault: true}, log) +} + func GetDefaultRegistryNamespace(log *zap.SugaredLogger) (*models.RegistryNamespace, error) { resp, err := mongodb.NewRegistryNamespaceColl().Find(&mongodb.FindRegOps{IsDefault: true}) if err != nil { - log.Errorf("get default registry error: %v", err) - return resp, fmt.Errorf("get default registry error: %v", err) + log.Errorf("get default registry error: %s", err) + return resp, fmt.Errorf("get default registry error: %s", err) } return resp, nil } @@ -70,25 +76,38 @@ func GetDefaultRegistryNamespace(log *zap.SugaredLogger) (*models.RegistryNamesp func ListRegistryNamespaces(log *zap.SugaredLogger) ([]*models.RegistryNamespace, error) { resp, err := mongodb.NewRegistryNamespaceColl().FindAll(&mongodb.FindRegOps{}) if err != nil { - log.Errorf("RegistryNamespace.List error: %v", err) - return resp, fmt.Errorf("RegistryNamespace.List error: %v", err) + log.Errorf("RegistryNamespace.List error: %s", err) + return resp, fmt.Errorf("RegistryNamespace.List error: %s", err) } return resp, nil } -func EnsureDefaultRegistrySecret(namespace string, kubeClient client.Client, log *zap.SugaredLogger) error { - reg, err := FindDefaultRegistry(log) - if err != nil { - log.Errorf( - "service.EnsureRegistrySecret: failed to find default candidate registry: %s %v", - namespace, err, - ) - return err +func EnsureDefaultRegistrySecret(namespace string, registryId string, kubeClient client.Client, log *zap.SugaredLogger) error { + var reg *models.RegistryNamespace + var err error + if len(registryId) > 0 { + reg, err = FindRegistryById(registryId, log) + if err != nil { + log.Errorf( + "service.EnsureRegistrySecret: failed to find registry: %s error msg:%s", + registryId, err, + ) + return err + } + } else { + reg, err = FindDefaultRegistry(log) + if err != nil { + log.Errorf( + "service.EnsureRegistrySecret: failed to find default candidate registry: %s %s", + namespace, err, + ) + return err + } } err = kube.CreateOrUpdateRegistrySecret(namespace, reg, kubeClient) if err != nil { - log.Errorf("[%s] CreateDockerSecret error: %v", namespace, err) + log.Errorf("[%s] CreateDockerSecret error: %s", namespace, err) return e.ErrUpdateSecret.AddDesc(e.CreateDefaultRegistryErrMsg) } diff --git a/pkg/microservice/aslan/core/common/service/s3/s3.go b/pkg/microservice/aslan/core/common/service/s3/s3.go index 24655148c8206bbc02653904d65c13b2e81f8581..fc363b616f8b1f63eb3ec30439a3f7a553cf85cb 100644 --- a/pkg/microservice/aslan/core/common/service/s3/s3.go +++ b/pkg/microservice/aslan/core/common/service/s3/s3.go @@ -148,6 +148,16 @@ func FindDefaultS3() (*S3, error) { return &S3{S3Storage: storage}, nil } +func FindS3ById(id string) (*S3, error) { + storage, err := commonrepo.NewS3StorageColl().Find(id) + if err != nil { + log.Warnf("Failed to find s3 in db, err: %s", err) + return nil, err + } + + return &S3{S3Storage: storage}, nil +} + // 获取内置的s3 func FindInternalS3() *S3 { storage := &models.S3Storage{ diff --git a/pkg/microservice/aslan/core/common/service/service.go b/pkg/microservice/aslan/core/common/service/service.go index 7804093148c816dd1bf605301a3d5b26cb790631..7715df4b813a1f9729fc41ed11e45cf63a2d8252 100644 --- a/pkg/microservice/aslan/core/common/service/service.go +++ b/pkg/microservice/aslan/core/common/service/service.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "net/url" "regexp" "strings" templ "text/template" @@ -90,8 +91,9 @@ type ServiceProductMap struct { } var ( - imageParseRegex = regexp.MustCompile(`(?P.+/)?(?P[^:]+){1}(:)?(?P.+)?`) - presetPatterns = []map[string]string{ + imageParseRegex = regexp.MustCompile(`(?P.+/)?(?P[^:]+){1}(:)?(?P.+)?`) + imageSpaceParseRegex = regexp.MustCompile(`\/(.*?)\/`) + presetPatterns = []map[string]string{ {setting.PathSearchComponentImage: "image.repository", setting.PathSearchComponentTag: "image.tag"}, {setting.PathSearchComponentImage: "image"}, } @@ -614,6 +616,55 @@ func ExtractImageName(imageURI string) string { return "" } +// ExtractImageRegistry extract registry url from total image uri +func ExtractImageRegistry(imageURI string) (string, error) { + subMatchAll := imageParseRegex.FindStringSubmatch(imageURI) + exNames := imageParseRegex.SubexpNames() + for i, matchedStr := range subMatchAll { + if i != 0 && matchedStr != "" && matchedStr != ":" { + if exNames[i] == "repo" { + u, err := url.Parse(matchedStr) + if err != nil { + return "", err + } + if len(u.Scheme) > 0 { + matchedStr = strings.TrimPrefix(matchedStr, fmt.Sprintf("%s://", u.Scheme)) + } + return matchedStr, nil + } + } + } + return "", fmt.Errorf("failed to extract registry url") +} + +// ExtractImageTag extract image tag from total image uri +func ExtractImageTag(imageURI string) string { + subMatchAll := imageParseRegex.FindStringSubmatch(imageURI) + exNames := imageParseRegex.SubexpNames() + for i, matchedStr := range subMatchAll { + if i != 0 && matchedStr != "" && matchedStr != ":" { + if exNames[i] == "tag" { + return matchedStr + } + } + } + return "" +} + +// ExtractRegistryNamespace extract registry namespace from image uri +func ExtractRegistryNamespace(imageURI string) string { + imageURI = strings.TrimPrefix(imageURI, "http://") + imageURI = strings.TrimPrefix(imageURI, "https://") + + imageComponent := strings.Split(imageURI, "/") + if len(imageComponent) <= 2 { + return "" + } + + nsComponent := imageComponent[1 : len(imageComponent)-1] + return strings.Join(nsComponent, "/") +} + func parseImagesByPattern(nested map[string]interface{}, patterns []map[string]string) ([]*models.Container, error) { flatMap, err := converter.Flatten(nested) if err != nil { diff --git a/pkg/microservice/aslan/core/common/service/webhook/client.go b/pkg/microservice/aslan/core/common/service/webhook/client.go index a51ee745500f82e21c8f2bfb784d43fbf5d721c3..6cce34ad7b8b8cb18356d3ea8119253f55da2d7c 100644 --- a/pkg/microservice/aslan/core/common/service/webhook/client.go +++ b/pkg/microservice/aslan/core/common/service/webhook/client.go @@ -26,6 +26,7 @@ const ( PipelinePrefix = "pipeline-" ColliePrefix = "collie-" ServicePrefix = "service-" + TestingPrefix = "testing-" taskTimeoutSecond = 10 ) diff --git a/pkg/microservice/aslan/core/common/service/workflow.go b/pkg/microservice/aslan/core/common/service/workflow.go index 6592413eadb1f8946554d5fda5aa5faa0efc18a8..fef9432311a6c5ec3f64431fe6dca3d4ce8cc754 100644 --- a/pkg/microservice/aslan/core/common/service/workflow.go +++ b/pkg/microservice/aslan/core/common/service/workflow.go @@ -273,6 +273,17 @@ func toHookSet(hooks interface{}) HookSet { codeHostID: h.CodeHostID, }) } + case []*models.TestingHook: + for _, h := range hs { + res.Insert(hookItem{ + hookUniqueID: hookUniqueID{ + name: h.MainRepo.Name, + owner: h.MainRepo.RepoOwner, + repo: h.MainRepo.RepoName, + }, + codeHostID: h.MainRepo.CodehostID, + }) + } } return res diff --git a/pkg/microservice/aslan/core/delivery/handler/artifact.go b/pkg/microservice/aslan/core/delivery/handler/artifact.go index a8eed0c3d75207244af2d38d7054337e5e5220b1..2e84ec6677d9e8212dc374b8332ed85adfbcdef9 100644 --- a/pkg/microservice/aslan/core/delivery/handler/artifact.go +++ b/pkg/microservice/aslan/core/delivery/handler/artifact.go @@ -77,6 +77,22 @@ func ListDeliveryArtifacts(c *gin.Context) { ctx.Resp, ctx.Err = artifacts, err } +func GetDeliveryArtifactIDByImage(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + image := c.Query("image") + if image == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("image can't be empty!") + return + } + args := &commonrepo.DeliveryArtifactArgs{ + Image: image, + } + + ctx.Resp, ctx.Err = deliveryservice.GetDeliveryArtifactIDByImage(args, ctx.Logger) +} + func GetDeliveryArtifact(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -89,7 +105,6 @@ func GetDeliveryArtifact(c *gin.Context) { args := &commonrepo.DeliveryArtifactArgs{ ID: id, } - args.ID = id ctx.Resp, ctx.Err = deliveryservice.GetDeliveryArtifact(args, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/delivery/handler/router.go b/pkg/microservice/aslan/core/delivery/handler/router.go index 889b7e7aadceb0559bce205afdfd098631f8730b..11141b24e72d71a272ed013a63fff3cd163b2301 100644 --- a/pkg/microservice/aslan/core/delivery/handler/router.go +++ b/pkg/microservice/aslan/core/delivery/handler/router.go @@ -30,6 +30,7 @@ func (*Router) Inject(router *gin.RouterGroup) { { deliveryArtifact.GET("", ListDeliveryArtifacts) deliveryArtifact.GET("/:id", GetDeliveryArtifact) + deliveryArtifact.GET("/image", GetDeliveryArtifactIDByImage) deliveryArtifact.POST("", CreateDeliveryArtifacts) deliveryArtifact.POST("/:id", UpdateDeliveryArtifact) deliveryArtifact.POST("/:id/activities", CreateDeliveryActivities) @@ -45,6 +46,14 @@ func (*Router) Inject(router *gin.RouterGroup) { deliveryRelease.GET("/:id", GetDeliveryVersion) deliveryRelease.GET("", ListDeliveryVersion) deliveryRelease.DELETE("/:id", GetProductNameByDelivery, gin2.UpdateOperationLogStatus, DeleteDeliveryVersion) + + deliveryRelease.POST("/helm", CreateHelmDeliveryVersion) + deliveryRelease.POST("/helm/global-variables", ApplyDeliveryGlobalVariables) + deliveryRelease.GET("/helm/charts", DownloadDeliveryChart) + deliveryRelease.GET("/helm/charts/version", GetChartVersionFromRepo) + deliveryRelease.GET("/helm/charts/preview", PreviewGetDeliveryChart) + deliveryRelease.GET("/helm/charts/filePath", GetDeliveryChartFilePath) + deliveryRelease.GET("/helm/charts/fileContent", GetDeliveryChartFileContent) } deliveryPackage := router.Group("packages") diff --git a/pkg/microservice/aslan/core/delivery/handler/version.go b/pkg/microservice/aslan/core/delivery/handler/version.go index 44ea5e99814a70fbf4977dd05dd5840a9fc412b0..39a4e5519179338530a6925d736a625a5417b17d 100644 --- a/pkg/microservice/aslan/core/delivery/handler/version.go +++ b/pkg/microservice/aslan/core/delivery/handler/version.go @@ -17,19 +17,15 @@ limitations under the License. package handler import ( - "encoding/json" "fmt" - "strconv" + "net/http" "strings" "github.com/gin-gonic/gin" "github.com/koderover/zadig/pkg/microservice/aslan/config" - commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" deliveryservice "github.com/koderover/zadig/pkg/microservice/aslan/core/delivery/service" - workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/log" @@ -65,189 +61,31 @@ func GetDeliveryVersion(c *gin.Context) { } version := new(commonrepo.DeliveryVersionArgs) version.ID = ID - ctx.Resp, ctx.Err = deliveryservice.GetDeliveryVersion(version, ctx.Logger) -} - -type ReleaseInfo struct { - VersionInfo *commonmodels.DeliveryVersion `json:"versionInfo"` - BuildInfo []*commonmodels.DeliveryBuild `json:"buildInfo"` - DeployInfo []*commonmodels.DeliveryDeploy `json:"deployInfo"` - TestInfo []*commonmodels.DeliveryTest `json:"testInfo"` - DistributeInfo []*commonmodels.DeliveryDistribute `json:"distributeInfo"` - SecurityInfo []*DeliverySecurityStats `json:"securityStatsInfo"` -} - -type DeliverySecurityStats struct { - ImageName string `json:"imageName"` - ImageID string `json:"imageId"` - DeliverySecurityStatsInfo DeliverySecurityStatsInfo `json:"deliverySecurityStatsInfo"` -} - -type DeliverySecurityStatsInfo struct { - Total int `json:"total"` - Unknown int `json:"unkown"` - Negligible int `json:"negligible"` - Low int `json:"low"` - Medium int `json:"medium"` - High int `json:"high"` - Critical int `json:"critical"` + ctx.Resp, ctx.Err = deliveryservice.GetDetailReleaseData(version, ctx.Logger) } func ListDeliveryVersion(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - taskIDStr := c.Query("taskId") - var taskID = 0 - var err error - if taskIDStr != "" { - taskID, err = strconv.Atoi(taskIDStr) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - return - } + args := new(deliveryservice.ListDeliveryVersionArgs) + err := c.BindQuery(args) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return } - perPageStr := c.Query("per_page") - pageStr := c.Query("page") - var ( - perPage int - page int - ) - if perPageStr == "" { - perPage = 20 - } else { - perPage, err = strconv.Atoi(perPageStr) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(fmt.Sprintf("perPage args err :%s", err)) - return - } + if args.Page <= 0 { + args.Page = 1 } - - if pageStr == "" { - page = 1 - } else { - page, err = strconv.Atoi(pageStr) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(fmt.Sprintf("page args err :%s", err)) - return - } + if args.PerPage <= 0 { + args.PerPage = 20 } - - serviceName := c.Query("serviceName") - - version := new(commonrepo.DeliveryVersionArgs) - version.ProductName = c.Query("projectName") - version.WorkflowName = c.Query("workflowName") - version.TaskID = taskID - version.PerPage = perPage - version.Page = page - deliveryVersions, err := deliveryservice.FindDeliveryVersion(version, ctx.Logger) - - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - return + if len(args.Verbosity) == 0 { + args.Verbosity = deliveryservice.VerbosityDetailed } - releaseInfos := make([]*ReleaseInfo, 0) - for _, deliveryVersion := range deliveryVersions { - releaseInfo := new(ReleaseInfo) - //versionInfo - releaseInfo.VersionInfo = deliveryVersion - - //deployInfo - deliveryDeployArgs := new(commonrepo.DeliveryDeployArgs) - deliveryDeployArgs.ReleaseID = deliveryVersion.ID.Hex() - deliveryDeploys, err := deliveryservice.FindDeliveryDeploy(deliveryDeployArgs, ctx.Logger) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - } - // 查询条件serviceName - if serviceName != "" { - match := false - for _, deliveryDeploy := range deliveryDeploys { - if deliveryDeploy.ServiceName == serviceName { - match = true - break - } - } - if !match { - continue - } - } - // 将serviceName替换为服务名/服务组件的形式，用于前端展示 - for _, deliveryDeploy := range deliveryDeploys { - if deliveryDeploy.ContainerName != "" { - deliveryDeploy.ServiceName = deliveryDeploy.ServiceName + "/" + deliveryDeploy.ContainerName - } - } - releaseInfo.DeployInfo = deliveryDeploys - - //buildInfo - deliveryBuildArgs := new(commonrepo.DeliveryBuildArgs) - deliveryBuildArgs.ReleaseID = deliveryVersion.ID.Hex() - deliveryBuilds, err := deliveryservice.FindDeliveryBuild(deliveryBuildArgs, ctx.Logger) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - } - releaseInfo.BuildInfo = deliveryBuilds - - //testInfo - deliveryTestArgs := new(commonrepo.DeliveryTestArgs) - deliveryTestArgs.ReleaseID = deliveryVersion.ID.Hex() - deliveryTests, err := deliveryservice.FindDeliveryTest(deliveryTestArgs, ctx.Logger) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - } - releaseInfo.TestInfo = deliveryTests - - //securityStatsInfo - deliverySecurityStatss := make([]*DeliverySecurityStats, 0) - if pipelineTask, err := workflowservice.GetPipelineTaskV2(int64(deliveryVersion.TaskID), deliveryVersion.WorkflowName, config.WorkflowType, ctx.Logger); err == nil { - for _, subStage := range pipelineTask.Stages { - if subStage.TaskType == config.TaskSecurity { - subSecurityTaskMap := subStage.SubTasks - for _, subTask := range subSecurityTaskMap { - securityInfo, _ := base.ToSecurityTask(subTask) - - deliverySecurityStats := new(DeliverySecurityStats) - deliverySecurityStats.ImageName = securityInfo.ImageName - deliverySecurityStats.ImageID = securityInfo.ImageID - deliverySecurityStatsMap, err := deliveryservice.FindDeliverySecurityStatistics(securityInfo.ImageID, ctx.Logger) - if err != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - } - var transErr error - b, err := json.Marshal(deliverySecurityStatsMap) - if err != nil { - transErr = fmt.Errorf("marshal task error: %v", err) - } - - if err := json.Unmarshal(b, &deliverySecurityStats.DeliverySecurityStatsInfo); err != nil { - transErr = fmt.Errorf("unmarshal task error: %v", err) - } - if transErr != nil { - ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) - } - - deliverySecurityStatss = append(deliverySecurityStatss, deliverySecurityStats) - } - break - } - } - releaseInfo.SecurityInfo = deliverySecurityStatss - } - - //distributeInfo - deliveryDistributeArgs := new(commonrepo.DeliveryDistributeArgs) - deliveryDistributeArgs.ReleaseID = deliveryVersion.ID.Hex() - deliveryDistributes, _ := deliveryservice.FindDeliveryDistribute(deliveryDistributeArgs, ctx.Logger) - - releaseInfo.DistributeInfo = deliveryDistributes - releaseInfos = append(releaseInfos, releaseInfo) - } - ctx.Err = err - ctx.Resp = releaseInfos + ctx.Resp, ctx.Err = deliveryservice.ListDeliveryVersion(args, ctx.Logger) } func getFileName(fileName string) string { @@ -326,6 +164,20 @@ func ListPackagesVersion(c *gin.Context) { ctx.Resp = fileInfoList } +func CreateHelmDeliveryVersion(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(deliveryservice.CreateHelmDeliveryVersionArgs) + err := c.ShouldBindJSON(args) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + args.CreateBy = ctx.UserName + ctx.Err = deliveryservice.CreateHelmDeliveryVersion(args, ctx.Logger) +} + func DeleteDeliveryVersion(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -371,3 +223,82 @@ func ListDeliveryServiceNames(c *gin.Context) { productName := c.Query("projectName") ctx.Resp, ctx.Err = deliveryservice.ListDeliveryServiceNames(productName, ctx.Logger) } + +func DownloadDeliveryChart(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + versionName := c.Query("version") + chartName := c.Query("chartName") + projectName := c.Query("projectName") + + fileBytes, fileName, err := deliveryservice.DownloadDeliveryChart(projectName, versionName, chartName, ctx.Logger) + if err != nil { + ctx.Err = err + return + } + + c.Writer.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, fileName)) + c.Data(http.StatusOK, "application/octet-stream", fileBytes) +} + +func GetChartVersionFromRepo(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + chartName := c.Query("chartName") + chartRepoName := c.Query("chartRepoName") + + ctx.Resp, ctx.Err = deliveryservice.GetChartVersions(chartName, chartRepoName) +} + +func PreviewGetDeliveryChart(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + versionName := c.Query("version") + chartName := c.Query("chartName") + projectName := c.Query("projectName") + + ctx.Resp, ctx.Err = deliveryservice.PreviewDeliveryChart(projectName, versionName, chartName, ctx.Logger) +} + +func GetDeliveryChartFilePath(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(deliveryservice.DeliveryChartFilePathArgs) + err := c.BindQuery(args) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + ctx.Resp, ctx.Err = deliveryservice.GetDeliveryChartFilePath(args, ctx.Logger) +} + +func GetDeliveryChartFileContent(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(deliveryservice.DeliveryChartFileContentArgs) + err := c.BindQuery(args) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = deliveryservice.GetDeliveryChartFileContent(args, ctx.Logger) +} + +func ApplyDeliveryGlobalVariables(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + args := new(deliveryservice.DeliveryVariablesApplyArgs) + err := c.BindJSON(args) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + ctx.Resp, ctx.Err = deliveryservice.ApplyDeliveryGlobalVariables(args, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/delivery/service/artifact.go b/pkg/microservice/aslan/core/delivery/service/artifact.go index ac941e49b7b1598099c659280194f6c99c6e91a6..cf8ee9683319581599e2788cf978304a53b4a3d0 100644 --- a/pkg/microservice/aslan/core/delivery/service/artifact.go +++ b/pkg/microservice/aslan/core/delivery/service/artifact.go @@ -168,3 +168,12 @@ func getDeliveryActivitiesMap(deliveryActivities []*commonmodels.DeliveryActivit } return artifactMapInfo } + +func GetDeliveryArtifactIDByImage(deliveryArtifactArgs *commonrepo.DeliveryArtifactArgs, log *zap.SugaredLogger) (string, error) { + deliveryArtifact, err := commonrepo.NewDeliveryArtifactColl().Get(deliveryArtifactArgs) + if err != nil { + log.Errorf("get deliveryArtifact error: %s", err) + return "", e.ErrFindArtifact.AddErr(err) + } + return deliveryArtifact.ID.Hex(), nil +} diff --git a/pkg/microservice/aslan/core/delivery/service/version.go b/pkg/microservice/aslan/core/delivery/service/version.go index 1c013f689596ebff19d325bee86642b25335ab40..7bba266c3160c80d8b37e4a7dbb8e898bb990a66 100644 --- a/pkg/microservice/aslan/core/delivery/service/version.go +++ b/pkg/microservice/aslan/core/delivery/service/version.go @@ -17,21 +17,189 @@ limitations under the License. package service import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + cm "github.com/chartmuseum/helm-push/pkg/chartmuseum" + "github.com/chartmuseum/helm-push/pkg/helm" + "github.com/hashicorp/go-multierror" + "github.com/otiai10/copy" + "github.com/pkg/errors" + "go.mongodb.org/mongo-driver/bson/primitive" "go.uber.org/zap" + chartloader "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/chartutil" "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/yaml" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" + workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" + "github.com/koderover/zadig/pkg/setting" e "github.com/koderover/zadig/pkg/tool/errors" + helmtool "github.com/koderover/zadig/pkg/tool/helmclient" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" + fsutil "github.com/koderover/zadig/pkg/util/fs" + yamlutil "github.com/koderover/zadig/pkg/util/yaml" +) + +const ( + VerbosityBrief string = "brief" // brief delivery data + VerbosityDetailed string = "detailed" // detailed delivery version with total data ) +type DeliveryVersionFilter struct { + ServiceName string +} + +type CreateHelmDeliveryVersionOption struct { + EnableOfflineDist bool `json:"enableOfflineDist"` + S3StorageID string `json:"s3StorageID"` +} + +type CreateHelmDeliveryVersionChartData struct { + ServiceName string `json:"serviceName"` + Version string `json:"version,omitempty"` + ValuesYamlContent string `json:"valuesYamlContent"` +} + +type CreateHelmDeliveryVersionArgs struct { + CreateBy string `json:"-"` + ProductName string `json:"productName"` + Retry bool `json:"retry"` + Version string `json:"version"` + Desc string `json:"desc"` + EnvName string `json:"envName"` + Labels []string `json:"labels"` + ImageRepoName string `json:"imageRepoName"` + *DeliveryVersionChartData +} + +type DeliveryVersionChartData struct { + GlobalVariables string `json:"globalVariables"` + ChartRepoName string `json:"chartRepoName"` + ImageRegistryID string `json:"imageRegistryID"` + ChartDatas []*CreateHelmDeliveryVersionChartData `json:"chartDatas"` + Options *CreateHelmDeliveryVersionOption `json:"options"` +} + +type DeliveryChartData struct { + ChartData *CreateHelmDeliveryVersionChartData + ServiceObj *commonmodels.Service + ProductService *commonmodels.ProductService + RenderChart *template.RenderChart + RenderSet *commonmodels.RenderSet +} + +type DeliveryChartResp struct { + FileInfos []*types.FileInfo `json:"fileInfos"` +} + +type DeliveryChartFilePathArgs struct { + Dir string `json:"dir"` + ProjectName string `json:"projectName"` + ChartName string `json:"chartName"` + Version string `json:"version"` +} + +type DeliveryChartFileContentArgs struct { + FilePath string `json:"filePath"` + FileName string `json:"fileName"` + ProjectName string `json:"projectName"` + ChartName string `json:"chartName"` + Version string `json:"version"` +} + +type DeliveryVariablesApplyArgs struct { + GlobalVariables string `json:"globalVariables,omitempty"` + ChartDatas []*CreateHelmDeliveryVersionChartData `json:"chartDatas"` +} + +type ListDeliveryVersionArgs struct { + Page int `form:"page"` + PerPage int `form:"per_page"` + TaskId int `form:"taskId"` + ServiceName string `form:"serviceName"` + Verbosity string `form:"verbosity"` + ProjectName string `form:"projectName"` + WorkflowName string `form:"workflowName"` +} + +type ReleaseInfo struct { + VersionInfo *commonmodels.DeliveryVersion `json:"versionInfo"` + BuildInfo []*commonmodels.DeliveryBuild `json:"buildInfo,omitempty"` + DeployInfo []*commonmodels.DeliveryDeploy `json:"deployInfo,omitempty"` + TestInfo []*commonmodels.DeliveryTest `json:"testInfo,omitempty"` + DistributeInfo []*commonmodels.DeliveryDistribute `json:"distributeInfo,omitempty"` + SecurityInfo []*DeliverySecurityStats `json:"securityStatsInfo,omitempty"` +} + +type DeliverySecurityStatsInfo struct { + Total int `json:"total"` + Unknown int `json:"unkown"` + Negligible int `json:"negligible"` + Low int `json:"low"` + Medium int `json:"medium"` + High int `json:"high"` + Critical int `json:"critical"` +} + +type DeliverySecurityStats struct { + ImageName string `json:"imageName"` + ImageID string `json:"imageId"` + DeliverySecurityStatsInfo DeliverySecurityStatsInfo `json:"deliverySecurityStatsInfo"` +} + +type ImageUrlDetail struct { + ImageUrl string + Name string + Registry string + Tag string +} + +type ServiceImageDetails struct { + ServiceName string + Images []*ImageUrlDetail + Registries []string +} + +type ChartVersionResp struct { + ChartName string `json:"chartName"` + ChartVersion string `json:"chartVersion"` +} + func GetDeliveryVersion(args *commonrepo.DeliveryVersionArgs, log *zap.SugaredLogger) (*commonmodels.DeliveryVersion, error) { - resp, err := commonrepo.NewDeliveryVersionColl().Get(args) + versionData, err := commonrepo.NewDeliveryVersionColl().Get(args) if err != nil { log.Errorf("get deliveryVersion error: %v", err) return nil, e.ErrGetDeliveryVersion } - return resp, err + return versionData, err +} + +func GetDetailReleaseData(args *commonrepo.DeliveryVersionArgs, log *zap.SugaredLogger) (*ReleaseInfo, error) { + versionData, err := commonrepo.NewDeliveryVersionColl().Get(args) + if err != nil { + log.Errorf("get deliveryVersion error: %v", err) + return nil, e.ErrGetDeliveryVersion + } + return buildListReleaseResp(VerbosityDetailed, versionData, nil, log) } func FindDeliveryVersion(args *commonrepo.DeliveryVersionArgs, log *zap.SugaredLogger) ([]*commonmodels.DeliveryVersion, error) { @@ -52,29 +220,1303 @@ func DeleteDeliveryVersion(args *commonrepo.DeliveryVersionArgs, log *zap.Sugare return nil } -func ListDeliveryServiceNames(productName string, log *zap.SugaredLogger) ([]string, error) { - serviceNames := sets.String{} +func filterReleases(filter *DeliveryVersionFilter, deliveryVersion *commonmodels.DeliveryVersion, logger *zap.SugaredLogger) bool { + if filter == nil { + return true + } + if filter.ServiceName != "" { + deliveryDeployArgs := new(commonrepo.DeliveryDeployArgs) + deliveryDeployArgs.ReleaseID = deliveryVersion.ID.Hex() + deliveryDeploys, err := FindDeliveryDeploy(deliveryDeployArgs, logger) + if err != nil { + return true + } + match := false + for _, deliveryDeploy := range deliveryDeploys { + if deliveryDeploy.ServiceName == filter.ServiceName { + match = true + break + } + } + if !match { + return false + } + } + return true +} - version := new(commonrepo.DeliveryVersionArgs) - version.ProductName = productName - deliveryVersions, err := FindDeliveryVersion(version, log) +func buildBriefRelease(deliveryVersion *commonmodels.DeliveryVersion, _ *zap.SugaredLogger) (*ReleaseInfo, error) { + return &ReleaseInfo{ + VersionInfo: deliveryVersion, + }, nil +} + +func buildDetailedRelease(deliveryVersion *commonmodels.DeliveryVersion, filterOpt *DeliveryVersionFilter, logger *zap.SugaredLogger) (*ReleaseInfo, error) { + releaseInfo := new(ReleaseInfo) + //versionInfo + releaseInfo.VersionInfo = deliveryVersion + + //deployInfo + deliveryDeployArgs := new(commonrepo.DeliveryDeployArgs) + deliveryDeployArgs.ReleaseID = deliveryVersion.ID.Hex() + deliveryDeploys, err := FindDeliveryDeploy(deliveryDeployArgs, logger) if err != nil { - log.Errorf("FindDeliveryVersion failed, err:%v", err) - return serviceNames.List(), err + return nil, err } + if filterOpt != nil { + if !filterReleases(filterOpt, deliveryVersion, logger) { + return nil, nil + } + } + // 将serviceName替换为服务名/服务组件的形式，用于前端展示 + for _, deliveryDeploy := range deliveryDeploys { + if deliveryDeploy.ContainerName != "" { + deliveryDeploy.ServiceName = deliveryDeploy.ServiceName + "/" + deliveryDeploy.ContainerName + } + } + releaseInfo.DeployInfo = deliveryDeploys + + //buildInfo + deliveryBuildArgs := new(commonrepo.DeliveryBuildArgs) + deliveryBuildArgs.ReleaseID = deliveryVersion.ID.Hex() + deliveryBuilds, err := FindDeliveryBuild(deliveryBuildArgs, logger) + if err != nil { + return nil, err + } + releaseInfo.BuildInfo = deliveryBuilds + + //testInfo + deliveryTestArgs := new(commonrepo.DeliveryTestArgs) + deliveryTestArgs.ReleaseID = deliveryVersion.ID.Hex() + deliveryTests, err := FindDeliveryTest(deliveryTestArgs, logger) + if err != nil { + return nil, err + } + releaseInfo.TestInfo = deliveryTests + + //securityStatsInfo + deliverySecurityStatss := make([]*DeliverySecurityStats, 0) + if pipelineTask, err := workflowservice.GetPipelineTaskV2(int64(deliveryVersion.TaskID), deliveryVersion.WorkflowName, config.WorkflowType, logger); err == nil { + for _, subStage := range pipelineTask.Stages { + if subStage.TaskType == config.TaskSecurity { + subSecurityTaskMap := subStage.SubTasks + for _, subTask := range subSecurityTaskMap { + securityInfo, _ := base.ToSecurityTask(subTask) + + deliverySecurityStats := new(DeliverySecurityStats) + deliverySecurityStats.ImageName = securityInfo.ImageName + deliverySecurityStats.ImageID = securityInfo.ImageID + deliverySecurityStatsMap, err := FindDeliverySecurityStatistics(securityInfo.ImageID, logger) + if err != nil { + return nil, err + } + var transErr error + b, err := json.Marshal(deliverySecurityStatsMap) + if err != nil { + transErr = fmt.Errorf("marshal task error: %v", err) + } + if err := json.Unmarshal(b, &deliverySecurityStats.DeliverySecurityStatsInfo); err != nil { + transErr = fmt.Errorf("unmarshal task error: %v", err) + } + if transErr != nil { + return nil, transErr + } + + deliverySecurityStatss = append(deliverySecurityStatss, deliverySecurityStats) + } + break + } + } + releaseInfo.SecurityInfo = deliverySecurityStatss + } + + //distributeInfo + deliveryDistributeArgs := new(commonrepo.DeliveryDistributeArgs) + deliveryDistributeArgs.ReleaseID = deliveryVersion.ID.Hex() + deliveryDistributes, _ := FindDeliveryDistribute(deliveryDistributeArgs, logger) + releaseInfo.DistributeInfo = deliveryDistributes + // fill some data for helm delivery releases + processReleaseRespData(releaseInfo) + + return releaseInfo, nil +} + +func buildListReleaseResp(verbosity string, deliveryVersion *commonmodels.DeliveryVersion, filterOpt *DeliveryVersionFilter, logger *zap.SugaredLogger) (*ReleaseInfo, error) { + switch verbosity { + case VerbosityBrief: + return buildBriefRelease(deliveryVersion, logger) + case VerbosityDetailed: + return buildDetailedRelease(deliveryVersion, filterOpt, logger) + default: + return buildDetailedRelease(deliveryVersion, filterOpt, logger) + } +} + +func ListDeliveryVersion(args *ListDeliveryVersionArgs, logger *zap.SugaredLogger) ([]*ReleaseInfo, error) { + versionListArgs := new(commonrepo.DeliveryVersionArgs) + versionListArgs.ProductName = args.ProjectName + versionListArgs.WorkflowName = args.WorkflowName + versionListArgs.TaskID = args.TaskId + versionListArgs.PerPage = args.PerPage + versionListArgs.Page = args.Page + deliveryVersions, err := FindDeliveryVersion(versionListArgs, logger) + if err != nil { + return nil, err + } + + releaseInfos := make([]*ReleaseInfo, 0) for _, deliveryVersion := range deliveryVersions { - deliveryDeployArgs := new(commonrepo.DeliveryDeployArgs) - deliveryDeployArgs.ReleaseID = deliveryVersion.ID.Hex() - deliveryDeploys, err := FindDeliveryDeploy(deliveryDeployArgs, log) + releaseInfo, err := buildListReleaseResp(args.Verbosity, deliveryVersion, &DeliveryVersionFilter{ServiceName: args.ServiceName}, logger) if err != nil { - log.Errorf("FindDeliveryDeploy failed, ReleaseID:%s, err:%v", deliveryVersion.ID, err) + return nil, err + } + if releaseInfo == nil { continue } - for _, deliveryDeploy := range deliveryDeploys { - serviceNames.Insert(deliveryDeploy.ServiceName) + releaseInfos = append(releaseInfos, releaseInfo) + } + return releaseInfos, nil +} + +// fill release +func processReleaseRespData(release *ReleaseInfo) { + if release.VersionInfo.Type != setting.DeliveryVersionTypeChart { + return + } + + distributeMap := make(map[string][]*commonmodels.DeliveryDistribute) + for _, distributeImage := range release.DistributeInfo { + if distributeImage.DistributeType != config.Image { + continue } + distributeMap[distributeImage.ChartName] = append(distributeMap[distributeImage.ChartName], distributeImage) } - return serviceNames.UnsortedList(), nil + chartDistributeCount := 0 + distributes := make([]*commonmodels.DeliveryDistribute, 0) + for _, distribute := range release.DistributeInfo { + if distribute.DistributeType == config.Image { + continue + } + switch distribute.DistributeType { + case config.Chart: + chartDistributeCount++ + distribute.SubDistributes = distributeMap[distribute.ChartName] + case config.File: + s3Storage, err := commonrepo.NewS3StorageColl().Find(distribute.S3StorageID) + if err != nil { + log.Errorf("failed to query s3 storageID: %s, err: %s", distribute.S3StorageID, err) + } else { + distribute.StorageURL = s3Storage.Endpoint + distribute.StorageBucket = s3Storage.Bucket + } + } + distributes = append(distributes, distribute) + } + release.DistributeInfo = distributes + + release.VersionInfo.Progress = buildDeliveryProgressInfo(release.VersionInfo, chartDistributeCount) +} + +func buildDeliveryProgressInfo(deliveryVersion *commonmodels.DeliveryVersion, successfulChartCount int) *commonmodels.DeliveryVersionProgress { + if deliveryVersion.Type != setting.DeliveryVersionTypeChart { + return nil + } + + progress := &commonmodels.DeliveryVersionProgress{ + SuccessChartCount: successfulChartCount, + TotalChartCount: 0, + PackageUploadStatus: "", + Error: "", + } + if deliveryVersion.Status == setting.DeliveryVersionStatusSuccess { + progress.TotalChartCount = successfulChartCount + progress.PackageUploadStatus = setting.DeliveryVersionPackageStatusSuccess + return progress + } + + argsBytes, err := json.Marshal(deliveryVersion.CreateArgument) + if err != nil { + log.Errorf("failed to marshal arguments, versionName: %s err %s", deliveryVersion.Version, err) + return progress + } + createArgs := new(DeliveryVersionChartData) + err = json.Unmarshal(argsBytes, createArgs) + if err != nil { + log.Errorf("failed to unMarshal arguments, versionName: %s err %s", deliveryVersion.Version, err) + return progress + } + + progress.TotalChartCount = len(createArgs.ChartDatas) + + if deliveryVersion.Status == setting.DeliveryVersionStatusFailed { + progress.PackageUploadStatus = setting.DeliveryVersionPackageStatusFailed + progress.Error = deliveryVersion.Error + return progress + } + + if len(createArgs.ChartDatas) > successfulChartCount { + progress.PackageUploadStatus = setting.DeliveryVersionPackageStatusWaiting + return progress + } + + progress.PackageUploadStatus = setting.DeliveryVersionPackageStatusUploading + return progress + +} + +func getChartTGZDir(productName, versionName string) string { + tmpDir := os.TempDir() + return filepath.Join(tmpDir, "chart-tgz", productName, versionName) +} + +func getChartExpandDir(productName, versionName string) string { + tmpDir := os.TempDir() + return filepath.Join(tmpDir, "chart", productName, versionName) +} + +func getProductEnvInfo(productName, envName string) (*commonmodels.Product, error) { + productInfo, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + log.Errorf("failed to query product info, productName: %s envName: %s err: %s", productName, envName, err) + return nil, fmt.Errorf("failed to query product info, productName: %s envName: %s", productName, envName) + } + return productInfo, nil +} + +func getChartRepoData(repoName string) (*commonmodels.HelmRepo, error) { + return commonrepo.NewHelmRepoColl().Find(&commonrepo.HelmRepoFindOption{RepoName: repoName}) +} + +func createChartRepoClient(repo *commonmodels.HelmRepo) (*cm.Client, error) { + client, err := cm.NewClient( + cm.URL(repo.URL), + cm.Username(repo.Username), + cm.Password(repo.Password), + // need support more auth types + ) + if err != nil { + return nil, errors.Wrapf(err, "failed to create chart repo client, repoName: %s", repo.RepoName) + } + return client, nil +} + +// find all images in one single chart +func extractImages(productService *commonmodels.ProductService, registryMap map[string]*commonmodels.RegistryNamespace) (*ServiceImageDetails, error) { + imageUrlsSet := sets.NewString() + for _, container := range productService.Containers { + imageUrlsSet.Insert(container.Image) + } + + ret := &ServiceImageDetails{ + ServiceName: productService.ServiceName, + Images: make([]*ImageUrlDetail, 0), + } + + registrySet := sets.NewString() + + for _, imageUrl := range imageUrlsSet.List() { + + registryUrl, err := commonservice.ExtractImageRegistry(imageUrl) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse registry from image uri: %s", imageUrl) + } + registryUrl = strings.TrimSuffix(registryUrl, "/") + + imageName := commonservice.ExtractImageName(imageUrl) + imageTag := commonservice.ExtractImageTag(imageUrl) + + registryID := "" + // used source registry + if registry, ok := registryMap[registryUrl]; ok { + registryID = registry.ID.Hex() + registrySet.Insert(registryID) + } + + ret.Images = append(ret.Images, &ImageUrlDetail{ + ImageUrl: imageUrl, + Name: imageName, + Tag: imageTag, + Registry: registryID, + }) + } + + ret.Registries = registrySet.List() + return ret, nil +} + +// ensure chart files exist +func ensureChartFiles(chartData *DeliveryChartData) (string, error) { + serviceObj := chartData.ServiceObj + revisionBasePath := config.LocalDeliveryChartPathWithRevision(serviceObj.ProductName, serviceObj.ServiceName, serviceObj.Revision) + deliveryChartPath := filepath.Join(revisionBasePath, serviceObj.ServiceName) + if exists, _ := fsutil.DirExists(deliveryChartPath); exists { + return deliveryChartPath, nil + } + + serviceName, revision := serviceObj.ServiceName, serviceObj.Revision + basePath := config.LocalServicePathWithRevision(serviceObj.ProductName, serviceName, revision) + if err := commonservice.PreloadServiceManifestsByRevision(basePath, serviceObj); err != nil { + log.Warnf("failed to get chart of revision: %d for service: %s, use latest version", revision, serviceName) + // use the latest version when it fails to download the specific version + basePath = config.LocalServicePath(serviceObj.ProductName, serviceName) + if err = commonservice.PreLoadServiceManifests(basePath, serviceObj); err != nil { + log.Errorf("failed to load chart info for service %v", serviceObj.ServiceName) + return "", err + } + } + + fullPath := filepath.Join(basePath, serviceObj.ServiceName) + err := copy.Copy(fullPath, deliveryChartPath) + if err != nil { + return "", err + } + + mergedValuesYaml, err := helmtool.MergeOverrideValues(chartData.RenderChart.ValuesYaml, chartData.RenderSet.DefaultValues, chartData.RenderChart.GetOverrideYaml(), chartData.RenderChart.OverrideValues) + if err != nil { + return "", err + } + err = os.WriteFile(filepath.Join(deliveryChartPath, setting.ValuesYaml), []byte(mergedValuesYaml), 0644) + if err != nil { + return "", errors.Wrapf(err, "failed to write values.yaml") + } + + return deliveryChartPath, nil +} + +func buildChartPackage(chartData *DeliveryChartData, chartRepo *commonmodels.HelmRepo, dir string, globalVariables string, registryMap map[string]*commonmodels.RegistryNamespace) error { + serviceObj := chartData.ServiceObj + + deliveryChartPath, err := ensureChartFiles(chartData) + if err != nil { + return err + } + + valuesYamlData := make(map[string]interface{}) + valuesFilePath := filepath.Join(deliveryChartPath, setting.ValuesYaml) + valueYamlContent, err := os.ReadFile(valuesFilePath) + if err != nil { + return errors.Wrapf(err, "failed to read values.yaml for service %s", serviceObj.ServiceName) + } + err = yaml.Unmarshal(valueYamlContent, &valuesYamlData) + if err != nil { + return errors.Wrapf(err, "failed to unmarshal values.yaml for service %s", serviceObj.ServiceName) + } + + // write values.yaml file before load + if len(chartData.ChartData.ValuesYamlContent) > 0 { // values.yaml was edited directly + if err = yaml.Unmarshal([]byte(chartData.ChartData.ValuesYamlContent), map[string]interface{}{}); err != nil { + log.Errorf("invalid yaml content, serviceName: %s, yamlContent: %s", serviceObj.ServiceName, chartData.ChartData.ValuesYamlContent) + return errors.Wrapf(err, "invalid yaml content for service: %s", serviceObj.ServiceName) + } + valueYamlContent = []byte(chartData.ChartData.ValuesYamlContent) + } else if len(globalVariables) > 0 { // merge global variables + valueYamlContent, err = yamlutil.Merge([][]byte{valueYamlContent, []byte(globalVariables)}) + if err != nil { + return errors.Wrapf(err, "failed to merge global variables for service: %s", serviceObj.ServiceName) + } + } + err = os.WriteFile(valuesFilePath, valueYamlContent, 0644) + if err != nil { + return errors.Wrapf(err, "failed to write values.yaml file for service %s", serviceObj.ServiceName) + } + + //load chart info from local storage + chartRequested, err := chartloader.Load(deliveryChartPath) + if err != nil { + return errors.Wrapf(err, "failed to load chart info, path %s", deliveryChartPath) + } + + //set metadata + chartRequested.Metadata.Name = chartData.ChartData.ServiceName + chartRequested.Metadata.Version = chartData.ChartData.Version + chartRequested.Metadata.AppVersion = chartData.ChartData.Version + + //create local chart package + chartPackagePath, err := helm.CreateChartPackage(&helm.Chart{Chart: chartRequested}, dir) + if err != nil { + return err + } + + client, err := createChartRepoClient(chartRepo) + if err != nil { + return errors.Wrapf(err, "failed to create chart repo client, repoName: %s", chartRepo.RepoName) + } + + log.Infof("pushing chart %s to %s...", filepath.Base(chartPackagePath), chartRepo.URL) + resp, err := client.UploadChartPackage(chartPackagePath, false) + if err != nil { + return errors.Wrapf(err, "failed to prepare pushing chart: %s", chartPackagePath) + } + err = handlePushResponse(resp) + if err != nil { + return errors.Wrapf(err, "failed to push chart: %s ", chartPackagePath) + } + return nil +} + +func handlePushResponse(resp *http.Response) error { + if resp.StatusCode != 201 && resp.StatusCode != 202 { + b, err := ioutil.ReadAll(resp.Body) + defer func(Body io.ReadCloser) { + _ = Body.Close() + }(resp.Body) + if err != nil { + return err + } + return getChartmuseumError(b, resp.StatusCode) + } + log.Infof("push chart to chart repo done") + return nil +} + +func getChartmuseumError(b []byte, code int) error { + var er struct { + Error string `json:"error"` + } + err := json.Unmarshal(b, &er) + if err != nil || er.Error == "" { + return errors.Errorf("%d: could not properly parse response JSON: %s", code, string(b)) + } + return errors.Errorf("%d: %s", code, er.Error) +} + +func makeChartTGZFileDir(productName, versionName string) (string, error) { + path := getChartTGZDir(productName, versionName) + if err := os.RemoveAll(path); err != nil { + if !os.IsExist(err) { + return "", errors.Wrapf(err, "failed to claer dir for chart tgz files") + } + } + err := os.MkdirAll(path, 0777) + if err != nil { + return "", errors.Wrapf(err, "failed to create chart tgz dir for version: %s", versionName) + } + return path, nil +} + +func CreateHelmDeliveryVersion(args *CreateHelmDeliveryVersionArgs, logger *zap.SugaredLogger) error { + if args.Retry { + return RetryCreateHelmDeliveryVersion(args.ProductName, args.Version, logger) + } else { + return CreateNewHelmDeliveryVersion(args, logger) + } +} + +// validate chartInfo, make sure service is in environment +// prepare data set for chart delivery +func prepareChartData(chartDatas []*CreateHelmDeliveryVersionChartData, productInfo *commonmodels.Product) (map[string]*DeliveryChartData, error) { + + renderSet, err := commonrepo.NewRenderSetColl().Find(&commonrepo.RenderSetFindOption{ + Revision: productInfo.Render.Revision, + Name: productInfo.Render.Name, + }) + if err != nil { + return nil, fmt.Errorf("failed to find renderSet: %s, revision: %d", productInfo.Render.Name, productInfo.Render.Revision) + } + chartMap := make(map[string]*template.RenderChart) + for _, rChart := range renderSet.ChartInfos { + chartMap[rChart.ServiceName] = rChart + } + + chartDataMap := make(map[string]*DeliveryChartData) + serviceMap := productInfo.GetServiceMap() + for _, chartData := range chartDatas { + if productService, ok := serviceMap[chartData.ServiceName]; ok { + serviceObj, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ServiceName: chartData.ServiceName, + Revision: productService.Revision, + Type: setting.HelmDeployType, + ProductName: productInfo.ProductName, + }) + if err != nil { + return nil, fmt.Errorf("failed to query service: %s", chartData.ServiceName) + } + renderChart, ok := chartMap[chartData.ServiceName] + if !ok { + return nil, fmt.Errorf("can't find renderChart for service: %s", chartData.ServiceName) + } + chartDataMap[chartData.ServiceName] = &DeliveryChartData{ + ChartData: chartData, + RenderChart: renderChart, + ServiceObj: serviceObj, + ProductService: productService, + RenderSet: renderSet, + } + } else { + return nil, fmt.Errorf("service %s not found in environment", chartData.ServiceName) + } + } + return chartDataMap, nil +} + +func buildRegistryMap() (map[string]*commonmodels.RegistryNamespace, error) { + registries, err := commonrepo.NewRegistryNamespaceColl().FindAll(&mongodb.FindRegOps{}) + if err != nil { + return nil, fmt.Errorf("failed to query registries") + } + ret := make(map[string]*commonmodels.RegistryNamespace) + for _, singleRegistry := range registries { + fullUrl := fmt.Sprintf("%s/%s", singleRegistry.RegAddr, singleRegistry.Namespace) + fullUrl = strings.TrimSuffix(fullUrl, "/") + u, _ := url.Parse(fullUrl) + if len(u.Scheme) > 0 { + fullUrl = strings.TrimPrefix(fullUrl, fmt.Sprintf("%s://", u.Scheme)) + } + ret[fullUrl] = singleRegistry + } + return ret, nil +} + +func buildArtifactTaskArgs(projectName, envName string, imagesMap *sync.Map) *commonmodels.ArtifactPackageTaskArgs { + imageArgs := make([]*commonmodels.ImagesByService, 0) + sourceRegistry := sets.NewString() + imagesMap.Range(func(key, value interface{}) bool { + imageDetail := value.(*ServiceImageDetails) + imagesByService := &commonmodels.ImagesByService{ + ServiceName: imageDetail.ServiceName, + Images: make([]*commonmodels.ImageData, 0), + } + for _, image := range imageDetail.Images { + imagesByService.Images = append(imagesByService.Images, &commonmodels.ImageData{ + ImageUrl: image.ImageUrl, + ImageName: image.Name, + ImageTag: image.Tag, + RegistryID: image.Registry, + }) + } + imageArgs = append(imageArgs, imagesByService) + sourceRegistry.Insert(imageDetail.Registries...) + return true + }) + + ret := &commonmodels.ArtifactPackageTaskArgs{ + ProjectName: projectName, + EnvName: envName, + Images: imageArgs, + SourceRegistries: sourceRegistry.List(), + } + return ret +} + +// insert delivery distribution data for single chart, include image and chart +func insertDeliveryDistributions(result *task.ServicePackageResult, chartVersion string, deliveryVersion *commonmodels.DeliveryVersion, args *DeliveryVersionChartData) error { + for _, image := range result.ImageData { + err := commonrepo.NewDeliveryDistributeColl().Insert(&commonmodels.DeliveryDistribute{ + ReleaseID: deliveryVersion.ID, + ServiceName: image.ImageName, // image name + ChartName: result.ServiceName, + DistributeType: config.Image, + RegistryName: image.ImageUrl, + Namespace: commonservice.ExtractRegistryNamespace(image.ImageUrl), + CreatedAt: time.Now().Unix(), + }) + if err != nil { + log.Errorf("failed to insert image distribute data, chartName: %s, err: %s", result.ServiceName, err) + return fmt.Errorf("failed to insert image distribute data, chartName: %s", result.ServiceName) + } + } + + err := commonrepo.NewDeliveryDistributeColl().Insert(&commonmodels.DeliveryDistribute{ + ReleaseID: deliveryVersion.ID, + DistributeType: config.Chart, + ChartName: result.ServiceName, + ChartVersion: chartVersion, + ChartRepoName: args.ChartRepoName, + SubDistributes: nil, + CreatedAt: time.Now().Unix(), + }) + if err != nil { + log.Errorf("failed to insert chart distribute data, chartName: %s, err: %s", result.ServiceName, err) + return fmt.Errorf("failed to insert chart distribute data, chartName: %s", result.ServiceName) + } + return nil +} + +func buildDeliveryCharts(chartDataMap map[string]*DeliveryChartData, deliveryVersion *commonmodels.DeliveryVersion, args *DeliveryVersionChartData, logger *zap.SugaredLogger) (err error) { + defer func() { + if err != nil { + deliveryVersion.Status = setting.DeliveryVersionStatusFailed + deliveryVersion.Error = err.Error() + } + err = commonrepo.NewDeliveryVersionColl().UpdateStatusByName(deliveryVersion.Version, deliveryVersion.Status, deliveryVersion.Error) + if err != nil { + logger.Errorf("failed to update delivery version data, name: %s error: %s", deliveryVersion.Version, err) + } + }() + + var errLock sync.Mutex + errorList := &multierror.Error{} + + appendError := func(err error) { + errLock.Lock() + defer errLock.Unlock() + errorList = multierror.Append(errorList, err) + } + + dir, err := makeChartTGZFileDir(deliveryVersion.ProductName, deliveryVersion.Version) + if err != nil { + return err + } + repoInfo, err := getChartRepoData(args.ChartRepoName) + if err != nil { + log.Errorf("failed to query chart-repo info, productName: %s, err: %s", deliveryVersion.ProductName, err) + return fmt.Errorf("failed to query chart-repo info, productName: %s, repoName: %s", deliveryVersion.ProductName, args.ChartRepoName) + } + + registryMap, err := buildRegistryMap() + if err != nil { + return fmt.Errorf("failed to build registry map") + } + + imagesDataMap := &sync.Map{} + + // push charts to repo + wg := sync.WaitGroup{} + for _, chartData := range chartDataMap { + wg.Add(1) + go func(cData *DeliveryChartData) { + defer wg.Done() + err := buildChartPackage(cData, repoInfo, dir, args.GlobalVariables, registryMap) + if err != nil { + logger.Errorf("failed to build chart package, serviceName: %s err: %s", cData.ChartData.ServiceName, err) + appendError(err) + return + } + imageData, err := extractImages(cData.ProductService, registryMap) + if err != nil { + logger.Errorf("failed to extract image data, serviceName: %s err: %s", cData.ChartData.ServiceName, err) + appendError(err) + return + } + imagesDataMap.Store(cData.ServiceObj.ServiceName, imageData) + }(chartData) + } + wg.Wait() + + if errorList.ErrorOrNil() != nil { + err = errorList.ErrorOrNil() + return + } + + // create task to deal with images + // offline docker images are not supported + taskArgs := buildArtifactTaskArgs(deliveryVersion.ProductName, deliveryVersion.ProductEnvInfo.EnvName, imagesDataMap) + taskArgs.TargetRegistries = []string{args.ImageRegistryID} + taskID, err := workflowservice.CreateArtifactPackageTask(taskArgs, deliveryVersion.Version, logger) + if err != nil { + return err + } + deliveryVersion.TaskID = int(taskID) + + // start a new routine to check task results + go waitVersionDone(deliveryVersion) + + return +} + +func updateVersionStatus(versionName, status, errStr string) { + err := commonrepo.NewDeliveryVersionColl().UpdateStatusByName(versionName, status, errStr) + if err != nil { + log.Errorf("failed to update version status, name: %s, err: %s", versionName, err) + } +} + +func taskFinished(status config.Status) bool { + return status == config.StatusPassed || status == config.StatusFailed || status == config.StatusTimeout || status == config.StatusCancelled +} + +func waitVersionDone(deliveryVersion *commonmodels.DeliveryVersion) { + waitTimeout := time.After(60 * time.Minute * 2) + for { + select { + case <-waitTimeout: + updateVersionStatus(deliveryVersion.Version, setting.DeliveryVersionStatusFailed, "timeout") + return + default: + done, err := checkVersionStatus(deliveryVersion) + if err != nil { + updateVersionStatus(deliveryVersion.Version, setting.DeliveryVersionStatusFailed, err.Error()) + return + } + if done { + return + } + } + time.Sleep(time.Second * 5) + } +} + +func checkVersionStatus(deliveryVersion *commonmodels.DeliveryVersion) (bool, error) { + if deliveryVersion.Status == setting.DeliveryVersionStatusSuccess || deliveryVersion.Status == setting.DeliveryVersionStatusFailed { + return true, nil + } + pipelineName := fmt.Sprintf("%s-%s-%s", deliveryVersion.ProductName, deliveryVersion.ProductEnvInfo.EnvName, "artifact") + taskData, err := commonrepo.NewTaskColl().Find(int64(deliveryVersion.TaskID), pipelineName, config.ArtifactType) + if err != nil { + return false, fmt.Errorf("failed to query taskData, id: %d, pipelineName: %s", deliveryVersion.TaskID) + } + + if len(taskData.Stages) != 1 { + return false, fmt.Errorf("invalid task data, stage length not leagal") + } + + argsBytes, err := json.Marshal(deliveryVersion.CreateArgument) + if err != nil { + return false, errors.Wrapf(err, "failed to marshal arguments, versionName: %s err: %s", deliveryVersion.Version, err) + } + createArgs := new(DeliveryVersionChartData) + err = json.Unmarshal(argsBytes, createArgs) + if err != nil { + return false, errors.Wrapf(err, "failed to unMarshal arguments, versionName: %s err: %s", deliveryVersion.Version, err) + } + + distributes, err := commonrepo.NewDeliveryDistributeColl().Find(&commonrepo.DeliveryDistributeArgs{ + DistributeType: config.Chart, + ReleaseID: deliveryVersion.ID.Hex(), + }) + if err != nil { + return false, errors.Wrapf(err, "failed to query distrubutes, versionName: %s", deliveryVersion.Version) + } + + // for charts has been successfully handled, download charts directly + successCharts := sets.NewString() + for _, distribute := range distributes { + successCharts.Insert(distribute.ChartName) + } + + stage := taskData.Stages[0] + errorList := &multierror.Error{} + + allTaskDone := true + for _, taskData := range stage.SubTasks { + artifactPackageArgs, err := base.ToArtifactPackageImageTask(taskData) + if err != nil { + return false, errors.Wrapf(err, "failed to generate origin artifact task data") + } + + progressData, err := artifactPackageArgs.GetProgress() + if err != nil { + return false, errors.Wrapf(err, "failed to get progress data from data") + } + progressDataMap := make(map[string]*task.ServicePackageResult) + for _, singleResult := range progressData { + progressDataMap[singleResult.ServiceName] = singleResult + } + + for _, chartData := range createArgs.ChartDatas { + // service artifact has been marked as success + if successCharts.Has(chartData.ServiceName) { + continue + } + if singleResult, ok := progressDataMap[chartData.ServiceName]; ok { + if singleResult.Result != "success" { + errorList = multierror.Append(errorList, fmt.Errorf("failed to build image distribute for service:%s ", singleResult.ServiceName)) + continue + } + err = insertDeliveryDistributions(singleResult, chartData.Version, deliveryVersion, createArgs) + if err != nil { + errorList = multierror.Append(errorList, fmt.Errorf("failed to insert distribute data for service:%s ", singleResult.ServiceName)) + continue + } + successCharts.Insert(chartData.ServiceName) + } + } + + if !taskFinished(artifactPackageArgs.TaskStatus) { + allTaskDone = false + } + if len(artifactPackageArgs.Error) > 0 { + errorList = multierror.Append(errorList, fmt.Errorf(artifactPackageArgs.Error)) + } + } + + if allTaskDone { + if successCharts.Len() == len(createArgs.ChartDatas) { + deliveryVersion.Status = setting.DeliveryVersionStatusSuccess + } else { + deliveryVersion.Status = setting.DeliveryVersionStatusFailed + } + } + if errorList.ErrorOrNil() != nil { + deliveryVersion.Error = errorList.Error() + } + updateVersionStatus(deliveryVersion.Version, deliveryVersion.Status, deliveryVersion.Error) + return allTaskDone, nil +} + +func CreateNewHelmDeliveryVersion(args *CreateHelmDeliveryVersionArgs, logger *zap.SugaredLogger) error { + // need appoint chart info + if len(args.ChartDatas) == 0 { + return e.ErrCreateDeliveryVersion.AddDesc("no chart info appointed") + } + + // prepare data + productInfo, err := getProductEnvInfo(args.ProductName, args.EnvName) + if err != nil { + log.Infof("failed to query product info, productName: %s envName %s, err: %s", args.ProductName, args.EnvName, err) + return e.ErrCreateDeliveryVersion.AddDesc(fmt.Sprintf("failed to query product info, procutName: %s envName %s", args.ProductName, args.EnvName)) + } + + // validate necessary params + if len(args.ChartRepoName) == 0 { + return e.ErrCreateDeliveryVersion.AddDesc("chart repo not appointed") + } + if len(args.ImageRegistryID) == 0 { + return e.ErrCreateDeliveryVersion.AddDesc("image registry not appointed") + } + + chartDataMap, err := prepareChartData(args.ChartDatas, productInfo) + if err != nil { + return e.ErrCreateDeliveryVersion.AddErr(err) + } + + productInfo.ID, _ = primitive.ObjectIDFromHex("") + + versionObj := &commonmodels.DeliveryVersion{ + Version: args.Version, + ProductName: args.ProductName, + Type: setting.DeliveryVersionTypeChart, + Desc: args.Desc, + Labels: args.Labels, + ProductEnvInfo: productInfo, + Status: setting.DeliveryVersionStatusCreating, + CreateArgument: args.DeliveryVersionChartData, + CreatedBy: args.CreateBy, + CreatedAt: time.Now().Unix(), + DeletedAt: 0, + } + + err = buildDeliveryCharts(chartDataMap, versionObj, args.DeliveryVersionChartData, logger) + if err != nil { + return err + } + + err = commonrepo.NewDeliveryVersionColl().Insert(versionObj) + if err != nil { + logger.Errorf("failed to insert version data, err: %s", err) + return e.ErrCreateDeliveryVersion.AddErr(fmt.Errorf("failed to insert delivery version: %s", versionObj.Version)) + } + + return nil +} + +func RetryCreateHelmDeliveryVersion(projectName, versionName string, logger *zap.SugaredLogger) error { + deliveryVersion, err := commonrepo.NewDeliveryVersionColl().Get(&commonrepo.DeliveryVersionArgs{ + ProductName: projectName, + Version: versionName, + }) + if err != nil { + logger.Errorf("failed to query delivery version data, verisonName: %s, error: %s", versionName, err) + return fmt.Errorf("failed to query delivery version data, verisonName: %s", versionName) + } + + if deliveryVersion.Status != setting.DeliveryVersionStatusFailed { + return fmt.Errorf("can't reCreate version with status:%s", deliveryVersion.Status) + } + + argsBytes, err := json.Marshal(deliveryVersion.CreateArgument) + if err != nil { + return errors.Wrapf(err, "failed to marshal arguments, versionName: %s err: %s", deliveryVersion.Version, err) + } + createArgs := new(DeliveryVersionChartData) + err = json.Unmarshal(argsBytes, createArgs) + if err != nil { + return errors.Wrapf(err, "failed to unMarshal arguments, versionName: %s err: %s", deliveryVersion.Version, err) + } + + productInfoSnap := deliveryVersion.ProductEnvInfo + + distributes, err := commonrepo.NewDeliveryDistributeColl().Find(&commonrepo.DeliveryDistributeArgs{ + DistributeType: config.Chart, + ReleaseID: deliveryVersion.ID.Hex(), + }) + if err != nil { + return errors.Wrapf(err, "failed to query distrubutes, versionName: %s", deliveryVersion.Version) + } + + // for charts has been successfully handled, download charts directly + successCharts := sets.NewString() + for _, distribute := range distributes { + if distribute.DistributeType != config.Chart { + continue + } + _, err := downloadChart(deliveryVersion, distribute) + if err != nil { + log.Errorf("failed to download chart from chart repo, chartName: %s, err: %s", distribute.ChartName, err) + continue + } + successCharts.Insert(distribute.ChartName) + } + + chartsToBeHandled := make([]*CreateHelmDeliveryVersionChartData, 0) + for _, chartConfig := range createArgs.ChartDatas { + if successCharts.Has(chartConfig.ServiceName) { + continue + } + chartsToBeHandled = append(chartsToBeHandled, chartConfig) + } + + chartDataMap, err := prepareChartData(chartsToBeHandled, productInfoSnap) + if err != nil { + return e.ErrCreateDeliveryVersion.AddErr(err) + } + + err = buildDeliveryCharts(chartDataMap, deliveryVersion, createArgs, logger) + if err != nil { + return err + } + + // update status + deliveryVersion.Status = setting.DeliveryVersionStatusRetrying + err = commonrepo.NewDeliveryVersionColl().UpdateStatusByName(deliveryVersion.Version, deliveryVersion.Status, "") + if err != nil { + logger.Errorf("failed to update delivery status, name: %s, err: %s", deliveryVersion.Version, err) + return fmt.Errorf("failed to update delivery status, name: %s", deliveryVersion.Version) + } + + return nil +} + +func ListDeliveryServiceNames(productName string, log *zap.SugaredLogger) ([]string, error) { + serviceNames := sets.String{} + + version := new(commonrepo.DeliveryVersionArgs) + version.ProductName = productName + deliveryVersions, err := FindDeliveryVersion(version, log) + if err != nil { + log.Errorf("FindDeliveryVersion failed, err:%v", err) + return serviceNames.List(), err + } + + for _, deliveryVersion := range deliveryVersions { + deliveryDeployArgs := new(commonrepo.DeliveryDeployArgs) + deliveryDeployArgs.ReleaseID = deliveryVersion.ID.Hex() + deliveryDeploys, err := FindDeliveryDeploy(deliveryDeployArgs, log) + if err != nil { + log.Errorf("FindDeliveryDeploy failed, ReleaseID:%s, err:%v", deliveryVersion.ID, err) + continue + } + for _, deliveryDeploy := range deliveryDeploys { + serviceNames.Insert(deliveryDeploy.ServiceName) + } + } + + return serviceNames.UnsortedList(), nil +} + +func downloadChart(deliveryVersion *commonmodels.DeliveryVersion, chartInfo *commonmodels.DeliveryDistribute) (string, error) { + productName, versionName := deliveryVersion.ProductName, deliveryVersion.Version + chartTGZName := fmt.Sprintf("%s-%s.tgz", chartInfo.ChartName, chartInfo.ChartVersion) + chartTGZFileParent := getChartTGZDir(productName, versionName) + chartTGZFilePath := filepath.Join(chartTGZFileParent, chartTGZName) + if _, err := os.Stat(chartTGZFilePath); err == nil { + // local cache exists + log.Infof("local cache exists, path %s", chartTGZFilePath) + return chartTGZFilePath, nil + } + + chartRepo, err := getChartRepoData(chartInfo.ChartRepoName) + if err != nil { + return "", fmt.Errorf("failed to query chart-repo info, repoName %s", chartInfo.ChartRepoName) + } + + client, err := createChartRepoClient(chartRepo) + if err != nil { + return "", err + } + + if err = os.MkdirAll(chartTGZFileParent, 0644); err != nil { + return "", errors.Wrapf(err, "failed to craete tgz parent dir") + } + + out, err := os.Create(chartTGZFilePath) + if err != nil { + _ = os.RemoveAll(chartTGZFilePath) + return "", errors.Wrapf(err, "failed to create chart tgz file") + } + + response, err := client.DownloadFile(fmt.Sprintf("charts/%s", chartTGZName)) + if err != nil { + return "", errors.Wrapf(err, "failed to download file") + } + + if response.StatusCode != 200 { + return "", errors.Wrapf(err, "download file failed %s", chartTGZName) + } + defer func() { _ = response.Body.Close() }() + + b, err := ioutil.ReadAll(response.Body) + if err != nil { + return "", errors.Wrapf(err, "failed to read response data") + } + + defer func(out *os.File) { + _ = out.Close() + }(out) + + err = ioutil.WriteFile(chartTGZFilePath, b, 0644) + if err != nil { + return "", err + } + return chartTGZFilePath, nil +} + +func getChartDistributeInfo(releaseID, chartName string, log *zap.SugaredLogger) (*commonmodels.DeliveryDistribute, error) { + distributes, _ := FindDeliveryDistribute(&commonrepo.DeliveryDistributeArgs{ + ReleaseID: releaseID, + ChartName: chartName, + DistributeType: config.Chart, + }, log) + + if len(distributes) != 1 { + log.Warnf("find chart %s failed, expect count %d, found count %d, release_id: %s", chartName, 1, len(distributes), releaseID) + return nil, fmt.Errorf("can't find target charts") + } + + chartInfo := distributes[0] + return chartInfo, nil +} + +func DownloadDeliveryChart(projectName, version string, chartName string, log *zap.SugaredLogger) ([]byte, string, error) { + + filePath, err := preDownloadChart(projectName, version, chartName, log) + if err != nil { + return nil, "", err + } + + fileBytes, err := os.ReadFile(filePath) + if err != nil { + return nil, "", err + } + + return fileBytes, filepath.Base(filePath), err +} + +func preDownloadChart(projectName, versionName, chartName string, log *zap.SugaredLogger) (string, error) { + deliveryInfo, err := GetDeliveryVersion(&commonrepo.DeliveryVersionArgs{ + ProductName: projectName, + Version: versionName, + }, log) + if err != nil { + return "", fmt.Errorf("failed to query delivery info") + } + + chartInfo, err := getChartDistributeInfo(deliveryInfo.ID.Hex(), chartName, log) + if err != nil { + return "", err + } + // prepare chart data + filePath, err := downloadChart(deliveryInfo, chartInfo) + if err != nil { + return "", err + } + return filePath, err +} + +func GetChartVersions(chartName, chartRepoName string) ([]*ChartVersionResp, error) { + + chartRepo, err := getChartRepoData(chartRepoName) + if err != nil { + return nil, fmt.Errorf("failed to query chart-repo info, repoName %s", chartRepoName) + } + + client, err := createChartRepoClient(chartRepo) + if err != nil { + return nil, errors.Wrapf(err, "failed to create chart repo client") + } + + resp, err := client.DownloadFile("index.yaml") + if err != nil { + return nil, errors.Wrapf(err, "failed to download index.yaml") + } + + defer func(Body io.ReadCloser) { + _ = Body.Close() + }(resp.Body) + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != 200 { + return nil, errors.Wrapf(getChartmuseumError(b, resp.StatusCode), "failed to download index.yaml") + } + + index, err := helm.LoadIndex(b) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse index.yaml") + } + + chartNameList := strings.Split(chartName, ",") + chartNameSet := sets.NewString(chartNameList...) + + ret := make([]*ChartVersionResp, 0) + + for name, entry := range index.Entries { + if !chartNameSet.Has(name) { + continue + } + if len(entry) == 0 { + continue + } + latestEntry := entry[0] + ret = append(ret, &ChartVersionResp{ + ChartName: name, + ChartVersion: latestEntry.Version, + }) + } + + return ret, nil +} + +func preDownloadAndUncompressChart(projectName, versionName, chartName string, log *zap.SugaredLogger) (string, error) { + + deliveryInfo, err := GetDeliveryVersion(&commonrepo.DeliveryVersionArgs{ + ProductName: projectName, + Version: versionName, + }, log) + if err != nil { + return "", fmt.Errorf("failed to query delivery info") + } + + chartDistribute, err := getChartDistributeInfo(deliveryInfo.ID.Hex(), chartName, log) + if err != nil { + return "", err + } + dstDir := getChartExpandDir(projectName, versionName) + dstDir = filepath.Join(dstDir, fmt.Sprintf("%s-%s", chartDistribute.ChartName, chartDistribute.ChartVersion)) + + filePath, err := preDownloadChart(projectName, versionName, chartName, log) + if err != nil { + return "", err + } + + file, err := os.Open(filePath) + if err != nil { + return "", errors.Wrap(err, "unable to open tarball") + } + defer func() { _ = file.Close() }() + + err = chartutil.Expand(dstDir, file) + if err != nil { + log.Errorf("failed to uncompress file: %s", filePath) + return "", errors.Wrapf(err, "failed to uncompress file") + } + return dstDir, nil +} + +func PreviewDeliveryChart(projectName, version, chartName string, log *zap.SugaredLogger) (*DeliveryChartResp, error) { + + dstDir, err := preDownloadAndUncompressChart(projectName, version, chartName, log) + if err != nil { + return nil, err + } + + ret := &DeliveryChartResp{ + FileInfos: make([]*types.FileInfo, 0), + } + + var fis []*types.FileInfo + files, err := os.ReadDir(filepath.Join(dstDir, chartName)) + if err != nil { + return nil, err + } + + for _, file := range files { + info, _ := file.Info() + if info == nil { + continue + } + fi := &types.FileInfo{ + Parent: "", + Name: file.Name(), + Size: info.Size(), + Mode: file.Type(), + ModTime: info.ModTime().Unix(), + IsDir: file.IsDir(), + } + + fis = append(fis, fi) + } + ret.FileInfos = fis + return ret, nil +} + +// load chart file infos +func loadChartFileInfos(fileDir, chartName string, dir string) ([]*types.FileInfo, error) { + var fis []*types.FileInfo + files, err := os.ReadDir(filepath.Join(fileDir, chartName, dir)) + if err != nil { + return nil, e.ErrFilePath.AddDesc(err.Error()) + } + + for _, file := range files { + info, _ := file.Info() + if info == nil { + continue + } + fi := &types.FileInfo{ + Parent: dir, + Name: file.Name(), + Size: info.Size(), + Mode: file.Type(), + ModTime: info.ModTime().Unix(), + IsDir: file.IsDir(), + } + fis = append(fis, fi) + } + return fis, nil +} + +func GetDeliveryChartFilePath(args *DeliveryChartFilePathArgs, log *zap.SugaredLogger) ([]*types.FileInfo, error) { + projectName, version, chartName := args.ProjectName, args.Version, args.ChartName + dstDir, err := preDownloadAndUncompressChart(projectName, version, chartName, log) + if err != nil { + return nil, nil + } + + fileInfos, err := loadChartFileInfos(dstDir, chartName, args.Dir) + if err != nil { + return nil, err + } + return fileInfos, nil +} + +func GetDeliveryChartFileContent(args *DeliveryChartFileContentArgs, log *zap.SugaredLogger) (string, error) { + projectName, version, chartName := args.ProjectName, args.Version, args.ChartName + dstDir, err := preDownloadAndUncompressChart(projectName, version, chartName, log) + if err != nil { + return "", nil + } + + file := filepath.Join(dstDir, chartName, args.FilePath, args.FileName) + fileContent, err := os.ReadFile(file) + if err != nil { + log.Errorf("Failed to read file %s, err: %s", file, err) + return "", e.ErrFileContent.AddDesc(err.Error()) + } + + return string(fileContent), nil +} + +func ApplyDeliveryGlobalVariables(args *DeliveryVariablesApplyArgs, logger *zap.SugaredLogger) (interface{}, error) { + ret := new(DeliveryVariablesApplyArgs) + for _, chartData := range args.ChartDatas { + mergedYaml, err := yamlutil.Merge([][]byte{[]byte(chartData.ValuesYamlContent), []byte(args.GlobalVariables)}) + if err != nil { + logger.Errorf("failed to merge gobal variables for service: %s", chartData.ServiceName) + return nil, errors.Wrapf(err, "failed to merge global variables for service: %s", chartData.ServiceName) + } + ret.ChartDatas = append(ret.ChartDatas, &CreateHelmDeliveryVersionChartData{ + ServiceName: chartData.ServiceName, + ValuesYamlContent: string(mergedYaml), + }) + } + return ret, nil } diff --git a/pkg/microservice/aslan/core/environment/handler/environment.go b/pkg/microservice/aslan/core/environment/handler/environment.go index ad200efdad4daa5159163846217df8f38b57261c..43627a1e679618ed3fe1942605bb9cff738b73ec 100644 --- a/pkg/microservice/aslan/core/environment/handler/environment.go +++ b/pkg/microservice/aslan/core/environment/handler/environment.go @@ -54,6 +54,11 @@ type NamespaceResource struct { Ingresses []resource.Ingress `json:"ingresses"` } +type UpdateProductRegistryRequest struct { + RegistryID string `json:"registry_id"` + Namespace string `json:"namespace"` +} + func ListProducts(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -130,7 +135,6 @@ func createHelmProduct(c *gin.Context, ctx *internalhandler.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("projectName can not be empty") return } - createArgs := make([]*service.CreateHelmProductArg, 0) data, err := c.GetRawData() if err != nil { @@ -204,11 +208,12 @@ func CreateProduct(c *gin.Context) { return } - args.UpdateBy = ctx.UserName - // don't save the local cluster id to db - if args.ClusterID == setting.LocalClusterID { - args.ClusterID = "" + if args.RegistryID == "" { + ctx.Err = e.ErrInvalidParam.AddDesc("RegistryId can not be null!") + return } + + args.UpdateBy = ctx.UserName ctx.Err = service.CreateProduct( ctx.UserName, ctx.RequestID, args, ctx.Logger, ) @@ -247,6 +252,36 @@ func UpdateProduct(c *gin.Context) { } } +func UpdateProductRegistry(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + projectName := c.Query("projectName") + args := new(UpdateProductRegistryRequest) + data, err := c.GetRawData() + if err != nil { + log.Errorf("UpdateProduct c.GetRawData() err : %v", err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + if err = json.Unmarshal(data, args); err != nil { + log.Errorf("UpdateProduct json.Unmarshal err : %v", err) + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + internalhandler.InsertOperationLog(c, ctx.UserName, projectName, "更新", "集成环境", args.Namespace, string(data), ctx.Logger) + c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + + if err := c.BindJSON(args); err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc(err.Error()) + return + } + ctx.Err = service.UpdateProductRegistry(args.Namespace, args.RegistryID, ctx.Logger) + if ctx.Err != nil { + ctx.Logger.Errorf("failed to update product %s %s: %v", args.Namespace, args.RegistryID, ctx.Err) + } +} + func UpdateProductRecycleDay(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() @@ -391,7 +426,7 @@ func GetHelmChartVersions(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - envName := c.Param("Name") + envName := c.Param("name") projectName := c.Query("projectName") ctx.Resp, ctx.Err = service.GetHelmChartVersions(projectName, envName, ctx.Logger) @@ -488,10 +523,6 @@ func ListWorkloads(c *gin.Context) { return } - if args.ClusterID == setting.LocalClusterID { - args.ClusterID = "" - } - count, services, err := commonservice.ListWorkloads("", args.ClusterID, args.Namespace, "", args.PerPage, args.Page, ctx.Logger, func(workloads []*commonservice.Workload) []*commonservice.Workload { workloadStat, _ := mongodb.NewWorkLoadsStatColl().Find(args.ClusterID, args.Namespace) workloadM := map[string]commonmodels.Workload{} diff --git a/pkg/microservice/aslan/core/environment/handler/helm.go b/pkg/microservice/aslan/core/environment/handler/helm.go new file mode 100644 index 0000000000000000000000000000000000000000..9e097cd91851bae463e6fec1e82e9574ff59a5b7 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/handler/helm.go @@ -0,0 +1,40 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package handler + +import ( + "github.com/gin-gonic/gin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" + internalhandler "github.com/koderover/zadig/pkg/shared/handler" +) + +func ListReleases(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + envName := c.Param("name") + projectName := c.Query("projectName") + ctx.Resp, ctx.Err = service.ListReleases(projectName, envName, ctx.Logger) +} + +func GetChartInfos(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + envName := c.Param("name") + servicesName := c.Query("serviceName") + projectName := c.Query("projectName") + ctx.Resp, ctx.Err = service.GetChartInfos(projectName, envName, servicesName, ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/environment/handler/kube.go b/pkg/microservice/aslan/core/environment/handler/kube.go index 691c1220856f085ac32420260d28d809adeeb473..869fae79646127fd482be0c6b12f45b137f2f970 100644 --- a/pkg/microservice/aslan/core/environment/handler/kube.go +++ b/pkg/microservice/aslan/core/environment/handler/kube.go @@ -55,9 +55,6 @@ func ListAvailableNamespaces(c *gin.Context) { defer func() { internalhandler.JSONResponse(c, ctx) }() clusterID := c.Query("clusterId") - if clusterID == setting.LocalClusterID { - clusterID = "" - } ctx.Resp, ctx.Err = service.ListAvailableNamespaces(clusterID, ctx.Logger) } @@ -101,3 +98,10 @@ func ListPodEvents(c *gin.Context) { ctx.Resp, ctx.Err = service.ListPodEvents(envName, productName, podName, ctx.Logger) } + +func ListNodes(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + ctx.Resp, ctx.Err = service.ListAvailableNodes(c.Query("clusterId"), ctx.Logger) +} diff --git a/pkg/microservice/aslan/core/environment/handler/router.go b/pkg/microservice/aslan/core/environment/handler/router.go index f5bb3dd991b0dac65048674aa5347895dcdc9559..1d598a141b9263c37d6e8e69a4cc7838d85556b7 100644 --- a/pkg/microservice/aslan/core/environment/handler/router.go +++ b/pkg/microservice/aslan/core/environment/handler/router.go @@ -86,6 +86,7 @@ func (*Router) Inject(router *gin.RouterGroup) { kube.DELETE("/pods/:podName", gin2.UpdateOperationLogStatus, DeletePod) kube.GET("/pods/:podName/events", ListPodEvents) kube.GET("/workloads", ListWorkloads) + kube.GET("/nodes", ListNodes) } // --------------------------------------------------------------------------------------- @@ -95,6 +96,7 @@ func (*Router) Inject(router *gin.RouterGroup) { { environments.GET("", ListProducts) environments.PUT("/:name", gin2.UpdateOperationLogStatus, UpdateProduct) + environments.PUT("/:name/registry", gin2.UpdateOperationLogStatus, UpdateProductRegistry) environments.PUT("", gin2.UpdateOperationLogStatus, UpdateMultiProducts) environments.POST("", gin2.UpdateOperationLogStatus, CreateProduct) environments.GET("/:name", GetProduct) @@ -107,6 +109,9 @@ func (*Router) Inject(router *gin.RouterGroup) { environments.GET("/:name/groups", ListGroups) environments.GET("/:name/workloads", ListWorkloadsInEnv) + environments.GET("/:name/helm/releases", ListReleases) + environments.GET("/:name/helm/charts", GetChartInfos) + environments.GET("/:name/services/:serviceName", GetService) environments.PUT("/:name/services/:serviceName", gin2.UpdateOperationLogStatus, UpdateService) environments.POST("/:name/services/:serviceName/restart", gin2.UpdateOperationLogStatus, RestartService) diff --git a/pkg/microservice/aslan/core/environment/service/configmap.go b/pkg/microservice/aslan/core/environment/service/configmap.go index bbb634c570cb998ecd42a6c60dc3b708e5c956cf..1982dea074599b6ea556544ac68039dec2b57d69 100644 --- a/pkg/microservice/aslan/core/environment/service/configmap.go +++ b/pkg/microservice/aslan/core/environment/service/configmap.go @@ -28,10 +28,12 @@ import ( "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/kube/getter" @@ -98,7 +100,7 @@ func ListConfigMaps(args *ListConfigMapArgs, log *zap.SugaredLogger) ([]*configM if err != nil { return nil, e.ErrListConfigMaps.AddErr(err) } - kubeClient, err := kube.GetKubeClient(product.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) if err != nil { return nil, e.ErrListConfigMaps.AddErr(err) } @@ -153,7 +155,7 @@ func UpdateConfigMap(envName string, args *UpdateConfigMapArgs, userName, userID if err != nil { return e.ErrUpdateConfigMap.AddErr(err) } - kubeClient, err := kube.GetKubeClient(product.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) if err != nil { return e.ErrUpdateConfigMap.AddErr(err) } @@ -230,7 +232,7 @@ func RollBackConfigMap(envName string, args *RollBackConfigMapArgs, userName, us if err != nil { return e.ErrUpdateConfigMap.AddErr(err) } - kubeClient, err := kube.GetKubeClient(product.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) if err != nil { return e.ErrUpdateConfigMap.AddErr(err) } diff --git a/pkg/microservice/aslan/core/environment/service/environment.go b/pkg/microservice/aslan/core/environment/service/environment.go index 71e03bfd22efff0276ce1041121b403a5a72ac3e..3e74fc9a8d9bc21be4479f54d9c935aacbed85c8 100644 --- a/pkg/microservice/aslan/core/environment/service/environment.go +++ b/pkg/microservice/aslan/core/environment/service/environment.go @@ -54,6 +54,7 @@ import ( commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" helmtool "github.com/koderover/zadig/pkg/tool/helmclient" @@ -93,8 +94,10 @@ type EnvResp struct { UpdateTime int64 `json:"updateTime"` IsPublic bool `json:"isPublic"` ClusterName string `json:"clusterName"` + ClusterID string `json:"cluster_id"` Production bool `json:"production"` Source string `json:"source"` + RegistryID string `json:"registry_id"` } type ProductResp struct { @@ -114,7 +117,9 @@ type ProductResp struct { ClusterName string `json:"cluster_name,omitempty"` RecycleDay int `json:"recycle_day"` IsProd bool `json:"is_prod"` + IsLocal bool `json:"is_local"` Source string `json:"source"` + RegisterID string `json:"registry_id"` } type ProductParams struct { @@ -145,6 +150,7 @@ type CreateHelmProductArg struct { Namespace string `json:"namespace"` ClusterID string `json:"clusterID"` DefaultValues string `json:"defaultValues"` + RegistryID string `json:"registry_id"` ChartValues []*commonservice.RenderChartArg `json:"chartValues"` } @@ -180,11 +186,19 @@ func ListProducts(projectName string, envNames []string, log *zap.SugaredLogger) } var res []*EnvResp + reg, err := commonservice.FindDefaultRegistry(log) + if err != nil { + log.Errorf("FindDefaultRegistry error: %v", err) + return nil, err + } for _, env := range envs { clusterID := env.ClusterID production := false clusterName := "" cluster, ok := clusterMap[clusterID] + if len(env.RegistryID) == 0 { + env.RegistryID = reg.ID.Hex() + } if ok { production = cluster.Production clusterName = cluster.Name @@ -201,6 +215,8 @@ func ListProducts(projectName string, envNames []string, log *zap.SugaredLogger) Error: env.Error, UpdateTime: env.UpdateTime, UpdateBy: env.UpdateBy, + RegistryID: env.RegistryID, + ClusterID: env.ClusterID, }) } @@ -275,7 +291,7 @@ func AutoUpdateProduct(envNames []string, productName, requestID string, force b continue } - kubeClient, err := kube.GetKubeClient(p.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), p.ClusterID) if err != nil { log.Errorf("Failed to get kube client for %s, error: %v", productName, err) continue @@ -387,7 +403,7 @@ func UpdateProduct(existedProd, updateProd *commonmodels.Product, renderSet *com return } - kubeClient, err := kube.GetKubeClient(existedProd.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), existedProd.ClusterID) if err != nil { return e.ErrUpdateEnv.AddErr(err) } @@ -522,6 +538,31 @@ func UpdateProduct(existedProd, updateProd *commonmodels.Product, renderSet *com return nil } +func UpdateProductRegistry(namespace, registryID string, log *zap.SugaredLogger) (err error) { + opt := &commonrepo.ProductFindOptions{Namespace: namespace} + exitedProd, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + log.Errorf("UpdateProductRegistry find product by namespace:%s,error: %v", namespace, err) + return e.ErrUpdateEnv.AddDesc(e.EnvNotFoundErrMsg) + } + err = commonrepo.NewProductColl().UpdateRegistry(namespace, registryID) + if err != nil { + log.Errorf("UpdateProductRegistry UpdateRegistry by namespace:%s registryID:%s error: %v", namespace, registryID, err) + return e.ErrUpdateEnv.AddErr(err) + } + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), exitedProd.ClusterID) + if err != nil { + return e.ErrUpdateEnv.AddErr(err) + } + err = ensureKubeEnv(exitedProd.Namespace, registryID, kubeClient, log) + + if err != nil { + log.Errorf("UpdateProductRegistry ensureKubeEnv by namespace:%s,error: %v", namespace, err) + return err + } + return nil +} + func UpdateProductV2(envName, productName, user, requestID string, force bool, kvs []*template.RenderKV, log *zap.SugaredLogger) (err error) { // 根据产品名称和产品创建者到数据库中查找已有产品记录 opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} @@ -531,7 +572,7 @@ func UpdateProductV2(envName, productName, user, requestID string, force bool, k return e.ErrUpdateEnv.AddDesc(e.EnvNotFoundErrMsg) } - kubeClient, err := kube.GetKubeClient(exitedProd.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), exitedProd.ClusterID) if err != nil { return e.ErrUpdateEnv.AddErr(err) } @@ -547,7 +588,7 @@ func UpdateProductV2(envName, productName, user, requestID string, force bool, k } } - err = ensureKubeEnv(exitedProd.Namespace, kubeClient, log) + err = ensureKubeEnv(exitedProd.Namespace, exitedProd.RegistryID, kubeClient, log) if err != nil { log.Errorf("[%s][P:%s] service.UpdateProductV2 create kubeEnv error: %v", envName, productName, err) @@ -698,7 +739,7 @@ func CreateHelmProduct(productName, userName, requestID string, args []*CreateHe errList := new(multierror.Error) for _, arg := range args { - err = createSingleHelmProduct(templateProduct, serviceGroup, requestID, userName, arg, log) + err = createSingleHelmProduct(templateProduct, serviceGroup, requestID, userName, arg.RegistryID, arg, log) if err != nil { errList = multierror.Append(errList, err) } @@ -706,7 +747,7 @@ func CreateHelmProduct(productName, userName, requestID string, args []*CreateHe return errList.ErrorOrNil() } -func createSingleHelmProduct(templateProduct *template.Product, serviceGroup [][]*commonmodels.ProductService, requestID, userName string, arg *CreateHelmProductArg, log *zap.SugaredLogger) error { +func createSingleHelmProduct(templateProduct *template.Product, serviceGroup [][]*commonmodels.ProductService, requestID, userName, registryID string, arg *CreateHelmProductArg, log *zap.SugaredLogger) error { productObj := &commonmodels.Product{ ProductName: templateProduct.ProductName, Revision: 1, @@ -721,6 +762,7 @@ func createSingleHelmProduct(templateProduct *template.Product, serviceGroup [][ IsOpenSource: templateProduct.IsOpensource, ChartInfos: templateProduct.ChartInfos, IsForkedProduct: false, + RegistryID: registryID, } customChartValueMap := make(map[string]*commonservice.RenderChartArg) @@ -859,6 +901,7 @@ func prepareEstimatedData(productName, envName, serviceName, usageScenario, defa return "", "", fmt.Errorf("failed to query renderset info, name %s", productInfo.Render.Name) } + // find target render chart from render set var targetChart *templatemodels.RenderChart for _, chart := range renderSet.ChartInfos { if chart.ServiceName == serviceName { @@ -867,27 +910,30 @@ func prepareEstimatedData(productName, envName, serviceName, usageScenario, defa } } - if targetChart == nil { - return "", "", fmt.Errorf("failed to find chart info, name: %s", serviceName) - } - switch usageScenario { case usageScenarioUpdateEnv: imageRelatedKey := sets.NewString() - if templateService != nil { - for _, container := range templateService.Containers { - if container.ImagePath != nil { - imageRelatedKey.Insert(container.ImagePath.Image, container.ImagePath.Repo, container.ImagePath.Tag) - } + for _, container := range templateService.Containers { + if container.ImagePath != nil { + imageRelatedKey.Insert(container.ImagePath.Image, container.ImagePath.Repo, container.ImagePath.Tag) } } + + curValuesYaml := "" + if targetChart != nil { // service has been applied into environment, use current values.yaml + curValuesYaml = targetChart.ValuesYaml + } + // merge environment values - mergedBs, err := overrideValues([]byte(targetChart.ValuesYaml), []byte(templateService.HelmChart.ValuesYaml), imageRelatedKey) + mergedBs, err := overrideValues([]byte(curValuesYaml), []byte(templateService.HelmChart.ValuesYaml), imageRelatedKey) if err != nil { return "", "", errors.Wrapf(err, "failed to override values") } return string(mergedBs), renderSet.DefaultValues, nil case usageScenarioUpdateRenderSet: + if targetChart == nil { + return "", "", fmt.Errorf("failed to find chart info, name: %s", serviceName) + } return targetChart.ValuesYaml, renderSet.DefaultValues, nil default: return "", "", fmt.Errorf("unrecognized usageScenario:%s", usageScenario) @@ -927,9 +973,15 @@ func checkOverrideValuesChange(source *template.RenderChart, args *commonservice } func UpdateHelmProductRenderset(productName, envName, userName, requestID string, args *EnvRendersetArg, log *zap.SugaredLogger) error { - renderSetName := commonservice.GetProductEnvNamespace(envName, productName, "") - - opt := &commonrepo.RenderSetFindOption{Name: renderSetName} + product, err := commonrepo.NewProductColl().Find(&commonrepo.ProductFindOptions{ + Name: productName, + EnvName: envName, + }) + if err != nil { + log.Errorf("UpdateHelmProductRenderset GetProductEnv envName:%s productName: %s error, error msg:%s", envName, productName, err) + return err + } + opt := &commonrepo.RenderSetFindOption{Name: product.Namespace} productRenderset, _, err := commonrepo.NewRenderSetColl().FindRenderSet(opt) if err != nil || productRenderset == nil { if err != nil { @@ -969,7 +1021,16 @@ func UpdateHelmProductRenderset(productName, envName, userName, requestID string updatedRcList = append(updatedRcList, updatedRc) } - return UpdateHelmProductVariable(productName, envName, userName, requestID, updatedRcList, productRenderset, log) + err = UpdateHelmProductVariable(productName, envName, userName, requestID, updatedRcList, productRenderset, log) + if err != nil { + return err + } + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) + if err != nil { + log.Errorf("UpdateHelmProductRenderset GetKubeClient error, error msg:%s", err) + return err + } + return ensureKubeEnv(product.Namespace, product.RegistryID, kubeClient, log) } func UpdateHelmProductVariable(productName, envName, username, requestID string, updatedRcs []*template.RenderChart, renderset *commonmodels.RenderSet, log *zap.SugaredLogger) error { @@ -1844,7 +1905,8 @@ func waitResourceRunning( }) } -func preCreateProduct(envName string, args *commonmodels.Product, kubeClient client.Client, log *zap.SugaredLogger) error { +func preCreateProduct(envName string, args *commonmodels.Product, kubeClient client.Client, + log *zap.SugaredLogger) error { var ( productTemplateName = args.ProductName renderSetName = commonservice.GetProductEnvNamespace(envName, args.ProductName, args.Namespace) @@ -1925,7 +1987,7 @@ func preCreateProduct(envName string, args *commonmodels.Product, kubeClient cli args.Render = tmpRenderInfo if preCreateNSAndSecret(productTmpl.ProductFeature) { - return ensureKubeEnv(args.Namespace, kubeClient, log) + return ensureKubeEnv(args.Namespace, args.RegistryID, kubeClient, log) } return nil } @@ -2037,7 +2099,7 @@ func applySystemImagePullSecrets(podSpec *corev1.PodSpec) { }) } -func ensureKubeEnv(namespace string, kubeClient client.Client, log *zap.SugaredLogger) error { +func ensureKubeEnv(namespace string, registryId string, kubeClient client.Client, log *zap.SugaredLogger) error { err := kube.CreateNamespace(namespace, kubeClient) if err != nil { log.Errorf("[%s] get or create namespace error: %v", namespace, err) @@ -2045,7 +2107,7 @@ func ensureKubeEnv(namespace string, kubeClient client.Client, log *zap.SugaredL } // 创建默认的镜像仓库secret - if err := commonservice.EnsureDefaultRegistrySecret(namespace, kubeClient, log); err != nil { + if err := commonservice.EnsureDefaultRegistrySecret(namespace, registryId, kubeClient, log); err != nil { log.Errorf("[%s] get or create namespace error: %v", namespace, err) return e.ErrCreateSecret.AddDesc(e.CreateDefaultRegistryErrMsg) } @@ -2577,7 +2639,7 @@ func overrideValues(currentValuesYaml, latestValuesYaml []byte, imageRelatedKey } if len(replaceMap) == 0 { - return nil, nil + return latestValuesYaml, nil } var replaceKV []string diff --git a/pkg/microservice/aslan/core/environment/service/environment_creator.go b/pkg/microservice/aslan/core/environment/service/environment_creator.go index f8c5a8d64dbe96f595a369384f5461ccca56f0e2..5c32ebfd74f496b1900f1ee71298360163e8744e 100644 --- a/pkg/microservice/aslan/core/environment/service/environment_creator.go +++ b/pkg/microservice/aslan/core/environment/service/environment_creator.go @@ -30,6 +30,7 @@ import ( commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/helmclient" "github.com/koderover/zadig/pkg/tool/kube/getter" @@ -41,6 +42,7 @@ type CreateProductParam struct { ProductName string EnvType string log *zap.SugaredLogger + RegistryID string } type AutoCreator struct { @@ -84,6 +86,7 @@ func (autoCreator *AutoCreator) Create(envName string) (string, error) { productObject.Namespace = commonservice.GetProductEnvNamespace(envName, productName, "") productObject.UpdateBy = autoCreator.Param.UserName productObject.EnvName = envName + productObject.RegistryID = autoCreator.Param.RegistryID if autoCreator.Param.EnvType == setting.HelmDeployType { productObject.Source = setting.SourceFromHelm } @@ -120,7 +123,7 @@ func newHelmProductCreator() *HelmProductCreator { } func (creator *HelmProductCreator) Create(user, requestID string, args *models.Product, log *zap.SugaredLogger) error { - kubeClient, err := kube.GetKubeClient(args.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), args.ClusterID) if err != nil { log.Errorf("[%s][%s] GetKubeClient error: %v", args.EnvName, args.ProductName, err) return e.ErrCreateEnv.AddErr(err) @@ -277,7 +280,7 @@ func newDefaultProductCreator() *DefaultProductCreator { } func (creator *DefaultProductCreator) Create(user, requestID string, args *models.Product, log *zap.SugaredLogger) error { - kubeClient, err := kube.GetKubeClient(args.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), args.ClusterID) if err != nil { return e.ErrCreateEnv.AddErr(err) } diff --git a/pkg/microservice/aslan/core/environment/service/environment_group.go b/pkg/microservice/aslan/core/environment/service/environment_group.go index 70e8f6890a044400f6a0897c1fb6d1539c3eac41..ac0fabe6a2945eb0687504d717046924ce416a9f 100644 --- a/pkg/microservice/aslan/core/environment/service/environment_group.go +++ b/pkg/microservice/aslan/core/environment/service/environment_group.go @@ -24,11 +24,13 @@ import ( "helm.sh/helm/v3/pkg/releaseutil" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" @@ -65,7 +67,7 @@ func ListGroups(serviceName, envName, productName string, perPage, page int, log //将获取到的所有服务按照名称进行排序 sort.SliceStable(allServices, func(i, j int) bool { return allServices[i].ServiceName < allServices[j].ServiceName }) - kubeClient, err := kube.GetKubeClient(productInfo.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), productInfo.ClusterID) if err != nil { log.Errorf("[%s][%s] error: %v", envName, productName, err) return resp, count, e.ErrListGroups.AddDesc(err.Error()) diff --git a/pkg/microservice/aslan/core/environment/service/export.go b/pkg/microservice/aslan/core/environment/service/export.go index 035c30d5b1c65abd80e5839ff1132d2bf4ba2798..acf9aa6b0959e5f7ecf2bc8e7cd9b8a52e1fa513 100644 --- a/pkg/microservice/aslan/core/environment/service/export.go +++ b/pkg/microservice/aslan/core/environment/service/export.go @@ -21,9 +21,10 @@ import ( "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/tool/kube/getter" ) @@ -39,7 +40,7 @@ func ExportYaml(envName, productName, serviceName string, log *zap.SugaredLogger } namespace := env.Namespace - kubeClient, err := kube.GetKubeClient(env.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), env.ClusterID) if err != nil { log.Errorf("cluster is not connected [%s][%s][%s]", env.EnvName, env.ProductName, env.ClusterID) return res diff --git a/pkg/microservice/aslan/core/environment/service/helm.go b/pkg/microservice/aslan/core/environment/service/helm.go new file mode 100644 index 0000000000000000000000000000000000000000..255a2ba7dfd4a58b9ce2d68742a9a46ca9b1a080 --- /dev/null +++ b/pkg/microservice/aslan/core/environment/service/helm.go @@ -0,0 +1,259 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/hashicorp/go-multierror" + "github.com/otiai10/copy" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" + "github.com/koderover/zadig/pkg/setting" + e "github.com/koderover/zadig/pkg/tool/errors" + helmtool "github.com/koderover/zadig/pkg/tool/helmclient" + "github.com/koderover/zadig/pkg/tool/log" + "github.com/koderover/zadig/pkg/types" + "github.com/koderover/zadig/pkg/util" +) + +type HelmReleaseResp struct { + ReleaseName string `json:"releaseName"` + ServiceName string `json:"serviceName"` + Revision int `json:"revision"` + Chart string `json:"chart"` + AppVersion string `json:"appVersion"` +} + +type ChartInfo struct { + ServiceName string `json:"serviceName"` + Revision int64 `json:"revision"` +} + +type HelmChartsResp struct { + ChartInfos []*ChartInfo `json:"chartInfos"` + FileInfos []*types.FileInfo `json:"fileInfos"` +} + +func ListReleases(productName, envName string, log *zap.SugaredLogger) ([]*HelmReleaseResp, error) { + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + prod, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + return nil, e.ErrCreateDeliveryVersion.AddDesc(err.Error()) + } + + restConfig, err := kube.GetRESTConfig(prod.ClusterID) + if err != nil { + log.Errorf("GetRESTConfig error: %v", err) + return nil, e.ErrCreateDeliveryVersion.AddDesc(err.Error()) + } + helmClient, err := helmtool.NewClientFromRestConf(restConfig, prod.Namespace) + if err != nil { + log.Errorf("[%s][%s] NewClientFromRestConf error: %v", envName, productName, err) + return nil, e.ErrCreateDeliveryVersion.AddErr(err) + } + + releases, err := helmClient.ListDeployedReleases() + if err != nil { + return nil, e.ErrCreateDeliveryVersion.AddErr(err) + } + + ret := make([]*HelmReleaseResp, 0, len(releases)) + for _, release := range releases { + ret = append(ret, &HelmReleaseResp{ + ReleaseName: release.Name, + ServiceName: util.ExtraServiceName(release.Name, prod.Namespace), + Revision: release.Version, + Chart: release.Chart.Name(), + AppVersion: release.Chart.AppVersion(), + }) + } + return ret, nil +} + +func loadChartFilesInfo(productName, serviceName string, revision int64, dir string) ([]*types.FileInfo, error) { + base := config.LocalServicePathWithRevision(productName, serviceName, revision) + + var fis []*types.FileInfo + files, err := os.ReadDir(filepath.Join(base, serviceName, dir)) + if err != nil { + log.Warnf("failed to read chart info for service %s with revision %d", serviceName, revision) + base = config.LocalServicePath(productName, serviceName) + files, err = os.ReadDir(filepath.Join(base, serviceName, dir)) + if err != nil { + return nil, err + } + } + + for _, file := range files { + info, _ := file.Info() + if info == nil { + continue + } + fi := &types.FileInfo{ + Parent: dir, + Name: file.Name(), + Size: info.Size(), + Mode: file.Type(), + ModTime: info.ModTime().Unix(), + IsDir: file.IsDir(), + } + + fis = append(fis, fi) + } + return fis, nil +} + +//prepare chart version data +func prepareChartVersionData(productName string, serviceObj *models.Service, renderChart *template.RenderChart, renderset *models.RenderSet) error { + serviceName, revision := serviceObj.ServiceName, serviceObj.Revision + base := config.LocalServicePathWithRevision(productName, serviceName, revision) + if err := commonservice.PreloadServiceManifestsByRevision(base, serviceObj); err != nil { + log.Warnf("failed to get chart of revision: %d for service: %s, use latest version", revision, serviceName) + // use the latest version when it fails to download the specific version + base = config.LocalServicePath(productName, serviceName) + if err = commonservice.PreLoadServiceManifests(base, serviceObj); err != nil { + log.Errorf("failed to load chart info for service %v", serviceObj.ServiceName) + return err + } + } + + fullPath := filepath.Join(base, serviceObj.ServiceName) + deliveryChartPath := filepath.Join(config.LocalDeliveryChartPathWithRevision(productName, serviceObj.ServiceName, serviceObj.Revision), serviceObj.ServiceName) + err := copy.Copy(fullPath, deliveryChartPath) + if err != nil { + return err + } + + mergedValuesYaml, err := helmtool.MergeOverrideValues(renderChart.ValuesYaml, renderset.DefaultValues, renderChart.GetOverrideYaml(), renderChart.OverrideValues) + if err != nil { + return err + } + + // write values.yaml + if err = os.WriteFile(filepath.Join(deliveryChartPath, setting.ValuesYaml), []byte(mergedValuesYaml), 0644); err != nil { + return err + } + + return nil +} + +func GetChartInfos(productName, envName, serviceName string, log *zap.SugaredLogger) (*HelmChartsResp, error) { + opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} + prod, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + return nil, e.ErrGetHelmCharts.AddErr(err) + } + renderSet, err := FindHelmRenderSet(productName, prod.Render.Name, log) + if err != nil { + log.Errorf("[%s][P:%s] find product renderset error: %v", envName, productName, err) + return nil, e.ErrGetHelmCharts.AddErr(err) + } + + chartMap := make(map[string]*template.RenderChart) + for _, chart := range renderSet.ChartInfos { + chartMap[chart.ServiceName] = chart + } + + allServiceMap := prod.GetServiceMap() + serviceMap := make(map[string]*models.ProductService) + + //validate data, make sure service and chart info exists + if len(serviceName) > 0 { + serviceList := strings.Split(serviceName, ",") + for _, singleService := range serviceList { + if service, ok := allServiceMap[singleService]; ok { + serviceMap[service.ServiceName] = service + } else { + return nil, e.ErrGetHelmCharts.AddDesc(fmt.Sprintf("failed to find service %s in target namespace", singleService)) + } + } + } else { + serviceMap = allServiceMap + } + + if len(serviceMap) == 0 { + return nil, nil + } + + ret := &HelmChartsResp{ + ChartInfos: make([]*ChartInfo, 0), + FileInfos: make([]*types.FileInfo, 0), + } + + errList := new(multierror.Error) + wg := sync.WaitGroup{} + + for _, service := range serviceMap { + ret.ChartInfos = append(ret.ChartInfos, &ChartInfo{ + ServiceName: service.ServiceName, + Revision: service.Revision, + }) + wg.Add(1) + // download chart info with particular version + go func(serviceName string, revision int64) { + defer wg.Done() + serviceObj, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ + ProductName: productName, + ServiceName: serviceName, + Revision: revision, + Type: setting.HelmDeployType, + }) + if err != nil { + log.Errorf("failed to query services name: %s, revision: %d, error: %s", serviceName, revision, err) + errList = multierror.Append(errList, fmt.Errorf("failed to query service, serviceName: %s, revision: %d", serviceName, revision)) + return + } + renderChart, ok := chartMap[serviceName] + if !ok { + errList = multierror.Append(errList, fmt.Errorf("failed to find render chart for service %s in target namespace", serviceName)) + return + } + err = prepareChartVersionData(productName, serviceObj, renderChart, renderSet) + if err != nil { + errList = multierror.Append(errList, fmt.Errorf("failed to prepare chart info for service %s", serviceObj.ServiceName)) + return + } + }(service.ServiceName, service.Revision) + } + wg.Wait() + + if errList.ErrorOrNil() != nil { + return nil, errList.ErrorOrNil() + } + + // expand file info for first service + serviceToExpand := ret.ChartInfos[0].ServiceName + fis, err := loadChartFilesInfo(productName, serviceToExpand, serviceMap[serviceToExpand].Revision, "") + if err != nil { + log.Errorf("Failed to load service file info, err: %s", err) + return nil, e.ErrListTemplate.AddErr(err) + } + ret.FileInfos = fis + + return ret, nil +} diff --git a/pkg/microservice/aslan/core/environment/service/image.go b/pkg/microservice/aslan/core/environment/service/image.go index b4312a5b6f015569abb123b32776b3e80e9f97e4..e6d344b0b4dd9ff4cb619365cf4ca84c2920f44a 100644 --- a/pkg/microservice/aslan/core/environment/service/image.go +++ b/pkg/microservice/aslan/core/environment/service/image.go @@ -27,12 +27,14 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/yaml" + "github.com/koderover/zadig/pkg/microservice/aslan/config" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" templatemodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/template" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" helmtool "github.com/koderover/zadig/pkg/tool/helmclient" "github.com/koderover/zadig/pkg/tool/kube/updater" @@ -291,7 +293,7 @@ func UpdateContainerImage(requestID string, args *UpdateContainerImageArgs, log } namespace := product.Namespace - kubeClient, err := kube.GetKubeClient(product.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) if err != nil { return e.ErrUpdateConainterImage.AddErr(err) } diff --git a/pkg/microservice/aslan/core/environment/service/k8s.go b/pkg/microservice/aslan/core/environment/service/k8s.go index 3f922b06ad1d2df98953311ec55ebb1d97cd7e82..24e4f5a1278c97184c9e4d02c9ef9d2c10455d9d 100644 --- a/pkg/microservice/aslan/core/environment/service/k8s.go +++ b/pkg/microservice/aslan/core/environment/service/k8s.go @@ -25,17 +25,16 @@ import ( "github.com/hashicorp/go-multierror" "go.uber.org/zap" - "sigs.k8s.io/controller-runtime/pkg/client" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" templaterepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb/template" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" ) @@ -84,7 +83,7 @@ func (k *K8sService) updateService(args *SvcOptArgs) error { return errors.New(e.UpsertServiceErrMsg) } - kubeClient, err := kube.GetKubeClient(exitedProd.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), exitedProd.ClusterID) if err != nil { return e.ErrUpdateEnv.AddErr(err) } diff --git a/pkg/microservice/aslan/core/environment/service/kube.go b/pkg/microservice/aslan/core/environment/service/kube.go index 7c8e0fa399b5ec7db6b531ee6da202e7127b5ab2..8bd9222b74db36c6bae334d371c016697bf4526e 100644 --- a/pkg/microservice/aslan/core/environment/service/kube.go +++ b/pkg/microservice/aslan/core/environment/service/kube.go @@ -21,14 +21,18 @@ import ( "time" "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" @@ -109,7 +113,7 @@ func ListPodEvents(envName, productName, podName string, log *zap.SugaredLogger) // ListAvailableNamespaces lists available namespaces created by non-koderover func ListAvailableNamespaces(clusterID string, log *zap.SugaredLogger) ([]*resource.Namespace, error) { resp := make([]*resource.Namespace, 0) - kubeClient, err := kube.GetKubeClient(clusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) if err != nil { log.Errorf("ListNamespaces clusterID:%s err:%v", clusterID, err) return resp, err @@ -118,11 +122,11 @@ func ListAvailableNamespaces(clusterID string, log *zap.SugaredLogger) ([]*resou if err != nil { log.Errorf("ListNamespaces err:%v", err) if apierrors.IsForbidden(err) { - return resp, nil + return resp, err } return resp, err } - + filterK8sNamespaces := sets.NewString("kube-node-lease", "kube-public", "kube-system") for _, namespace := range namespaces { if value, IsExist := namespace.Labels[setting.EnvCreatedBy]; IsExist { if value == setting.EnvCreator { @@ -130,6 +134,10 @@ func ListAvailableNamespaces(clusterID string, log *zap.SugaredLogger) ([]*resou } } + if filterK8sNamespaces.Has(namespace.Name) { + continue + } + resp = append(resp, wrapper.Namespace(namespace).Resource()) } @@ -146,7 +154,7 @@ func ListServicePods(productName, envName string, serviceName string, log *zap.S if err != nil { return res, e.ErrListServicePod.AddErr(err) } - kubeClient, err := kube.GetKubeClient(product.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) if err != nil { return res, e.ErrListServicePod.AddErr(err) } @@ -173,7 +181,7 @@ func DeletePod(envName, productName, podName string, log *zap.SugaredLogger) err if err != nil { return e.ErrDeletePod.AddErr(err) } - kubeClient, err := kube.GetKubeClient(product.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) if err != nil { return e.ErrDeletePod.AddErr(err) } @@ -223,3 +231,56 @@ func getModifiedServiceFromObjectMeta(om metav1.Object) *serviceInfo { LastUpdateTime: t, } } + +func ListAvailableNodes(clusterID string, log *zap.SugaredLogger) (*NodeResp, error) { + resp := new(NodeResp) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) + if err != nil { + log.Errorf("ListAvailableNodes clusterID:%s err:%s", clusterID, err) + return resp, err + } + + nodes, err := getter.ListNodes(kubeClient) + if err != nil { + log.Errorf("ListNodes err:%s", err) + if apierrors.IsForbidden(err) { + return resp, err + } + return resp, err + } + + nodeInfos := make([]*resource.Node, 0) + labels := sets.NewString() + for _, node := range nodes { + nodeResource := &resource.Node{ + Ready: nodeReady(node), + Labels: nodeLabel(node), + IP: node.Name, + } + nodeInfos = append(nodeInfos, nodeResource) + labels.Insert(nodeResource.Labels...) + } + resp.Nodes = nodeInfos + resp.Labels = labels.List() + return resp, nil +} + +// Ready indicates that the node is ready for traffic. +func nodeReady(node *corev1.Node) bool { + cs := node.Status.Conditions + for _, c := range cs { + if c.Type == corev1.NodeReady && c.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +func nodeLabel(node *corev1.Node) []string { + labels := make([]string, 0, len(node.Labels)) + labelM := node.Labels + for key, value := range labelM { + labels = append(labels, fmt.Sprintf("%s:%s", key, value)) + } + return labels +} diff --git a/pkg/microservice/aslan/core/environment/service/product.go b/pkg/microservice/aslan/core/environment/service/product.go index 88d38802cb64fe89299e1c7a99b16a81db4b01c3..7bd248130dff0f8b3f9d08bea377e7e9a356ff38 100644 --- a/pkg/microservice/aslan/core/environment/service/product.go +++ b/pkg/microservice/aslan/core/environment/service/product.go @@ -158,7 +158,7 @@ func GetProduct(username, envName, productName string, log *zap.SugaredLogger) ( opt := &commonrepo.ProductFindOptions{Name: productName, EnvName: envName} prod, err := commonrepo.NewProductColl().Find(opt) if err != nil { - log.Errorf("[User:%s][EnvName:%s][Product:%s] Product.FindByOwner error: %v", username, envName, productName, err) + log.Errorf("[User:%s][EnvName:%s][Product:%s] Product.FindByOwner error: %s", username, envName, productName, err) return nil, e.ErrGetEnv } @@ -169,6 +169,14 @@ func GetProduct(username, envName, productName string, log *zap.SugaredLogger) ( } } + if len(prod.RegistryID) == 0 { + reg, err := commonservice.FindDefaultRegistry(log) + if err != nil { + log.Errorf("[User:%s][EnvName:%s][Product:%s] FindDefaultRegistry error: %s", username, envName, productName, err) + return nil, err + } + prod.RegistryID = reg.ID.Hex() + } resp := buildProductResp(prod.EnvName, prod, log) return resp, nil } @@ -190,6 +198,7 @@ func buildProductResp(envName string, prod *commonmodels.Product, log *zap.Sugar ClusterID: prod.ClusterID, RecycleDay: prod.RecycleDay, Source: prod.Source, + RegisterID: prod.RegistryID, } if prod.ClusterID != "" { @@ -207,12 +216,15 @@ func buildProductResp(envName string, prod *commonmodels.Product, log *zap.Sugar } prodResp.IsProd = cluster.Production prodResp.ClusterName = cluster.Name + prodResp.IsLocal = cluster.Local - if !clusterService.ClusterConnected(prod.ClusterID) { + if !prodResp.IsLocal && !clusterService.ClusterConnected(prod.ClusterID) { prodResp.Status = setting.ClusterDisconnected prodResp.Error = "集群未连接" return prodResp } + } else { + prodResp.IsLocal = true } if prod.Status == setting.ProductStatusCreating { diff --git a/pkg/microservice/aslan/core/environment/service/service.go b/pkg/microservice/aslan/core/environment/service/service.go index 630d376478aaf4bba2daec1a57e03b2a1d872116..572d6d878b8be3966f8681a12f4a3869eb03fac3 100644 --- a/pkg/microservice/aslan/core/environment/service/service.go +++ b/pkg/microservice/aslan/core/environment/service/service.go @@ -27,11 +27,13 @@ import ( "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" internalresource "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" @@ -59,7 +61,7 @@ func ScaleService(envName, productName, serviceName string, number int, log *zap return e.ErrScaleService.AddErr(err) } - kubeClient, err := kube.GetKubeClient(prod.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), prod.ClusterID) if err != nil { return e.ErrScaleService.AddErr(err) } @@ -110,7 +112,7 @@ func Scale(args *ScaleArgs, logger *zap.SugaredLogger) error { return e.ErrScaleService.AddErr(err) } - kubeClient, err := kube.GetKubeClient(prod.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), prod.ClusterID) if err != nil { return e.ErrScaleService.AddErr(err) } @@ -140,7 +142,7 @@ func RestartScale(args *RestartScaleArgs, _ *zap.SugaredLogger) error { return err } - kubeClient, err := kube.GetKubeClient(prod.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), prod.ClusterID) if err != nil { return err } @@ -176,7 +178,7 @@ func GetService(envName, productName, serviceName string, workLoadType string, l return nil, e.ErrGetService.AddErr(err) } - kubeClient, err := kube.GetKubeClient(env.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), env.ClusterID) if err != nil { return nil, e.ErrGetService.AddErr(err) } @@ -322,7 +324,7 @@ func RestartService(envName string, args *SvcOptArgs, log *zap.SugaredLogger) (e if err != nil { return err } - kubeClient, err := kube.GetKubeClient(productObj.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), productObj.ClusterID) if err != nil { return err } @@ -421,7 +423,7 @@ func validateServiceContainer(envName, productName, serviceName, container strin return "", err } - kubeClient, err := kube.GetKubeClient(prod.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), prod.ClusterID) if err != nil { return "", err } diff --git a/pkg/microservice/aslan/core/environment/service/types.go b/pkg/microservice/aslan/core/environment/service/types.go index 60942bf80bce957cee213812eec0f89f41914fc3..f3c0af942b654ff2374dc19fb709bd9b06f64d23 100644 --- a/pkg/microservice/aslan/core/environment/service/types.go +++ b/pkg/microservice/aslan/core/environment/service/types.go @@ -115,3 +115,8 @@ type ContainerNotFound struct { func (c *ContainerNotFound) Error() string { return fmt.Sprintf("serviceName:%s,container:%s", c.ServiceName, c.Container) } + +type NodeResp struct { + Nodes []*internalresource.Node `json:"data"` + Labels []string `json:"labels"` +} diff --git a/pkg/microservice/aslan/core/log/service/sse.go b/pkg/microservice/aslan/core/log/service/sse.go index eb7872dd3f3c0d74a72efac60f4419204a8d69ea..1cd19e6d3dad11320038180b40ee15b4fc416a3b 100644 --- a/pkg/microservice/aslan/core/log/service/sse.go +++ b/pkg/microservice/aslan/core/log/service/sse.go @@ -32,7 +32,7 @@ import ( commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" - krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/tool/kube/containerlog" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/kube/watcher" @@ -53,6 +53,7 @@ type GetContainerOptions struct { TestName string EnvName string ProductName string + ClusterID string } func ContainerLogStream(ctx context.Context, streamChan chan interface{}, envName, productName, podName, containerName string, follow bool, tailLines int64, log *zap.SugaredLogger) { @@ -119,13 +120,29 @@ func TaskContainerLogStream(ctx context.Context, streamChan chan interface{}, op return } log.Debugf("Start to get task container log.") - if options.EnvName != "" && options.ProductName != "" { + // Cloud host scenario reads real-time logs from the environment, so pipelineName is empty + if options.EnvName != "" && options.ProductName != "" && options.PipelineName == "" { //修改pipelineName，判断pipelineName是否为空，为空代表是来自环境里面请求，不为空代表是来自工作流任务的请求 - if options.PipelineName == "" { - options.PipelineName = fmt.Sprintf("%s-%s-%s", options.ServiceName, options.EnvName, "job") - if taskObj, err := commonrepo.NewTaskColl().FindTask(options.PipelineName, config.ServiceType); err == nil { - options.TaskID = taskObj.TaskID - } + options.PipelineName = fmt.Sprintf("%s-%s-%s", options.ServiceName, options.EnvName, "job") + if taskObj, err := commonrepo.NewTaskColl().FindTask(options.PipelineName, config.ServiceType); err == nil { + options.TaskID = taskObj.TaskID + } + // Need to get build info based on the project name and service component name, then get clusterID and namespace + } else if options.ProductName != "" { + build, err := commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{ + ProductName: options.ProductName, + Targets: []string{options.ServiceName}, + }) + if err != nil { + // Maybe this service is a shared service + build, err = commonrepo.NewBuildColl().Find(&commonrepo.BuildFindOption{ + Targets: []string{options.ServiceName}, + }) + } + // Compatible with the situation where the old data has not been modified + if build != nil && build.PreBuild != nil && build.PreBuild.ClusterID != "" && build.PreBuild.Namespace != "" { + options.ClusterID = build.PreBuild.ClusterID + options.Namespace = build.PreBuild.Namespace } } @@ -137,24 +154,45 @@ func TaskContainerLogStream(ctx context.Context, streamChan chan interface{}, op } func TestJobContainerLogStream(ctx context.Context, streamChan chan interface{}, options *GetContainerOptions, log *zap.SugaredLogger) { - options.SubTask = string(config.TaskTestingV2) selector := getPipelineSelector(options) + // get cluster ID + testing, _ := commonrepo.NewTestingColl().Find(getTestName(options.ServiceName), "") + // Compatible with the situation where the old data has not been modified + if testing != nil && testing.PreTest != nil && testing.PreTest.ClusterID != "" && testing.PreTest.Namespace != "" { + options.ClusterID = testing.PreTest.ClusterID + options.Namespace = testing.PreTest.Namespace + } waitAndGetLog(ctx, streamChan, selector, options, log) } +func getTestName(serviceName string) string { + testName := strings.TrimRight(serviceName, "-job") + return testName +} + func waitAndGetLog(ctx context.Context, streamChan chan interface{}, selector labels.Selector, options *GetContainerOptions, log *zap.SugaredLogger) { PodCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() log.Debugf("Waiting until pod is running before establishing the stream.") - err := watcher.WaitUntilPodRunning(PodCtx, options.Namespace, selector, krkubeclient.Clientset()) + clientSet, err := kubeclient.GetClientset(config.HubServerAddress(), options.ClusterID) + if err != nil { + log.Errorf("GetContainerLogs, get client set error: %s", err) + return + } + err = watcher.WaitUntilPodRunning(PodCtx, options.Namespace, selector, clientSet) + if err != nil { + log.Errorf("GetContainerLogs, wait pod running error: %s", err) + return + } + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), options.ClusterID) if err != nil { - log.Errorf("GetContainerLogs, wait pod running error: %+v", err) + log.Errorf("GetContainerLogs, get kube client error: %s", err) return } - pods, err := getter.ListPods(options.Namespace, selector, krkubeclient.Client()) + pods, err := getter.ListPods(options.Namespace, selector, kubeClient) if err != nil { log.Errorf("GetContainerLogs, get pod error: %+v", err) return @@ -169,7 +207,7 @@ func waitAndGetLog(ctx context.Context, streamChan chan interface{}, selector la pods[0].Name, options.SubTask, true, options.TailLines, - krkubeclient.Clientset(), + clientSet, log, ) } diff --git a/pkg/microservice/aslan/core/multicluster/handler/clusters.go b/pkg/microservice/aslan/core/multicluster/handler/clusters.go index 42617c9798797cfcfc8ccbc176cd977d20653794..ca1ed7448909620026f5a61c67dd4eea3acec7eb 100644 --- a/pkg/microservice/aslan/core/multicluster/handler/clusters.go +++ b/pkg/microservice/aslan/core/multicluster/handler/clusters.go @@ -22,7 +22,6 @@ import ( "github.com/gin-gonic/gin" - commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/aslan/core/multicluster/service" internalhandler "github.com/koderover/zadig/pkg/shared/handler" e "github.com/koderover/zadig/pkg/tool/errors" @@ -52,7 +51,7 @@ func CreateCluster(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - args := new(commonmodels.K8SCluster) + args := new(service.K8SCluster) if err := c.BindJSON(args); err != nil { ctx.Err = e.ErrInvalidParam.AddErr(err) return @@ -73,7 +72,7 @@ func UpdateCluster(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - args := new(commonmodels.K8SCluster) + args := new(service.K8SCluster) if err := c.BindJSON(args); err != nil { ctx.Err = e.ErrInvalidParam.AddErr(err) return diff --git a/pkg/microservice/aslan/core/multicluster/service/bundle.go b/pkg/microservice/aslan/core/multicluster/service/bundle.go index 6a5d2084a510582a0cda5cc1e044329d93b04911..5eb49f1643241d5a63cf688c0351210c67ba671e 100644 --- a/pkg/microservice/aslan/core/multicluster/service/bundle.go +++ b/pkg/microservice/aslan/core/multicluster/service/bundle.go @@ -20,7 +20,6 @@ import ( "strconv" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" - "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/log" ) @@ -37,14 +36,7 @@ func GetBundleResources() ([]*resourceSpec, error) { return nil, err } - res := []*resourceSpec{ - { - ResourceID: setting.LocalClusterID, - Spec: map[string]interface{}{ - "production": "false", - }, - }, - } + var res []*resourceSpec for _, cluster := range clusters { res = append(res, &resourceSpec{ ResourceID: cluster.ID.Hex(), diff --git a/pkg/microservice/aslan/core/multicluster/service/clusters.go b/pkg/microservice/aslan/core/multicluster/service/clusters.go index ddc196cb109f94002700996d0e035b4e9d02b344..dc7f954a35285e1bf44c1229378198cab4333bcc 100644 --- a/pkg/microservice/aslan/core/multicluster/service/clusters.go +++ b/pkg/microservice/aslan/core/multicluster/service/clusters.go @@ -17,10 +17,14 @@ limitations under the License. package service import ( + "fmt" "net/http" + "regexp" + "strings" "github.com/gin-gonic/gin" "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" configbase "github.com/koderover/zadig/pkg/config" @@ -32,21 +36,39 @@ import ( e "github.com/koderover/zadig/pkg/tool/errors" ) +var namePattern = regexp.MustCompile(`^[0-9a-zA-Z_.-]{1,32}$`) + type K8SCluster struct { - ID string `json:"id"` - Name string `json:"name"` - Description string `json:"description"` - Status setting.K8SClusterStatus `json:"status"` - Production bool `json:"production"` - CreatedAt int64 `json:"createdAt"` - CreatedBy string `json:"createdBy"` - Provider int8 `json:"provider"` + ID string `json:"id,omitempty"` + Name string `json:"name"` + Description string `json:"description"` + AdvancedConfig *AdvancedConfig `json:"advanced_config,omitempty"` + Status setting.K8SClusterStatus `json:"status"` + Production bool `json:"production"` + CreatedAt int64 `json:"createdAt"` + CreatedBy string `json:"createdBy"` + Provider int8 `json:"provider"` + Local bool `json:"local"` +} + +type AdvancedConfig struct { + Strategy string `json:"strategy,omitempty" bson:"strategy,omitempty"` + NodeLabels []string `json:"node_labels,omitempty" bson:"node_labels,omitempty"` +} + +func (k *K8SCluster) Clean() error { + k.Name = strings.TrimSpace(k.Name) + k.Description = strings.TrimSpace(k.Description) + + if !namePattern.MatchString(k.Name) { + return fmt.Errorf("The cluster name does not meet the rules") + } + + return nil } func ListClusters(ids []string, logger *zap.SugaredLogger) ([]*K8SCluster, error) { idSet := sets.NewString(ids...) - localClusterIncluded := idSet.Has(setting.LocalClusterID) - idSet = idSet.Delete(setting.LocalClusterID) cs, err := commonrepo.NewK8SClusterColl().List(&commonrepo.ClusterListOpts{IDs: idSet.UnsortedList()}) if err != nil { logger.Errorf("Failed to list clusters, err: %s", err) @@ -54,25 +76,25 @@ func ListClusters(ids []string, logger *zap.SugaredLogger) ([]*K8SCluster, error } var res []*K8SCluster - if len(ids) == 0 || localClusterIncluded { - res = append(res, &K8SCluster{ - ID: setting.LocalClusterID, - Name: setting.LocalClusterName, - Production: false, - Status: setting.Normal, - }) - } - for _, c := range cs { + var advancedConfig *AdvancedConfig + if c.AdvancedConfig != nil { + advancedConfig = &AdvancedConfig{ + Strategy: c.AdvancedConfig.Strategy, + NodeLabels: convertToNodeLabels(c.AdvancedConfig.NodeLabels), + } + } res = append(res, &K8SCluster{ - ID: c.ID.Hex(), - Name: c.Name, - Description: c.Description, - Status: c.Status, - Production: c.Production, - CreatedBy: c.CreatedBy, - CreatedAt: c.CreatedAt, - Provider: c.Provider, + ID: c.ID.Hex(), + Name: c.Name, + Description: c.Description, + Status: c.Status, + Production: c.Production, + CreatedBy: c.CreatedBy, + CreatedAt: c.CreatedAt, + Provider: c.Provider, + Local: c.Local, + AdvancedConfig: advancedConfig, }) } @@ -85,15 +107,74 @@ func GetCluster(id string, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, return s.GetCluster(id, logger) } -func CreateCluster(cluster *commonmodels.K8SCluster, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, error) { - s, _ := kube.NewService("") +func convertToNodeLabels(nodeSelectorRequirements []*commonmodels.NodeSelectorRequirement) []string { + var nodeLabels []string + for _, nodeSelectorRequirement := range nodeSelectorRequirements { + for _, labelValue := range nodeSelectorRequirement.Value { + nodeLabels = append(nodeLabels, fmt.Sprintf("%s:%s", nodeSelectorRequirement.Key, labelValue)) + } + } - return s.CreateCluster(cluster, logger) + return nodeLabels } -func UpdateCluster(id string, cluster *commonmodels.K8SCluster, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, error) { +func convertToNodeSelectorRequirements(nodeLabels []string) []*commonmodels.NodeSelectorRequirement { + var nodeSelectorRequirements []*commonmodels.NodeSelectorRequirement + nodeLabelM := make(map[string][]string) + for _, nodeLabel := range nodeLabels { + if !strings.Contains(nodeLabel, ":") || len(strings.Split(nodeLabel, ":")) != 2 { + continue + } + key := strings.Split(nodeLabel, ":")[0] + value := strings.Split(nodeLabel, ":")[1] + nodeLabelM[key] = append(nodeLabelM[key], value) + } + for key, value := range nodeLabelM { + nodeSelectorRequirements = append(nodeSelectorRequirements, &commonmodels.NodeSelectorRequirement{ + Key: key, + Value: value, + Operator: corev1.NodeSelectorOpIn, + }) + } + return nodeSelectorRequirements +} + +func CreateCluster(args *K8SCluster, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, error) { s, _ := kube.NewService("") + var advancedConfig *commonmodels.AdvancedConfig + if args.AdvancedConfig != nil { + advancedConfig = &commonmodels.AdvancedConfig{ + Strategy: args.AdvancedConfig.Strategy, + NodeLabels: convertToNodeSelectorRequirements(args.AdvancedConfig.NodeLabels), + } + } + cluster := &commonmodels.K8SCluster{ + Name: args.Name, + Description: args.Description, + AdvancedConfig: advancedConfig, + Status: args.Status, + Production: args.Production, + Provider: args.Provider, + CreatedAt: args.CreatedAt, + CreatedBy: args.CreatedBy, + } + + return s.CreateCluster(cluster, args.ID, logger) +} +func UpdateCluster(id string, args *K8SCluster, logger *zap.SugaredLogger) (*commonmodels.K8SCluster, error) { + s, _ := kube.NewService("") + advancedConfig := new(commonmodels.AdvancedConfig) + if args.AdvancedConfig != nil { + advancedConfig.Strategy = args.AdvancedConfig.Strategy + advancedConfig.NodeLabels = convertToNodeSelectorRequirements(args.AdvancedConfig.NodeLabels) + } + cluster := &commonmodels.K8SCluster{ + Name: args.Name, + Description: args.Description, + AdvancedConfig: advancedConfig, + Production: args.Production, + } return s.UpdateCluster(id, cluster, logger) } diff --git a/pkg/microservice/aslan/core/project/service/project.go b/pkg/microservice/aslan/core/project/service/project.go index 4c9e1fb92efc8a573b1f5ed04dc30bf9629c8ca6..196d63db09dbf082dd9144621e03ac3b09fcca04 100644 --- a/pkg/microservice/aslan/core/project/service/project.go +++ b/pkg/microservice/aslan/core/project/service/project.go @@ -41,12 +41,13 @@ type ProjectListOptions struct { type ProjectDetailedRepresentation struct { *ProjectBriefRepresentation - Alias string `json:"alias"` - Desc string `json:"desc"` - UpdatedAt int64 `json:"updatedAt"` - UpdatedBy string `json:"updatedBy"` - Onboard bool `json:"onboard"` - Public bool `json:"public"` + Alias string `json:"alias"` + Desc string `json:"desc"` + UpdatedAt int64 `json:"updatedAt"` + UpdatedBy string `json:"updatedBy"` + Onboard bool `json:"onboard"` + Public bool `json:"public"` + DeployType string `json:"deployType"` } type ProjectBriefRepresentation struct { @@ -98,12 +99,13 @@ func listDetailedProjectInfos(opts *ProjectListOptions, logger *zap.SugaredLogge ProjectMinimalRepresentation: &ProjectMinimalRepresentation{Name: name}, Envs: nameWithEnvMap[name], }, - Alias: info.Alias, - Desc: info.Desc, - UpdatedAt: info.UpdatedAt, - UpdatedBy: info.UpdatedBy, - Onboard: info.OnboardStatus != 0, - Public: info.Public, + Alias: info.Alias, + Desc: info.Desc, + UpdatedAt: info.UpdatedAt, + UpdatedBy: info.UpdatedBy, + Onboard: info.OnboardStatus != 0, + Public: info.Public, + DeployType: info.DeployType, }) } diff --git a/pkg/microservice/aslan/core/service/handler/helm.go b/pkg/microservice/aslan/core/service/handler/helm.go index 0573b0989f9aab1bdee972c218cba3452efe9c30..d1462e369fe6a9409520e1bdaed977469305c2eb 100644 --- a/pkg/microservice/aslan/core/service/handler/helm.go +++ b/pkg/microservice/aslan/core/service/handler/helm.go @@ -46,13 +46,30 @@ func GetHelmServiceModule(c *gin.Context) { func GetFilePath(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = svcservice.GetFilePath(c.Param("serviceName"), c.Param("productName"), c.Query("dir"), ctx.Logger) + revision := int64(0) + var err error + if len(c.Query("revision")) > 0 { + revision, err = strconv.ParseInt(c.Query("revision"), 10, 64) + } + if err != nil { + ctx.Err = e.ErrInvalidParam.AddDesc("invalid revision number") + return + } + ctx.Resp, ctx.Err = svcservice.GetFilePath(c.Param("serviceName"), c.Param("productName"), revision, c.Query("dir"), ctx.Logger) } func GetFileContent(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - ctx.Resp, ctx.Err = svcservice.GetFileContent(c.Param("serviceName"), c.Param("productName"), c.Query("filePath"), c.Query("fileName"), ctx.Logger) + + param := new(svcservice.GetFileContentParam) + err := c.ShouldBindQuery(param) + if err != nil { + ctx.Err = e.ErrInvalidParam.AddErr(err) + return + } + + ctx.Resp, ctx.Err = svcservice.GetFileContent(c.Param("serviceName"), c.Param("productName"), param, ctx.Logger) } func CreateOrUpdateHelmService(c *gin.Context) { diff --git a/pkg/microservice/aslan/core/service/handler/service.go b/pkg/microservice/aslan/core/service/handler/service.go index bfa9de83f4a8c217411a06c72ea3287b435a78e7..495551d7423f84e0ddd0b35b8d76f9210cf3b4df 100644 --- a/pkg/microservice/aslan/core/service/handler/service.go +++ b/pkg/microservice/aslan/core/service/handler/service.go @@ -191,9 +191,6 @@ func CreateK8sWorkloads(c *gin.Context) { return } - if args.ClusterID == setting.LocalClusterID { - args.ClusterID = "" - } ctx.Err = svcservice.CreateK8sWorkLoads(c, ctx.RequestID, ctx.UserName, args.ProductName, args.WorkLoads, args.ClusterID, args.Namespace, args.EnvName, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/service/service/helm.go b/pkg/microservice/aslan/core/service/service/helm.go index 562b385238f6ca49169b205fbb4460ae635b6f66..a9d2dec2ca004746d394ca885b1f24cdea080ee9 100644 --- a/pkg/microservice/aslan/core/service/service/helm.go +++ b/pkg/microservice/aslan/core/service/service/helm.go @@ -113,6 +113,13 @@ type ChartTemplateData struct { DefaultValuesYAML []byte // content of values.yaml in template } +type GetFileContentParam struct { + FilePath string `json:"filePath" form:"filePath"` + FileName string `json:"fileName" form:"fileName"` + Revision int64 `json:"revision" form:"revision"` + DeliveryVersion bool `json:"deliveryVersion" form:"deliveryVersion"` +} + func ListHelmServices(productName string, log *zap.SugaredLogger) (*HelmService, error) { helmService := &HelmService{ ServiceInfos: []*models.Service{}, @@ -133,7 +140,7 @@ func ListHelmServices(productName string, log *zap.SugaredLogger) (*HelmService, helmService.ServiceInfos = services if len(services) > 0 { - fis, err := loadServiceFileInfos(services[0].ProductName, services[0].ServiceName, "") + fis, err := loadServiceFileInfos(services[0].ProductName, services[0].ServiceName, 0, "") if err != nil { log.Errorf("Failed to load service file info, err: %s", err) return nil, e.ErrListTemplate.AddErr(err) @@ -172,24 +179,39 @@ func GetHelmServiceModule(serviceName, productName string, revision int64, log * return helmServiceModule, err } -func GetFilePath(serviceName, productName, dir string, _ *zap.SugaredLogger) ([]*types.FileInfo, error) { - return loadServiceFileInfos(productName, serviceName, dir) +func GetFilePath(serviceName, productName string, revision int64, dir string, _ *zap.SugaredLogger) ([]*types.FileInfo, error) { + return loadServiceFileInfos(productName, serviceName, revision, dir) } -func GetFileContent(serviceName, productName, filePath, fileName string, log *zap.SugaredLogger) (string, error) { - base := config.LocalServicePath(productName, serviceName) - +func GetFileContent(serviceName, productName string, param *GetFileContentParam, log *zap.SugaredLogger) (string, error) { + filePath, fileName, revision, forDelivery := param.FilePath, param.FileName, param.Revision, param.DeliveryVersion svc, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ ProductName: productName, ServiceName: serviceName, + Revision: revision, }) if err != nil { return "", e.ErrFileContent.AddDesc(err.Error()) } - err = commonservice.PreLoadServiceManifests(base, svc) - if err != nil { - return "", e.ErrFileContent.AddDesc(err.Error()) + base := config.LocalServicePath(productName, serviceName) + if revision > 0 { + base = config.LocalServicePathWithRevision(productName, serviceName, revision) + if err = commonservice.PreloadServiceManifestsByRevision(base, svc); err != nil { + log.Warnf("failed to get chart of revision: %d for service: %s, use latest version", + svc.Revision, svc.ServiceName) + } + } + if err != nil || revision == 0 { + base = config.LocalServicePath(productName, serviceName) + err = commonservice.PreLoadServiceManifests(base, svc) + if err != nil { + return "", e.ErrFileContent.AddDesc(err.Error()) + } + } + + if forDelivery { + base = config.LocalDeliveryChartPathWithRevision(productName, serviceName, revision) } file := filepath.Join(base, serviceName, filePath, fileName) @@ -912,9 +934,7 @@ func createOrUpdateHelmService(fsTree fs.FS, args *helmServiceCreationArgs, logg return serviceObj, nil } -func loadServiceFileInfos(productName, serviceName, dir string) ([]*types.FileInfo, error) { - base := config.LocalServicePath(productName, serviceName) - +func loadServiceFileInfos(productName, serviceName string, revision int64, dir string) ([]*types.FileInfo, error) { svc, err := commonrepo.NewServiceColl().Find(&commonrepo.ServiceFindOption{ ProductName: productName, ServiceName: serviceName, @@ -923,6 +943,22 @@ func loadServiceFileInfos(productName, serviceName, dir string) ([]*types.FileIn return nil, e.ErrFilePath.AddDesc(err.Error()) } + base := config.LocalServicePath(productName, serviceName) + if revision > 0 { + base = config.LocalServicePathWithRevision(productName, serviceName, revision) + if err = commonservice.PreloadServiceManifestsByRevision(base, svc); err != nil { + log.Warnf("failed to get chart of revision: %d for service: %s, use latest version", + svc.Revision, svc.ServiceName) + } + } + if err != nil || revision == 0 { + base = config.LocalServicePath(productName, serviceName) + err = commonservice.PreLoadServiceManifests(base, svc) + if err != nil { + return nil, e.ErrFilePath.AddDesc(err.Error()) + } + } + err = commonservice.PreLoadServiceManifests(base, svc) if err != nil { return nil, e.ErrFilePath.AddDesc(err.Error()) diff --git a/pkg/microservice/aslan/core/service/service/service.go b/pkg/microservice/aslan/core/service/service/service.go index b664967decb3ac8ce7eb5be56bb6966e88697337..1bd83639e2ca2e016625bc62d7237883a2d44ea5 100644 --- a/pkg/microservice/aslan/core/service/service/service.go +++ b/pkg/microservice/aslan/core/service/service/service.go @@ -43,10 +43,10 @@ import ( commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/command" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/fs" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/microservice/aslan/core/environment/service" "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/shared/client/systemconfig" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/gerrit" "github.com/koderover/zadig/pkg/tool/httpclient" @@ -256,7 +256,7 @@ func GetServiceOption(args *commonmodels.Service, log *zap.SugaredLogger) (*Serv } func CreateK8sWorkLoads(ctx context.Context, requestID, username string, productName string, workLoads []models.Workload, clusterID, namespace string, envName string, log *zap.SugaredLogger) error { - kubeClient, err := kube.GetKubeClient(clusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), clusterID) if err != nil { log.Errorf("[%s] error: %v", namespace, err) return err @@ -385,7 +385,7 @@ type UpdateWorkloadsArgs struct { } func UpdateWorkloads(ctx context.Context, requestID, username, productName, envName string, args UpdateWorkloadsArgs, log *zap.SugaredLogger) error { - kubeClient, err := kube.GetKubeClient(args.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), args.ClusterID) if err != nil { log.Errorf("[%s] error: %s", args.Namespace, err) return err diff --git a/pkg/microservice/aslan/core/system/handler/helm.go b/pkg/microservice/aslan/core/system/handler/helm.go index 31a84a53daa99dc9ec778705ea19d41741dc6bc9..7f92809e67cd1d89fee9839566813008f15c5052 100644 --- a/pkg/microservice/aslan/core/system/handler/helm.go +++ b/pkg/microservice/aslan/core/system/handler/helm.go @@ -49,6 +49,7 @@ func CreateHelmRepo(c *gin.Context) { ctx.Err = e.ErrInvalidParam.AddDesc("invalid url") return } + ctx.Err = service.CreateHelmRepo(args, ctx.Logger) } diff --git a/pkg/microservice/aslan/core/system/handler/registry.go b/pkg/microservice/aslan/core/system/handler/registry.go index 7a419d0cbd173adfbb610c09fbee92d35aecf000..b9a77ed8be921a432dc197d43016104b54b05b1c 100644 --- a/pkg/microservice/aslan/core/system/handler/registry.go +++ b/pkg/microservice/aslan/core/system/handler/registry.go @@ -60,6 +60,24 @@ func GetDefaultRegistryNamespace(c *gin.Context) { } } +func GetRegistryNamespace(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + + reg, err := commonservice.FindRegistryById(c.Param("id"), ctx.Logger) + if err != nil { + ctx.Err = err + return + } + + ctx.Resp = &Registry{ + ID: reg.ID.Hex(), + RegAddr: reg.RegAddr, + IsDefault: reg.IsDefault, + Namespace: reg.Namespace, + } +} + func ListRegistryNamespaces(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() diff --git a/pkg/microservice/aslan/core/system/handler/router.go b/pkg/microservice/aslan/core/system/handler/router.go index 9de790a64336842bcd2ad797ed92519733fa1a2b..253728546dab6cb45b0ebb50fbbd0273da015ba5 100644 --- a/pkg/microservice/aslan/core/system/handler/router.go +++ b/pkg/microservice/aslan/core/system/handler/router.go @@ -61,6 +61,7 @@ func (*Router) Inject(router *gin.RouterGroup) { registry.GET("", ListRegistries) // 获取默认的镜像仓库配置，用于kodespace CLI调用 registry.GET("/namespaces/default", GetDefaultRegistryNamespace) + registry.GET("/namespaces/specific/:id", GetRegistryNamespace) registry.GET("/namespaces", ListRegistryNamespaces) registry.POST("/namespaces", gin2.UpdateOperationLogStatus, CreateRegistryNamespace) registry.PUT("/namespaces/:id", gin2.UpdateOperationLogStatus, UpdateRegistryNamespace) diff --git a/pkg/microservice/aslan/core/system/service/registry.go b/pkg/microservice/aslan/core/system/service/registry.go index 0aa8288eea2bfb3bfed246e8ec4f924d3ffd4758..192f5fbdbe81b95f7488cdfe639ab5d728f7fc76 100644 --- a/pkg/microservice/aslan/core/system/service/registry.go +++ b/pkg/microservice/aslan/core/system/service/registry.go @@ -22,12 +22,13 @@ import ( "go.uber.org/zap" + "github.com/koderover/zadig/pkg/microservice/aslan/config" commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/registry" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/util" ) @@ -86,12 +87,12 @@ func CreateRegistryNamespace(username string, args *commonmodels.RegistryNamespa } for _, env := range envs { go func(prod *commonmodels.Product) { - kubeClient, err := kube.GetKubeClient(prod.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), prod.ClusterID) if err != nil { log.Errorf("[updateRegistry] Failed to get kubecli for namespace: %s", prod.Namespace) return } - err = commonservice.EnsureDefaultRegistrySecret(prod.Namespace, kubeClient, log) + err = commonservice.EnsureDefaultRegistrySecret(prod.Namespace, args.ID.Hex(), kubeClient, log) if err != nil { log.Errorf("[updateRegistry] Failed to update registry secret for namespace: %s, the error is: %+v", prod.Namespace, err) } @@ -161,12 +162,12 @@ func UpdateRegistryNamespace(username, id string, args *commonmodels.RegistryNam } for _, env := range envs { go func(prod *commonmodels.Product) { - kubeClient, err := kube.GetKubeClient(prod.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), prod.ClusterID) if err != nil { log.Errorf("[updateRegistry] Failed to get kubecli for namespace: %s", prod.Namespace) return } - err = commonservice.EnsureDefaultRegistrySecret(prod.Namespace, kubeClient, log) + err = commonservice.EnsureDefaultRegistrySecret(prod.Namespace, id, kubeClient, log) if err != nil { log.Errorf("[updateRegistry] Failed to update registry secret for namespace: %s, the error is: %+v", prod.Namespace, err) } diff --git a/pkg/microservice/aslan/core/workflow/handler/webhook.go b/pkg/microservice/aslan/core/workflow/handler/webhook.go index fb9b8a500c93a9bcc52decd90d8ae7a84e9d171c..0a6dcea6081f328a53068cc3d063d4df15e6dd33 100644 --- a/pkg/microservice/aslan/core/workflow/handler/webhook.go +++ b/pkg/microservice/aslan/core/workflow/handler/webhook.go @@ -72,6 +72,11 @@ func processGithub(payload []byte, req *http.Request, requestID string, log *zap log.Errorf("error happens to trigger workflow %v", err) errs = multierror.Append(errs, err) } - + //测试管理webhook + err = webhook.ProcessGithubWebHookForTest(payload, req, requestID, log) + if err != nil { + log.Errorf("error happens to trigger ProcessGithubWebHookForTest %v", err) + errs = multierror.Append(errs, err) + } return errs.ErrorOrNil() } diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/github.go b/pkg/microservice/aslan/core/workflow/service/webhook/github.go index 19553661294269d029960b136a7895fff85721f1..595710dc8746192ba67107892d75921fefe85456 100644 --- a/pkg/microservice/aslan/core/workflow/service/webhook/github.go +++ b/pkg/microservice/aslan/core/workflow/service/webhook/github.go @@ -381,6 +381,42 @@ func pushEventCommitsFiles(e *github.PushEvent) []string { return files } + +func ProcessGithubWebHookForTest(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger)error{ + hookType := github.WebHookType(req) + if hookType == "integration_installation" || hookType == "installation" || hookType == "ping" { + return nil + } + + err := validateSecret(payload, []byte(gitservice.GetHookSecret()), req) + if err != nil { + return err + } + + event, err := github.ParseWebHook(github.WebHookType(req), payload) + if err != nil { + return err + } + + switch et := event.(type) { + case *github.PullRequestEvent: + err = TriggerTestByGithubEvent(et, requestID, log) + if err != nil { + log.Errorf("TriggerTestByGithubEvent error: %v", err) + return e.ErrGithubWebHook.AddErr(err) + } + + case *github.PushEvent: + err = TriggerTestByGithubEvent(et, requestID, log) + if err != nil { + log.Infof("TriggerTestByGithubEvent error: %v", err) + return e.ErrGithubWebHook.AddErr(err) + } + } + return nil +} + + func ProcessGithubWebHook(payload []byte, req *http.Request, requestID string, log *zap.SugaredLogger) error { forwardedProto := req.Header.Get("X-Forwarded-Proto") forwardedHost := req.Header.Get("X-Forwarded-Host") diff --git a/pkg/microservice/aslan/core/workflow/service/webhook/github_testing_task.go b/pkg/microservice/aslan/core/workflow/service/webhook/github_testing_task.go new file mode 100644 index 0000000000000000000000000000000000000000..fcd494c97a3c26e5ce0f803d2e4bbee977f58eaf --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/webhook/github_testing_task.go @@ -0,0 +1,190 @@ +package webhook + +import ( + "regexp" + "strconv" + + "github.com/google/go-github/v35/github" + "github.com/hashicorp/go-multierror" + "go.uber.org/zap" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + testingservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/testing/service" + "github.com/koderover/zadig/pkg/setting" +) + +func TriggerTestByGithubEvent(event interface{}, requestID string, log *zap.SugaredLogger) error { + //1.find configured testing + testingList, err := commonrepo.NewTestingColl().List(&commonrepo.ListTestOption{}) + if err != nil { + log.Errorf("failed to list testing %v", err) + return err + } + mErr := &multierror.Error{} + diffSrv := func(pullRequestEvent *github.PullRequestEvent, codehostId int) ([]string, error) { + return findChangedFilesOfPullRequest(pullRequestEvent, codehostId) + } + for _, testing := range testingList { + if testing.HookCtl != nil && testing.HookCtl.Enabled { + for _, item := range testing.HookCtl.Items { + if item.TestArgs == nil { + continue + } + matcher := createGithubEventMatcherForTesting(event, diffSrv, testing, log) + if matcher == nil { + continue + } + if matches, err := matcher.Match(item.MainRepo); err != nil { + mErr = multierror.Append(err) + } else if matches { + log.Infof("event match hook %v of %s", item.MainRepo, testing.Name) + var mergeRequestID, commitID string + if ev, isPr := event.(*github.PullRequestEvent); isPr { + mergeRequestID = strconv.Itoa(*ev.PullRequest.Number) + commitID = *ev.PullRequest.Head.SHA + } + args := matcher.UpdateTaskArgs(item.TestArgs, requestID) + args.MergeRequestID = mergeRequestID + args.CommitID = commitID + args.Source = setting.SourceFromGitlab + args.CodehostID = item.MainRepo.CodehostID + args.RepoOwner = item.MainRepo.RepoOwner + args.RepoName = item.MainRepo.RepoName + if resp, err := testingservice.CreateTestTask(args, log); err != nil { + log.Errorf("failed to create testing task when receive event %v due to %v ", event, err) + mErr = multierror.Append(mErr, err) + } else { + log.Infof("succeed to create task %v", resp) + } + } else { + log.Debugf("event not matches %v", item.MainRepo) + } + } + } + } + return mErr.ErrorOrNil() +} + +type githubPushEventMatcherForTesting struct { + log *zap.SugaredLogger + testing *commonmodels.Testing + event *github.PushEvent +} + +type githubMergeEventMatcherForTesting struct { + diffFunc githubPullRequestDiffFunc + log *zap.SugaredLogger + testing *commonmodels.Testing + event *github.PullRequestEvent +} + +func (gpem *githubPushEventMatcherForTesting) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gpem.event + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == *ev.Repo.FullName { + if !EventConfigured(hookRepo, config.HookEventPush) { + return false, nil + } + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != getBranchFromRef(*ev.Ref) { + return false, nil + } + + if isRegular { + if matched, _ := regexp.MatchString(hookRepo.Branch, getBranchFromRef(*ev.Ref)); !matched { + return false, nil + } + } + hookRepo.Branch = getBranchFromRef(*ev.Ref) + var changedFiles []string + for _, commit := range ev.Commits { + changedFiles = append(changedFiles, commit.Added...) + changedFiles = append(changedFiles, commit.Removed...) + changedFiles = append(changedFiles, commit.Modified...) + } + + return MatchChanges(hookRepo, changedFiles), nil + } + + return false, nil +} + +func (gpem *githubPushEventMatcherForTesting) UpdateTaskArgs(args *commonmodels.TestTaskArgs, requestID string) *commonmodels.TestTaskArgs { + factory := &testArgsFactory{ + testing: gpem.testing, + reqID: requestID, + } + + factory.Update(args) + return args +} + +func createGithubEventMatcherForTesting( + event interface{}, diffSrv githubPullRequestDiffFunc, testing *commonmodels.Testing, log *zap.SugaredLogger, +) gitEventMatcherForTesting { + switch evt := event.(type) { + case *github.PushEvent: + return &githubPushEventMatcherForTesting{ + testing: testing, + log: log, + event: evt, + } + case *github.PullRequestEvent: + return &githubMergeEventMatcherForTesting{ + diffFunc: diffSrv, + log: log, + event: evt, + testing: testing, + } + } + + return nil +} + +func (gmem *githubMergeEventMatcherForTesting) Match(hookRepo *commonmodels.MainHookRepo) (bool, error) { + ev := gmem.event + // TODO: match codehost + if (hookRepo.RepoOwner + "/" + hookRepo.RepoName) == *ev.PullRequest.Base.Repo.FullName { + if !EventConfigured(hookRepo, config.HookEventPr) { + return false, nil + } + + isRegular := hookRepo.IsRegular + if !isRegular && hookRepo.Branch != *ev.PullRequest.Base.Ref { + return false, nil + } + + if isRegular { + if matched, _ := regexp.MatchString(hookRepo.Branch, *ev.PullRequest.Base.Ref); !matched { + return false, nil + } + } + hookRepo.Branch = *ev.PullRequest.Base.Ref + + if *ev.PullRequest.State == "open" { + var changedFiles []string + changedFiles, err := gmem.diffFunc(ev, hookRepo.CodehostID) + if err != nil { + gmem.log.Warnf("failed to get changes of event %v", ev) + return false, err + } + gmem.log.Debugf("succeed to get %d changes in merge event", len(changedFiles)) + + return MatchChanges(hookRepo, changedFiles), nil + } + + } + return false, nil +} + +func (gmem *githubMergeEventMatcherForTesting) UpdateTaskArgs(args *commonmodels.TestTaskArgs, requestID string) *commonmodels.TestTaskArgs { + factory := &testArgsFactory{ + testing: gmem.testing, + reqID: requestID, + } + + args = factory.Update(args) + + return args +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/artifact_task.go b/pkg/microservice/aslan/core/workflow/service/workflow/artifact_task.go new file mode 100644 index 0000000000000000000000000000000000000000..decf797e921acfa612cbfbc7db3530239ce7e3e6 --- /dev/null +++ b/pkg/microservice/aslan/core/workflow/service/workflow/artifact_task.go @@ -0,0 +1,141 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workflow + +import ( + "fmt" + "sort" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/koderover/zadig/pkg/microservice/aslan/config" + commonmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + taskmodels "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models/task" + commonrepo "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/mongodb" + commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" + "github.com/koderover/zadig/pkg/setting" + e "github.com/koderover/zadig/pkg/tool/errors" + "github.com/koderover/zadig/pkg/util" +) + +// get global config payload +func CreateArtifactPackageTask(args *commonmodels.ArtifactPackageTaskArgs, taskCreator string, log *zap.SugaredLogger) (int64, error) { + + configPayload := commonservice.GetConfigPayload(0) + repos, err := commonrepo.NewRegistryNamespaceColl().FindAll(&commonrepo.FindRegOps{}) + + if err != nil { + log.Errorf("CreateArtifactPackageTask query registries failed, err: %s", err) + return 0, fmt.Errorf("failed to query registries") + } + + registriesInvolved := sets.NewString() + registriesInvolved.Insert(args.SourceRegistries...) + registriesInvolved.Insert(args.TargetRegistries...) + + configPayload.RepoConfigs = make(map[string]*commonmodels.RegistryNamespace) + for _, repo := range repos { + if !registriesInvolved.Has(repo.ID.Hex()) { + continue + } + // if the registry is SWR, we need to modify ak/sk according to the rule + if repo.RegProvider == config.SWRProvider { + ak := fmt.Sprintf("%s@%s", repo.Region, repo.AccessKey) + sk := util.ComputeHmacSha256(repo.AccessKey, repo.SecretKey) + repo.AccessKey = ak + repo.SecretKey = sk + } + configPayload.RepoConfigs[repo.ID.Hex()] = repo + } + + defaultS3, err := s3.FindDefaultS3() + if err != nil { + err = e.ErrFindDefaultS3Storage.AddDesc("default storage is required by distribute task") + return 0, err + } + + defaultURL, err := defaultS3.GetEncryptedURL() + if err != nil { + err = e.ErrS3Storage.AddErr(err) + return 0, err + } + + task := &task.Task{ + Type: config.ArtifactType, + ProductName: args.ProjectName, + Status: config.StatusCreated, + ArtifactPackageTaskArgs: args, + TaskCreator: taskCreator, + ConfigPayload: configPayload, + StorageURI: defaultURL, + } + + subTask, err := (&taskmodels.ArtifactPackage{ + TaskType: config.TaskArtifactPackage, + Enabled: true, + TaskStatus: "", + Timeout: 0, + StartTime: 0, + EndTime: 0, + LogFile: "", + Images: args.Images, + SourceRegistries: args.SourceRegistries, + TargetRegistries: args.TargetRegistries, + }).ToSubTask() + + if err != nil { + return 0, err + } + + task.SubTasks = []map[string]interface{}{subTask} + + if err := ensurePipelineTask(task, "", log); err != nil { + log.Errorf("CreateServiceTask ensurePipelineTask err : %v", err) + return 0, err + } + + stages := make([]*commonmodels.Stage, 0) + for _, subTask := range task.SubTasks { + AddSubtaskToStage(&stages, subTask, args.EnvName) + } + sort.Sort(ByStageKind(stages)) + task.Stages = stages + if len(task.Stages) == 0 { + return 0, e.ErrCreateTask.AddDesc(e.PipelineSubTaskNotFoundErrMsg) + } + + pipelineName := fmt.Sprintf("%s-%s-%s", args.ProjectName, args.EnvName, "artifact") + nextTaskID, err := commonrepo.NewCounterColl().GetNextSeq(fmt.Sprintf(setting.ServiceTaskFmt, pipelineName)) + if err != nil { + log.Errorf("CreateServiceTask Counter.GetNextSeq error: %v", err) + return 0, e.ErrGetCounter.AddDesc(err.Error()) + } + + task.SubTasks = []map[string]interface{}{} + task.TaskID = nextTaskID + task.PipelineName = pipelineName + + if err := CreateTask(task); err != nil { + log.Error(err) + return 0, e.ErrCreateTask + } + + return nextTaskID, nil +} diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/convert.go b/pkg/microservice/aslan/core/workflow/service/workflow/convert.go index 4003a1c6cfbee242608823338ca73528d0353496..9c8dc8c6e95c3ce2e9f0b9b6edaa412adc22c780 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/convert.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/convert.go @@ -31,91 +31,93 @@ import ( func ConvertQueueToTask(queueTask *commonmodels.Queue) *task.Task { return &task.Task{ - TaskID: queueTask.TaskID, - ProductName: queueTask.ProductName, - PipelineName: queueTask.PipelineName, - Type: queueTask.Type, - Status: queueTask.Status, - Description: queueTask.Description, - TaskCreator: queueTask.TaskCreator, - TaskRevoker: queueTask.TaskRevoker, - CreateTime: queueTask.CreateTime, - StartTime: queueTask.StartTime, - EndTime: queueTask.EndTime, - SubTasks: queueTask.SubTasks, - Stages: queueTask.Stages, - ReqID: queueTask.ReqID, - AgentHost: queueTask.AgentHost, - DockerHost: queueTask.DockerHost, - TeamName: queueTask.TeamName, - IsDeleted: queueTask.IsDeleted, - IsArchived: queueTask.IsArchived, - AgentID: queueTask.AgentID, - MultiRun: queueTask.MultiRun, - Target: queueTask.Target, - BuildModuleVer: queueTask.BuildModuleVer, - ServiceName: queueTask.ServiceName, - TaskArgs: queueTask.TaskArgs, - WorkflowArgs: queueTask.WorkflowArgs, - TestArgs: queueTask.TestArgs, - ServiceTaskArgs: queueTask.ServiceTaskArgs, - ConfigPayload: queueTask.ConfigPayload, - Error: queueTask.Error, - Services: queueTask.Services, - Render: queueTask.Render, - StorageURI: queueTask.StorageURI, - TestReports: queueTask.TestReports, - RwLock: queueTask.RwLock, - ResetImage: queueTask.ResetImage, - TriggerBy: queueTask.TriggerBy, - Features: queueTask.Features, - IsRestart: queueTask.IsRestart, - StorageEndpoint: queueTask.StorageEndpoint, + TaskID: queueTask.TaskID, + ProductName: queueTask.ProductName, + PipelineName: queueTask.PipelineName, + Type: queueTask.Type, + Status: queueTask.Status, + Description: queueTask.Description, + TaskCreator: queueTask.TaskCreator, + TaskRevoker: queueTask.TaskRevoker, + CreateTime: queueTask.CreateTime, + StartTime: queueTask.StartTime, + EndTime: queueTask.EndTime, + SubTasks: queueTask.SubTasks, + Stages: queueTask.Stages, + ReqID: queueTask.ReqID, + AgentHost: queueTask.AgentHost, + DockerHost: queueTask.DockerHost, + TeamName: queueTask.TeamName, + IsDeleted: queueTask.IsDeleted, + IsArchived: queueTask.IsArchived, + AgentID: queueTask.AgentID, + MultiRun: queueTask.MultiRun, + Target: queueTask.Target, + BuildModuleVer: queueTask.BuildModuleVer, + ServiceName: queueTask.ServiceName, + TaskArgs: queueTask.TaskArgs, + WorkflowArgs: queueTask.WorkflowArgs, + TestArgs: queueTask.TestArgs, + ServiceTaskArgs: queueTask.ServiceTaskArgs, + ArtifactPackageTaskArgs: queueTask.ArtifactPackageTaskArgs, + ConfigPayload: queueTask.ConfigPayload, + Error: queueTask.Error, + Services: queueTask.Services, + Render: queueTask.Render, + StorageURI: queueTask.StorageURI, + TestReports: queueTask.TestReports, + RwLock: queueTask.RwLock, + ResetImage: queueTask.ResetImage, + TriggerBy: queueTask.TriggerBy, + Features: queueTask.Features, + IsRestart: queueTask.IsRestart, + StorageEndpoint: queueTask.StorageEndpoint, } } func ConvertTaskToQueue(task *task.Task) *commonmodels.Queue { return &commonmodels.Queue{ - TaskID: task.TaskID, - ProductName: task.ProductName, - PipelineName: task.PipelineName, - Type: task.Type, - Status: task.Status, - Description: task.Description, - TaskCreator: task.TaskCreator, - TaskRevoker: task.TaskRevoker, - CreateTime: task.CreateTime, - StartTime: task.StartTime, - EndTime: task.EndTime, - SubTasks: task.SubTasks, - Stages: task.Stages, - ReqID: task.ReqID, - AgentHost: task.AgentHost, - DockerHost: task.DockerHost, - TeamName: task.TeamName, - IsDeleted: task.IsDeleted, - IsArchived: task.IsArchived, - AgentID: task.AgentID, - MultiRun: task.MultiRun, - Target: task.Target, - BuildModuleVer: task.BuildModuleVer, - ServiceName: task.ServiceName, - TaskArgs: task.TaskArgs, - WorkflowArgs: task.WorkflowArgs, - TestArgs: task.TestArgs, - ServiceTaskArgs: task.ServiceTaskArgs, - ConfigPayload: task.ConfigPayload, - Error: task.Error, - Services: task.Services, - Render: task.Render, - StorageURI: task.StorageURI, - TestReports: task.TestReports, - RwLock: task.RwLock, - ResetImage: task.ResetImage, - TriggerBy: task.TriggerBy, - Features: task.Features, - IsRestart: task.IsRestart, - StorageEndpoint: task.StorageEndpoint, + TaskID: task.TaskID, + ProductName: task.ProductName, + PipelineName: task.PipelineName, + Type: task.Type, + Status: task.Status, + Description: task.Description, + TaskCreator: task.TaskCreator, + TaskRevoker: task.TaskRevoker, + CreateTime: task.CreateTime, + StartTime: task.StartTime, + EndTime: task.EndTime, + SubTasks: task.SubTasks, + Stages: task.Stages, + ReqID: task.ReqID, + AgentHost: task.AgentHost, + DockerHost: task.DockerHost, + TeamName: task.TeamName, + IsDeleted: task.IsDeleted, + IsArchived: task.IsArchived, + AgentID: task.AgentID, + MultiRun: task.MultiRun, + Target: task.Target, + BuildModuleVer: task.BuildModuleVer, + ServiceName: task.ServiceName, + TaskArgs: task.TaskArgs, + WorkflowArgs: task.WorkflowArgs, + TestArgs: task.TestArgs, + ServiceTaskArgs: task.ServiceTaskArgs, + ArtifactPackageTaskArgs: task.ArtifactPackageTaskArgs, + ConfigPayload: task.ConfigPayload, + Error: task.Error, + Services: task.Services, + Render: task.Render, + StorageURI: task.StorageURI, + TestReports: task.TestReports, + RwLock: task.RwLock, + ResetImage: task.ResetImage, + TriggerBy: task.TriggerBy, + Features: task.Features, + IsRestart: task.IsRestart, + StorageEndpoint: task.StorageEndpoint, } } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_task.go b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_task.go index 31feb5d750016f0b6c81f54911804e4992d4047d..f60fea433bfb04e1e68a664552c415613365fe36 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_task.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_task.go @@ -567,6 +567,7 @@ func TestArgsToTestSubtask(args *commonmodels.TestTaskArgs, pt *task.Task, log * TestName: "test", Timeout: testModule.Timeout, } + testTask.TestModuleName = testModule.Name testTask.JobCtx.TestType = testModule.TestType testTask.JobCtx.Builds = testModule.Repos @@ -588,6 +589,8 @@ func TestArgsToTestSubtask(args *commonmodels.TestTaskArgs, pt *task.Task, log * testTask.InstallItems = testModule.PreTest.Installs testTask.JobCtx.CleanWorkspace = testModule.PreTest.CleanWorkspace testTask.JobCtx.EnableProxy = testModule.PreTest.EnableProxy + testTask.Namespace = testModule.PreTest.Namespace + testTask.ClusterID = testModule.PreTest.ClusterID envs := testModule.PreTest.Envs[:] diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_validation.go b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_validation.go index 1ab541b8bf0fd800d449a1a9204dc57f44eb684c..1c33ddcc4bc79f2a2ff5e48ee3fba2aaea258628 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_validation.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/pipeline_validation.go @@ -43,9 +43,9 @@ import ( "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/base" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/codehub" git "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/github" - "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/kube" "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/shared/client/systemconfig" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/wrapper" e "github.com/koderover/zadig/pkg/tool/errors" "github.com/koderover/zadig/pkg/tool/kube/getter" @@ -688,7 +688,7 @@ func validateServiceContainer(envName, productName, serviceName, container strin return "", err } - kubeClient, err := kube.GetKubeClient(product.ClusterID) + kubeClient, err := kubeclient.GetKubeClient(config.HubServerAddress(), product.ClusterID) if err != nil { return "", err } diff --git a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task.go b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task.go index 06232049203336004c41b49242d477a55073c1c4..08014a9cab1fd481f93196cb27dac256df7eb9b2 100644 --- a/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task.go +++ b/pkg/microservice/aslan/core/workflow/service/workflow/workflow_task.go @@ -451,17 +451,21 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, } } - productTempl, err := template.NewProductColl().Find(args.ProductTmplName) + workflow, err := commonrepo.NewWorkflowColl().Find(args.WorkflowName) if err != nil { - log.Errorf("productTempl.Find error: %v", err) + log.Errorf("Workflow.Find error: %v", err) return nil, e.ErrFindWorkflow.AddDesc(err.Error()) } - workflow, err := commonrepo.NewWorkflowColl().Find(args.WorkflowName) + project, err := template.NewProductColl().Find(workflow.ProductTmplName) if err != nil { - log.Errorf("Workflow.Find error: %v", err) + log.Errorf("project.Find error: %v", err) return nil, e.ErrFindWorkflow.AddDesc(err.Error()) } + // developer don't pass args.ProductTmplName + if args.ProductTmplName == "" { + args.ProductTmplName = workflow.ProductTmplName + } args.IsParallel = workflow.IsParallel var env *commonmodels.Product @@ -494,6 +498,18 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, // 获取全局configpayload configPayload := commonservice.GetConfigPayload(args.CodehostID) + if len(env.RegistryID) == 0 { + op := &commonrepo.FindRegOps{ + IsDefault: true, + } + reg, err := commonrepo.NewRegistryNamespaceColl().Find(op) + if err != nil { + log.Errorf("get default registry error: %v", err) + return nil, e.ErrGetCounter.AddDesc(err.Error()) + } + env.RegistryID = reg.ID.Hex() + } + configPayload.RegistryID = env.RegistryID repos, err := commonrepo.NewRegistryNamespaceColl().FindAll(&commonrepo.FindRegOps{}) if err == nil { configPayload.RepoConfigs = make(map[string]*commonmodels.RegistryNamespace) @@ -526,7 +542,8 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, buildModuleArgs := &commonmodels.BuildModuleArgs{ Target: target.Name, ServiceName: target.ServiceName, - ProductName: target.ProductName, + //ProductName: target.ProductName, // TODO productName may be nil in some situation, need to figure out reason + ProductName: args.ProductTmplName, Variables: target.Envs, Env: env, } @@ -550,7 +567,7 @@ func CreateWorkflowTask(args *commonmodels.WorkflowTaskArgs, taskCreator string, if deployEnv.Type == setting.PMDeployType { continue } - deployTask, err := deployEnvToSubTasks(deployEnv, env, productTempl.Timeout) + deployTask, err := deployEnvToSubTasks(deployEnv, env, project.Timeout) if err != nil { log.Errorf("deploy env to subtask error: %v", err) return nil, e.ErrCreateTask.AddErr(err) @@ -770,11 +787,14 @@ func AddDataToArgs(args *commonmodels.WorkflowTaskArgs, log *zap.SugaredLogger) for _, target := range args.Target { target.HasBuild = true // openAPI模式，传入的name是服务名称 - // 只支持k8s服务 + serviceType, err := getValidServiceType(target.ServiceType) + if err != nil { + return err + } opt := &commonrepo.ServiceFindOption{ ServiceName: target.Name, ProductName: workflow.ProductTmplName, - Type: setting.K8SDeployType, + Type: serviceType, ExcludeStatus: setting.ProductStatusDeleting} serviceTmpl, err := commonrepo.NewServiceColl().Find(opt) if err != nil { @@ -880,6 +900,19 @@ func AddDataToArgs(args *commonmodels.WorkflowTaskArgs, log *zap.SugaredLogger) return nil } +// Only supports k8s and helm two service types currently +func getValidServiceType(serviceType string) (string, error) { + switch serviceType { + // Compatible when the service_type is equal to empty + case setting.K8SDeployType, "": + return setting.K8SDeployType, nil + case setting.HelmDeployType: + return setting.HelmDeployType, nil + default: + return "", fmt.Errorf("Unsupported service type") + } +} + func dealWithNamespace(args *commonmodels.WorkflowTaskArgs) { args.Namespace = strings.TrimPrefix(args.Namespace, ",") args.Namespace = strings.TrimSuffix(args.Namespace, ",") @@ -1158,6 +1191,8 @@ func workFlowArgsToTaskArgs(target string, workflowArgs *commonmodels.WorkflowTa // TODO 和validation中转化testsubtask合并为一个方法 func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log *zap.SugaredLogger) ([]*task.Testing, error) { var resp []*task.Testing + var servicesArray []string + var services string // 创建任务的测试参数为脱敏数据，需要转换为实际数据 for _, test := range args.Tests { @@ -1167,6 +1202,11 @@ func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log * } } + for _, service := range args.Target { + servicesArray = append(servicesArray, service.ServiceName) + } + services = strings.Join(servicesArray, ",") + testArgs := args.Tests testCreator := args.WorkflowTaskCreator @@ -1211,6 +1251,8 @@ func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log * testTask.InstallItems = testModule.PreTest.Installs testTask.JobCtx.CleanWorkspace = testModule.PreTest.CleanWorkspace testTask.JobCtx.EnableProxy = testModule.PreTest.EnableProxy + testTask.Namespace = testModule.PreTest.Namespace + testTask.ClusterID = testModule.PreTest.ClusterID envs := testModule.PreTest.Envs[:] @@ -1224,10 +1266,14 @@ func testArgsToSubtask(args *commonmodels.WorkflowTaskArgs, pt *task.Task, log * } } envs = append(envs, &commonmodels.KeyVal{Key: "TEST_URL", Value: GetLink(pt, configbase.SystemAddress(), config.WorkflowType)}) + envs = append(envs, &commonmodels.KeyVal{Key: "SERVICES", Value: services}) + testTask.JobCtx.EnvVars = envs testTask.ImageID = testModule.PreTest.ImageID testTask.BuildOS = testModule.PreTest.BuildOS testTask.ImageFrom = testModule.PreTest.ImageFrom + testTask.ClusterID = testModule.PreTest.ClusterID + testTask.Namespace = testModule.PreTest.Namespace // 自定义基础镜像的镜像名称可能会被更新，需要使用ID获取最新的镜像名称 if testModule.PreTest.ImageID != "" { basicImage, err := commonrepo.NewBasicImageColl().Find(testModule.PreTest.ImageID) @@ -1577,6 +1623,8 @@ func BuildModuleToSubTasks(args *commonmodels.BuildModuleArgs, log *zap.SugaredL Timeout: module.Timeout, Registries: registries, ProductName: args.ProductName, + Namespace: module.PreBuild.Namespace, + ClusterID: module.PreBuild.ClusterID, } if args.TaskType != "" { @@ -1598,7 +1646,6 @@ func BuildModuleToSubTasks(args *commonmodels.BuildModuleArgs, log *zap.SugaredL } if serviceTmpl != nil { - build.Namespace = args.Env.Namespace build.ServiceType = setting.PMDeployType envHost := make(map[string][]string) for _, envConfig := range serviceTmpl.EnvConfigs { @@ -1774,7 +1821,6 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e // } // } //} - // 设置Pipeline对应的服务名称 if t.ServiceName != "" { pt.ServiceName = t.ServiceName @@ -1794,16 +1840,33 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e setManunalBuilds(t.JobCtx.Builds, pt.TaskArgs.Builds, log) } + opt := &commonrepo.ProductFindOptions{Name: pt.ProductName} + exitedProd, err := commonrepo.NewProductColl().Find(opt) + if err != nil { + log.Errorf("can't find product by name:%s error msg: %v", pt.ProductName, err) + return e.ErrFindRegistry.AddDesc(err.Error()) + } + // 生成默认镜像tag后缀 //pt.TaskArgs.Deploy.Tag = releaseCandidate(t, pt.TaskID, pt.ProductName, pt.EnvName, "image") // 设置镜像名称 // 编译任务使用 t.JobCtx.Image // 注意: 其他任务从 pt.TaskArgs.Deploy.Image 获取, 必须要有编译任务 - reg, err := commonservice.FindDefaultRegistry(log) - if err != nil { - log.Errorf("can't find default candidate registry: %v", err) - return e.ErrFindRegistry.AddDesc(err.Error()) + var reg *commonmodels.RegistryNamespace + if len(exitedProd.RegistryID) > 0 { + reg, err = commonservice.FindRegistryById(exitedProd.RegistryID, log) + if err != nil { + log.Errorf("service.EnsureRegistrySecret: failed to find registry: %s error msg:%v", + exitedProd.RegistryID, err) + return e.ErrFindRegistry.AddDesc(err.Error()) + } + } else { + reg, err = commonservice.FindDefaultRegistry(log) + if err != nil { + log.Errorf("can't find default candidate registry: %v", err) + return e.ErrFindRegistry.AddDesc(err.Error()) + } } t.JobCtx.Image = GetImage(reg, releaseCandidate(t, pt.TaskID, pt.ProductName, envName, "image")) @@ -2165,6 +2228,8 @@ func ensurePipelineTask(pt *task.Task, envName string, log *zap.SugaredLogger) e } case config.TaskDistribute: // 为了兼容历史数据类型，目前什么都不用做，避免出错 + case config.TaskArtifactPackage: + // do nothing default: return e.NewErrInvalidTaskType(string(pre.TaskType)) } diff --git a/pkg/microservice/aslan/core/workflow/testing/service/testing.go b/pkg/microservice/aslan/core/workflow/testing/service/testing.go index 2647c29c51f2251ca77265c4fcc9e74955843d0d..78d6daf60f8dd6d3842fe33de4a37fbb77b77523 100644 --- a/pkg/microservice/aslan/core/workflow/testing/service/testing.go +++ b/pkg/microservice/aslan/core/workflow/testing/service/testing.go @@ -32,6 +32,7 @@ import ( commonservice "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/nsq" "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/s3" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/service/webhook" commonutil "github.com/koderover/zadig/pkg/microservice/aslan/core/common/util" workflowservice "github.com/koderover/zadig/pkg/microservice/aslan/core/workflow/service/workflow" "github.com/koderover/zadig/pkg/setting" @@ -52,6 +53,11 @@ func CreateTesting(username string, testing *commonmodels.Testing, log *zap.Suga return e.ErrCreateTestModule.AddErr(err) } + err = commonservice.ProcessWebhook(testing.HookCtl.Items,nil,webhook.TestingPrefix+testing.Name,log) + if err != nil { + return e.ErrUpdateTestModule.AddErr(err) + } + testing.UpdateBy = username err = commonrepo.NewTestingColl().Create(testing) if err != nil { @@ -112,6 +118,11 @@ func UpdateTesting(username string, testing *commonmodels.Testing, log *zap.Suga commonservice.EnsureSecretEnvs(existed.PreTest.Envs, testing.PreTest.Envs) } + err = commonservice.ProcessWebhook(testing.HookCtl.Items,existed.HookCtl.Items,webhook.TestingPrefix+testing.Name,log) + if err != nil { + return e.ErrUpdateTestModule.AddErr(err) + } + testing.UpdateBy = username testing.UpdateTime = time.Now().Unix() diff --git a/pkg/microservice/hubserver/core/repository/models/models.go b/pkg/microservice/hubserver/core/repository/models/models.go index 4bfefe013db24ce09ce1143aeb05ca85d37390d2..04c47b98cab5e635b16cc31a7cb12a869c3b1dfb 100644 --- a/pkg/microservice/hubserver/core/repository/models/models.go +++ b/pkg/microservice/hubserver/core/repository/models/models.go @@ -44,6 +44,7 @@ type K8SCluster struct { CreatedBy string `json:"createdBy" bson:"createdBy"` Disconnected bool `json:"-" bson:"disconnected"` Token string `json:"token" bson:"-"` + Local bool `json:"local" bson:"local"` } func (K8SCluster) TableName() string { diff --git a/pkg/microservice/hubserver/core/service/service.go b/pkg/microservice/hubserver/core/service/service.go index 2acae6b2ec821611b3ebce14874ac1860b8a86f4..9b84f2bc2a50364f4ada8fac48669093d2f5c888 100644 --- a/pkg/microservice/hubserver/core/service/service.go +++ b/pkg/microservice/hubserver/core/service/service.go @@ -235,7 +235,7 @@ func Reset() { } for _, cluster := range clusters { - if cluster.Status == config.Normal { + if cluster.Status == config.Normal && !cluster.Local { cluster.Status = config.Abnormal err := mongodb.NewK8sClusterColl().UpdateStatus(cluster) if err != nil { @@ -271,7 +271,7 @@ func Sync(server *remotedialer.Server, stopCh <-chan struct{}) { statusChanged = true } } else { - if cluster.Status == config.Normal { + if cluster.Status == config.Normal && !cluster.Local { log.Infof( "cluster %s disconnected changed %s => %s", cluster.Name, cluster.Status, config.Abnormal, diff --git a/pkg/microservice/packager/config/config.go b/pkg/microservice/packager/config/config.go new file mode 100644 index 0000000000000000000000000000000000000000..76ed9e11392ee3c6a6fdffcddf2fb3a07bc4474e --- /dev/null +++ b/pkg/microservice/packager/config/config.go @@ -0,0 +1,33 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "github.com/spf13/viper" + + // init the config first + _ "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/setting" +) + +func JobConfigFile() string { + return viper.GetString(setting.JobConfigFile) +} + +func Home() string { + return viper.GetString(setting.Home) +} diff --git a/pkg/microservice/packager/core/service/packager.go b/pkg/microservice/packager/core/service/packager.go new file mode 100644 index 0000000000000000000000000000000000000000..8f9e41da9cdb8dd7730e7b90a7cfc5b32c93b6c4 --- /dev/null +++ b/pkg/microservice/packager/core/service/packager.go @@ -0,0 +1,48 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +type ImageData struct { + ImageUrl string `yaml:"image_url" json:"image_url"` + ImageName string `yaml:"image_name" json:"image_name"` + ImageTag string `yaml:"image_tag" json:"image_tag"` + RegistryID string `yaml:"registry_id" json:"registry_id"` +} + +// ImagesByService defines all images in a service +type ImagesByService struct { + ServiceName string `yaml:"service_name"` + Images []*ImageData `yaml:"images"` +} + +//DockerRegistry registry host/user/password +type DockerRegistry struct { + RegistryID string `yaml:"registry_id"` + Host string `yaml:"host"` + UserName string `yaml:"username"` + Password string `yaml:"password"` + Namespace string `yaml:"namespace"` +} + +// Context parameters for job to run with +type Context struct { + JobType string `yaml:"job_type"` + ProgressFile string `yaml:"progress_file"` + Images []*ImagesByService `yaml:"images"` + SourceRegistries []*DockerRegistry `yaml:"source_registries"` + TargetRegistries []*DockerRegistry `yaml:"target_registries"` +} diff --git a/pkg/microservice/packager/core/service/service.go b/pkg/microservice/packager/core/service/service.go new file mode 100644 index 0000000000000000000000000000000000000000..ba4e036e5040e097d1ad0da12caa57e59152d7c6 --- /dev/null +++ b/pkg/microservice/packager/core/service/service.go @@ -0,0 +1,297 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/pkg/errors" + "gopkg.in/yaml.v3" + + "github.com/koderover/zadig/pkg/microservice/packager/config" + "github.com/koderover/zadig/pkg/tool/log" +) + +type Packager struct { + Ctx *Context +} + +type PackageResult struct { + ServiceName string `json:"service_name"` + Result string `json:"result"` + ErrorMsg string `json:"error_msg"` + ImageData []*ImageData `json:"image_data"` +} + +func NewPackager() (*Packager, error) { + context, err := os.ReadFile(config.JobConfigFile()) + if err != nil { + return nil, err + } + + var ctx *Context + if err := yaml.Unmarshal(context, &ctx); err != nil { + return nil, err + } + + packager := &Packager{ + Ctx: ctx, + } + + return packager, nil +} + +func buildRegistryMap(registries []*DockerRegistry) map[string]*DockerRegistry { + ret := make(map[string]*DockerRegistry) + for _, registry := range registries { + ret[registry.RegistryID] = registry + } + return ret +} + +func base64EncodeAuth(auth *types.AuthConfig) (string, error) { + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(auth); err != nil { + return "", err + } + return base64.URLEncoding.EncodeToString(buf.Bytes()), nil +} + +func buildTargetImage(imageName, imageTag, host, nameSpace string) string { + ret := "" + if len(nameSpace) > 0 { + ret = fmt.Sprintf("%s/%s/%s:%s", host, nameSpace, imageName, imageTag) + } else { + ret = fmt.Sprintf("%s/%s:%s", host, imageName, imageTag) + } + ret = strings.TrimPrefix(ret, "http://") + ret = strings.TrimPrefix(ret, "https://") + return ret +} + +func pullImage(dockerClient *client.Client, imageUrl string, options *types.ImagePullOptions) error { + // pull image + pullResponse, err := dockerClient.ImagePull(context.TODO(), imageUrl, *options) + if err != nil { + return err + } + + defer func(pullResponse io.ReadCloser) { + err := pullResponse.Close() + if err != nil { + log.Errorf("failed to close response reader") + } + }(pullResponse) + + bs, err := io.ReadAll(pullResponse) + if err != nil { + return err + } + + if strings.Contains(string(bs), "error") { + log.Errorf("image push failed: %s", string(bs)) + return fmt.Errorf("failed to push image") + } + return nil +} + +func pushImage(dockerClient *client.Client, targetImageUrl string, options *types.ImagePushOptions) error { + pushResponse, err := dockerClient.ImagePush(context.TODO(), targetImageUrl, *options) + if err != nil { + return errors.Wrapf(err, "failed to push image: %s", targetImageUrl) + } + + defer func(pullResponse io.ReadCloser) { + err := pullResponse.Close() + if err != nil { + log.Errorf("failed to close response reader") + } + }(pushResponse) + + bs, err := io.ReadAll(pushResponse) + if err != nil { + return err + } + + if strings.Contains(string(bs), "error") { + log.Errorf("image push failed: %s", string(bs)) + return fmt.Errorf("failed to push image") + } + return nil +} + +func handleSingleService(imageByService *ImagesByService, allRegistries map[string]*DockerRegistry, targetRegistries []*DockerRegistry, dockerClient *client.Client) ([]*ImageData, error) { + targetImageUrlByRepo := make(map[string][]string) + retImages := make([]*ImageData, 0) + + for _, singleImage := range imageByService.Images { + options := types.ImagePullOptions{} + // for images from public repo，registryID won't be appointed + if len(singleImage.RegistryID) > 0 { + registryInfo, ok := allRegistries[singleImage.RegistryID] + if !ok { + return nil, fmt.Errorf("failed to find source registry for image: %s", singleImage.ImageUrl) + } + encodedAuth, err := base64EncodeAuth(&types.AuthConfig{ + Username: registryInfo.UserName, + Password: registryInfo.Password, + ServerAddress: registryInfo.Host, + }) + if err != nil { + return nil, errors.Wrapf(err, "faied to create docker pull auth data") + } + options.RegistryAuth = encodedAuth + } + + // pull image + err := pullImage(dockerClient, singleImage.ImageUrl, &options) + if err != nil { + return nil, errors.Wrapf(err, "failed to pull image: %s", singleImage.ImageUrl) + } + + // tag image + for _, registry := range targetRegistries { + targetImage := buildTargetImage(singleImage.ImageName, singleImage.ImageTag, registry.Host, registry.Namespace) + err = dockerClient.ImageTag(context.TODO(), singleImage.ImageUrl, targetImage) + if err != nil { + return nil, errors.Wrapf(err, "failed to tag image from: %s to: %s", singleImage.ImageUrl, targetImage) + } + targetImageUrlByRepo[registry.RegistryID] = append(targetImageUrlByRepo[registry.RegistryID], targetImage) + retImages = append(retImages, &ImageData{ + ImageUrl: targetImage, + ImageName: singleImage.ImageName, + ImageTag: singleImage.ImageTag, + RegistryID: singleImage.RegistryID, + }) + } + } + + // push image + for _, registry := range targetRegistries { + encodedAuth, err := base64EncodeAuth(&types.AuthConfig{ + Username: registry.UserName, + Password: registry.Password, + ServerAddress: registry.Host, + }) + if err != nil { + return nil, errors.Wrapf(err, "faied to create docker push auth data") + } + options := types.ImagePushOptions{ + RegistryAuth: encodedAuth, + } + + for _, targetImageUrl := range targetImageUrlByRepo[registry.RegistryID] { + err = pushImage(dockerClient, targetImageUrl, &options) + if err != nil { + return nil, errors.Wrapf(err, "failed to push image: %s", targetImageUrl) + } + } + } + return retImages, nil +} + +func (p *Packager) Exec() error { + + // init docker client + cli, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + log.Errorf("failed to init docker client %s", err) + return err + } + + defer func() { + err = cli.Close() + if err != nil { + log.Errorf("failed to close client: %s", err) + } + }() + + // run docker info to ensure docker daemon connection + for i := 0; i < 120; i++ { + _, err := cli.Info(context.TODO()) + if err == nil { + break + } + log.Errorf("failed tor run docker info, try index: %d, err: %s", i, err) + time.Sleep(time.Second * 1) + } + + // create log file + if len(p.Ctx.ProgressFile) >= 0 { + if err := os.MkdirAll(filepath.Dir(p.Ctx.ProgressFile), 0770); err != nil { + return errors.Wrapf(err, "failed to create progress file dir") + } + file, err := os.Create(p.Ctx.ProgressFile) + if err != nil { + return errors.Wrapf(err, "failed to create progress file") + } + if err = file.Close(); err != nil { + return errors.Wrapf(err, "failed to close progress file") + } + } + + allRegistries := append(p.Ctx.SourceRegistries, p.Ctx.TargetRegistries...) + realTimeProgress := make([]*PackageResult, 0) + + for _, imageByService := range p.Ctx.Images { + result := &PackageResult{ + ServiceName: imageByService.ServiceName, + } + images, err := handleSingleService(imageByService, buildRegistryMap(allRegistries), p.Ctx.TargetRegistries, cli) + + if err != nil { + result.Result = "failed" + result.ErrorMsg = err.Error() + log.Errorf("[result][fail][%s][%s]", imageByService.ServiceName, err) + } else { + result.Result = "success" + result.ImageData = images + log.Infof("[result][success][%s]", imageByService.ServiceName) + } + + realTimeProgress = append(realTimeProgress, result) + + if len(p.Ctx.ProgressFile) > 0 { + bs, err := json.Marshal(realTimeProgress) + if err != nil { + log.Errorf("failed to marshal progress data %s", err) + continue + } + err = os.WriteFile(p.Ctx.ProgressFile, bs, 0644) + if err != nil { + log.Errorf("failed to write progress data %s", err) + continue + } + } + } + + // keep job alive for extra 10 seconds to make the runner be able to catch all progress info + // TODO need optimize + time.Sleep(time.Second * 10) + return nil +} diff --git a/pkg/microservice/packager/executor/executor.go b/pkg/microservice/packager/executor/executor.go new file mode 100644 index 0000000000000000000000000000000000000000..840ca0134d8973b543237c7f4310a166acf3753a --- /dev/null +++ b/pkg/microservice/packager/executor/executor.go @@ -0,0 +1,46 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package executor + +import ( + commonconfig "github.com/koderover/zadig/pkg/config" + "github.com/koderover/zadig/pkg/microservice/packager/core/service" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/log" +) + +func Execute() error { + log.Init(&log.Config{ + Level: commonconfig.LogLevel(), + NoCaller: true, + NoLogLevel: true, + Development: commonconfig.Mode() != setting.ReleaseMode, + }) + + packager, err := service.NewPackager() + if err != nil { + log.Errorf("Failed to start packager, error: %s", err) + return err + } + + if err := packager.Exec(); err != nil { + log.Errorf("Failed to run packager, error: %s", err) + return err + } + + return nil +} diff --git a/pkg/microservice/picket/client/aslan/artifact.go b/pkg/microservice/picket/client/aslan/artifact.go new file mode 100644 index 0000000000000000000000000000000000000000..9a5042f941b7174fe6f77f76002ff34cfa21bc78 --- /dev/null +++ b/pkg/microservice/picket/client/aslan/artifact.go @@ -0,0 +1,31 @@ +package aslan + +import ( + "fmt" + "net/http" + "net/url" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +func (c *Client) GetArtifactByImage(header http.Header, qs url.Values, image string) ([]byte, error) { + url := fmt.Sprintf("/delivery/artifacts/image") + + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs), httpclient.SetQueryParam(image, image)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (c *Client) GetArtifactInfo(header http.Header, qs url.Values, id string) ([]byte, error) { + url := fmt.Sprintf("/delivery/artifacts/%s", id) + + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} diff --git a/pkg/microservice/picket/client/aslan/delivery.go b/pkg/microservice/picket/client/aslan/delivery.go new file mode 100644 index 0000000000000000000000000000000000000000..58e11011c6b0617c8a91f90349f228de4519e4d3 --- /dev/null +++ b/pkg/microservice/picket/client/aslan/delivery.go @@ -0,0 +1,25 @@ +package aslan + +import ( + "fmt" + "net/http" + "net/url" + + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +func (c *Client) ListDelivery(header http.Header, qs url.Values, productName, workflowName, taskId, perPage, page string) ([]byte, error) { + url := fmt.Sprintf("/delivery/releases") + queryParams := make(map[string]string) + queryParams["productName"] = productName + queryParams["workflowName"] = workflowName + queryParams["taskId"] = taskId + queryParams["per_page"] = perPage + queryParams["page"] = page + res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs), httpclient.SetQueryParams(queryParams)) + if err != nil { + return nil, err + } + + return res.Body(), nil +} diff --git a/pkg/microservice/picket/client/aslan/kubeconfig.go b/pkg/microservice/picket/client/aslan/kubeconfig.go index b463271b3b23320cee320aa592c222a380cd11bc..6373970ad925efb5668c98fa514a204db7595adc 100644 --- a/pkg/microservice/picket/client/aslan/kubeconfig.go +++ b/pkg/microservice/picket/client/aslan/kubeconfig.go @@ -1,3 +1,19 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package aslan import ( diff --git a/pkg/microservice/picket/client/aslan/workflow.go b/pkg/microservice/picket/client/aslan/workflow.go index b1a50012eafd6b0bed687e1a1fa8d58c36aa50f1..40bd045dcf39fdd95a0f7d85548fb8b743b77bf9 100644 --- a/pkg/microservice/picket/client/aslan/workflow.go +++ b/pkg/microservice/picket/client/aslan/workflow.go @@ -90,14 +90,3 @@ func (c *Client) ListWorkflowTask(header http.Header, qs url.Values, commitId st return res.Body(), nil } - -func (c *Client) ListDelivery(header http.Header, qs url.Values, productName, workflowName, taskId, perPage, page string) ([]byte, error) { - url := fmt.Sprintf("/delivery/releases?orgId=1&productName=%s&workflowName=%s&taskId=%s&per_page=%s&page=%s", productName, workflowName, taskId, perPage, page) - - res, err := c.Get(url, httpclient.SetHeadersFromHTTPHeader(header), httpclient.SetQueryParamsFromValues(qs)) - if err != nil { - return nil, err - } - - return res.Body(), nil -} diff --git a/pkg/microservice/picket/config/consts.go b/pkg/microservice/picket/config/consts.go index 06e1924e5eaa51319062d711f778e270bb7d1f8c..9441030192842fcc7ef09716bd5f1661ffdc5dad 100644 --- a/pkg/microservice/picket/config/consts.go +++ b/pkg/microservice/picket/config/consts.go @@ -1,3 +1,19 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package config type RulesLogicalOperator string diff --git a/pkg/microservice/picket/core/filter/handler/kubeconfig.go b/pkg/microservice/picket/core/filter/handler/kubeconfig.go index 5cac861faf413fbbc37e048d8e19836c437ffa4e..365e75e5f2e3ab18ce96b70b10d44670ce30386a 100644 --- a/pkg/microservice/picket/core/filter/handler/kubeconfig.go +++ b/pkg/microservice/picket/core/filter/handler/kubeconfig.go @@ -1,3 +1,19 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package handler import ( diff --git a/pkg/microservice/picket/core/filter/service/kubeconfig.go b/pkg/microservice/picket/core/filter/service/kubeconfig.go index 773fc8cb49c017a97ec27f698ebc767a08293a89..b020f970761fc518005fa0e089e8a39b0aa396e8 100644 --- a/pkg/microservice/picket/core/filter/service/kubeconfig.go +++ b/pkg/microservice/picket/core/filter/service/kubeconfig.go @@ -1,3 +1,19 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package service import ( diff --git a/pkg/microservice/picket/core/public/handler/public.go b/pkg/microservice/picket/core/public/handler/public.go index 17c209e3afbcd6bfce665d05f7df77aa99f1d760..602ee5e397525e312857bc312704c3f6c8571434 100644 --- a/pkg/microservice/picket/core/public/handler/public.go +++ b/pkg/microservice/picket/core/public/handler/public.go @@ -22,6 +22,7 @@ import ( "github.com/gin-gonic/gin" + "github.com/koderover/zadig/pkg/microservice/aslan/core/common/repository/models" "github.com/koderover/zadig/pkg/microservice/picket/core/public/service" internalhandler "github.com/koderover/zadig/pkg/shared/handler" ) @@ -29,10 +30,24 @@ import ( func CreateWorkflowTask(c *gin.Context) { ctx := internalhandler.NewContext(c) defer func() { internalhandler.JSONResponse(c, ctx) }() - body, _ := c.GetRawData() + req := new(models.WorkflowTaskArgs) + if err := c.ShouldBindJSON(req); err != nil { + ctx.Err = err + ctx.Logger.Errorf("ShouldBindJSON err:%s", err) + return + } + req.Namespace = req.EnvName + req.RequestMode = "openAPI" + body, err := json.Marshal(req) + if err != nil { + ctx.Err = err + ctx.Logger.Errorf("Marshal err:%s", err) + return + } res, err := service.CreateWorkflowTask(c.Request.Header, c.Request.URL.Query(), body, ctx.Logger) if err != nil { ctx.Err = err + ctx.Logger.Errorf("CreateWorkflowTask err:%s", err) return } // to avoid customer feel confused ，return workflow_name instead of pipline_name @@ -40,6 +55,7 @@ func CreateWorkflowTask(c *gin.Context) { err = json.Unmarshal(res, &resp) if err != nil { ctx.Err = err + ctx.Logger.Errorf("Unmarshal err:%s", err) return } resp.WorkflowName = resp.PipelineName @@ -123,3 +139,10 @@ func ListDelivery(c *gin.Context) { pageStr := c.Query("page") ctx.Resp, ctx.Err = service.ListDelivery(c.Request.Header, c.Request.URL.Query(), productName, workflowName, taskIDStr, perPageStr, pageStr, ctx.Logger) } + +func GetArtifactInfo(c *gin.Context) { + ctx := internalhandler.NewContext(c) + defer func() { internalhandler.JSONResponse(c, ctx) }() + image := c.Query("image") + ctx.Resp, ctx.Err = service.GetArtifactInfo(c.Request.Header, c.Request.URL.Query(), image, ctx.Logger) +} diff --git a/pkg/microservice/picket/core/public/handler/router.go b/pkg/microservice/picket/core/public/handler/router.go index a4eeca51f44cdcf031ffc821d7786037d7e194d3..177292def84db4630fe5f6a73efe7520fdb89bae 100644 --- a/pkg/microservice/picket/core/public/handler/router.go +++ b/pkg/microservice/picket/core/public/handler/router.go @@ -27,9 +27,10 @@ func (*Router) Inject(router *gin.RouterGroup) { dev := router.Group("") { dev.POST("/workflowTask/create", CreateWorkflowTask) - dev.DELETE("workflowTask/id/:id/pipelines/:name/cancel", CancelWorkflowTask) + dev.POST("/workflowTask/id/:id/pipelines/:name/cancel", CancelWorkflowTask) dev.POST("/workflowTask/id/:id/pipelines/:name/restart", RestartWorkflowTask) dev.GET("/workflowTask", ListWorkflowTask) dev.GET("/dc/releases", ListDelivery) + dev.GET("/dc/artifact", GetArtifactInfo) } } diff --git a/pkg/microservice/picket/core/public/service/public.go b/pkg/microservice/picket/core/public/service/public.go index f3d9ef9eefcd3c17afda130201fc3418f36f729f..632d96e094debea1cb74e79d919704c1f244ede5 100644 --- a/pkg/microservice/picket/core/public/service/public.go +++ b/pkg/microservice/picket/core/public/service/public.go @@ -17,6 +17,8 @@ limitations under the License. package service import ( + "encoding/json" + "fmt" "net/http" "net/url" @@ -44,3 +46,73 @@ func ListWorkflowTask(header http.Header, qs url.Values, commitId string, _ *zap func ListDelivery(header http.Header, qs url.Values, productName string, workflowName string, taskId string, perPage string, page string, _ *zap.SugaredLogger) ([]byte, error) { return aslan.New().ListDelivery(header, qs, productName, workflowName, taskId, perPage, page) } + +type DeliveryArtifact struct { + Name string `json:"name"` + Type string `json:"type"` + Source string `json:"source"` + Image string `json:"image,omitempty"` + ImageHash string `json:"image_hash,omitempty"` + ImageTag string `json:"image_tag"` + ImageDigest string `json:"image_digest,omitempty"` + ImageSize int64 `json:"image_size,omitempty"` + Architecture string `json:"architecture,omitempty"` + Os string `json:"os,omitempty"` + PackageFileLocation string `json:"package_file_location,omitempty"` + PackageStorageURI string `json:"package_storage_uri,omitempty"` + CreatedBy string `json:"created_by"` + CreatedTime int64 `json:"created_time"` +} + +type DeliveryActivity struct { + Type string `json:"type"` + Content string `json:"content,omitempty"` + URL string `json:"url,omitempty"` + Commits []*ActivityCommit `json:"commits,omitempty"` + Issues []string `json:"issues,omitempty"` + Namespace string `json:"namespace,omitempty"` + EnvName string `json:"env_name,omitempty"` + PublishHosts []string `json:"publish_hosts,omitempty"` + PublishNamespaces []string `json:"publish_namespaces,omitempty"` + RemoteFileKey string `json:"remote_file_key,omitempty"` + DistStorageURL string `json:"dist_storage_url,omitempty"` + SrcStorageURL string `json:"src_storage_url,omitempty"` + StartTime int64 `json:"start_time,omitempty"` + EndTime int64 `json:"end_time,omitempty"` + CreatedBy string `json:"created_by"` + CreatedTime int64 `json:"created_time"` +} + +type ActivityCommit struct { + Address string `json:"address"` + Source string `json:"source,omitempty"` + RepoOwner string `json:"repo_owner"` + RepoName string `json:"repo_name"` + Branch string `json:"branch"` + PR int `json:"pr,omitempty"` + Tag string `json:"tag,omitempty"` + CommitID string `json:"commit_id,omitempty"` + CommitMessage string `json:"commit_message,omitempty"` + AuthorName string `json:"author_name,omitempty"` +} + +type DeliveryArtifactInfo struct { + *DeliveryArtifact + DeliveryActivities []*DeliveryActivity `json:"activities"` + DeliveryActivitiesMap map[string][]*DeliveryActivity `json:"sortedActivities,omitempty"` +} + +func GetArtifactInfo(header http.Header, qs url.Values, image string, _ *zap.SugaredLogger) (*DeliveryArtifactInfo, error) { + artifactID, err := aslan.New().GetArtifactByImage(header, qs, image) + if err != nil { + return nil, fmt.Errorf("failed to get artifact by image err:%s", err) + } + + body, err := aslan.New().GetArtifactInfo(header, qs, string(artifactID)) + var deliveryArtifactInfo *DeliveryArtifactInfo + if err = json.Unmarshal(body, &deliveryArtifactInfo); err != nil { + return nil, fmt.Errorf("failed to unmarshal deliveryArtifactInfo err:%s", err) + } + + return deliveryArtifactInfo, nil +} diff --git a/pkg/microservice/podexec/core/service/ws_terminal.go b/pkg/microservice/podexec/core/service/ws_terminal.go index dfb805ecc0e128001136a0041fa956bf02126314..003ad1f5dfffccc46f130cb7131aa04ce1782f2c 100644 --- a/pkg/microservice/podexec/core/service/ws_terminal.go +++ b/pkg/microservice/podexec/core/service/ws_terminal.go @@ -35,6 +35,7 @@ import ( "k8s.io/client-go/tools/remotecommand" conf "github.com/koderover/zadig/pkg/microservice/podexec/config" + "github.com/koderover/zadig/pkg/setting" "github.com/koderover/zadig/pkg/tool/log" ) @@ -206,7 +207,7 @@ func NewKubeOutClusterClient(clusterID string) (kubernetes.Interface, *rest.Conf var config *rest.Config var err error - if clusterID == "" { + if clusterID == "" || clusterID == setting.LocalClusterID { kubeConfig := "" config, err = clientcmd.BuildConfigFromFlags("", kubeConfig) if err != nil { diff --git a/pkg/microservice/reaper/core/service/archive/file_archive.go b/pkg/microservice/reaper/core/service/archive/file_archive.go index 96ee051c8b865ef890773e9e819f658968f2c4ac..5857925a1f0436ebed8a8c3341a556d360ff5cf4 100644 --- a/pkg/microservice/reaper/core/service/archive/file_archive.go +++ b/pkg/microservice/reaper/core/service/archive/file_archive.go @@ -42,12 +42,14 @@ type WorkspaceAchiever struct { PipelineName string ServiceName string Envs []string + aesKey string } -func NewWorkspaceAchiever(storageURI, pipelineName, serviceName, wd string, caches, gitFolders, envs []string) *WorkspaceAchiever { +func NewWorkspaceAchiever(storageURI, pipelineName, serviceName, wd, aesKey string, caches, gitFolders, envs []string) *WorkspaceAchiever { return &WorkspaceAchiever{ paths: caches, wd: wd, + aesKey: aesKey, gitFolders: gitFolders, StorageURI: storageURI, PipelineName: pipelineName, @@ -224,7 +226,7 @@ func (c *WorkspaceAchiever) Achieve(target string) ([]string, error) { // return err //} - if store, err := s3.NewS3StorageFromEncryptedURI(c.StorageURI); err == nil { + if store, err := s3.NewS3StorageFromEncryptedURI(c.StorageURI, c.aesKey); err == nil { forcedPathStyle := true if store.Provider == setting.ProviderSourceAli { forcedPathStyle = false diff --git a/pkg/microservice/reaper/core/service/meta/types.go b/pkg/microservice/reaper/core/service/meta/types.go index 4a587d48159b20edcdef3ef2f3ac9a12d12f2ad9..86d69329ea37b5a92f9fac79fe364f6e669d81b2 100644 --- a/pkg/microservice/reaper/core/service/meta/types.go +++ b/pkg/microservice/reaper/core/service/meta/types.go @@ -115,6 +115,7 @@ type Context struct { StorageBucket string `yaml:"storage_bucket"` StorageProvider int `yaml:"storage_provider"` ArtifactInfo *ArtifactInfo `yaml:"artifact_info"` + AesKey string `yaml:"aes_key"` } type ArtifactInfo struct { diff --git a/pkg/microservice/reaper/core/service/reaper/archive.go b/pkg/microservice/reaper/core/service/reaper/archive.go index b729ce665dd8d55920a05aaf12889db5b7e99158..c2c136bff4220fbb69c6d394602ea15cd3f091fc 100644 --- a/pkg/microservice/reaper/core/service/reaper/archive.go +++ b/pkg/microservice/reaper/core/service/reaper/archive.go @@ -34,7 +34,7 @@ func (r *Reaper) archiveS3Files() (err error) { if r.Ctx.FileArchiveCtx != nil && r.Ctx.StorageURI != "" { var store *s3.S3 - if store, err = s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI); err != nil { + if store, err = s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI, r.Ctx.AesKey); err != nil { log.Errorf("failed to create s3 storage %s", r.Ctx.StorageURI) return } @@ -77,7 +77,7 @@ func (r *Reaper) archiveTestFiles() error { return nil } - store, err := s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI) + store, err := s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI, r.Ctx.AesKey) if err != nil { log.Errorf("failed to create s3 storage %s, err: %s", r.Ctx.StorageURI, err) return err @@ -130,7 +130,7 @@ func (r *Reaper) archiveHTMLTestReportFile() error { return nil } - store, err := s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI) + store, err := s3.NewS3StorageFromEncryptedURI(r.Ctx.StorageURI, r.Ctx.AesKey) if err != nil { log.Errorf("failed to create s3 storage %s, err: %s", r.Ctx.StorageURI, err) return err diff --git a/pkg/microservice/reaper/core/service/reaper/artifact.go b/pkg/microservice/reaper/core/service/reaper/artifact.go index ff93b4ff84faccbf28a460def596944ac00b5ea4..09af436c748ca67ee52f9833b5a42b5ab3d25355 100644 --- a/pkg/microservice/reaper/core/service/reaper/artifact.go +++ b/pkg/microservice/reaper/core/service/reaper/artifact.go @@ -38,7 +38,7 @@ func artifactsUpload(ctx *meta.Context, activeWorkspace string) error { ) if ctx.StorageURI != "" { - if store, err = s3.NewS3StorageFromEncryptedURI(ctx.StorageURI); err != nil { + if store, err = s3.NewS3StorageFromEncryptedURI(ctx.StorageURI, ctx.AesKey); err != nil { log.Errorf("artifactsUpload failed to create s3 storage err:%v", err) return err } diff --git a/pkg/microservice/reaper/core/service/reaper/cachemanager.go b/pkg/microservice/reaper/core/service/reaper/cachemanager.go index ae4eb53787c1ef056df1ea39a467b283957c3d15..a4eba4d9ac9da5b4b6596c0671600450261f321f 100644 --- a/pkg/microservice/reaper/core/service/reaper/cachemanager.go +++ b/pkg/microservice/reaper/core/service/reaper/cachemanager.go @@ -80,13 +80,15 @@ type TarCacheManager struct { StorageURI string PipelineName string ServiceName string + aesKey string } -func NewTarCacheManager(storageURI, pipelineName, serviceName string) *TarCacheManager { +func NewTarCacheManager(storageURI, pipelineName, serviceName, aesKey string) *TarCacheManager { return &TarCacheManager{ StorageURI: storageURI, PipelineName: pipelineName, ServiceName: serviceName, + aesKey: aesKey, } } @@ -180,7 +182,7 @@ func (gcm *TarCacheManager) Unarchive(source, dest string) error { func (gcm *TarCacheManager) getS3Storage() (*s3.S3, error) { var err error var store *s3.S3 - if store, err = s3.NewS3StorageFromEncryptedURI(gcm.StorageURI); err != nil { + if store, err = s3.NewS3StorageFromEncryptedURI(gcm.StorageURI, gcm.aesKey); err != nil { log.Errorf("Archive failed to create s3 storage %s", gcm.StorageURI) return nil, err } diff --git a/pkg/microservice/reaper/core/service/reaper/reaper.go b/pkg/microservice/reaper/core/service/reaper/reaper.go index 136d0d39419280f04a02305328fd58b15b036c3d..d85d4a6742d2ab28c81e7f37c9ef2c41a85e6775 100644 --- a/pkg/microservice/reaper/core/service/reaper/reaper.go +++ b/pkg/microservice/reaper/core/service/reaper/reaper.go @@ -74,7 +74,7 @@ func NewReaper() (*Reaper, error) { reaper := &Reaper{ Ctx: ctx, - cm: NewTarCacheManager(ctx.StorageURI, ctx.PipelineName, ctx.ServiceName), + cm: NewTarCacheManager(ctx.StorageURI, ctx.PipelineName, ctx.ServiceName, ctx.AesKey), } return reaper, nil @@ -85,7 +85,7 @@ func (r *Reaper) GetCacheFile() string { } func (r *Reaper) archiveCustomCaches(wd, dest string, caches []string) ([]string, error) { - fileAchiever := archive.NewWorkspaceAchiever(r.Ctx.StorageURI, r.Ctx.PipelineName, r.Ctx.ServiceName, wd, caches, []string{}, r.getUserEnvs()) + fileAchiever := archive.NewWorkspaceAchiever(r.Ctx.StorageURI, r.Ctx.PipelineName, r.Ctx.ServiceName, wd, r.Ctx.AesKey, caches, []string{}, r.getUserEnvs()) // list files matches caches return fileAchiever.Achieve(dest) diff --git a/pkg/microservice/reaper/core/service/reaper/script.go b/pkg/microservice/reaper/core/service/reaper/script.go index 47cbf24092e75285c36750dd640342c42ab3a122..43fadc53247039c5618b57e54b7338b674379f30 100644 --- a/pkg/microservice/reaper/core/service/reaper/script.go +++ b/pkg/microservice/reaper/core/service/reaper/script.go @@ -206,6 +206,7 @@ func (r *Reaper) runScripts() error { // avoid non-blocking IO for stdout to workaround "stdout: write error" for _, script := range r.Ctx.Scripts { scripts = append(scripts, script) + // TODO: This may cause nodejs compilation problems, but it is not completely determined, so keep it for now. if strings.Contains(script, "yarn ") || strings.Contains(script, "npm ") || strings.Contains(script, "bower ") { scripts = append(scripts, "echo 'turn off O_NONBLOCK after using node'") scripts = append(scripts, "python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'") @@ -229,33 +230,22 @@ func (r *Reaper) runScripts() error { if err != nil { return err } - outScanner := bufio.NewScanner(cmdOutReader) - go func() { - for outScanner.Scan() { - fmt.Printf("%s\n", r.maskSecretEnvs(outScanner.Text())) - if len(r.Ctx.PostScripts) > 0 { - util.WriteFile(fileName, []byte(outScanner.Text()+"\n"), 0700) - } - } - }() - cmdErrReader, err := cmd.StderrPipe() - if err != nil { + if err := cmd.Start(); err != nil { return err } - errScanner := bufio.NewScanner(cmdErrReader) go func() { - for errScanner.Scan() { - fmt.Printf("%s\n", r.maskSecretEnvs(errScanner.Text())) + for outScanner.Scan() { + fmt.Printf("%s\n", r.maskSecretEnvs(outScanner.Text())) if len(r.Ctx.PostScripts) > 0 { - util.WriteFile(fileName, []byte(errScanner.Text()+"\n"), 0700) + util.WriteFile(fileName, []byte(outScanner.Text()+"\n"), 0700) } } }() - return cmd.Run() + return cmd.Wait() } func (r *Reaper) prepareScriptsEnv() []string { @@ -377,7 +367,7 @@ func (r *Reaper) RunPMDeployScripts() error { func (r *Reaper) downloadArtifactFile() error { var err error var store *s3.S3 - if store, err = s3.NewS3StorageFromEncryptedURI(r.Ctx.ArtifactInfo.URL); err != nil { + if store, err = s3.NewS3StorageFromEncryptedURI(r.Ctx.ArtifactInfo.URL, r.Ctx.AesKey); err != nil { log.Errorf("Archive failed to create s3 storage %s", r.Ctx.ArtifactInfo.URL) return err } diff --git a/pkg/microservice/reaper/internal/s3/s3.go b/pkg/microservice/reaper/internal/s3/s3.go index 3c15cc4b871e9ae053f8f5df28b72328b9dcf603..ed87e9c7b2bf31908dde1aa3c3332e05cec3ab7d 100644 --- a/pkg/microservice/reaper/internal/s3/s3.go +++ b/pkg/microservice/reaper/internal/s3/s3.go @@ -75,8 +75,8 @@ func NewS3StorageFromURL(uri string) (*S3, error) { return ret, nil } -func NewS3StorageFromEncryptedURI(encryptedURI string) (*S3, error) { - uri, err := crypto.AesDecrypt(encryptedURI) +func NewS3StorageFromEncryptedURI(encryptedURI, aesKey string) (*S3, error) { + uri, err := crypto.AesDecrypt(encryptedURI, aesKey) if err != nil { return nil, err } diff --git a/pkg/microservice/warpdrive/config/const.go b/pkg/microservice/warpdrive/config/const.go index 6e4fcaca3b6423ad88dbff395188e0043d9ce667..380b211f5baf53e2ac0b92b9321b3971fa9c394c 100644 --- a/pkg/microservice/warpdrive/config/const.go +++ b/pkg/microservice/warpdrive/config/const.go @@ -19,20 +19,21 @@ package config type TaskType string const ( - TaskPipeline TaskType = "pipeline" - TaskBuild TaskType = "buildv2" - TaskArtifactDeploy TaskType = "artifact_deploy" - TaskJenkinsBuild TaskType = "jenkins_build" - TaskArtifact TaskType = "artifact" - TaskDeploy TaskType = "deploy" - TaskTestingV2 TaskType = "testingv2" - TaskDistributeToS3 TaskType = "distribute2kodo" - TaskReleaseImage TaskType = "release_image" - TaskJira TaskType = "jira" - TaskDockerBuild TaskType = "docker_build" - TaskSecurity TaskType = "security" - TaskResetImage TaskType = "reset_image" - TaskDistribute TaskType = "distribute" + TaskPipeline TaskType = "pipeline" + TaskBuild TaskType = "buildv2" + TaskArtifactDeploy TaskType = "artifact_deploy" + TaskJenkinsBuild TaskType = "jenkins_build" + TaskArtifact TaskType = "artifact" + TaskDeploy TaskType = "deploy" + TaskTestingV2 TaskType = "testingv2" + TaskDistributeToS3 TaskType = "distribute2kodo" + TaskReleaseImage TaskType = "release_image" + TaskJira TaskType = "jira" + TaskDockerBuild TaskType = "docker_build" + TaskSecurity TaskType = "security" + TaskResetImage TaskType = "reset_image" + TaskDistribute TaskType = "distribute" + TaskArtifactPackage TaskType = "artifact_package" ) type Status string @@ -55,16 +56,18 @@ const ( type PipelineType string const ( - // SingleType 单服务工作流 + // SingleType single-service workflow SingleType PipelineType = "single" - // WorkflowType 多服务工作流 + // WorkflowType multi-service workflow WorkflowType PipelineType = "workflow" - // FreestyleType 自由编排工作流 + // FreestyleType freeStyle workflow FreestyleType PipelineType = "freestyle" - // TestType 测试 + // TestType testing TestType PipelineType = "test" - // ServiceType 服务 + // ServiceType pipeline ServiceType PipelineType = "service" + // ArtifactType artifact build + ArtifactType PipelineType = "artifact" ) type NotifyType int diff --git a/pkg/microservice/warpdrive/core/service/taskcontroller/task_handler.go b/pkg/microservice/warpdrive/core/service/taskcontroller/task_handler.go index 301be7bf65044b4a5752bd43d188703c1fc8101f..db98e0842198af860c41de3d2c8c8a9b3d1eeabf 100644 --- a/pkg/microservice/warpdrive/core/service/taskcontroller/task_handler.go +++ b/pkg/microservice/warpdrive/core/service/taskcontroller/task_handler.go @@ -411,6 +411,9 @@ func (h *ExecHandler) executeTask(taskCtx context.Context, plugin plugins.TaskPl } else if pipelineTask.Type == config.ServiceType { fileName = strings.Replace(strings.ToLower(fmt.Sprintf("%s-%s-%d-%s-%s", config.ServiceType, pipelineTask.PipelineName, pipelineTask.TaskID, plugin.Type(), servicename)), "_", "-", -1) + } else if pipelineTask.Type == config.ArtifactType { + fileName = strings.Replace(strings.ToLower(fmt.Sprintf("%s-%s-%d-%s", config.ArtifactType, pipelineTask.PipelineName, pipelineTask.TaskID, plugin.Type())), + "_", "-", -1) } plugin.Init(jobName, fileName, xl) diff --git a/pkg/microservice/warpdrive/core/service/taskcontroller/task_helper.go b/pkg/microservice/warpdrive/core/service/taskcontroller/task_helper.go index b0a13595e7a9bed971ac39b6d125f4861f78e302..f8b7a57ee491c966de3b6d36c9db86eddd90b072 100644 --- a/pkg/microservice/warpdrive/core/service/taskcontroller/task_helper.go +++ b/pkg/microservice/warpdrive/core/service/taskcontroller/task_helper.go @@ -136,6 +136,9 @@ func updatePipelineSubTask(t interface{}, pipelineTask *task.Task, pos int, serv } else if pipelineTask.Type == config.ServiceType { xl.Info("pipeline type is service type: pipeline 3.0") pipelineTask.Stages[pos].SubTasks[servicename] = subTask + } else if pipelineTask.Type == config.ArtifactType { + xl.Info("pipeline type is artifact-package type: pipeline 3.0") + pipelineTask.Stages[pos].SubTasks[servicename] = subTask } } @@ -538,17 +541,18 @@ func getSubTaskTypeAndIsRestart(subTask map[string]interface{}) bool { func initTaskPlugins(execHandler *ExecHandler) { pluginConf := map[config.TaskType]plugins.Initiator{ - config.TaskJira: plugins.InitializeJiraTaskPlugin, - config.TaskBuild: plugins.InitializeBuildTaskPlugin, - config.TaskArtifactDeploy: plugins.InitializeArtifactTaskPlugin, - config.TaskJenkinsBuild: plugins.InitializeJenkinsBuildPlugin, - config.TaskDockerBuild: plugins.InitializeDockerBuildTaskPlugin, - config.TaskDeploy: plugins.InitializeDeployTaskPlugin, - config.TaskTestingV2: plugins.InitializeTestTaskPlugin, - config.TaskSecurity: plugins.InitializeSecurityPlugin, - config.TaskReleaseImage: plugins.InitializeReleaseImagePlugin, - config.TaskDistributeToS3: plugins.InitializeDistribute2S3TaskPlugin, - config.TaskResetImage: plugins.InitializeDeployTaskPlugin, + config.TaskJira: plugins.InitializeJiraTaskPlugin, + config.TaskBuild: plugins.InitializeBuildTaskPlugin, + config.TaskArtifactDeploy: plugins.InitializeArtifactTaskPlugin, + config.TaskJenkinsBuild: plugins.InitializeJenkinsBuildPlugin, + config.TaskDockerBuild: plugins.InitializeDockerBuildTaskPlugin, + config.TaskDeploy: plugins.InitializeDeployTaskPlugin, + config.TaskTestingV2: plugins.InitializeTestTaskPlugin, + config.TaskSecurity: plugins.InitializeSecurityPlugin, + config.TaskReleaseImage: plugins.InitializeReleaseImagePlugin, + config.TaskDistributeToS3: plugins.InitializeDistribute2S3TaskPlugin, + config.TaskResetImage: plugins.InitializeDeployTaskPlugin, + config.TaskArtifactPackage: plugins.InitializeArtifactPackagePlugin, } for name, pluginInitiator := range pluginConf { registerTaskPlugin(execHandler, name, pluginInitiator) diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/artifact_deploy.go b/pkg/microservice/warpdrive/core/service/taskplugin/artifact_deploy.go index b37306e7082745d722a75cf2b07663e940e31b7f..a21b0fd9c03f14111e14452c898a12533e9b814d 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/artifact_deploy.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/artifact_deploy.go @@ -215,7 +215,7 @@ func (p *ArtifactDeployTaskPlugin) Run(ctx context.Context, pipelineTask *task.T } //Resource request default value is LOW - job, err := buildJob(p.Type(), jobImage, p.JobName, serviceName, p.Task.ResReq, p.Task.ResReqSpec, pipelineCtx, pipelineTask, p.Task.Registries) + job, err := buildJob(p.Type(), jobImage, p.JobName, serviceName, "", pipelineTask.ConfigPayload.Build.KubeNamespace, p.Task.ResReq, p.Task.ResReqSpec, pipelineCtx, pipelineTask, p.Task.Registries) if err != nil { msg := fmt.Sprintf("create build job context error: %v", err) p.Log.Error(msg) @@ -237,7 +237,7 @@ func (p *ArtifactDeployTaskPlugin) Run(ctx context.Context, pipelineTask *task.T } // 将集成到KodeRover的私有镜像仓库的访问权限设置到namespace中 - if err := createOrUpdateRegistrySecrets(p.KubeNamespace, p.Task.Registries, p.kubeClient); err != nil { + if err := createOrUpdateRegistrySecrets(p.KubeNamespace, pipelineTask.ConfigPayload.RegistryID, p.Task.Registries, p.kubeClient); err != nil { msg := fmt.Sprintf("create secret error: %v", err) p.Log.Error(msg) p.Task.TaskStatus = config.StatusFailed @@ -307,7 +307,7 @@ func (p *ArtifactDeployTaskPlugin) Complete(ctx context.Context, pipelineTask *t } }() - err := saveContainerLog(pipelineTask, p.KubeNamespace, p.FileName, jobLabel, p.kubeClient) + err := saveContainerLog(pipelineTask, p.KubeNamespace, "", p.FileName, jobLabel, p.kubeClient) if err != nil { p.Log.Error(err) p.Task.Error = err.Error() diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/artifact_package.go b/pkg/microservice/warpdrive/core/service/taskplugin/artifact_package.go new file mode 100644 index 0000000000000000000000000000000000000000..a2eec1c000d08bc2a822f1c7e0ef903a665eeaa2 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/taskplugin/artifact_package.go @@ -0,0 +1,422 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package taskplugin + +import ( + "context" + "fmt" + "time" + + "go.uber.org/zap" + "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types" + "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/shared/kube/wrapper" + krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" + "github.com/koderover/zadig/pkg/tool/kube/getter" + "github.com/koderover/zadig/pkg/tool/kube/podexec" + "github.com/koderover/zadig/pkg/tool/kube/updater" +) + +// InitializeArtifactPackagePlugin to ini +func InitializeArtifactPackagePlugin(taskType config.TaskType) TaskPlugin { + return &ArtifactPackageTaskPlugin{ + Name: taskType, + kubeClient: krkubeclient.Client(), + } +} + +type ArtifactPackageTaskPlugin struct { + Name config.TaskType + KubeNamespace string + JobName string + FileName string + kubeClient client.Client + Task *task.ArtifactPackage + AckFunc func() + Log *zap.SugaredLogger + + RealTimeProgress string +} + +const ( + // ArtifactPackageBuildImageTaskTimeout ... + ArtifactPackageBuildImageTaskTimeout = 60 * 5 // 5 minutes +) + +// Init ... +func (p *ArtifactPackageTaskPlugin) Init(jobname, filename string, xl *zap.SugaredLogger) { + p.JobName = jobname + p.FileName = filename + // SetLogger ... + p.Log = xl +} + +func (p *ArtifactPackageTaskPlugin) SetAckFunc(f func()) { + p.AckFunc = f +} + +// Type ... +func (p *ArtifactPackageTaskPlugin) Type() config.TaskType { + return p.Name +} + +// Status ... +func (p *ArtifactPackageTaskPlugin) Status() config.Status { + return p.Task.TaskStatus +} + +// SetStatus ... +func (p *ArtifactPackageTaskPlugin) SetStatus(status config.Status) { + p.Task.TaskStatus = status +} + +func (p *ArtifactPackageTaskPlugin) SetProgress(progress string) { + p.Task.Progress = progress +} + +// TaskTimeout ... +func (p *ArtifactPackageTaskPlugin) TaskTimeout() int { + if p.Task.Timeout == 0 { + p.Task.Timeout = ArtifactPackageBuildImageTaskTimeout + } + return p.Task.Timeout +} + +func (p *ArtifactPackageTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineCtx *task.PipelineCtx, serviceName string) { + p.KubeNamespace = pipelineTask.ConfigPayload.Build.KubeNamespace + + sourceRegistries := make([]*types.DockerRegistry, 0) + for _, registryID := range pipelineTask.ArtifactPackageTaskArgs.SourceRegistries { + if registry, ok := pipelineTask.ConfigPayload.RepoConfigs[registryID]; ok { + sourceRegistries = append(sourceRegistries, &types.DockerRegistry{ + RegistryID: registryID, + Host: registry.RegAddr, + Namespace: registry.Namespace, + UserName: registry.AccessKey, + Password: registry.SecretKey, + }) + } + } + + targetRegistries := make([]*types.DockerRegistry, 0) + for _, registryID := range pipelineTask.ArtifactPackageTaskArgs.TargetRegistries { + if registry, ok := pipelineTask.ConfigPayload.RepoConfigs[registryID]; ok { + targetRegistries = append(targetRegistries, &types.DockerRegistry{ + RegistryID: registryID, + Host: registry.RegAddr, + Namespace: registry.Namespace, + UserName: registry.AccessKey, + Password: registry.SecretKey, + }) + } + } + + jobCtx := &types.ArtifactPackagerContext{ + JobType: setting.BuildChartPackage, + ProgressFile: setting.ProgressFile, + Images: pipelineTask.ArtifactPackageTaskArgs.Images, + SourceRegistries: sourceRegistries, + TargetRegistries: targetRegistries, + } + + jobCtxBytes, err := yaml.Marshal(jobCtx) + if err != nil { + msg := fmt.Sprintf("cannot mashal predetor.Context data: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + return + } + + p.Task.Error = "" + + jobLabel := &JobLabel{ + PipelineName: pipelineTask.PipelineName, + TaskID: pipelineTask.TaskID, + TaskType: string(p.Type()), + PipelineType: string(pipelineTask.Type), + } + + if err := ensureDeleteConfigMap(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + msg := fmt.Sprintf("ensureDeleteConfigMap error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + return + } + + if err := createJobConfigMap( + p.KubeNamespace, p.JobName, jobLabel, string(jobCtxBytes), p.kubeClient); err != nil { + msg := fmt.Sprintf("createJobConfigMap error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + return + } + p.Log.Infof("succeed to create cm for artifact package job %s", p.JobName) + + job, err := buildJob(p.Type(), pipelineTask.ConfigPayload.Release.PackagerImage, p.JobName, serviceName, "", pipelineTask.ConfigPayload.Build.KubeNamespace, setting.MinRequest, setting.LowRequestSpec, pipelineCtx, pipelineTask, []*task.RegistryNamespace{}) + if err != nil { + msg := fmt.Sprintf("create release artifact package job context error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + return + } + + if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + msg := fmt.Sprintf("delete release artifact package job error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + return + } + + job.Namespace = p.KubeNamespace + if err := updater.CreateJob(job, p.kubeClient); err != nil { + msg := fmt.Sprintf("create release image job error: %v", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + return + } + p.Log.Infof("succeed to create artifact package job %s", p.JobName) +} + +// SetTask ... +func (p *ArtifactPackageTaskPlugin) SetTask(t map[string]interface{}) error { + task, err := ToArtifactTask(t) + if err != nil { + return err + } + p.Task = task + return nil +} + +// GetTask ... +func (p *ArtifactPackageTaskPlugin) GetTask() interface{} { + return p.Task +} + +// Wait ... +func (p *ArtifactPackageTaskPlugin) Wait(ctx context.Context) { + + status := p.waitJobStart() + if status != config.StatusRunning { + p.SetStatus(status) + return + } + + status = p.waitJobEnd(ctx, p.TaskTimeout()) + p.SetProgress(p.RealTimeProgress) + p.SetStatus(status) +} + +func (p *ArtifactPackageTaskPlugin) waitJobStart() config.Status { + namespace, jobName, kubeClient, xl := p.KubeNamespace, p.JobName, p.kubeClient, p.Log + xl.Infof("wait job to start: %s/%s", namespace, jobName) + podTimeout := time.After(120 * time.Second) + var started bool + for { + select { + case <-podTimeout: + return config.StatusTimeout + default: + job, _, err := getter.GetJob(namespace, jobName, kubeClient) + if err != nil { + xl.Errorf("get job failed, namespace:%s, jobName:%s, err:%v", namespace, jobName, err) + } + if job != nil { + started = job.Status.Active > 0 + } + } + if started { + break + } + + time.Sleep(time.Second) + } + return config.StatusRunning +} + +func (p *ArtifactPackageTaskPlugin) waitJobEnd(ctx context.Context, taskTimeout int) (status config.Status) { + namespace, jobName, kubeClient, xl := p.KubeNamespace, p.JobName, p.kubeClient, p.Log + xl.Infof("wait job to start: %s/%s", namespace, jobName) + timeout := time.After(time.Duration(taskTimeout) * time.Second) + + // 等待job 运行结束 + xl.Infof("wait job to end: %s %s", namespace, jobName) + for { + select { + case <-ctx.Done(): + return config.StatusCancelled + + case <-timeout: + return config.StatusTimeout + + default: + job, found, err := getter.GetJob(namespace, jobName, kubeClient) + if err != nil || !found { + xl.Errorf("failed to get pod with label job-name=%s %v", jobName, err) + return config.StatusFailed + } + // pod is still running + if job.Status.Active != 0 { + + pods, err := getter.ListPods(namespace, labels.Set{"job-name": jobName}.AsSelector(), kubeClient) + if err != nil { + xl.Errorf("failed to find pod with label job-name=%s %v", jobName, err) + return config.StatusFailed + } + + var done bool + for _, pod := range pods { + ipod := wrapper.Pod(pod) + if ipod.Pending() { + continue + } + if ipod.Failed() { + return config.StatusFailed + } + + if !ipod.Finished() { + progressInfo, err := getProgressInfo(namespace, ipod.Name, ipod.ContainerNames()[0], setting.ProgressFile) + if err != nil { + xl.Infof("failed to check dog food file %s %v", pods[0].Name, err) + break + } + p.RealTimeProgress = progressInfo + xl.Infof("find progress info %s", progressInfo) + p.SetProgress(progressInfo) + p.SetStatus(config.StatusRunning) + + if p.AckFunc != nil { + p.AckFunc() + } + } else { + done = true + } + } + + if done { + return config.StatusPassed + } + } else if job.Status.Succeeded != 0 { + return config.StatusPassed + } else { + return config.StatusFailed + } + } + + time.Sleep(time.Second * 5) + } + +} + +// Complete ... +func (p *ArtifactPackageTaskPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serviceName string) { + jobLabel := &JobLabel{ + PipelineName: pipelineTask.PipelineName, + ServiceName: serviceName, + TaskID: pipelineTask.TaskID, + TaskType: string(p.Type()), + PipelineType: string(pipelineTask.Type), + } + + // 清理用户取消和超时的任务 + defer func() { + if p.Task.TaskStatus == config.StatusCancelled || p.Task.TaskStatus == config.StatusTimeout { + if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + p.Task.Error = err.Error() + } + return + } + }() + + // 保存实时日志到s3 + err := saveContainerLog(pipelineTask, p.KubeNamespace, "", p.FileName, jobLabel, p.kubeClient) + if err != nil { + p.Log.Error(err) + p.Task.Error = err.Error() + return + } + + p.Task.LogFile = p.JobName +} + +// IsTaskDone ... +func (p *ArtifactPackageTaskPlugin) IsTaskDone() bool { + if p.Task.TaskStatus != config.StatusCreated && p.Task.TaskStatus != config.StatusRunning { + return true + } + return false +} + +// IsTaskFailed ... +func (p *ArtifactPackageTaskPlugin) IsTaskFailed() bool { + if p.Task.TaskStatus == config.StatusFailed || p.Task.TaskStatus == config.StatusTimeout || p.Task.TaskStatus == config.StatusCancelled { + return true + } + return false +} + +// SetStartTime ... +func (p *ArtifactPackageTaskPlugin) SetStartTime() { + p.Task.StartTime = time.Now().Unix() +} + +// SetEndTime ... +func (p *ArtifactPackageTaskPlugin) SetEndTime() { + p.Task.EndTime = time.Now().Unix() +} + +// IsTaskEnabled ... +func (p *ArtifactPackageTaskPlugin) IsTaskEnabled() bool { + return p.Task.Enabled +} + +// ResetError ... +func (p *ArtifactPackageTaskPlugin) ResetError() { + p.Task.Error = "" +} + +func getProgressInfo(namespace string, pod string, container string, progressInfoFile string) (string, error) { + stdOut, stdErr, success, err := podexec.ExecWithOptions(podexec.ExecOptions{ + Command: []string{"cat", progressInfoFile}, + Namespace: namespace, + PodName: pod, + ContainerName: container, + }) + + if err != nil { + return "", err + } + if len(stdErr) != 0 { + return "", fmt.Errorf(stdErr) + } + if !success { + return "", fmt.Errorf("pod exec execute failed") + } + return stdOut, nil +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/build.go b/pkg/microservice/warpdrive/core/service/taskplugin/build.go index 8374fdb0366fd34ee9ee421bd997662d7d27950b..a3a625e1321f8d51558523445d07a8eaf4b07850 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/build.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/build.go @@ -32,6 +32,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/warpdrive/config" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/updater" ) @@ -106,6 +107,39 @@ func (p *BuildTaskPlugin) SetBuildStatusCompleted(status config.Status) { //TODO: Binded Archive File logic func (p *BuildTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineCtx *task.PipelineCtx, serviceName string) { + p.KubeNamespace = pipelineTask.ConfigPayload.Build.KubeNamespace + if p.Task.Namespace != "" { + p.KubeNamespace = p.Task.Namespace + kubeClient, err := kubeclient.GetKubeClient(pipelineTask.ConfigPayload.HubServerAddr, p.Task.ClusterID) + if err != nil { + msg := fmt.Sprintf("failed to get kube client: %s", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + p.SetBuildStatusCompleted(config.StatusFailed) + return + } + p.kubeClient = kubeClient + } + + // not local cluster + replaceDindServer := "." + DindServer + if p.Task.ClusterID != "" && p.Task.ClusterID != setting.LocalClusterID { + if strings.Contains(pipelineTask.DockerHost, pipelineTask.ConfigPayload.Build.KubeNamespace) { + // replace namespace only + pipelineTask.DockerHost = strings.Replace(pipelineTask.DockerHost, pipelineTask.ConfigPayload.Build.KubeNamespace, KoderoverAgentNamespace, 1) + } else { + // add namespace + pipelineTask.DockerHost = strings.Replace(pipelineTask.DockerHost, replaceDindServer, replaceDindServer+"."+KoderoverAgentNamespace, 1) + } + } else if p.Task.ClusterID == "" || p.Task.ClusterID == setting.LocalClusterID { + if !strings.Contains(pipelineTask.DockerHost, pipelineTask.ConfigPayload.Build.KubeNamespace) { + // add namespace + pipelineTask.DockerHost = strings.Replace(pipelineTask.DockerHost, replaceDindServer, replaceDindServer+"."+pipelineTask.ConfigPayload.Build.KubeNamespace, 1) + } + } + pipelineCtx.DockerHost = pipelineTask.DockerHost + if pipelineTask.Type == config.WorkflowType { envName := pipelineTask.WorkflowArgs.Namespace envNameVar := &task.KeyVal{Key: "ENV_NAME", Value: envName, IsCredential: false} @@ -143,7 +177,6 @@ func (p *BuildTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipe p.Task.JobCtx.EnvVars = append(p.Task.JobCtx.EnvVars, artifactKeysVar) } - p.KubeNamespace = pipelineTask.ConfigPayload.Build.KubeNamespace for _, repo := range p.Task.JobCtx.Builds { repoName := strings.Replace(repo.RepoName, "-", "_", -1) if len(repo.Branch) > 0 { @@ -230,7 +263,7 @@ func (p *BuildTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipe } //Resource request default value is LOW - job, err := buildJob(p.Type(), jobImage, p.JobName, serviceName, p.Task.ResReq, p.Task.ResReqSpec, pipelineCtx, pipelineTask, p.Task.Registries) + job, err := buildJob(p.Type(), jobImage, p.JobName, serviceName, p.Task.ClusterID, pipelineTask.ConfigPayload.Build.KubeNamespace, p.Task.ResReq, p.Task.ResReqSpec, pipelineCtx, pipelineTask, p.Task.Registries) if err != nil { msg := fmt.Sprintf("create build job context error: %v", err) p.Log.Error(msg) @@ -252,7 +285,7 @@ func (p *BuildTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipe } // 将集成到KodeRover的私有镜像仓库的访问权限设置到namespace中 - if err := createOrUpdateRegistrySecrets(p.KubeNamespace, p.Task.Registries, p.kubeClient); err != nil { + if err := createOrUpdateRegistrySecrets(p.KubeNamespace, pipelineTask.ConfigPayload.RegistryID, p.Task.Registries, p.kubeClient); err != nil { msg := fmt.Sprintf("create secret error: %v", err) p.Log.Error(msg) p.Task.TaskStatus = config.StatusFailed @@ -313,16 +346,20 @@ func (p *BuildTaskPlugin) Complete(ctx context.Context, pipelineTask *task.Task, // 清理用户取消和超时的任务 defer func() { - if p.Task.TaskStatus == config.StatusCancelled || p.Task.TaskStatus == config.StatusTimeout { - if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { - p.Log.Error(err) - p.Task.Error = err.Error() - } - return + if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + p.Task.Error = err.Error() } + + if err := ensureDeleteConfigMap(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + p.Task.Error = err.Error() + } + + return }() - err := saveContainerLog(pipelineTask, p.KubeNamespace, p.FileName, jobLabel, p.kubeClient) + err := saveContainerLog(pipelineTask, p.KubeNamespace, p.Task.ClusterID, p.FileName, jobLabel, p.kubeClient) if err != nil { p.Log.Error(err) p.Task.Error = err.Error() diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/deploy.go b/pkg/microservice/warpdrive/core/service/taskplugin/deploy.go index 34b72acab4d348e20d893e37c048e24f84b24ac8..3c5d364d734691b9f8806528082ba50a0ef2be88 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/deploy.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/deploy.go @@ -39,13 +39,13 @@ import ( "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/shared/kube/wrapper" helmtool "github.com/koderover/zadig/pkg/tool/helmclient" "github.com/koderover/zadig/pkg/tool/httpclient" krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/getter" - "github.com/koderover/zadig/pkg/tool/kube/multicluster" "github.com/koderover/zadig/pkg/tool/kube/updater" s3tool "github.com/koderover/zadig/pkg/tool/s3" "github.com/koderover/zadig/pkg/util" @@ -202,13 +202,13 @@ func (p *DeployTaskPlugin) Run(ctx context.Context, pipelineTask *task.Task, _ * }() if pipelineTask.ConfigPayload.DeployClusterID != "" { - p.restConfig, err = multicluster.GetRESTConfig(pipelineTask.ConfigPayload.HubServerAddr, pipelineTask.ConfigPayload.DeployClusterID) + p.restConfig, err = kubeclient.GetRESTConfig(pipelineTask.ConfigPayload.HubServerAddr, pipelineTask.ConfigPayload.DeployClusterID) if err != nil { err = errors.WithMessage(err, "can't get k8s rest config") return } - p.kubeClient, err = multicluster.GetKubeClient(pipelineTask.ConfigPayload.HubServerAddr, pipelineTask.ConfigPayload.DeployClusterID) + p.kubeClient, err = kubeclient.GetKubeClient(pipelineTask.ConfigPayload.HubServerAddr, pipelineTask.ConfigPayload.DeployClusterID) if err != nil { err = errors.WithMessage(err, "can't init k8s client") return diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/docker_build.go b/pkg/microservice/warpdrive/core/service/taskplugin/docker_build.go index 7824478d07fc04a2b7dcfa74f271b361e438c9b4..d9abed532db9a1de1fed888b95d1cb8665978b50 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/docker_build.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/docker_build.go @@ -196,7 +196,7 @@ func (p *DockerBuildPlugin) Run(ctx context.Context, pipelineTask *task.Task, pi return } - job, err := buildJob(p.Type(), pipelineTask.ConfigPayload.Release.PredatorImage, p.JobName, serviceName, setting.MinRequest, setting.MinRequestSpec, pipelineCtx, pipelineTask, []*task.RegistryNamespace{}) + job, err := buildJob(p.Type(), pipelineTask.ConfigPayload.Release.PredatorImage, p.JobName, serviceName, "", pipelineTask.ConfigPayload.Build.KubeNamespace, setting.MinRequest, setting.MinRequestSpec, pipelineCtx, pipelineTask, []*task.RegistryNamespace{}) if err != nil { msg := fmt.Sprintf("create build job context error: %v", err) p.Log.Error(msg) @@ -244,7 +244,7 @@ func (p *DockerBuildPlugin) Complete(ctx context.Context, pipelineTask *task.Tas }() // 保存实时日志到s3 - err := saveContainerLog(pipelineTask, p.KubeNamespace, p.FileName, jobLabel, p.kubeClient) + err := saveContainerLog(pipelineTask, p.KubeNamespace, "", p.FileName, jobLabel, p.kubeClient) if err != nil { p.Log.Error(err) p.Task.Error = err.Error() diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/jenkins_plugin.go b/pkg/microservice/warpdrive/core/service/taskplugin/jenkins_plugin.go index 2d5c475cf36dc093647678b0db1bde67c072ab6e..1879bb0a9c559fc5b645fb27882ba7791e97a22c 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/jenkins_plugin.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/jenkins_plugin.go @@ -157,8 +157,7 @@ func (j *JenkinsBuildPlugin) Run(ctx context.Context, pipelineTask *task.Task, p } j.Log.Infof("succeed to create cm for jenkins build job %s", j.JobName) - j.Log.Infof("JenkinsBuildConfig : %+v", pipelineTask.ConfigPayload.JenkinsBuildConfig) - job, err := buildJob(j.Type(), pipelineTask.ConfigPayload.JenkinsBuildConfig.JenkinsBuildImage, j.JobName, serviceName, setting.MinRequest, setting.MinRequestSpec, pipelineCtx, pipelineTask, []*task.RegistryNamespace{}) + job, err := buildJob(j.Type(), pipelineTask.ConfigPayload.JenkinsBuildConfig.JenkinsBuildImage, j.JobName, serviceName, "", pipelineTask.ConfigPayload.Build.KubeNamespace, setting.MinRequest, setting.MinRequestSpec, pipelineCtx, pipelineTask, []*task.RegistryNamespace{}) if err != nil { msg := fmt.Sprintf("create jenkins build job context error: %v", err) j.Log.Error(msg) @@ -234,7 +233,7 @@ func (j *JenkinsBuildPlugin) Complete(ctx context.Context, pipelineTask *task.Ta }() // 保存实时日志到s3 - err := saveContainerLog(pipelineTask, j.KubeNamespace, j.FileName, jobLabel, j.kubeClient) + err := saveContainerLog(pipelineTask, j.KubeNamespace, "", j.FileName, jobLabel, j.kubeClient) if err != nil { j.Log.Error(err) j.Task.Error = err.Error() diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/job.go b/pkg/microservice/warpdrive/core/service/taskplugin/job.go index 4835fa8184ab7c2f0bc291b8c91e8dcb6ba2e9dd..521c55eaef63e1a281b646aa7ce1b2fb7a713424 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/job.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/job.go @@ -43,8 +43,8 @@ import ( "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" "github.com/koderover/zadig/pkg/shared/kube/wrapper" - krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/containerlog" "github.com/koderover/zadig/pkg/tool/kube/getter" "github.com/koderover/zadig/pkg/tool/kube/podexec" @@ -58,10 +58,15 @@ const ( defaultSecretEmail = "bot@koderover.com" PredatorPlugin = "predator-plugin" JenkinsPlugin = "jenkins-plugin" -) - -const ( - registrySecretSuffix = "-registry-secret" + PackagerPlugin = "packager-plugin" + NormalSchedule = "normal" + RequiredSchedule = "required" + PreferredSchedule = "preferred" + + registrySecretSuffix = "-registry-secret" + ResourceServer = "resource-server" + DindServer = "dind" + KoderoverAgentNamespace = "koderover-agent" ) func saveFile(src io.Reader, localFile string) error { @@ -76,7 +81,7 @@ func saveFile(src io.Reader, localFile string) error { return err } -func saveContainerLog(pipelineTask *task.Task, namespace, fileName string, jobLabel *JobLabel, kubeClient client.Client) error { +func saveContainerLog(pipelineTask *task.Task, namespace, clusterID, fileName string, jobLabel *JobLabel, kubeClient client.Client) error { selector := labels.Set(getJobLabels(jobLabel)).AsSelector() pods, err := getter.ListPods(namespace, selector, kubeClient) if err != nil { @@ -96,7 +101,14 @@ func saveContainerLog(pipelineTask *task.Task, namespace, fileName string, jobLa sort.SliceStable(pods, func(i, j int) bool { return pods[i].CreationTimestamp.Before(&pods[j].CreationTimestamp) }) - if err := containerlog.GetContainerLogs(namespace, pods[0].Name, pods[0].Spec.Containers[0].Name, false, int64(0), buf, krkubeclient.Clientset()); err != nil { + + clientSet, err := kubeclient.GetClientset(pipelineTask.ConfigPayload.HubServerAddr, clusterID) + if err != nil { + log.Errorf("saveContainerLog, get client set error: %s", err) + return err + } + + if err := containerlog.GetContainerLogs(namespace, pods[0].Name, pods[0].Spec.Containers[0].Name, false, int64(0), buf, clientSet); err != nil { return err } @@ -107,6 +119,7 @@ func saveContainerLog(pipelineTask *task.Task, namespace, fileName string, jobLa if err = saveFile(buf, tempFileName); err == nil { var store *s3.S3 if store, err = s3.NewS3StorageFromEncryptedURI(pipelineTask.StorageURI); err != nil { + log.Errorf("failed to NewS3StorageFromEncryptedURI ") return err } if store.Subfolder != "" { @@ -210,6 +223,7 @@ func (b *JobCtxBuilder) BuildReaperContext(pipelineTask *task.Task, serviceName TaskID: pipelineTask.TaskID, ServiceName: serviceName, StorageEndpoint: pipelineTask.StorageEndpoint, + AesKey: pipelineTask.ConfigPayload.AesKey, } for _, install := range b.Installs { inst := &types.Install{ @@ -352,12 +366,19 @@ const ( // getJobLabels get labels k-v map from JobLabel struct func getJobLabels(jobLabel *JobLabel) map[string]string { - return map[string]string{ + retMap := map[string]string{ jobLabelTaskKey: fmt.Sprintf("%s-%d", strings.ToLower(jobLabel.PipelineName), jobLabel.TaskID), jobLabelServiceKey: strings.ToLower(jobLabel.ServiceName), jobLabelSTypeKey: strings.Replace(jobLabel.TaskType, "_", "-", -1), jobLabelPTypeKey: jobLabel.PipelineType, } + // no need to add labels with empty value to a job + for k, v := range retMap { + if len(v) == 0 { + delete(retMap, k) + } + } + return retMap } func createJobConfigMap(namespace, jobName string, jobLabel *JobLabel, jobCtx string, kubeClient client.Client) error { @@ -382,12 +403,14 @@ func createJobConfigMap(namespace, jobName string, jobLabel *JobLabel, jobCtx st //"s-job": pipelinename-taskid-tasktype-servicename, //"s-task": pipelinename-taskid, //"s-type": tasktype, -func buildJob(taskType config.TaskType, jobImage, jobName, serviceName string, resReq setting.Request, resReqSpec setting.RequestSpec, ctx *task.PipelineCtx, pipelineTask *task.Task, registries []*task.RegistryNamespace) (*batchv1.Job, error) { +func buildJob(taskType config.TaskType, jobImage, jobName, serviceName, clusterID, currentNamespace string, resReq setting.Request, resReqSpec setting.RequestSpec, ctx *task.PipelineCtx, pipelineTask *task.Task, registries []*task.RegistryNamespace) (*batchv1.Job, error) { return buildJobWithLinkedNs( taskType, jobImage, jobName, serviceName, + clusterID, + currentNamespace, resReq, resReqSpec, ctx, @@ -398,15 +421,24 @@ func buildJob(taskType config.TaskType, jobImage, jobName, serviceName string, r ) } -func buildJobWithLinkedNs(taskType config.TaskType, jobImage, jobName, serviceName string, resReq setting.Request, resReqSpec setting.RequestSpec, ctx *task.PipelineCtx, pipelineTask *task.Task, registries []*task.RegistryNamespace, execNs, linkedNs string) (*batchv1.Job, error) { - var reaperBootingScript string +func buildJobWithLinkedNs(taskType config.TaskType, jobImage, jobName, serviceName, clusterID, currentNamespace string, resReq setting.Request, resReqSpec setting.RequestSpec, ctx *task.PipelineCtx, pipelineTask *task.Task, registries []*task.RegistryNamespace, execNs, linkedNs string) (*batchv1.Job, error) { + var ( + reaperBootingScript string + reaperBinaryFile = pipelineTask.ConfigPayload.Release.ReaperBinaryFile + ) + // not local cluster + if clusterID != "" && clusterID != setting.LocalClusterID { + reaperBinaryFile = strings.Replace(reaperBinaryFile, ResourceServer, ResourceServer+".koderover-agent", -1) + } else { + reaperBinaryFile = strings.Replace(reaperBinaryFile, ResourceServer, ResourceServer+"."+currentNamespace, -1) + } - if !strings.Contains(jobImage, PredatorPlugin) && !strings.Contains(jobImage, JenkinsPlugin) { - reaperBootingScript = fmt.Sprintf("curl -m 60 --retry-delay 5 --retry 3 -sL %s -o reaper && chmod +x reaper && mv reaper /usr/local/bin && /usr/local/bin/reaper", pipelineTask.ConfigPayload.Release.ReaperBinaryFile) + if !strings.Contains(jobImage, PredatorPlugin) && !strings.Contains(jobImage, JenkinsPlugin) && !strings.Contains(jobImage, PackagerPlugin) { + reaperBootingScript = fmt.Sprintf("curl -m 60 --retry-delay 5 --retry 3 -sL %s -o reaper && chmod +x reaper && mv reaper /usr/local/bin && /usr/local/bin/reaper", reaperBinaryFile) if pipelineTask.ConfigPayload.Proxy.EnableApplicationProxy && pipelineTask.ConfigPayload.Proxy.Type == "http" { reaperBootingScript = fmt.Sprintf("curl -m 60 --retry-delay 5 --retry 3 -sL --proxy %s %s -o reaper && chmod +x reaper && mv reaper /usr/local/bin && /usr/local/bin/reaper", pipelineTask.ConfigPayload.Proxy.GetProxyURL(), - pipelineTask.ConfigPayload.Release.ReaperBinaryFile, + reaperBinaryFile, ) } } @@ -482,11 +514,15 @@ func buildJobWithLinkedNs(taskType config.TaskType, jobImage, jobName, serviceNa }, } - if !strings.Contains(jobImage, PredatorPlugin) && !strings.Contains(jobImage, JenkinsPlugin) { + if !strings.Contains(jobImage, PredatorPlugin) && !strings.Contains(jobImage, JenkinsPlugin) && !strings.Contains(jobImage, PackagerPlugin) { job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c"} job.Spec.Template.Spec.Containers[0].Args = []string{reaperBootingScript} } + if affinity := addNodeAffinity(clusterID, pipelineTask.ConfigPayload.K8SClusters); affinity != nil { + job.Spec.Template.Spec.Affinity = affinity + } + if linkedNs != "" && execNs != "" && pipelineTask.ConfigPayload.CustomDNSSupported { job.Spec.Template.Spec.DNSConfig = &corev1.PodDNSConfig{ Searches: []string{ @@ -514,14 +550,7 @@ func buildJobWithLinkedNs(taskType config.TaskType, jobImage, jobName, serviceNa return job, nil } -func createOrUpdateRegistrySecrets(namespace string, registries []*task.RegistryNamespace, kubeClient client.Client) error { - defaultRegistry := &task.RegistryNamespace{ - RegAddr: config.DefaultRegistryAddr(), - AccessKey: config.DefaultRegistryAK(), - SecretKey: config.DefaultRegistrySK(), - } - registries = append(registries, defaultRegistry) - +func createOrUpdateRegistrySecrets(namespace, registryID string, registries []*task.RegistryNamespace, kubeClient client.Client) error { for _, reg := range registries { if reg.AccessKey == "" { continue @@ -533,7 +562,7 @@ func createOrUpdateRegistrySecrets(namespace string, registries []*task.Registry if reg.RegType != "" { secretName = namespaceInRegistry + "-" + reg.RegType + registrySecretSuffix } - if reg.RegAddr == config.DefaultRegistryAddr() { + if reg.ID == registryID { secretName = setting.DefaultImagePullSecret } @@ -591,11 +620,6 @@ func getVolumeMounts(ctx *task.PipelineCtx) []corev1.VolumeMount { Name: "job-config", MountPath: ctx.ConfigMapMountDir, }) - resp = append(resp, corev1.VolumeMount{ - Name: "aes-key", - ReadOnly: true, - MountPath: "/etc/encryption", - }) return resp } @@ -612,18 +636,6 @@ func getVolumes(jobName string) []corev1.Volume { }, }, }) - resp = append(resp, corev1.Volume{ - Name: "aes-key", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "zadig-aes-key", - Items: []corev1.KeyToPath{{ - Key: "aesKey", - Path: "aes", - }}, - }, - }, - }) return resp } @@ -809,3 +821,73 @@ func checkDogFoodExistsInContainer(namespace string, pod string, container strin return success, err } + +func addNodeAffinity(clusterID string, K8SClusters []*task.K8SCluster) *corev1.Affinity { + clusterConfig := findClusterConfig(clusterID, K8SClusters) + if clusterConfig == nil { + return nil + } + + if len(clusterConfig.NodeLabels) == 0 { + return nil + } + + switch clusterConfig.Strategy { + case RequiredSchedule: + nodeSelectorTerms := make([]corev1.NodeSelectorTerm, 0) + for _, nodeLabel := range clusterConfig.NodeLabels { + var matchExpressions []corev1.NodeSelectorRequirement + matchExpressions = append(matchExpressions, corev1.NodeSelectorRequirement{ + Key: nodeLabel.Key, + Operator: nodeLabel.Operator, + Values: nodeLabel.Value, + }) + nodeSelectorTerms = append(nodeSelectorTerms, corev1.NodeSelectorTerm{ + MatchExpressions: matchExpressions, + }) + } + + affinity := &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: nodeSelectorTerms, + }, + }, + } + return affinity + case PreferredSchedule: + preferredScheduleTerms := make([]corev1.PreferredSchedulingTerm, 0) + for _, nodeLabel := range clusterConfig.NodeLabels { + var matchExpressions []corev1.NodeSelectorRequirement + matchExpressions = append(matchExpressions, corev1.NodeSelectorRequirement{ + Key: nodeLabel.Key, + Operator: nodeLabel.Operator, + Values: nodeLabel.Value, + }) + nodeSelectorTerm := corev1.NodeSelectorTerm{ + MatchExpressions: matchExpressions, + } + preferredScheduleTerms = append(preferredScheduleTerms, corev1.PreferredSchedulingTerm{ + Weight: 10, + Preference: nodeSelectorTerm, + }) + } + affinity := &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: preferredScheduleTerms, + }, + } + return affinity + default: + return nil + } +} + +func findClusterConfig(clusterID string, K8SClusters []*task.K8SCluster) *task.AdvancedConfig { + for _, K8SCluster := range K8SClusters { + if K8SCluster.ID == clusterID { + return K8SCluster.AdvancedConfig + } + } + return nil +} diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/release_image.go b/pkg/microservice/warpdrive/core/service/taskplugin/release_image.go index 373e3068b3266cfaff39119dce976b91a4b0f0a0..fd34f27bdd479e43947f0f96b9e593772c34a93c 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/release_image.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/release_image.go @@ -176,7 +176,7 @@ func (p *ReleaseImagePlugin) Run(ctx context.Context, pipelineTask *task.Task, p } p.Log.Infof("succeed to create cm for image job %s", p.JobName) - job, err := buildJob(p.Type(), pipelineTask.ConfigPayload.Release.PredatorImage, p.JobName, serviceName, setting.MinRequest, setting.MinRequestSpec, pipelineCtx, pipelineTask, []*task.RegistryNamespace{}) + job, err := buildJob(p.Type(), pipelineTask.ConfigPayload.Release.PredatorImage, p.JobName, serviceName, "", pipelineTask.ConfigPayload.Build.KubeNamespace, setting.MinRequest, setting.MinRequestSpec, pipelineCtx, pipelineTask, []*task.RegistryNamespace{}) if err != nil { msg := fmt.Sprintf("create release image job context error: %v", err) p.Log.Error(msg) @@ -232,7 +232,7 @@ func (p *ReleaseImagePlugin) Complete(ctx context.Context, pipelineTask *task.Ta }() // 保存实时日志到s3 - err := saveContainerLog(pipelineTask, p.KubeNamespace, p.FileName, jobLabel, p.kubeClient) + err := saveContainerLog(pipelineTask, p.KubeNamespace, "", p.FileName, jobLabel, p.kubeClient) if err != nil { p.Log.Error(err) p.Task.Error = err.Error() diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/testing.go b/pkg/microservice/warpdrive/core/service/taskplugin/testing.go index 07ab9732c719a28f6951ed815b39b96448a60d60..5afb0e7a8ca8a8c1db6dbd83b4bdde4969cc1d52 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/testing.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/testing.go @@ -34,6 +34,7 @@ import ( "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types" "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" "github.com/koderover/zadig/pkg/setting" + kubeclient "github.com/koderover/zadig/pkg/shared/kube/client" krkubeclient "github.com/koderover/zadig/pkg/tool/kube/client" "github.com/koderover/zadig/pkg/tool/kube/updater" s3tool "github.com/koderover/zadig/pkg/tool/s3" @@ -104,6 +105,35 @@ func (p *TestPlugin) TaskTimeout() int { func (p *TestPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineCtx *task.PipelineCtx, serviceName string) { p.KubeNamespace = pipelineTask.ConfigPayload.Test.KubeNamespace + if p.Task.Namespace != "" { + p.KubeNamespace = p.Task.Namespace + kubeClient, err := kubeclient.GetKubeClient(pipelineTask.ConfigPayload.HubServerAddr, p.Task.ClusterID) + if err != nil { + msg := fmt.Sprintf("failed to get kube client: %s", err) + p.Log.Error(msg) + p.Task.TaskStatus = config.StatusFailed + p.Task.Error = msg + return + } + p.kubeClient = kubeClient + } + // not local cluster + replaceDindServer := "." + DindServer + if p.Task.ClusterID != "" && p.Task.ClusterID != setting.LocalClusterID { + if strings.Contains(pipelineTask.DockerHost, pipelineTask.ConfigPayload.Build.KubeNamespace) { + // replace namespace only + pipelineTask.DockerHost = strings.Replace(pipelineTask.DockerHost, pipelineTask.ConfigPayload.Build.KubeNamespace, KoderoverAgentNamespace, 1) + } else { + // add namespace + pipelineTask.DockerHost = strings.Replace(pipelineTask.DockerHost, replaceDindServer, replaceDindServer+"."+KoderoverAgentNamespace, 1) + } + } else if p.Task.ClusterID == "" || p.Task.ClusterID == setting.LocalClusterID { + if !strings.Contains(pipelineTask.DockerHost, pipelineTask.ConfigPayload.Build.KubeNamespace) { + // add namespace + pipelineTask.DockerHost = strings.Replace(pipelineTask.DockerHost, replaceDindServer, replaceDindServer+"."+pipelineTask.ConfigPayload.Build.KubeNamespace, 1) + } + } + pipelineCtx.DockerHost = pipelineTask.DockerHost // 重置错误信息 p.Task.Error = "" // 获取测试相关的namespace @@ -200,7 +230,7 @@ func (p *TestPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineC // search namespace should also include desired namespace job, err := buildJobWithLinkedNs( - p.Type(), jobImage, p.JobName, serviceName, p.Task.ResReq, p.Task.ResReqSpec, pipelineCtx, pipelineTask, p.Task.Registries, + p.Type(), jobImage, p.JobName, serviceName, p.Task.ClusterID, pipelineTask.ConfigPayload.Test.KubeNamespace, p.Task.ResReq, p.Task.ResReqSpec, pipelineCtx, pipelineTask, p.Task.Registries, p.KubeNamespace, linkedNamespace, ) @@ -224,7 +254,7 @@ func (p *TestPlugin) Run(ctx context.Context, pipelineTask *task.Task, pipelineC } // 将集成到KodeRover的私有镜像仓库的访问权限设置到namespace中 - if err := createOrUpdateRegistrySecrets(p.KubeNamespace, p.Task.Registries, p.kubeClient); err != nil { + if err := createOrUpdateRegistrySecrets(p.KubeNamespace, pipelineTask.ConfigPayload.RegistryID, p.Task.Registries, p.kubeClient); err != nil { p.Log.Errorf("create secret error: %v", err) } if err := updater.CreateJob(job, p.kubeClient); err != nil { @@ -257,16 +287,18 @@ func (p *TestPlugin) Complete(ctx context.Context, pipelineTask *task.Task, serv // 日志保存失败与否都清理job defer func() { - if p.Task.TaskStatus == config.StatusCancelled || p.Task.TaskStatus == config.StatusTimeout { - if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { - p.Log.Error(err) - p.Task.Error = err.Error() - } - return + if err := ensureDeleteJob(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + p.Task.Error = err.Error() + } + if err := ensureDeleteConfigMap(p.KubeNamespace, jobLabel, p.kubeClient); err != nil { + p.Log.Error(err) + p.Task.Error = err.Error() } + return }() - err := saveContainerLog(pipelineTask, p.KubeNamespace, p.FileName, jobLabel, p.kubeClient) + err := saveContainerLog(pipelineTask, p.KubeNamespace, p.Task.ClusterID, p.FileName, jobLabel, p.kubeClient) if err != nil { p.Log.Error(err) p.Task.Error = err.Error() diff --git a/pkg/microservice/warpdrive/core/service/taskplugin/utils.go b/pkg/microservice/warpdrive/core/service/taskplugin/utils.go index 6d41110151f5ad42fcade3424f17542a7daac8a5..35236b81330b0730e853c38f1cbde875ac02c5ca 100644 --- a/pkg/microservice/warpdrive/core/service/taskplugin/utils.go +++ b/pkg/microservice/warpdrive/core/service/taskplugin/utils.go @@ -98,7 +98,7 @@ type Preview struct { func ToPreview(sb map[string]interface{}) (*Preview, error) { var pre *Preview if err := IToi(sb, &pre); err != nil { - return nil, fmt.Errorf("convert interface to SubTaskPreview error: %v", err) + return nil, fmt.Errorf("convert interface to SubTaskPreview error: %s", err) } return pre, nil } @@ -106,15 +106,15 @@ func ToPreview(sb map[string]interface{}) (*Preview, error) { func ToBuildTask(sb map[string]interface{}) (*task.Build, error) { var t *task.Build if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to BuildTaskV2 error: %v", err) + return nil, fmt.Errorf("convert interface to BuildTaskV2 error: %s", err) } return t, nil } -func ToArtifactTask(sb map[string]interface{}) (*task.Artifact, error) { - var t *task.Artifact +func ToArtifactTask(sb map[string]interface{}) (*task.ArtifactPackage, error) { + var t *task.ArtifactPackage if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to ArtifactTask error: %v", err) + return nil, fmt.Errorf("convert interface to ArtifactTask error: %s", err) } return t, nil } @@ -122,7 +122,7 @@ func ToArtifactTask(sb map[string]interface{}) (*task.Artifact, error) { func ToDockerBuildTask(sb map[string]interface{}) (*task.DockerBuild, error) { var t *task.DockerBuild if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to DockerBuildTask error: %v", err) + return nil, fmt.Errorf("convert interface to DockerBuildTask error: %s", err) } return t, nil } @@ -130,7 +130,7 @@ func ToDockerBuildTask(sb map[string]interface{}) (*task.DockerBuild, error) { func ToDeployTask(sb map[string]interface{}) (*task.Deploy, error) { var t *task.Deploy if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to DeployTask error: %v", err) + return nil, fmt.Errorf("convert interface to DeployTask error: %s", err) } return t, nil } @@ -138,7 +138,7 @@ func ToDeployTask(sb map[string]interface{}) (*task.Deploy, error) { func ToTestingTask(sb map[string]interface{}) (*task.Testing, error) { var t *task.Testing if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to Testing error: %v", err) + return nil, fmt.Errorf("convert interface to Testing error: %s", err) } return t, nil } @@ -146,7 +146,7 @@ func ToTestingTask(sb map[string]interface{}) (*task.Testing, error) { func ToDistributeToS3Task(sb map[string]interface{}) (*task.DistributeToS3, error) { var t *task.DistributeToS3 if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to DistributeToS3Task error: %v", err) + return nil, fmt.Errorf("convert interface to DistributeToS3Task error: %s", err) } return t, nil } @@ -154,15 +154,23 @@ func ToDistributeToS3Task(sb map[string]interface{}) (*task.DistributeToS3, erro func ToReleaseImageTask(sb map[string]interface{}) (*task.ReleaseImage, error) { var t *task.ReleaseImage if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to ReleaseImageTask error: %v", err) + return nil, fmt.Errorf("convert interface to ReleaseImageTask error: %s", err) } return t, nil } +func ToArtifactPackageTask(sb map[string]interface{}) (*task.ArtifactPackageTaskArgs, error) { + var ret *task.ArtifactPackageTaskArgs + if err := IToi(sb, &ret); err != nil { + return nil, fmt.Errorf("convert interface to ArtifactPackageTaskArgs error: %s", err) + } + return ret, nil +} + func ToJiraTask(sb map[string]interface{}) (*task.Jira, error) { var t *task.Jira if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to JiraTask error: %v", err) + return nil, fmt.Errorf("convert interface to JiraTask error: %s", err) } return t, nil } @@ -170,7 +178,7 @@ func ToJiraTask(sb map[string]interface{}) (*task.Jira, error) { func ToSecurityTask(sb map[string]interface{}) (*task.Security, error) { var t *task.Security if err := IToi(sb, &t); err != nil { - return nil, fmt.Errorf("convert interface to securityTask error: %v", err) + return nil, fmt.Errorf("convert interface to securityTask error: %s", err) } return t, nil } @@ -178,7 +186,7 @@ func ToSecurityTask(sb map[string]interface{}) (*task.Security, error) { func ToJenkinsBuildTask(sb map[string]interface{}) (*task.JenkinsBuild, error) { var task *task.JenkinsBuild if err := IToi(sb, &task); err != nil { - return nil, fmt.Errorf("convert interface to JenkinsBuildTask error: %v", err) + return nil, fmt.Errorf("convert interface to JenkinsBuildTask error: %s", err) } return task, nil } @@ -186,11 +194,11 @@ func ToJenkinsBuildTask(sb map[string]interface{}) (*task.JenkinsBuild, error) { func IToi(before interface{}, after interface{}) error { b, err := json.Marshal(before) if err != nil { - return fmt.Errorf("marshal task error: %v", err) + return fmt.Errorf("marshal task error: %s", err) } if err := json.Unmarshal(b, &after); err != nil { - return fmt.Errorf("unmarshal task error: %v", err) + return fmt.Errorf("unmarshal task error: %s", err) } return nil diff --git a/pkg/microservice/warpdrive/core/service/types/packager.go b/pkg/microservice/warpdrive/core/service/types/packager.go new file mode 100644 index 0000000000000000000000000000000000000000..19aac50ae5cf4003e2e114c85effc29bee11c7d2 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/types/packager.go @@ -0,0 +1,27 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import "github.com/koderover/zadig/pkg/microservice/warpdrive/core/service/types/task" + +type ArtifactPackagerContext struct { + JobType string `yaml:"job_type"` + ProgressFile string `yaml:"progress_file"` + Images []*task.ImagesByService `yaml:"images"` + SourceRegistries []*DockerRegistry `yaml:"source_registries"` + TargetRegistries []*DockerRegistry `yaml:"target_registries"` +} diff --git a/pkg/microservice/warpdrive/core/service/types/reaper.go b/pkg/microservice/warpdrive/core/service/types/reaper.go index 9ed4b2fdd9e18459c33abebfe354af557d36019f..ca4f4ec1d53a5aab3f292b20a9ced0fe66cb237c 100644 --- a/pkg/microservice/warpdrive/core/service/types/reaper.go +++ b/pkg/microservice/warpdrive/core/service/types/reaper.go @@ -113,6 +113,7 @@ type Context struct { StorageBucket string `yaml:"storage_bucket"` StorageProvider int8 `yaml:"storage_provider"` ArtifactInfo *ArtifactInfo `yaml:"artifact_info"` + AesKey string `yaml:"aes_key"` } type ArtifactInfo struct { @@ -269,10 +270,11 @@ type GinkgoTest struct { // DockerRegistry 推送镜像到 docker registry 配置 type DockerRegistry struct { - Host string `yaml:"host"` - Namespace string `yaml:"namespace"` - UserName string `yaml:"username"` - Password string `yaml:"password"` + RegistryID string `yaml:"registry_id"` + Host string `yaml:"host"` + Namespace string `yaml:"namespace"` + UserName string `yaml:"username"` + Password string `yaml:"password"` } // Git ... diff --git a/pkg/microservice/warpdrive/core/service/types/task/artifact_package.go b/pkg/microservice/warpdrive/core/service/types/task/artifact_package.go new file mode 100644 index 0000000000000000000000000000000000000000..5ebe24fb0eea78b73cdd0bfdc46df29f65640f25 --- /dev/null +++ b/pkg/microservice/warpdrive/core/service/types/task/artifact_package.go @@ -0,0 +1,50 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "fmt" + + "github.com/koderover/zadig/pkg/microservice/warpdrive/config" +) + +type ArtifactPackage struct { + TaskType config.TaskType `bson:"type" json:"type"` + Enabled bool `bson:"enabled" json:"enabled"` + TaskStatus config.Status `bson:"status" json:"status"` + Progress string `bson:"progress" json:"progress"` + Timeout int `bson:"timeout,omitempty" json:"timeout,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + StartTime int64 `bson:"start_time,omitempty" json:"start_time,omitempty"` + EndTime int64 `bson:"end_time,omitempty" json:"end_time,omitempty"` + LogFile string `bson:"log_file" json:"log_file"` + + // source images + Images []*ImagesByService `bson:"images" json:"images"` + // target registries to push images + SourceRegistries []string `bson:"source_registries" json:"source_registries"` + // target registries to push images + TargetRegistries []string `bson:"target_registries" json:"target_registries"` +} + +func (ri *ArtifactPackage) ToSubTask() (map[string]interface{}, error) { + var task map[string]interface{} + if err := IToi(ri, &task); err != nil { + return nil, fmt.Errorf("convert ReleaseImageTask to interface error: %v", err) + } + return task, nil +} diff --git a/pkg/microservice/warpdrive/core/service/types/task/build.go b/pkg/microservice/warpdrive/core/service/types/task/build.go index 42a36ee6e4b7d9f89d9cbe8eebd672ae3245a823..5882537896a3c9c59976733ce3639a699d6b1bfc 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/build.go +++ b/pkg/microservice/warpdrive/core/service/types/task/build.go @@ -58,6 +58,7 @@ type Build struct { // Get the host bound to the environment of the cloud host service configuration EnvHostInfo map[string][]string `bson:"env_host_info,omitempty" json:"env_host_info,omitempty"` ArtifactInfo *ArtifactInfo `bson:"artifact_info,omitempty" json:"artifact_info,omitempty"` + ClusterID string `bson:"cluster_id,omitempty" json:"cluster_id,omitempty"` } type ArtifactInfo struct { @@ -86,6 +87,7 @@ type Install struct { } type RegistryNamespace struct { + ID string `bson:"_id,omitempty" json:"id,omitempty"` RegAddr string `bson:"reg_addr" json:"reg_addr"` RegType string `bson:"reg_type" json:"reg_type"` RegProvider string `bson:"reg_provider" json:"reg_provider"` diff --git a/pkg/microservice/warpdrive/core/service/types/task/config_payload.go b/pkg/microservice/warpdrive/core/service/types/task/config_payload.go index 3b3e6e0a8f735c0007ab176b1e5a77a135294cd3..61f65238f9db62d2ef77077b4c81feaaa2636058 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/config_payload.go +++ b/pkg/microservice/warpdrive/core/service/types/task/config_payload.go @@ -18,6 +18,8 @@ package task import ( "fmt" + + corev1 "k8s.io/api/core/v1" ) type ConfigPayload struct { @@ -40,6 +42,7 @@ type ConfigPayload struct { CustomDNSSupported bool `json:"custom_dns_supported"` HubServerAddr string DeployClusterID string + AesKey string `json:"aes_key"` RepoConfigs map[string]*RegistryNamespace @@ -49,6 +52,9 @@ type ConfigPayload struct { // ResetCache means ignore workspace cache ResetCache bool `json:"reset_cache"` PrivateKeys []*PrivateKey `json:"private_keys"` + K8SClusters []*K8SCluster `json:"k8s_clusters"` + + RegistryID string `json:"registry_id"` } func (cp *ConfigPayload) GetGitKnownHost() string { @@ -82,15 +88,15 @@ type S3Config struct { type GithubConfig struct { // github API access token - AccessToken string + AccessToken string `json:"access_token"` // github ssh key with base64 encoded - SSHKey string + SSHKey string `json:"ssh_key"` // github knownhost - KnownHost string + KnownHost string `json:"known_host"` // github app private key - AppKey string + AppKey string `json:"app_key"` // gihhub app id - AppID int + AppID int `json:"app_id"` } type GitlabConfig struct { @@ -129,6 +135,9 @@ type ReleaseConfig struct { // PredatorImage sets docker build image // e.g. xxx.com/resources/predator-plugin:v0.1.0 PredatorImage string + // PackagerImage sets docker build image + // e.g. xxx.com/resources/packager-plugin:v0.1.0 + PackagerImage string } type ImageReleaseConfig struct { @@ -148,3 +157,20 @@ type JenkinsBuildConfig struct { type PrivateKey struct { Name string `json:"name"` } + +type K8SCluster struct { + ID string `json:"id,omitempty" bson:"_id,omitempty"` + Name string `json:"name" bson:"name"` + AdvancedConfig *AdvancedConfig `json:"advanced_config,omitempty" bson:"advanced_config,omitempty"` +} + +type AdvancedConfig struct { + Strategy string `json:"strategy,omitempty" bson:"strategy,omitempty"` + NodeLabels []*NodeSelectorRequirement `json:"node_labels,omitempty" bson:"node_labels,omitempty"` +} + +type NodeSelectorRequirement struct { + Key string `json:"key"` + Value []string `json:"value"` + Operator corev1.NodeSelectorOperator `json:"operator"` +} diff --git a/pkg/microservice/warpdrive/core/service/types/task/model.go b/pkg/microservice/warpdrive/core/service/types/task/model.go index d6a0d9751efdae6481adb4bc075e187a31094112..e3f776067ad185fa37feea06e5211faea021cb59 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/model.go +++ b/pkg/microservice/warpdrive/core/service/types/task/model.go @@ -64,6 +64,8 @@ type Task struct { TestArgs *TestTaskArgs `bson:"test_args,omitempty" json:"test_args,omitempty"` // ServiceTaskArgs 脚本部署工作流任务参数 ServiceTaskArgs *ServiceTaskArgs `bson:"service_args,omitempty" json:"service_args,omitempty"` + // ArtifactPackageTaskArgs arguments for artifact-package type tasks + ArtifactPackageTaskArgs *ArtifactPackageTaskArgs `bson:"artifact_package_args,omitempty" json:"artifact_package_args,omitempty"` // ConfigPayload 系统配置信息 ConfigPayload *ConfigPayload `json:"config_payload,omitempty"` Error string `bson:"error,omitempty" json:"error,omitempty"` @@ -243,6 +245,25 @@ type ServiceTaskArgs struct { Updatable bool `bson:"updatable" json:"updatable"` } +type ImageData struct { + ImageUrl string `bson:"image_url" json:"image_url" yaml:"image_url"` + ImageName string `bson:"image_name" json:"image_name" yaml:"image_name"` + ImageTag string `bson:"image_tag" json:"image_tag" yaml:"image_tag"` + RegistryID string `bson:"registry_id" json:"registry_id" yaml:"registry_id"` +} + +type ImagesByService struct { + ServiceName string `bson:"service_name" json:"service_name" yaml:"service_name"` + Images []*ImageData `bson:"images" json:"images" yaml:"images"` +} + +type ArtifactPackageTaskArgs struct { + ProductName string `bson:"product_name" json:"product_name"` + Images []*ImagesByService `bson:"images" json:"images"` + SourceRegistries []string `bson:"source_registries" json:"source_registries"` + TargetRegistries []string `bson:"target_registries" json:"target_registries"` +} + type ProductService struct { ServiceName string `bson:"service_name" json:"service_name"` ProductName string `bson:"product_name" json:"product_name"` diff --git a/pkg/microservice/warpdrive/core/service/types/task/testing.go b/pkg/microservice/warpdrive/core/service/types/task/testing.go index 74aa27c4bdec558e15ef87b3c9c4f9a09bad2951..d443b7185de8d50e2444158b753d7bb3030e9b0c 100644 --- a/pkg/microservice/warpdrive/core/service/types/task/testing.go +++ b/pkg/microservice/warpdrive/core/service/types/task/testing.go @@ -46,6 +46,8 @@ type Testing struct { ReportReady bool `bson:"report_ready" json:"report_ready"` IsRestart bool `bson:"is_restart" json:"is_restart"` Registries []*RegistryNamespace `bson:"-" json:"registries"` + ClusterID string `bson:"cluster_id" json:"cluster_id"` + Namespace string `bson:"namespace" json:"namespace"` } func (t *Testing) ToSubTask() (map[string]interface{}, error) { diff --git a/pkg/setting/consts.go b/pkg/setting/consts.go index 582fbe972e4ae3bcc9b461f1eef0f1b46910671b..250cc4b4bb81c3a39c387387b0633c11cc6734a3 100644 --- a/pkg/setting/consts.go +++ b/pkg/setting/consts.go @@ -50,6 +50,7 @@ const ( ENVReaperImage = "REAPER_IMAGE" ENVReaperBinaryFile = "REAPER_BINARY_FILE" ENVPredatorImage = "PREDATOR_IMAGE" + EnvPackagerImage = "PACKAGER_IMAGE" ENVDockerHosts = "DOCKER_HOSTS" @@ -271,6 +272,16 @@ const ( FunctionTestType = "function" ) +const ( + DeliveryVersionTypeChart = "HelmChart" + DeliveryVersionTypeK8SWorkflow = "K8SWorkflow" +) + +const ( + DeliveryDeployTypeImage = "image" + DeliveryDeployTypeChart = "chart" +) + const ( AuthorizationHeader = "Authorization" ) @@ -299,6 +310,10 @@ const ( ReleaseImageJob = "docker-release" ) +const ( + BuildChartPackage = "chart-package" +) + const ( JenkinsBuildJob = "jenkins-build" ) @@ -322,6 +337,21 @@ const ( ProductStatusUnstable = "Unstable" ) +// DeliveryVersion status +const ( + DeliveryVersionStatusSuccess = "success" + DeliveryVersionStatusFailed = "failed" + DeliveryVersionStatusCreating = "creating" + DeliveryVersionStatusRetrying = "retrying" +) + +const ( + DeliveryVersionPackageStatusSuccess = "success" + DeliveryVersionPackageStatusFailed = "failed" + DeliveryVersionPackageStatusWaiting = "waiting" + DeliveryVersionPackageStatusUploading = "uploading" +) + const ( NormalModeProduct = "normal" ) @@ -525,6 +555,8 @@ const MaxTries = 1 const DogFood = "/var/run/koderover-dog-food" +const ProgressFile = "/var/log/job-progress" + const ( ResponseError = "error" ResponseData = "response" @@ -551,11 +583,6 @@ const ( ResourcesHeader = "Resources" ) -const ( - LocalClusterID = "local" // special(fake) id of the local cluster - LocalClusterName = "local" // special(fake) name of the local cluster -) - type K8SClusterStatus string const ( @@ -564,3 +591,5 @@ const ( Normal K8SClusterStatus = "normal" Abnormal K8SClusterStatus = "abnormal" ) + +const LocalClusterID = "0123456789abcdef12345678" diff --git a/pkg/shared/client/aslan/healthz.go b/pkg/shared/client/aslan/healthz.go new file mode 100644 index 0000000000000000000000000000000000000000..d47bcee3b959bb3d590ca8e46d9328540a174a4e --- /dev/null +++ b/pkg/shared/client/aslan/healthz.go @@ -0,0 +1,24 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +// Healthz api/health +func (c *Client) Healthz() error { + url := "/health" + _, err := c.Get(url) + return err +} diff --git a/pkg/shared/client/aslan/multicluster.go b/pkg/shared/client/aslan/multicluster.go new file mode 100644 index 0000000000000000000000000000000000000000..0611bcd85d8a69a0c827ce9a6008ac47fb51cf76 --- /dev/null +++ b/pkg/shared/client/aslan/multicluster.go @@ -0,0 +1,78 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aslan + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/httpclient" +) + +type cluster struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + Status setting.K8SClusterStatus `json:"status"` + Local bool `json:"local"` +} + +func (c *Client) AddLocalCluster() error { + url := "/cluster/clusters" + req := cluster{ + ID: setting.LocalClusterID, + Name: fmt.Sprintf("%s-%s", "local", time.Now().Format("20060102150405")), + } + + _, err := c.Post(url, httpclient.SetBody(req)) + if err != nil { + return fmt.Errorf("Failed to add multi cluster, error: %s", err) + } + + return nil +} + +type clusterResp struct { + Name string `json:"name"` + Local bool `json:"local"` +} + +type ErrorMessage struct { + Type string `json:"type"` + Code int `json:"code"` +} + +func (c *Client) GetLocalCluster() (*clusterResp, error) { + url := fmt.Sprintf("/cluster/clusters/%s", setting.LocalClusterID) + + clusterResp := &clusterResp{} + resp, err := c.Get(url, httpclient.SetResult(clusterResp)) + if err != nil { + errorMessage := new(ErrorMessage) + err := json.Unmarshal(resp.Body(), errorMessage) + if err != nil { + return nil, fmt.Errorf("Failed to get cluster, error: %s", err) + } + if errorMessage.Code == 6643 { + return nil, nil + } + return nil, fmt.Errorf("Failed to get cluster, error: %s", err) + } + + return clusterResp, nil +} diff --git a/pkg/shared/kube/client/client.go b/pkg/shared/kube/client/client.go new file mode 100644 index 0000000000000000000000000000000000000000..8374adb512070869156989e0aa3198b30aa766cf --- /dev/null +++ b/pkg/shared/kube/client/client.go @@ -0,0 +1,56 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/koderover/zadig/pkg/setting" + "github.com/koderover/zadig/pkg/tool/kube/multicluster" +) + +func GetKubeClient(hubServerAddr, clusterID string) (client.Client, error) { + if clusterID == setting.LocalClusterID { + clusterID = "" + } + + return multicluster.GetKubeClient(hubServerAddr, clusterID) +} + +func GetKubeAPIReader(hubServerAddr, clusterID string) (client.Reader, error) { + if clusterID == setting.LocalClusterID { + clusterID = "" + } + return multicluster.GetKubeAPIReader(hubServerAddr, clusterID) +} + +func GetRESTConfig(hubServerAddr, clusterID string) (*rest.Config, error) { + if clusterID == setting.LocalClusterID { + clusterID = "" + } + return multicluster.GetRESTConfig(hubServerAddr, clusterID) +} + +func GetClientset(hubServerAddr, clusterID string) (kubernetes.Interface, error) { + if clusterID == setting.LocalClusterID { + clusterID = "" + } + + return multicluster.GetClientset(hubServerAddr, clusterID) +} diff --git a/pkg/shared/kube/resource/namespace.go b/pkg/shared/kube/resource/namespace.go index 7fad232c187ab457fff39ed8448543ffb6e180ec..50fc6e1bea7806aaad84c34a856e5793211fdff5 100644 --- a/pkg/shared/kube/resource/namespace.go +++ b/pkg/shared/kube/resource/namespace.go @@ -21,4 +21,6 @@ type Namespace struct { Status string `json:"status"` Age string `json:"age"` Labels map[string]string `json:"labels"` + // Whether it is the namespace currently installed by zadig + Current bool `json:"current"` } diff --git a/pkg/shared/kube/resource/node.go b/pkg/shared/kube/resource/node.go new file mode 100644 index 0000000000000000000000000000000000000000..733ef420426875e31d72dcc799853a565531a26d --- /dev/null +++ b/pkg/shared/kube/resource/node.go @@ -0,0 +1,23 @@ +/* +Copyright 2021 The KodeRover Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resource + +type Node struct { + Labels []string `json:"labels"` + Ready bool `json:"ready"` + IP string `json:"ip"` +} diff --git a/pkg/shared/kube/wrapper/namespace.go b/pkg/shared/kube/wrapper/namespace.go index 4faa5d727bcdcd310f7b56e37b14914c512fd027..3fd58ef1eb237c9d479cdda02c132ef84722f4e2 100644 --- a/pkg/shared/kube/wrapper/namespace.go +++ b/pkg/shared/kube/wrapper/namespace.go @@ -19,6 +19,7 @@ package wrapper import ( corev1 "k8s.io/api/core/v1" + "github.com/koderover/zadig/pkg/config" "github.com/koderover/zadig/pkg/shared/kube/resource" "github.com/koderover/zadig/pkg/util" ) @@ -53,9 +54,10 @@ func (w *namespace) Terminating() bool { func (w *namespace) Resource() *resource.Namespace { return &resource.Namespace{ - Name: w.Name, - Status: string(w.Status.Phase), - Age: util.Age(w.CreationTimestamp.Unix()), - Labels: w.Labels, + Name: w.Name, + Status: string(w.Status.Phase), + Age: util.Age(w.CreationTimestamp.Unix()), + Labels: w.Labels, + Current: w.Name == config.Namespace(), } } diff --git a/pkg/tool/crypto/aes.go b/pkg/tool/crypto/aes.go index 65e69ce07140e5f084114efa774544299f7fc4de..9889f609d9038895b81b15d920e7b6afedbbd998 100644 --- a/pkg/tool/crypto/aes.go +++ b/pkg/tool/crypto/aes.go @@ -51,6 +51,10 @@ func getAESKey() string { return aesKey } +func GetAesKey() string { + return getAESKey() +} + func AesEncrypt(src string) (string, error) { client, err := NewAes(getAESKey()) if err != nil { @@ -63,8 +67,16 @@ func AesEncrypt(src string) (string, error) { return dest, nil } -func AesDecrypt(src string) (string, error) { - client, err := NewAes(getAESKey()) +func AesDecrypt(src string, aesKey ...string) (string, error) { + var ( + err error + client *Aes + ) + if len(aesKey) > 0 { + client, err = NewAes(aesKey[0]) + } else { + client, err = NewAes(getAESKey()) + } if err != nil { return "", err } diff --git a/pkg/tool/errors/http_errors.go b/pkg/tool/errors/http_errors.go index 45554ea7610ee1996efd9d34fa7ec6a7280409ca..e0ffc2eed6b6994c21aea78cb9df04cb9381fc23 100644 --- a/pkg/tool/errors/http_errors.go +++ b/pkg/tool/errors/http_errors.go @@ -680,4 +680,10 @@ var ( ErrUpdateExternalLink = NewHTTPError(6842, "更新链接失败") ErrDeleteExternalLink = NewHTTPError(6843, "删除链接失败") ErrListExternalLink = NewHTTPError(6844, "获取链接列表失败") + + //----------------------------------------------------------------------------------------------- + // helm releated Error Range: 6850 - 6869 + //----------------------------------------------------------------------------------------------- + ErrListHelmReleases = NewHTTPError(6850, "获取release失败") + ErrGetHelmCharts = NewHTTPError(6851, "获取chart信息失败") ) diff --git a/pkg/tool/kube/client/cluster.go b/pkg/tool/kube/client/cluster.go index 927cd411525ecff7c5352bbc133ab7f88181a7b8..d4b70be033481acd6ded5e2a0efb2ae753d4a8c6 100644 --- a/pkg/tool/kube/client/cluster.go +++ b/pkg/tool/kube/client/cluster.go @@ -38,7 +38,11 @@ var c cluster.Cluster // Cluster is a singleton, it will be initialized only once. func Cluster() cluster.Cluster { once.Do(func() { - c = initCluster(ctrl.GetConfigOrDie()) + var err error + c, err = initCluster(ctrl.GetConfigOrDie()) + if err != nil { + panic(err) + } }) return c @@ -77,7 +81,10 @@ func NewClientFromAPIConfig(cfg *api.Config) (client.Client, error) { return nil, err } - cls := initCluster(restConfig) + cls, err := initCluster(restConfig) + if err != nil { + return nil, err + } return newAPIClient(cls.GetClient(), cls.GetAPIReader()), nil } @@ -105,7 +112,7 @@ func (c *apiClient) List(ctx context.Context, list client.ObjectList, opts ...cl return c.apiReader.List(ctx, list, opts...) } -func initCluster(restConfig *rest.Config) cluster.Cluster { +func initCluster(restConfig *rest.Config) (cluster.Cluster, error) { scheme := runtime.NewScheme() // add all known types @@ -116,8 +123,8 @@ func initCluster(restConfig *rest.Config) cluster.Cluster { clusterOptions.Scheme = scheme }) if err != nil { - panic(errors.Wrap(err, "unable to init client")) + return nil, errors.Wrap(err, "unable to init client") } - return c + return c, nil }