From 4b0d1b7a14f7f770671121f5068d95bfcfacf2b8 Mon Sep 17 00:00:00 2001 From: "wingo.wen" Date: Wed, 13 Oct 2021 11:22:51 +0800 Subject: [PATCH 001/100] global config internet auto fill --- common/python/common/consts.py | 4 +++ common/python/db/global_config_dao.py | 15 +++++++++ flow/service/config/__init__.py | 0 .../config/global_config_init_service.py | 32 +++++++++++++++++++ 4 files changed, 51 insertions(+) create mode 100644 flow/service/config/__init__.py create mode 100644 flow/service/config/global_config_init_service.py diff --git a/common/python/common/consts.py b/common/python/common/consts.py index d0d2288e5..0b522ae18 100644 --- a/common/python/common/consts.py +++ b/common/python/common/consts.py @@ -77,6 +77,10 @@ ENV_CONF_KEY_CK_HOST = "CK_HOST" ENV_CONF_KEY_CK_PORT = "CK_PORT" ENV_CONF_KEY_CONFIG = "config" +# Network Config +ENV_CONF_KEY_INTRANET_IP = "INTRANET_IP" +ENV_CONF_KEY_EXTRANET_IP = "EXTRANET_IP" + class TransferAction(object): """ diff --git a/common/python/db/global_config_dao.py b/common/python/db/global_config_dao.py index 81eaf64b1..011f1f256 100644 --- a/common/python/db/global_config_dao.py +++ b/common/python/db/global_config_dao.py @@ -166,3 +166,18 @@ class GlobalConfigDao: result[item.name] = item.value return result + + @staticmethod + def fill_intranet_base_uri(config_dict): + """ + :param config_dict: {service.var_name : intranet_base_uri} + :return: + """ + with DB.connection_context(): + items = GlobalConfigModel.select().where( + GlobalConfigModel.name == "intranet_base_uri" + ) + for item in items: + if item['value'] is None: + pass + diff --git a/flow/service/config/__init__.py b/flow/service/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/flow/service/config/global_config_init_service.py b/flow/service/config/global_config_init_service.py new file mode 100644 index 000000000..86fa9cae1 --- /dev/null +++ b/flow/service/config/global_config_init_service.py @@ -0,0 +1,32 @@ +# Copyright 2021 Tianmian Tech. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from common.python.common import consts +from common.python.utils.conf_utils import get_env_config + +INTRANET_IP = get_env_config(consts.ENV_CONF_KEY_INTRANET_IP) + + +class GlobalSettingInit: + """ + Initialize the relevant configuration based on the environment variables + """ + + def __init__(self): + """ + wefe_board.intranet_base_uri + wefe_flow.intranet_base_uri + wefe_gateway.intranet_base_uri + wefe_serving.intranet_base_uri + """ + -- Gitee From 431cb2757cfdd0549b9d30f8a67c40134eb45807 Mon Sep 17 00:00:00 2001 From: "wingo.wen" Date: Wed, 13 Oct 2021 16:52:27 +0800 Subject: [PATCH 002/100] global config internet auto fill --- common/python/common/consts.py | 3 ++ common/python/db/global_config_dao.py | 10 +++--- flow/app_launcher.py | 13 ++++++++ .../config/global_config_init_service.py | 31 ++++++++++++++++-- .../resources/docker-compose.yml | 2 ++ .../resources/variables.env | 7 ++++ .../wefe_python_service_start.sh | 32 +++++++++++-------- 7 files changed, 79 insertions(+), 19 deletions(-) create mode 100644 release/docker/docker_service/wefe_python_service/resources/variables.env diff --git a/common/python/common/consts.py b/common/python/common/consts.py index 0b522ae18..c2b6910bd 100644 --- a/common/python/common/consts.py +++ b/common/python/common/consts.py @@ -70,6 +70,9 @@ COMM_CONF_KEY_FC_OSS_INTERNAL_ENDPOINT = "fc.oss.internal_endpoint" COMM_CONF_KEY_WEB_IP = "flow.web.ip" COMM_CONF_KEY_WEB_PORT = "flow.web.port" +ENV_CONF_KEY_FLOW_PORT = "FLOW_PORT" +ENV_CONF_KEY_NGINX_PORT = "NGINX_PORT" + # Configuration of environment variables ENV_CONF_KEY_GATEWAY_HOST = "GATEWAY_HOST" ENV_CONF_KEY_GATEWAY_PORT = "GATEWAY_PORT" diff --git a/common/python/db/global_config_dao.py b/common/python/db/global_config_dao.py index 011f1f256..15bd250dc 100644 --- a/common/python/db/global_config_dao.py +++ b/common/python/db/global_config_dao.py @@ -170,7 +170,7 @@ class GlobalConfigDao: @staticmethod def fill_intranet_base_uri(config_dict): """ - :param config_dict: {service.var_name : intranet_base_uri} + :param config_dict: {service.group : intranet_base_uri} :return: """ with DB.connection_context(): @@ -178,6 +178,8 @@ class GlobalConfigDao: GlobalConfigModel.name == "intranet_base_uri" ) for item in items: - if item['value'] is None: - pass - + if item.value is None: + if config_dict[item.group] is not None and \ + len(config_dict[item.group]) != 0: + item.value = config_dict[item.group] + item.save() diff --git a/flow/app_launcher.py b/flow/app_launcher.py index cff7d5b3a..3cf11f718 100644 --- a/flow/app_launcher.py +++ b/flow/app_launcher.py @@ -22,6 +22,7 @@ from common.python.db.db_models import DB, Job from common.python.utils.log_utils import LoggerFactory, schedule_logger from flow.cycle_actions.flow_action_queue.flow_action_queue_consumer import FlowActionQueueConsumer from flow.cycle_actions.guard.job_guard import JobGuard +from flow.service.config.global_config_init_service import GlobalSettingInit from flow.service.gateway.gateway_service import GatewayService from flow.service.job_scheduler.clear_job_middle_data_scheduler import ClearJobMiddleDataScheduler from flow.service.job_scheduler.job_stop_action import JobStopAction @@ -39,6 +40,18 @@ class AppLauncher: def start_app(): AppLauncher.logger.info("AppLauncher start ") + # Init Config + AppLauncher.logger.info("GlobalSetting Init") + init_config = GlobalSettingInit() + start = time.time() + while not init_config.is_ready_to_boot(): + time.sleep(3) + end = time.time() + if (end - start) >= 60: + AppLauncher.logger.info("The Member ID in Empty, Wait to Init") + start = end + AppLauncher.logger.info("GlobalSetting Init Success")\ + # Update gateway IP whitelist AppLauncher.logger.info("repost ip to white list") GatewayService.report_ip_to_white_list() diff --git a/flow/service/config/global_config_init_service.py b/flow/service/config/global_config_init_service.py index 86fa9cae1..79998bb52 100644 --- a/flow/service/config/global_config_init_service.py +++ b/flow/service/config/global_config_init_service.py @@ -12,9 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. from common.python.common import consts +from common.python.db.global_config_dao import GlobalConfigDao from common.python.utils.conf_utils import get_env_config -INTRANET_IP = get_env_config(consts.ENV_CONF_KEY_INTRANET_IP) +ENV_INTRANET_IP = get_env_config(consts.ENV_CONF_KEY_INTRANET_IP) or 'ip' +ENV_GATEWAY_PORT = get_env_config(consts.ENV_CONF_KEY_GATEWAY_PORT) or 'port' +ENV_FLOW_PORT = get_env_config(consts.ENV_CONF_KEY_FLOW_PORT) or 'port' +ENV_NGINX_PORT = get_env_config(consts.ENV_CONF_KEY_NGINX_PORT) or 'port' class GlobalSettingInit: @@ -25,8 +29,31 @@ class GlobalSettingInit: def __init__(self): """ wefe_board.intranet_base_uri - wefe_flow.intranet_base_uri +, wefe_flow.intranet_base_uri wefe_gateway.intranet_base_uri wefe_serving.intranet_base_uri """ + intranet_dict = { + 'wefe_board': 'http://' + ENV_INTRANET_IP + ':' + ENV_NGINX_PORT + '/board-service', + 'wefe_flow': 'http://' + ENV_INTRANET_IP + ':' + ENV_FLOW_PORT, + 'wefe_gateway': ENV_INTRANET_IP + ':' + ENV_GATEWAY_PORT, + 'wefe_serving': '' + } + GlobalConfigDao.fill_intranet_base_uri(intranet_dict) + @staticmethod + def is_ready_to_boot(): + """ + The Member must to be initialized + Check the variable global-config.member_id + :return: bool + """ + member_info = GlobalConfigDao.getMemberInfo() + if member_info.member_id is None: + return False + else: + return True + + +if __name__ == '__main__': + print(GlobalSettingInit().is_ready_to_boot()) diff --git a/release/docker/docker_service/wefe_python_service/resources/docker-compose.yml b/release/docker/docker_service/wefe_python_service/resources/docker-compose.yml index 9b8a5fcbf..13b313875 100644 --- a/release/docker/docker_service/wefe_python_service/resources/docker-compose.yml +++ b/release/docker/docker_service/wefe_python_service/resources/docker-compose.yml @@ -4,6 +4,8 @@ services: wefe_python_service: image: wefe_python_service:v.2.1 # wefe_version tty: true + env_file: + - variables.env ports: - 8888:5000 # flow_port privileged: true diff --git a/release/docker/docker_service/wefe_python_service/resources/variables.env b/release/docker/docker_service/wefe_python_service/resources/variables.env new file mode 100644 index 000000000..458995b11 --- /dev/null +++ b/release/docker/docker_service/wefe_python_service/resources/variables.env @@ -0,0 +1,7 @@ +# define python service container env + +FLOW_PORT=P +NGINX_PORT= +GATEWAY_PORT= + +INTRANET_IP= \ No newline at end of file diff --git a/release/docker/docker_service/wefe_python_service/wefe_python_service_start.sh b/release/docker/docker_service/wefe_python_service/wefe_python_service_start.sh index de17e0573..079821f65 100644 --- a/release/docker/docker_service/wefe_python_service/wefe_python_service_start.sh +++ b/release/docker/docker_service/wefe_python_service/wefe_python_service_start.sh @@ -4,17 +4,23 @@ # 导入配置 source ../wefe.cfg -# 修改服务启动配置 -sed -i "/wefe_version/s/python_service:.*#/python_service:$WEFE_VERSION #/g" ./resources/docker-compose.yml -sed -i "/flow_logs/s@-.*:@- $DATA_PATH/logs/flow:@g" ./resources/docker-compose.yml +# 填充环境变量 +sed -i "/FLOW_PORT/s/=.*/=$PYTHON_SERVICE_PORT/g" ./resources/variables.env +sed -i "/NGINX_PORT/s/=.*/=$NGINX_PORT/g" ./resources/variables.env +sed -i "/GATEWAY_PORT/s/=.*/=$GATEWAY_SERVICE_PORT/g" ./resources/variables.env +sed -i "/INTRANET_IP/s/=.*/=$INTRANET_IP/g" ./resources/variables.env -# 修改 flow 端口 -sed -i "/flow_port/s/-.*:/- $PYTHON_SERVICE_PORT:/g" ./resources/docker-compose.yml - -# 加载本地离线镜像包 -echo "开始加载 flow 离线镜像" -docker load < resources/wefe_python_service_$WEFE_VERSION\.tar -echo "加载 flow 离线镜像完成" - -# 启动 flow 镜像 -docker-compose -p $WEFE_ENV -f resources/docker-compose.yml up -d +## 修改服务启动配置 +#sed -i "/wefe_version/s/python_service:.*#/python_service:$WEFE_VERSION #/g" ./resources/docker-compose.yml +#sed -i "/flow_logs/s@-.*:@- $DATA_PATH/logs/flow:@g" ./resources/docker-compose.yml +# +## 修改 flow 端口 +#sed -i "/flow_port/s/-.*:/- $PYTHON_SERVICE_PORT:/g" ./resources/docker-compose.yml +# +## 加载本地离线镜像包 +#echo "开始加载 flow 离线镜像" +#docker load < resources/wefe_python_service_$WEFE_VERSION\.tar +#echo "加载 flow 离线镜像完成" +# +## 启动 flow 镜像 +#docker-compose -p $WEFE_ENV -f resources/docker-compose.yml up -d -- Gitee From d27e4478cdf208fc0226154a6dc6740620ffecfe Mon Sep 17 00:00:00 2001 From: "wingo.wen" Date: Thu, 14 Oct 2021 11:50:44 +0800 Subject: [PATCH 003/100] global config internet auto fill --- flow/service.sh | 1 + .../config/fill_config_shell_service.py | 44 +++++++++++++++++++ .../config/global_config_init_service.py | 18 +------- 3 files changed, 47 insertions(+), 16 deletions(-) create mode 100644 flow/service/config/fill_config_shell_service.py diff --git a/flow/service.sh b/flow/service.sh index 54b381aad..50f941dc1 100644 --- a/flow/service.sh +++ b/flow/service.sh @@ -54,6 +54,7 @@ start() { pip install -r ${PYTHON_ROOT}/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple + nohup python3 ${PYTHON_ROOT}/flow/service/config/fill_config_shell_service.py >> "${log_dir}/console.log" 2>>"${log_dir}/error.log" & nohup python3 ${PYTHON_ROOT}/flow/app_launcher.py >> "${log_dir}/console.log" 2>>"${log_dir}/error.log" & sleep 3 diff --git a/flow/service/config/fill_config_shell_service.py b/flow/service/config/fill_config_shell_service.py new file mode 100644 index 000000000..58b61b658 --- /dev/null +++ b/flow/service/config/fill_config_shell_service.py @@ -0,0 +1,44 @@ +# Copyright 2021 Tianmian Tech. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from common.python.common import consts +from common.python.db.global_config_dao import GlobalConfigDao +from common.python.utils.conf_utils import get_env_config +from common.python.utils.log_utils import LoggerFactory + +ENV_INTRANET_IP = get_env_config(consts.ENV_CONF_KEY_INTRANET_IP) or 'ip' +ENV_GATEWAY_PORT = get_env_config(consts.ENV_CONF_KEY_GATEWAY_PORT) or 'port' +ENV_FLOW_PORT = get_env_config(consts.ENV_CONF_KEY_FLOW_PORT) or 'port' +ENV_NGINX_PORT = get_env_config(consts.ENV_CONF_KEY_NGINX_PORT) or 'port' + +logger = LoggerFactory.get_logger("InitConfig") + +if __name__ == '__main__': + """ + Initialize the configuration of the system based on the profile + + wefe_board.intranet_base_uri +, wefe_flow.intranet_base_uri + wefe_gateway.intranet_base_uri + wefe_serving.intranet_base_uri + """ + logger.info('Init Config') + intranet_dict = { + 'wefe_board': 'http://' + ENV_INTRANET_IP + ':' + ENV_NGINX_PORT + '/board-service', + 'wefe_flow': 'http://' + ENV_INTRANET_IP + ':' + ENV_FLOW_PORT, + 'wefe_gateway': ENV_INTRANET_IP + ':' + ENV_GATEWAY_PORT, + 'wefe_serving': '' + } + GlobalConfigDao.fill_intranet_base_uri(intranet_dict) + logger.info('Init Success') diff --git a/flow/service/config/global_config_init_service.py b/flow/service/config/global_config_init_service.py index 79998bb52..061c580cf 100644 --- a/flow/service/config/global_config_init_service.py +++ b/flow/service/config/global_config_init_service.py @@ -26,21 +26,6 @@ class GlobalSettingInit: Initialize the relevant configuration based on the environment variables """ - def __init__(self): - """ - wefe_board.intranet_base_uri -, wefe_flow.intranet_base_uri - wefe_gateway.intranet_base_uri - wefe_serving.intranet_base_uri - """ - intranet_dict = { - 'wefe_board': 'http://' + ENV_INTRANET_IP + ':' + ENV_NGINX_PORT + '/board-service', - 'wefe_flow': 'http://' + ENV_INTRANET_IP + ':' + ENV_FLOW_PORT, - 'wefe_gateway': ENV_INTRANET_IP + ':' + ENV_GATEWAY_PORT, - 'wefe_serving': '' - } - GlobalConfigDao.fill_intranet_base_uri(intranet_dict) - @staticmethod def is_ready_to_boot(): """ @@ -49,7 +34,8 @@ class GlobalSettingInit: :return: bool """ member_info = GlobalConfigDao.getMemberInfo() - if member_info.member_id is None: + gateway_config = GlobalConfigDao.getGatewayConfig() + if (member_info.member_id is None) or (gateway_config.intranet_base_uri is None): return False else: return True -- Gitee From c85e8c85b03abe1a4e6ba2c63606218bcbbd716c Mon Sep 17 00:00:00 2001 From: "wingo.wen" Date: Thu, 14 Oct 2021 11:50:53 +0800 Subject: [PATCH 004/100] global config internet auto fill --- common/python/db/global_config_dao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/python/db/global_config_dao.py b/common/python/db/global_config_dao.py index 15bd250dc..69b49be0a 100644 --- a/common/python/db/global_config_dao.py +++ b/common/python/db/global_config_dao.py @@ -178,7 +178,7 @@ class GlobalConfigDao: GlobalConfigModel.name == "intranet_base_uri" ) for item in items: - if item.value is None: + if item.value is None or len(item.value) == 0: if config_dict[item.group] is not None and \ len(config_dict[item.group]) != 0: item.value = config_dict[item.group] -- Gitee From faef8d13db447a01ff688a07578a3b1d3493e0ae Mon Sep 17 00:00:00 2001 From: "wingo.wen" Date: Thu, 14 Oct 2021 14:17:25 +0800 Subject: [PATCH 005/100] change nginx version --- release/docker/docker_image/app/wefe_board_website/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/docker/docker_image/app/wefe_board_website/Dockerfile b/release/docker/docker_image/app/wefe_board_website/Dockerfile index 21ba989a6..87924c9aa 100644 --- a/release/docker/docker_image/app/wefe_board_website/Dockerfile +++ b/release/docker/docker_image/app/wefe_board_website/Dockerfile @@ -1,4 +1,4 @@ -FROM nginx:1.19.2 +FROM nginx:1.14.2 WORKDIR /opt/website -- Gitee From 90cd44a67939903e0ac955334718b2308ed3b0cf Mon Sep 17 00:00:00 2001 From: "wingo.wen" Date: Thu, 14 Oct 2021 15:27:24 +0800 Subject: [PATCH 006/100] docker shell --- release/docker/deploy_shell/tool/service.sh | 1 + .../wefe_python_service_start.sh | 28 +++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/release/docker/deploy_shell/tool/service.sh b/release/docker/deploy_shell/tool/service.sh index db74d881a..d6186eee3 100755 --- a/release/docker/deploy_shell/tool/service.sh +++ b/release/docker/deploy_shell/tool/service.sh @@ -56,6 +56,7 @@ start() { pip install -r ${PYTHON_ROOT}/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple + nohup python3 ${PYTHON_ROOT}/flow/service/config/fill_config_shell_service.py >> "${log_dir}/console.log" 2>>"${log_dir}/error.log" & nohup python3 ${PYTHON_ROOT}/flow/app_launcher.py >> "${log_dir}/console.log" 2>>"${log_dir}/error.log" & sleep 3 diff --git a/release/docker/docker_service/wefe_python_service/wefe_python_service_start.sh b/release/docker/docker_service/wefe_python_service/wefe_python_service_start.sh index 079821f65..8088bb9be 100644 --- a/release/docker/docker_service/wefe_python_service/wefe_python_service_start.sh +++ b/release/docker/docker_service/wefe_python_service/wefe_python_service_start.sh @@ -10,17 +10,17 @@ sed -i "/NGINX_PORT/s/=.*/=$NGINX_PORT/g" ./resources/variables.env sed -i "/GATEWAY_PORT/s/=.*/=$GATEWAY_SERVICE_PORT/g" ./resources/variables.env sed -i "/INTRANET_IP/s/=.*/=$INTRANET_IP/g" ./resources/variables.env -## 修改服务启动配置 -#sed -i "/wefe_version/s/python_service:.*#/python_service:$WEFE_VERSION #/g" ./resources/docker-compose.yml -#sed -i "/flow_logs/s@-.*:@- $DATA_PATH/logs/flow:@g" ./resources/docker-compose.yml -# -## 修改 flow 端口 -#sed -i "/flow_port/s/-.*:/- $PYTHON_SERVICE_PORT:/g" ./resources/docker-compose.yml -# -## 加载本地离线镜像包 -#echo "开始加载 flow 离线镜像" -#docker load < resources/wefe_python_service_$WEFE_VERSION\.tar -#echo "加载 flow 离线镜像完成" -# -## 启动 flow 镜像 -#docker-compose -p $WEFE_ENV -f resources/docker-compose.yml up -d +# 修改服务启动配置 +sed -i "/wefe_version/s/python_service:.*#/python_service:$WEFE_VERSION #/g" ./resources/docker-compose.yml +sed -i "/flow_logs/s@-.*:@- $DATA_PATH/logs/flow:@g" ./resources/docker-compose.yml + +# 修改 flow 端口 +sed -i "/flow_port/s/-.*:/- $PYTHON_SERVICE_PORT:/g" ./resources/docker-compose.yml + +# 加载本地离线镜像包 +echo "开始加载 flow 离线镜像" +docker load < resources/wefe_python_service_$WEFE_VERSION\.tar +echo "加载 flow 离线镜像完成" + +# 启动 flow 镜像 +docker-compose -p $WEFE_ENV -f resources/docker-compose.yml up -d -- Gitee From 65fccb6cad124779e221bd80d694aa420662ff02 Mon Sep 17 00:00:00 2001 From: "wingo.wen" Date: Thu, 14 Oct 2021 15:47:29 +0800 Subject: [PATCH 007/100] change nginx version --- release/docker/docker_image/app/wefe_board_website/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/docker/docker_image/app/wefe_board_website/Dockerfile b/release/docker/docker_image/app/wefe_board_website/Dockerfile index 87924c9aa..21ba989a6 100644 --- a/release/docker/docker_image/app/wefe_board_website/Dockerfile +++ b/release/docker/docker_image/app/wefe_board_website/Dockerfile @@ -1,4 +1,4 @@ -FROM nginx:1.14.2 +FROM nginx:1.19.2 WORKDIR /opt/website -- Gitee From 400e89fa3893fca84494776a507752a81d699dbc Mon Sep 17 00:00:00 2001 From: "wingo.wen" Date: Thu, 14 Oct 2021 17:25:05 +0800 Subject: [PATCH 008/100] Get member id wrong --- flow/service.sh | 2 +- flow/service/gateway/gateway_service.py | 4 ++-- flow/service/job_scheduler/job_stop_action.py | 4 ++-- flow/service/job_scheduler/run_task_action.py | 4 +++- flow/settings.py | 7 +++++-- release/docker/deploy_shell/tool/service.sh | 2 +- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/flow/service.sh b/flow/service.sh index 50f941dc1..d402e5965 100644 --- a/flow/service.sh +++ b/flow/service.sh @@ -54,7 +54,7 @@ start() { pip install -r ${PYTHON_ROOT}/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple - nohup python3 ${PYTHON_ROOT}/flow/service/config/fill_config_shell_service.py >> "${log_dir}/console.log" 2>>"${log_dir}/error.log" & + python3 ${PYTHON_ROOT}/flow/service/config/fill_config_shell_service.py nohup python3 ${PYTHON_ROOT}/flow/app_launcher.py >> "${log_dir}/console.log" 2>>"${log_dir}/error.log" & sleep 3 diff --git a/flow/service/gateway/gateway_service.py b/flow/service/gateway/gateway_service.py index ad6d5843a..21f366390 100644 --- a/flow/service/gateway/gateway_service.py +++ b/flow/service/gateway/gateway_service.py @@ -23,7 +23,7 @@ from common.python.protobuf.pyproto.gateway_meta_pb2 import TransferMeta from common.python.utils import network_utils from common.python.utils.core_utils import get_commit_id from common.python.utils.log_utils import LoggerFactory -from flow.settings import MEMBER_ID, JOB_GRPC, GATEWAY_INTRANET_HOST, GATEWAY_INTRANET_PORT +from flow.settings import JOB_GRPC, GATEWAY_INTRANET_HOST, GATEWAY_INTRANET_PORT, MemberInfo from flow.web.utils.const import * @@ -70,7 +70,7 @@ class GatewayService: """ Send messages to self gateway service """ - return GatewayService.send(MEMBER_ID, processor) + return GatewayService.send(MemberInfo.MEMBER_ID, processor) @staticmethod def send(dst_member_id, processor, data="") -> dict: diff --git a/flow/service/job_scheduler/job_stop_action.py b/flow/service/job_scheduler/job_stop_action.py index 76ad5da26..f0beeeca7 100644 --- a/flow/service/job_scheduler/job_stop_action.py +++ b/flow/service/job_scheduler/job_stop_action.py @@ -27,7 +27,7 @@ from common.python.utils.log_utils import schedule_logger from flow.alert_service.job_error_mail_warn_scheduler import JobErrorMailWarnScheduler from flow.service.board.board_service import BoardService from flow.service.job_scheduler.job_service import JobService -from flow.settings import MEMBER_ID +from flow.settings import MemberInfo class JobStopAction: @@ -134,7 +134,7 @@ class JobStopAction: def log_job_info(self, message, exception=None): running_job = self.job.job_id + '_' + self.job.my_role logger = schedule_logger(running_job) - message = '{} {} on kill job {} |{}'.format(self.job.my_role, MEMBER_ID, self.job.job_id, message) + message = '{} {} on kill job {} |{}'.format(self.job.my_role, MemberInfo.MEMBER_ID, self.job.job_id, message) if exception: logger.exception(message, exception) else: diff --git a/flow/service/job_scheduler/run_task_action.py b/flow/service/job_scheduler/run_task_action.py index 303da9cc2..df37624e4 100644 --- a/flow/service/job_scheduler/run_task_action.py +++ b/flow/service/job_scheduler/run_task_action.py @@ -28,10 +28,12 @@ from common.python.utils import conf_utils from common.python.utils.core_utils import current_datetime from common.python.utils.log_utils import LoggerFactory, schedule_logger from flow.service.job_scheduler.job_service import JobService -from flow.settings import MEMBER_ID +from flow.settings import MemberInfo from flow.utils import job_utils from kernel.task_executor import TaskExecutor +MEMBER_ID = MemberInfo.MEMBER_ID + class RunTaskAction: logger = LoggerFactory.get_logger("RunTaskAction") diff --git a/flow/settings.py b/flow/settings.py index 280b7cd87..e5374345a 100644 --- a/flow/settings.py +++ b/flow/settings.py @@ -27,8 +27,11 @@ stat_logger = log_utils.get_logger("wefe_flow_stat") detect_logger = log_utils.get_logger("wefe_flow_detect") access_logger = log_utils.get_logger("wefe_flow_access") -MEMBER_ID = GlobalSetting.get_member_id() -MEMBER_NAME = GlobalSetting.get_member_name() + +class MemberInfo: + MEMBER_ID = GlobalSetting.get_member_id() + MEMBER_NAME = GlobalSetting.get_member_name() + gateway_intranet = GlobalConfigDao.getGatewayConfig().intranet_base_uri.split(":") GATEWAY_INTRANET_HOST = gateway_intranet[0] diff --git a/release/docker/deploy_shell/tool/service.sh b/release/docker/deploy_shell/tool/service.sh index d6186eee3..7a5889cdc 100755 --- a/release/docker/deploy_shell/tool/service.sh +++ b/release/docker/deploy_shell/tool/service.sh @@ -56,7 +56,7 @@ start() { pip install -r ${PYTHON_ROOT}/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple - nohup python3 ${PYTHON_ROOT}/flow/service/config/fill_config_shell_service.py >> "${log_dir}/console.log" 2>>"${log_dir}/error.log" & + python3 ${PYTHON_ROOT}/flow/service/config/fill_config_shell_service.py nohup python3 ${PYTHON_ROOT}/flow/app_launcher.py >> "${log_dir}/console.log" 2>>"${log_dir}/error.log" & sleep 3 -- Gitee From 4f2a87e32a57c4b321297226ecdb718c8ec74d29 Mon Sep 17 00:00:00 2001 From: "ivenn.zheng" Date: Thu, 14 Oct 2021 18:02:19 +0800 Subject: [PATCH 009/100] =?UTF-8?q?=E4=BF=AE=E6=94=B9=20s.yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/python/calculation/fc/function/wefe-fc/s.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/common/python/calculation/fc/function/wefe-fc/s.yaml b/common/python/calculation/fc/function/wefe-fc/s.yaml index 0cc394cc1..ae31221aa 100644 --- a/common/python/calculation/fc/function/wefe-fc/s.yaml +++ b/common/python/calculation/fc/function/wefe-fc/s.yaml @@ -6,12 +6,12 @@ vars: wefe-fc: name: wefe-fc description: function index - role: acs:ram::145***590:role/wefe-fc-role + role: acs:ram::1458598865203590:role/wefe-fc-role internetAccess: true - vpcConfig: - vpcId: vpc-wz9x2vv0i1sk4d9t2ra02 - vswitchIds: ["vsw-wz94re9ev2qlrcseobwfg"] - securityGroupId: sg-wz92tp0qnjlhnqhrw298 +# vpcConfig: +# vpcId: vpc-wz9x2vv0i1sk4d9t2ra02 +# vswitchIds: ["vsw-wz94re9ev2qlrcseobwfg"] +# securityGroupId: sg-wz92tp0qnjlhnqhrw298 nasConfig: Auto services: fc-wefe-fc-index: -- Gitee From e494177f2d3ce81db9e90e2a2bc6bafbf0b4ba55 Mon Sep 17 00:00:00 2001 From: "ivenn.zheng" Date: Thu, 14 Oct 2021 18:03:50 +0800 Subject: [PATCH 010/100] =?UTF-8?q?=E4=BF=AE=E6=94=B9=20s.yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/python/calculation/fc/function/wefe-fc/s.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/python/calculation/fc/function/wefe-fc/s.yaml b/common/python/calculation/fc/function/wefe-fc/s.yaml index ae31221aa..8d349db8b 100644 --- a/common/python/calculation/fc/function/wefe-fc/s.yaml +++ b/common/python/calculation/fc/function/wefe-fc/s.yaml @@ -6,7 +6,7 @@ vars: wefe-fc: name: wefe-fc description: function index - role: acs:ram::1458598865203590:role/wefe-fc-role + role: acs:ram::145***590:role/wefe-fc-role internetAccess: true # vpcConfig: # vpcId: vpc-wz9x2vv0i1sk4d9t2ra02 -- Gitee From 226871ab5e6f0908d258f4d3b6de4622a5f99858 Mon Sep 17 00:00:00 2001 From: "ivenn.zheng" Date: Thu, 14 Oct 2021 19:04:20 +0800 Subject: [PATCH 011/100] =?UTF-8?q?=E4=BF=AE=E6=94=B9=20s.yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/python/calculation/fc/function/wefe-fc/s.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/python/calculation/fc/function/wefe-fc/s.yaml b/common/python/calculation/fc/function/wefe-fc/s.yaml index 8d349db8b..aae0928d0 100644 --- a/common/python/calculation/fc/function/wefe-fc/s.yaml +++ b/common/python/calculation/fc/function/wefe-fc/s.yaml @@ -8,10 +8,10 @@ vars: description: function index role: acs:ram::145***590:role/wefe-fc-role internetAccess: true -# vpcConfig: -# vpcId: vpc-wz9x2vv0i1sk4d9t2ra02 -# vswitchIds: ["vsw-wz94re9ev2qlrcseobwfg"] -# securityGroupId: sg-wz92tp0qnjlhnqhrw298 + vpcConfig: + vpcId: vpc-wz9****ytzsqd66su8 + vswitchIds: ["vsw-wz9a***qencd9juh0"] + securityGroupId: sg-wz***rmmzeoua4s nasConfig: Auto services: fc-wefe-fc-index: -- Gitee From cd17116e7c30f33eb067016502371a1bab39b447 Mon Sep 17 00:00:00 2001 From: "ivenn.zheng" Date: Thu, 14 Oct 2021 19:13:55 +0800 Subject: [PATCH 012/100] =?UTF-8?q?=E4=BF=AE=E6=94=B9=20README=5FFC.md=20?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- release/docker/README_FC.md | 1 + 1 file changed, 1 insertion(+) diff --git a/release/docker/README_FC.md b/release/docker/README_FC.md index f7564ebb1..92b4fa3a8 100644 --- a/release/docker/README_FC.md +++ b/release/docker/README_FC.md @@ -235,6 +235,7 @@ AliyunSTSAssumeRoleAccess AliyunLogFullAccess AliyunFCFullAccess AliyunNASFullAccess +AliyunVPCFullAccess ``` 注：系统权限添加一次只能5个，剩下的请再次添加进去，如下： -- Gitee From 88b2edca6a4ff9c9d484e27ca21d15bead8dbdc4 Mon Sep 17 00:00:00 2001 From: "Nova.li" Date: Fri, 15 Oct 2021 11:36:35 +0800 Subject: [PATCH 013/100] =?UTF-8?q?feat:=20=E3=80=90ID1074592=E3=80=91=20f?= =?UTF-8?q?usion-=E7=94=9F=E6=88=90=E6=95=B0=E6=8D=AE=E9=9B=86=E3=80=81?= =?UTF-8?q?=E8=BF=87=E6=BB=A4=E5=99=A8=E4=BF=AE=E6=94=B9=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/views/components/progressBar.vue | 9 ++-- .../src/views/index/data-set-view.vue | 50 ++++++++++--------- .../src/views/index/filter-view.vue | 50 ++++++++++--------- 3 files changed, 58 insertions(+), 51 deletions(-) diff --git a/fusion/fusion-website/src/views/components/progressBar.vue b/fusion/fusion-website/src/views/components/progressBar.vue index ea979123e..e51e39c53 100644 --- a/fusion/fusion-website/src/views/components/progressBar.vue +++ b/fusion/fusion-website/src/views/components/progressBar.vue @@ -1,8 +1,8 @@