From f8090d2dc4be1aad8af1e8d24bb6a270e90af60c Mon Sep 17 00:00:00 2001 From: "zhongling.h" Date: Wed, 28 Feb 2024 15:03:34 +0800 Subject: [PATCH 01/10] add anolis23-modelscope1.12.0 configuration --- AI/modelscope/1.12.0/23/Dockerfile | 45 +++++++++++ .../1.12.0/23/requirements/audio.txt | 5 ++ .../23/requirements/audio/audio_asr.txt | 1 + .../23/requirements/audio/audio_codec.txt | 1 + .../23/requirements/audio/audio_kws.txt | 7 ++ .../23/requirements/audio/audio_signal.txt | 12 +++ .../23/requirements/audio/audio_tts.txt | 28 +++++++ AI/modelscope/1.12.0/23/requirements/cv.txt | 74 +++++++++++++++++++ AI/modelscope/1.12.0/23/requirements/docs.txt | 7 ++ .../1.12.0/23/requirements/framework.txt | 22 ++++++ .../1.12.0/23/requirements/multi-modal.txt | 29 ++++++++ AI/modelscope/1.12.0/23/requirements/nlp.txt | 26 +++++++ .../1.12.0/23/requirements/science.txt | 8 ++ AI/modelscope/1.12.0/23/requirements/svr.txt | 4 + .../1.12.0/23/requirements/tensorflow1x.txt | 1 + .../1.12.0/23/requirements/tests.txt | 5 ++ AI/modelscope/buildspec.yml | 20 ++++- 17 files changed, 293 insertions(+), 2 deletions(-) create mode 100644 AI/modelscope/1.12.0/23/Dockerfile create mode 100644 AI/modelscope/1.12.0/23/requirements/audio.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/audio/audio_asr.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/audio/audio_codec.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/audio/audio_kws.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/audio/audio_signal.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/audio/audio_tts.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/cv.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/docs.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/framework.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/multi-modal.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/nlp.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/science.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/svr.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/tensorflow1x.txt create mode 100644 AI/modelscope/1.12.0/23/requirements/tests.txt diff --git a/AI/modelscope/1.12.0/23/Dockerfile b/AI/modelscope/1.12.0/23/Dockerfile new file mode 100644 index 0000000..d060841 --- /dev/null +++ b/AI/modelscope/1.12.0/23/Dockerfile @@ -0,0 +1,45 @@ +FROM anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/cuda:12.1.1-23-devel + +RUN yum install -y \ + automake \ + libtool \ + system-rpm-config \ + wget \ + git git-lfs \ + blas \ + glibc-locale-source \ + libSM \ + langpacks-zh_CN && yum clean all + +RUN mkdir -p /var/modelscope +COPY requirements /var/modelscope + +RUN yum install -y pytorch-devel pytorch python3-librosa jupyterlab python3-fairseq python3-moviepy python3-nvdiffrast python3-fasttext lapack http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8714/788714/tbb-2021.11.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8714/788714/tbb-bind-2021.11.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8714/788714/tbb-devel-2021.11.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/4114/794114/python3-datasets-2.16.1-2.an23.noarch.rpm && yum clean all +RUN yum install -y python3-modelscope-1.12.0 && yum clean all + +RUN pip3 freeze > /var/modelscope/base.txt + +RUN sed -i '/datasets/d' /var/modelscope/framework.txt && \ + sed -i '/yapf/d' /var/modelscope/tests.txt && \ + sed -i '/timm/d' /var/modelscope/base.txt && \ + sed -i '/builddir/d' /var/modelscope/base.txt && \ + sed -i '/ffmpeg/d' /var/modelscope/cv.txt && \ + sed -i '/protobuf/d' /var/modelscope/nlp.txt && \ + sed -i '/decorator/d' /var/modelscope/*.txt && \ + sed -i '/scikit-image/d' /var/modelscope/*.txt + +RUN pip install \ + -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html \ + -r /var/modelscope/framework.txt \ + -r /var/modelscope/audio.txt \ + -r /var/modelscope/cv.txt \ + -r /var/modelscope/multi-modal.txt \ + -r /var/modelscope/nlp.txt \ + -r /var/modelscope/science.txt \ + -r /var/modelscope/tests.txt \ + uvicorn text2sql_lgesql faiss + +RUN curl -L https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/modelscope/modelscope-1.12.0.tar.gz | tar -xz -C /root/ +RUN git clone https://www.modelscope.cn/ModelScope_Developer/modelscope-library-test-data.git /tmp/modelscope_test && \ + mv /tmp/modelscope_test/* /root/modelscope-1.12.0/data/test/ && \ + rm -rf /tmp/modelscope_test \ No newline at end of file diff --git a/AI/modelscope/1.12.0/23/requirements/audio.txt b/AI/modelscope/1.12.0/23/requirements/audio.txt new file mode 100644 index 0000000..88e469c --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/audio.txt @@ -0,0 +1,5 @@ +-r audio/audio_asr.txt +-r audio/audio_kws.txt +-r audio/audio_signal.txt +-r audio/audio_tts.txt +-r audio/audio_codec.txt diff --git a/AI/modelscope/1.12.0/23/requirements/audio/audio_asr.txt b/AI/modelscope/1.12.0/23/requirements/audio/audio_asr.txt new file mode 100644 index 0000000..a63614f --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/audio/audio_asr.txt @@ -0,0 +1 @@ +funasr>=1.0.0 diff --git a/AI/modelscope/1.12.0/23/requirements/audio/audio_codec.txt b/AI/modelscope/1.12.0/23/requirements/audio/audio_codec.txt new file mode 100644 index 0000000..c7ac8b2 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/audio/audio_codec.txt @@ -0,0 +1 @@ +funcodec>=0.2.0 diff --git a/AI/modelscope/1.12.0/23/requirements/audio/audio_kws.txt b/AI/modelscope/1.12.0/23/requirements/audio/audio_kws.txt new file mode 100644 index 0000000..276a0a2 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/audio/audio_kws.txt @@ -0,0 +1,7 @@ +kaldiio +kwsbp>=0.0.6 +matplotlib +py_sound_connect>=0.1 +scipy +SoundFile>0.10 +tensorboardX diff --git a/AI/modelscope/1.12.0/23/requirements/audio/audio_signal.txt b/AI/modelscope/1.12.0/23/requirements/audio/audio_signal.txt new file mode 100644 index 0000000..65f1ec6 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/audio/audio_signal.txt @@ -0,0 +1,12 @@ +hdbscan +hyperpyyaml +librosa==0.10.1 +MinDAEC +mir_eval>=0.7 +rotary_embedding_torch>=0.1.5 +scipy +SoundFile>0.10 +speechbrain>=0.5.12 +torchaudio +tqdm +umap-learn diff --git a/AI/modelscope/1.12.0/23/requirements/audio/audio_tts.txt b/AI/modelscope/1.12.0/23/requirements/audio/audio_tts.txt new file mode 100644 index 0000000..5cff1b2 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/audio/audio_tts.txt @@ -0,0 +1,28 @@ +bitstring +greenlet>=1.1.2 +inflect +jedi>=0.18.1 +kantts +librosa==0.10.1 +lxml +matplotlib +msgpack>=1.0.4 +parso>=0.8.3 +pexpect>=4.8.0 +pickleshare>=0.7.5 +prompt-toolkit>=3.0.30 +protobuf +ptflops +ptyprocess>=0.7.0 +pygments>=2.12.0 +pysptk>=0.1.15,<0.1.19 +pytorch_wavelets +PyWavelets>=1.0.0 +scikit-learn +sox +tensorboardx +tqdm +traitlets>=5.3.0 +ttsfrd>=0.1.2 +unidecode +wcwidth>=0.2.5 diff --git a/AI/modelscope/1.12.0/23/requirements/cv.txt b/AI/modelscope/1.12.0/23/requirements/cv.txt new file mode 100644 index 0000000..c8edb67 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/cv.txt @@ -0,0 +1,74 @@ +accelerate +albumentations>=1.0.3 +av>=9.2.0 +bmt_clipit>=1.0 +chumpy +clip>=1.0 +control_ldm +ddpm_guided_diffusion +diffusers +easydict +easyrobust +edit_distance +face_alignment>=1.3.5 +fairscale>=0.4.1 +fastai>=1.0.51 +ffmpeg>=1.4 +ffmpeg-python>=0.2.0 +ftfy +fvcore +# remove for windows support +# healpy +imageio>=2.9.0 +imageio-ffmpeg>=0.4.2 +imgaug>=0.4.0 +kornia>=0.5.0 +lap +lmdb +lpips +ml_collections +mmcls>=0.21.0 +mmdet>=2.25.0,<=2.28.2 +# mmdet3d-1.0.0rc6 remove networkx and numba version restriction +mmdet3d==1.0.0a1 +mmsegmentation<=0.30.0 +moviepy>=1.0.3 +nerfacc==0.2.2 +networkx +numba +omegaconf +onnx +onnxruntime>=1.10 +onnxsim +open-clip-torch>=2.7.0 +opencv-python +paint_ldm +pandas +panopticapi +plyfile>=0.7.4 +psutil +pyclipper +PyMCubes +pytorch-lightning +regex +# <0.20.0 for compatible python3.7 python3.8 +scikit-image>=0.19.3,<0.20.0 +scikit-learn>=0.20.1 +shapely +shotdetect_scenedetect_lgss>=0.0.4 +smplx +tensorflow-estimator>=1.15.1 +tf_slim +thop +timm>=0.4.9 +torch-scatter +torchmetrics>=0.6.2 +torchsummary>=1.5.1 +torchvision +tqdm +transformers>=4.26.0 +trimesh +ujson +utils +videofeatures_clipit>=1.0 +yacs diff --git a/AI/modelscope/1.12.0/23/requirements/docs.txt b/AI/modelscope/1.12.0/23/requirements/docs.txt new file mode 100644 index 0000000..3b35383 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/docs.txt @@ -0,0 +1,7 @@ +docutils>=0.16.0 +myst_parser +recommonmark +sphinx>=5.3.0 +sphinx-book-theme +sphinx-copybutton +sphinx_markdown_tables diff --git a/AI/modelscope/1.12.0/23/requirements/framework.txt b/AI/modelscope/1.12.0/23/requirements/framework.txt new file mode 100644 index 0000000..8804fe8 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/framework.txt @@ -0,0 +1,22 @@ +addict +attrs +datasets>=2.14.5 +einops +filelock>=3.3.0 +gast>=0.2.2 +numpy +oss2 +pandas +Pillow>=6.2.0 +# pyarrow 9.0.0 introduced event_loop core dump +pyarrow>=6.0.0,!=9.0.0 +python-dateutil>=2.1 +pyyaml +requests>=2.25 +scipy +setuptools +simplejson>=3.3.0 +sortedcontainers>=1.5.9 +tqdm>=4.64.0 +urllib3>=1.26 +yapf diff --git a/AI/modelscope/1.12.0/23/requirements/multi-modal.txt b/AI/modelscope/1.12.0/23/requirements/multi-modal.txt new file mode 100644 index 0000000..b4d551c --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/multi-modal.txt @@ -0,0 +1,29 @@ +accelerate +cloudpickle +decord>=0.6.0 +diffusers>=0.25.0 +fairseq +ftfy>=6.0.3 +librosa==0.10.1 +opencv-python +pycocoevalcap>=1.2 +pycocotools>=2.0.4 +pydot +# compatible with taming-transformers-rom1504 +pytorch_lightning<=1.7.7 +rapidfuzz +# rough-score was just recently updated from 0.0.4 to 0.0.7 +# which introduced compatability issues that are being investigated +rouge_score<=0.0.4 +sacrebleu +safetensors +# scikit-video +soundfile +taming-transformers-rom1504 +timm +tokenizers +torchvision +transformers>=4.27.1 +# triton==2.0.0.dev20221120 +unicodedata2 +zhconv diff --git a/AI/modelscope/1.12.0/23/requirements/nlp.txt b/AI/modelscope/1.12.0/23/requirements/nlp.txt new file mode 100644 index 0000000..f69f869 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/nlp.txt @@ -0,0 +1,26 @@ +boto3 +embeddings +filelock +ftfy +jieba>=0.42.1 +matplotlib +megatron_util +nltk +pandas +# protobuf version beyond 3.20.0 is not compatible with TensorFlow 1.x, therefore is discouraged. +protobuf>=3.19.0,<3.21.0 +pythainlp +pyvi +regex +rouge +sacremoses>=0.0.41 +scikit_learn +sentencepiece +seqeval +spacy>=2.3.5 +stanza +subword_nmt>=0.3.8 +termcolor +tokenizers +transformers>=4.12.0 +zhconv diff --git a/AI/modelscope/1.12.0/23/requirements/science.txt b/AI/modelscope/1.12.0/23/requirements/science.txt new file mode 100644 index 0000000..c30ff64 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/science.txt @@ -0,0 +1,8 @@ +biopython +iopath +ipdb +lmdb +ml_collections +scipy +tensorboardX +tokenizers diff --git a/AI/modelscope/1.12.0/23/requirements/svr.txt b/AI/modelscope/1.12.0/23/requirements/svr.txt new file mode 100644 index 0000000..ea439c6 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/svr.txt @@ -0,0 +1,4 @@ +fastapi +requests +sse-starlette +uvicorn diff --git a/AI/modelscope/1.12.0/23/requirements/tensorflow1x.txt b/AI/modelscope/1.12.0/23/requirements/tensorflow1x.txt new file mode 100644 index 0000000..c808f28 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/tensorflow1x.txt @@ -0,0 +1 @@ +numpy<=1.18.5 diff --git a/AI/modelscope/1.12.0/23/requirements/tests.txt b/AI/modelscope/1.12.0/23/requirements/tests.txt new file mode 100644 index 0000000..5ec4df7 --- /dev/null +++ b/AI/modelscope/1.12.0/23/requirements/tests.txt @@ -0,0 +1,5 @@ +expecttest +flake8 +isort>=4.3.21 +pre-commit +yapf==0.30.0 # use fix version to ensure consistent auto-styling diff --git a/AI/modelscope/buildspec.yml b/AI/modelscope/buildspec.yml index 8604d20..f592f48 100644 --- a/AI/modelscope/buildspec.yml +++ b/AI/modelscope/buildspec.yml @@ -26,14 +26,30 @@ images: # 定义是否构建参数 build: true test: false - platform: [linux/amd64] + platform: [linux/amd64] docker_file: path: AI/modelscope/1.10.0/8/Dockerfile + variable: + - &GPU [(USE_GPU, "True")] + scene: + args: [*GPU] + tags: + - [1.10.0-an8] + registry: [*ACR_PROD] + test_config: [*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''] + + BuildA23ModelscopeDockerImage: + # 定义是否构建参数 + build: true + test: false + platform: [linux/amd64] + docker_file: + path: AI/modelscope/1.12.0/23/Dockerfile variable: - &GPU [(USE_GPU, "True")] scene: args: [*GPU] tags: - - [1.10.0-an8] + - [1.12.0-an23] registry: [*ACR_PROD] test_config: [*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''] -- Gitee From ca3eb2039222ede31f36f3ad5c34d4b08cc06624 Mon Sep 17 00:00:00 2001 From: "zhongling.h" Date: Tue, 5 Mar 2024 17:10:58 +0800 Subject: [PATCH 02/10] upgrade modelscope --- AI/modelscope/1.12.0/23/Dockerfile | 76 ++++++++++++++++++++++++++++-- AI/modelscope/buildspec.yml | 5 +- 2 files changed, 75 insertions(+), 6 deletions(-) diff --git a/AI/modelscope/1.12.0/23/Dockerfile b/AI/modelscope/1.12.0/23/Dockerfile index d060841..3f9bab7 100644 --- a/AI/modelscope/1.12.0/23/Dockerfile +++ b/AI/modelscope/1.12.0/23/Dockerfile @@ -14,8 +14,77 @@ RUN yum install -y \ RUN mkdir -p /var/modelscope COPY requirements /var/modelscope -RUN yum install -y pytorch-devel pytorch python3-librosa jupyterlab python3-fairseq python3-moviepy python3-nvdiffrast python3-fasttext lapack http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8714/788714/tbb-2021.11.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8714/788714/tbb-bind-2021.11.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8714/788714/tbb-devel-2021.11.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/4114/794114/python3-datasets-2.16.1-2.an23.noarch.rpm && yum clean all -RUN yum install -y python3-modelscope-1.12.0 && yum clean all +RUN yum install -y jupyterlab && yum clean all +RUN yum install -y python3-modelscope \ + && yum clean all + +RUN yum install -y 'python3dist(sacremoses)' \ + 'python3dist(pycryptodomex)' \ + 'python3dist(easydict)' \ + 'python3dist(pywavelets)' \ + 'python3dist(virtualenv)' \ + 'python3dist(unicodedata2)' \ + 'python3dist(munkres)' \ + 'python3dist(jamo)' \ + 'python3dist(flatbuffers)' \ + 'python3dist(distlib)' \ + 'python3dist(cymem)' \ + 'python3dist(widgetsnbextension)' \ + 'python3dist(wasabi)' \ + 'python3dist(virtualenv)' \ + 'python3dist(unidecode)' \ + 'python3dist(terminaltables)' \ + 'python3dist(tensorflow-estimator)' \ + 'python3dist(tenacity)' \ + 'python3dist(smart-open)' \ + 'python3dist(ruamel.yaml.clib)' \ + 'python3dist(pyquaternion)' \ + 'python3dist(pyflakes)' \ + 'python3dist(pycodestyle)' \ + 'python3dist(prettytable)' \ + 'python3dist(nodeenv)' \ + 'python3dist(murmurhash)' \ + 'python3dist(mock)' \ + 'python3dist(mccabe)' \ + 'python3dist(langcodes)' \ + 'python3dist(itsdangerous)' \ + 'python3dist(isort)' \ + 'python3dist(iopath)' \ + 'python3dist(identify)' \ + 'python3dist(humanfriendly)' \ + 'python3dist(h5py)' \ + 'python3dist(h11)' \ + 'python3dist(greenlet)' \ + 'python3dist(fastprogress)' \ + 'python3dist(fastcore)' \ + 'python3dist(et-xmlfile)' \ + 'python3dist(dnspython)' \ + 'python3dist(cfgv)' \ + 'python3dist(blinker)' \ + 'python3dist(biopython)' \ + 'python3dist(beartype)' \ + 'python3dist(ruamel.yaml)' \ + 'python3dist(pre-commit)' \ + 'python3dist(plotly)' \ + 'python3dist(openpyxl)' \ + 'python3dist(fvcore)' \ + 'python3dist(flake8)' \ + 'python3dist(fastdownload)' \ + 'python3dist(eventlet)' \ + 'python3dist(coloredlogs)' \ + 'python3dist(botocore)' \ + 'python3dist(seqeval)' \ + 'python3dist(s3transfer)' \ + 'python3dist(qudida)' \ + 'python3dist(ptflops)' \ + 'python3dist(lyft-dataset-sdk)' \ + 'python3dist(hyperpyyaml)' \ + 'python3dist(descartes)' \ + 'python3dist(speechbrain)' \ + 'python3dist(pycocoevalcap)' \ + 'python3dist(ipywidgets)' \ + 'python3dist(boto3)' \ + 'python3dist(notebook)' && yum clean all RUN pip3 freeze > /var/modelscope/base.txt @@ -37,7 +106,8 @@ RUN pip install \ -r /var/modelscope/nlp.txt \ -r /var/modelscope/science.txt \ -r /var/modelscope/tests.txt \ - uvicorn text2sql_lgesql faiss + -r /var/modelscope/base.txt \ + spacy==3.7.2 torchmetrics==0.11.4 uvicorn text2sql_lgesql faiss mmdet control-ldm subword_nmt ddpm_guided_diffusion ujson faiss diffusers face_alignment bitsandbytes cpm_kernels py_sound_connect triton unicodedata2 mmcls spacy RUN curl -L https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/modelscope/modelscope-1.12.0.tar.gz | tar -xz -C /root/ RUN git clone https://www.modelscope.cn/ModelScope_Developer/modelscope-library-test-data.git /tmp/modelscope_test && \ diff --git a/AI/modelscope/buildspec.yml b/AI/modelscope/buildspec.yml index f592f48..f15ffbc 100644 --- a/AI/modelscope/buildspec.yml +++ b/AI/modelscope/buildspec.yml @@ -16,7 +16,7 @@ t-one: test_suite: &TEST_SUITE image-ci-test # 执行测试 case, 多个用数组表示 test_conf: &TEST_CONF group=ai_container - test_case: &TEST_CASE pytorch_ai_container,tensorflow2_ai_container,jupyterlab_ai_container + test_case: &TEST_CASE pytorch_ai_container cloud_server_tag: &CLOUD_SERVER_TAG [anolis-container-ci-x86, anolis-container-ci-arm] # 构建镜像配置 @@ -41,7 +41,6 @@ images: BuildA23ModelscopeDockerImage: # 定义是否构建参数 build: true - test: false platform: [linux/amd64] docker_file: path: AI/modelscope/1.12.0/23/Dockerfile @@ -52,4 +51,4 @@ images: tags: - [1.12.0-an23] registry: [*ACR_PROD] - test_config: [*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''] + test_config: [*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''] \ No newline at end of file -- Gitee From 4f78adf432a344cd4f5c7f7677f69b5f8ce3fe11 Mon Sep 17 00:00:00 2001 From: ZHe Date: Tue, 12 Mar 2024 14:04:48 +0800 Subject: [PATCH 03/10] update cloud internal mirror address --- AI/modelscope/1.12.0/23/Dockerfile | 14 ++++++++++++-- AI/modelscope/1.12.0/23/pip.conf | 3 +++ 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 AI/modelscope/1.12.0/23/pip.conf diff --git a/AI/modelscope/1.12.0/23/Dockerfile b/AI/modelscope/1.12.0/23/Dockerfile index 3f9bab7..8f10d84 100644 --- a/AI/modelscope/1.12.0/23/Dockerfile +++ b/AI/modelscope/1.12.0/23/Dockerfile @@ -1,5 +1,11 @@ FROM anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/cuda:12.1.1-23-devel +RUN yum install -y python3-pip +RUN cp -r /etc/yum.repos.d/ /etc/yum.repos.d.bak && cp /etc/pip.conf /etc/pip.conf.bak + +RUN sed -i 's#https://mirrors.openanolis.cn#http://mirrors.cloud.aliyuncs.com#g' /etc/yum.repos.d/* +COPY pip.conf /etc/pip.conf + RUN yum install -y \ automake \ libtool \ @@ -86,6 +92,8 @@ RUN yum install -y 'python3dist(sacremoses)' \ 'python3dist(boto3)' \ 'python3dist(notebook)' && yum clean all +RUN yum install -y http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8416/798416/python3-faiss-1.8.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8416/798416/libfaiss-1.8.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8420/798420/python3-torchmetrics-0.11.4-1.an23.noarch.rpm + RUN pip3 freeze > /var/modelscope/base.txt RUN sed -i '/datasets/d' /var/modelscope/framework.txt && \ @@ -107,9 +115,11 @@ RUN pip install \ -r /var/modelscope/science.txt \ -r /var/modelscope/tests.txt \ -r /var/modelscope/base.txt \ - spacy==3.7.2 torchmetrics==0.11.4 uvicorn text2sql_lgesql faiss mmdet control-ldm subword_nmt ddpm_guided_diffusion ujson faiss diffusers face_alignment bitsandbytes cpm_kernels py_sound_connect triton unicodedata2 mmcls spacy + spacy==3.7.2 torchmetrics==0.11.4 uvicorn text2sql_lgesql faiss mmdet control-ldm subword_nmt ddpm_guided_diffusion ujson diffusers face_alignment bitsandbytes cpm_kernels py_sound_connect triton unicodedata2 mmcls spacy RUN curl -L https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/modelscope/modelscope-1.12.0.tar.gz | tar -xz -C /root/ RUN git clone https://www.modelscope.cn/ModelScope_Developer/modelscope-library-test-data.git /tmp/modelscope_test && \ mv /tmp/modelscope_test/* /root/modelscope-1.12.0/data/test/ && \ - rm -rf /tmp/modelscope_test \ No newline at end of file + rm -rf /tmp/modelscope_test + +RUN cp -r /etc/yum.repos.d.bak/* /etc/yum.repos.d/ && cp /etc/pip.conf.bak /etc/pip.conf \ No newline at end of file diff --git a/AI/modelscope/1.12.0/23/pip.conf b/AI/modelscope/1.12.0/23/pip.conf new file mode 100644 index 0000000..ad0aefa --- /dev/null +++ b/AI/modelscope/1.12.0/23/pip.conf @@ -0,0 +1,3 @@ +[global] +index-url = http://mirrors.cloud.aliyuncs.com/pypi/web/simple +trusted-host = mirrors.cloud.aliyuncs.com \ No newline at end of file -- Gitee From 17346f6e24e342c27293cc975ef64403e2a59a0e Mon Sep 17 00:00:00 2001 From: ZHe Date: Tue, 12 Mar 2024 15:02:27 +0800 Subject: [PATCH 04/10] update tag --- AI/modelscope/buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AI/modelscope/buildspec.yml b/AI/modelscope/buildspec.yml index f15ffbc..4d6a87b 100644 --- a/AI/modelscope/buildspec.yml +++ b/AI/modelscope/buildspec.yml @@ -49,6 +49,6 @@ images: scene: args: [*GPU] tags: - - [1.12.0-an23] + - [1.12.0-an23-cu121-torch201] registry: [*ACR_PROD] test_config: [*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''] \ No newline at end of file -- Gitee From be721915fb57b6c42878b17c995cf36ec95693c3 Mon Sep 17 00:00:00 2001 From: ZHe Date: Wed, 13 Mar 2024 11:04:41 +0800 Subject: [PATCH 05/10] update gstreamer --- AI/modelscope/1.12.0/23/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AI/modelscope/1.12.0/23/Dockerfile b/AI/modelscope/1.12.0/23/Dockerfile index 8f10d84..586c00a 100644 --- a/AI/modelscope/1.12.0/23/Dockerfile +++ b/AI/modelscope/1.12.0/23/Dockerfile @@ -92,7 +92,7 @@ RUN yum install -y 'python3dist(sacremoses)' \ 'python3dist(boto3)' \ 'python3dist(notebook)' && yum clean all -RUN yum install -y http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8416/798416/python3-faiss-1.8.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8416/798416/libfaiss-1.8.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8420/798420/python3-torchmetrics-0.11.4-1.an23.noarch.rpm +RUN yum install -y http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8416/798416/python3-faiss-1.8.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8416/798416/libfaiss-1.8.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8420/798420/python3-torchmetrics-0.11.4-1.an23.noarch.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/574/800574/gstreamer1-plugin-libav-1.22.6-1.an23.x86_64.rpm gstreamer1-plugins-good RUN pip3 freeze > /var/modelscope/base.txt -- Gitee From 4549a35742015913727f15f693dbdec403245658 Mon Sep 17 00:00:00 2001 From: ZHe Date: Wed, 13 Mar 2024 14:34:40 +0800 Subject: [PATCH 06/10] add torchsde --- AI/modelscope/1.12.0/23/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AI/modelscope/1.12.0/23/Dockerfile b/AI/modelscope/1.12.0/23/Dockerfile index 586c00a..be5387a 100644 --- a/AI/modelscope/1.12.0/23/Dockerfile +++ b/AI/modelscope/1.12.0/23/Dockerfile @@ -3,7 +3,7 @@ FROM anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/cuda:12.1.1-23-de RUN yum install -y python3-pip RUN cp -r /etc/yum.repos.d/ /etc/yum.repos.d.bak && cp /etc/pip.conf /etc/pip.conf.bak -RUN sed -i 's#https://mirrors.openanolis.cn#http://mirrors.cloud.aliyuncs.com#g' /etc/yum.repos.d/* +RUN sed -i 's#https://mirrors.openanolis.cn#http://mirrors.cloud.aliyuncs.com#g' /etc/yum.repos.d/*.repo COPY pip.conf /etc/pip.conf RUN yum install -y \ @@ -105,7 +105,7 @@ RUN sed -i '/datasets/d' /var/modelscope/framework.txt && \ sed -i '/decorator/d' /var/modelscope/*.txt && \ sed -i '/scikit-image/d' /var/modelscope/*.txt -RUN pip install \ +RUN pip install --no-cache \ -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html \ -r /var/modelscope/framework.txt \ -r /var/modelscope/audio.txt \ @@ -115,7 +115,7 @@ RUN pip install \ -r /var/modelscope/science.txt \ -r /var/modelscope/tests.txt \ -r /var/modelscope/base.txt \ - spacy==3.7.2 torchmetrics==0.11.4 uvicorn text2sql_lgesql faiss mmdet control-ldm subword_nmt ddpm_guided_diffusion ujson diffusers face_alignment bitsandbytes cpm_kernels py_sound_connect triton unicodedata2 mmcls spacy + spacy==3.7.2 torchmetrics==0.11.4 uvicorn text2sql_lgesql faiss mmdet control-ldm subword_nmt ddpm_guided_diffusion ujson diffusers face_alignment bitsandbytes cpm_kernels py_sound_connect triton unicodedata2 mmcls spacy torchsde RUN curl -L https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/modelscope/modelscope-1.12.0.tar.gz | tar -xz -C /root/ RUN git clone https://www.modelscope.cn/ModelScope_Developer/modelscope-library-test-data.git /tmp/modelscope_test && \ -- Gitee From d80984bbc9ee1be467bb14cc1d3b15065aa6f1e4 Mon Sep 17 00:00:00 2001 From: ZHe Date: Fri, 15 Mar 2024 10:01:51 +0800 Subject: [PATCH 07/10] update modelscope --- AI/modelscope/1.12.0/23/Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/AI/modelscope/1.12.0/23/Dockerfile b/AI/modelscope/1.12.0/23/Dockerfile index be5387a..2b13990 100644 --- a/AI/modelscope/1.12.0/23/Dockerfile +++ b/AI/modelscope/1.12.0/23/Dockerfile @@ -90,9 +90,10 @@ RUN yum install -y 'python3dist(sacremoses)' \ 'python3dist(pycocoevalcap)' \ 'python3dist(ipywidgets)' \ 'python3dist(boto3)' \ - 'python3dist(notebook)' && yum clean all + 'python3dist(notebook)' \ + 'python3dist(pytorch3d)' && yum clean all -RUN yum install -y http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8416/798416/python3-faiss-1.8.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8416/798416/libfaiss-1.8.0-1.an23.x86_64.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/8420/798420/python3-torchmetrics-0.11.4-1.an23.noarch.rpm http://8.131.87.1/kojifiles/repos/dist-an23-build/latest/x86_64/toplink/work/tasks/574/800574/gstreamer1-plugin-libav-1.22.6-1.an23.x86_64.rpm gstreamer1-plugins-good +RUN yum install -y python3-faiss python3-torchmetrics-0.11.4 gstreamer1-plugin-libav gstreamer1-plugins-good RUN pip3 freeze > /var/modelscope/base.txt @@ -115,7 +116,7 @@ RUN pip install --no-cache \ -r /var/modelscope/science.txt \ -r /var/modelscope/tests.txt \ -r /var/modelscope/base.txt \ - spacy==3.7.2 torchmetrics==0.11.4 uvicorn text2sql_lgesql faiss mmdet control-ldm subword_nmt ddpm_guided_diffusion ujson diffusers face_alignment bitsandbytes cpm_kernels py_sound_connect triton unicodedata2 mmcls spacy torchsde + spacy==3.7.2 torchmetrics==0.11.4 uvicorn text2sql_lgesql faiss mmdet control-ldm subword_nmt ddpm_guided_diffusion ujson diffusers face_alignment bitsandbytes cpm_kernels py_sound_connect triton unicodedata2 mmcls spacy torchsde healpy RUN curl -L https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/modelscope/modelscope-1.12.0.tar.gz | tar -xz -C /root/ RUN git clone https://www.modelscope.cn/ModelScope_Developer/modelscope-library-test-data.git /tmp/modelscope_test && \ -- Gitee From 3d7eb89fd9c4da01039d2bfdc6b813ebc56edd4a Mon Sep 17 00:00:00 2001 From: ZHe Date: Fri, 15 Mar 2024 10:05:22 +0800 Subject: [PATCH 08/10] update modelscope --- AI/modelscope/1.12.0/23/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AI/modelscope/1.12.0/23/Dockerfile b/AI/modelscope/1.12.0/23/Dockerfile index 2b13990..7af7a86 100644 --- a/AI/modelscope/1.12.0/23/Dockerfile +++ b/AI/modelscope/1.12.0/23/Dockerfile @@ -15,7 +15,8 @@ RUN yum install -y \ blas \ glibc-locale-source \ libSM \ - langpacks-zh_CN && yum clean all + langpacks-zh_CN \ + gcc && yum clean all RUN mkdir -p /var/modelscope COPY requirements /var/modelscope -- Gitee From 51a784e8fc6810a3a46cbaefe9d94a6bc88f8e86 Mon Sep 17 00:00:00 2001 From: ZHe Date: Fri, 15 Mar 2024 10:13:09 +0800 Subject: [PATCH 09/10] update modelscope --- AI/modelscope/1.12.0/23/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/AI/modelscope/1.12.0/23/Dockerfile b/AI/modelscope/1.12.0/23/Dockerfile index 7af7a86..19654fd 100644 --- a/AI/modelscope/1.12.0/23/Dockerfile +++ b/AI/modelscope/1.12.0/23/Dockerfile @@ -94,8 +94,6 @@ RUN yum install -y 'python3dist(sacremoses)' \ 'python3dist(notebook)' \ 'python3dist(pytorch3d)' && yum clean all -RUN yum install -y python3-faiss python3-torchmetrics-0.11.4 gstreamer1-plugin-libav gstreamer1-plugins-good - RUN pip3 freeze > /var/modelscope/base.txt RUN sed -i '/datasets/d' /var/modelscope/framework.txt && \ -- Gitee From 3b3667c629331714d1f723af5d1e13f337de6a23 Mon Sep 17 00:00:00 2001 From: ZHe Date: Mon, 18 Mar 2024 15:11:28 +0800 Subject: [PATCH 10/10] update modelscope --- AI/modelscope/1.12.0/23/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AI/modelscope/1.12.0/23/Dockerfile b/AI/modelscope/1.12.0/23/Dockerfile index 19654fd..d66cd21 100644 --- a/AI/modelscope/1.12.0/23/Dockerfile +++ b/AI/modelscope/1.12.0/23/Dockerfile @@ -16,7 +16,7 @@ RUN yum install -y \ glibc-locale-source \ libSM \ langpacks-zh_CN \ - gcc && yum clean all + gcc gcc-c++ && yum clean all RUN mkdir -p /var/modelscope COPY requirements /var/modelscope @@ -122,4 +122,4 @@ RUN git clone https://www.modelscope.cn/ModelScope_Developer/modelscope-library- mv /tmp/modelscope_test/* /root/modelscope-1.12.0/data/test/ && \ rm -rf /tmp/modelscope_test -RUN cp -r /etc/yum.repos.d.bak/* /etc/yum.repos.d/ && cp /etc/pip.conf.bak /etc/pip.conf \ No newline at end of file +RUN cp -r /etc/yum.repos.d.bak/* /etc/yum.repos.d/ && cp /etc/pip.conf.bak /etc/pip.conf -- Gitee