From 1d8dabb39689a82b1cc6432820a71611467d6c0b Mon Sep 17 00:00:00 2001 From: appleinsky <2164757809@qq.com> Date: Thu, 24 Oct 2024 00:55:06 +0800 Subject: [PATCH 01/12] =?UTF-8?q?=E3=80=90S2=E7=AE=97=E5=AD=90=E6=8F=90?= =?UTF-8?q?=E4=BA=A4=E3=80=91ThreeNN=E7=AE=97=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 提交ThreeNN算子代码及readme --- .../AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 182 ++++ .../AclNNInvocation/inc/operator_desc.h | 57 ++ .../FrameworkLaunch/AclNNInvocation/run.sh | 120 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 72 ++ .../AclNNInvocation/scripts/verify_result.py | 26 + .../scripts/verify_result_int.py | 26 + .../AclNNInvocation/src/CMakeLists.txt | 68 ++ .../AclNNInvocation/src/common.cpp | 79 ++ .../AclNNInvocation/src/main.cpp | 171 ++++ .../AclNNInvocation/src/op_runner.cpp | 486 +++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 56 ++ .../FrameworkLaunch/ThreeNN/CMakeLists.txt | 69 ++ .../FrameworkLaunch/ThreeNN/CMakePresets.json | 63 ++ .../FrameworkLaunch/ThreeNN/ThreeNN.json | 50 ++ .../FrameworkLaunch/ThreeNN/build.sh | 80 ++ .../ThreeNN/cmake/config.cmake | 25 + .../FrameworkLaunch/ThreeNN/cmake/func.cmake | 192 ++++ .../FrameworkLaunch/ThreeNN/cmake/intf.cmake | 26 + .../ThreeNN/cmake/makeself.cmake | 17 + .../ThreeNN/cmake/three_nn.cpp | 127 +++ .../ThreeNN/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 127 +++ .../ThreeNN/cmake/util/ascendc_impl_build.py | 501 +++++++++++ .../ThreeNN/cmake/util/ascendc_ops_config.py | 120 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../ThreeNN/cmake/util/batch_replay_impl.temp | 117 +++ .../ThreeNN/cmake/util/code_channel_infer.py | 58 ++ .../ThreeNN/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 20 + .../ThreeNN/cmake/util/gen_ops_filter.sh | 62 ++ .../ThreeNN/cmake/util/gen_version_info.sh | 6 + .../ThreeNN/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 242 ++++++ .../ThreeNN/cmake/util/kernel_entry.py | 115 +++ .../ThreeNN/cmake/util/kernel_impl.temp | 10 + .../ThreeNN/cmake/util/makeself/COPYING | 339 ++++++++ .../ThreeNN/cmake/util/makeself/README.md | 246 ++++++ .../ThreeNN/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../ThreeNN/cmake/util/makeself/makeself.1 | 110 +++ .../ThreeNN/cmake/util/makeself/makeself.lsm | 16 + .../ThreeNN/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../ThreeNN/cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../ThreeNN/cmake/util/opdesc_parser.py | 298 +++++++ .../ThreeNN/cmake/util/parse_ini_to_json.py | 338 +++++++ .../ThreeNN/cmake/util/preset_parse.py | 23 + .../ThreeNN/cmake/util/replay_codegen.py | 105 +++ .../ThreeNN/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 87 ++ .../ThreeNN/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_three_nn_plugin.cc | 23 + .../ThreeNN/op_host/CMakeLists.txt | 82 ++ .../ThreeNN/op_host/three_nn.cpp | 109 +++ .../ThreeNN/op_host/three_nn_tiling.h | 16 + .../ThreeNN/op_kernel/CMakeLists.txt | 68 ++ .../ThreeNN/op_kernel/three_nn.cpp | 166 ++++ .../ThreeNN/scripts/install.sh | 318 +++++++ .../ThreeNN/scripts/upgrade.sh | 151 ++++ operator_contrib/ThreeNNSample/README.en.md | 160 ++++ operator_contrib/ThreeNNSample/README.md | 145 +++ 65 files changed, 8034 insertions(+) create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh create mode 100644 operator_contrib/ThreeNNSample/README.en.md create mode 100644 operator_contrib/ThreeNNSample/README.md diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..7036813a9 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** +* @file common.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..bf2a9ef43 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,182 @@ +/** +* @file op_runner.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "aclnn/acl_meta.h" +#include "acl/acl.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template + T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template + const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..4cbdf07e0 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,57 @@ +/** +* @file operator_desc.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh new file mode 100644 index 000000000..96ca9c3b1 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh @@ -0,0 +1,120 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while : +do + case "$1" in + # float16, float, int32 + (-v | --dtype) + DTYPE="$2" + shift 2;; + (--) + shift; + break;; + (*) + echo "[ERROR] Unexpected option: $1"; + break;; + esac +done + +if [ ! $ASCEND_HOME_DIR ]; then + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + export ASCEND_HOME_DIR=$HOME/Ascend/ascend-toolkit/latest + else + export ASCEND_HOME_DIR=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $ASCEND_HOME_DIR/bin/setenv.bash + +export DDK_PATH=$ASCEND_HOME_DIR +arch=$(uname -m) +export NPU_HOST_LIB=$ASCEND_HOME_DIR/${arch}-linux/lib64 + +function main { + # 1. 清除算子输出和日志文件 + + rm ./output/execute_op + rm ./input/*.bin + rm ./output/*.bin + # rm -rf ./output/output*.bin > /dev/null + + # 2. 生成或复用输入数据和真值数据 + if [ -d "./input" ]; then + if [ "$(ls -A "./input")" ]; then + echo "已存在测试数据" + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译或复用acl可执行文件 + if [ -e "./output/execute_op" ]; then + echo "可执行存在" + else + echo "可执行不存在" + cd $CURRENT_DIR; rm -rf build; mkdir -p build; cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + fi + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + timeout 30 ./execute_op + + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + ret_dist=`python3 scripts/verify_result.py output/output_dist.bin output/golden_dist.bin` + ret_idx=`python3 scripts/verify_result_int.py output/output_idx.bin output/golden_idx.bin` + echo "verify dist $ret_dist" + echo "verify idx $ret_idx" + if [ "x$ret_dist" == "xtest pass" ] && [ "x$ret_idx" == "xtest pass" ]; then + echo "" + echo "#####################################" + echo "INFO: you have passed the Precision!" + echo "#####################################" + echo "" + fi +} + +main diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..c723df735 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,72 @@ +#!/usr/bin/python3 +# -*- coding:utf-8 -*- +# Copyright 2022-2023 Huawei Technologies Co., Ltd +import numpy as np +import os +import torch + +def run_three_nn(b, n, m, xyz1, xyz2): + dist = np.zeros_like(xyz1, dtype=np.float32) + idx = np.zeros_like(xyz1, dtype=np.int32) + + for ii in range(b): + for jj in range(n): + x1 = xyz1[ii, jj, 0] + y1 = xyz1[ii, jj, 1] + z1 = xyz1[ii, jj, 2] + best1 = 1e40 + best2 = 1e40 + best3 = 1e40 + besti1 = 0 + besti2 = 0 + besti3 = 0 + for kk in range(m): + x2 = xyz2[ii, kk, 0] + y2 = xyz2[ii, kk, 1] + z2 = xyz2[ii, kk, 2] + d = (x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1) + (z2 - z1) * (z2 - z1) + if (d < best1): + best3 = best2 + besti3 = besti2 + best2 = best1 + besti2 = besti1 + best1 = d + besti1 = kk + elif (d < best2): + best3 = best2 + besti3 = besti2 + best2 = d + besti2 = kk + elif (d < best3): + best3 = d + besti3 = kk + dist[ii, jj, 0] = best1 + idx[ii, jj, 0] = besti1 + dist[ii, jj, 1] = best2 + idx[ii, jj, 1] = besti2 + dist[ii, jj, 2] = best3 + idx[ii, jj, 2] = besti3 + + return dist, idx + +def gen_golden_data_simple(): + b = 5 + n = 160 + m = 256 + xyz1_shape = [b,n,3] + xyz2_shape = [b,m,3] + input_xyz1 = np.random.uniform(-10, 10, xyz1_shape).astype(np.float32) + input_xyz2 = np.random.uniform(-10, 10, xyz2_shape).astype(np.float32) + dist, idx = run_three_nn(b, n, m, input_xyz1, input_xyz2) + os.system("mkdir -p input") + os.system("mkdir -p output") + print("dist = ",dist) + print("idx = ",idx) + input_xyz1.tofile("./input/input_xyz1.bin") + input_xyz2.tofile("./input/input_xyz2.bin") + dist.tofile("./output/golden_dist.bin") + idx.tofile("./output/golden_idx.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..d917ab413 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,26 @@ +import os +import sys +import numpy as np +np.set_printoptions(suppress=True) + +loss = 1e-4 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 +minimum = 10e-10 + +def verify_result(real_result, golden): + real_result = np.fromfile(real_result, dtype=np.float32) # 从bin文件读取实际运算结果 + golden = np.fromfile(golden, dtype=np.float32) # 从bin文件读取预期运算结果 + result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 + deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 + result_atol = np.less_equal(result, loss) # 计算绝对误差 + result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 + if not result_rtol.all() and not result_atol.all(): + if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 + print("[ERROR] result error") + print(real_result) + print(golden) + return False + print("test pass") + return True + +if __name__ == '__main__': + verify_result(sys.argv[1],sys.argv[2]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py new file mode 100644 index 000000000..9cbc3482b --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py @@ -0,0 +1,26 @@ +import os +import sys +import numpy as np +np.set_printoptions(suppress=True) + +loss = 1e-4 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 +minimum = 10e-10 + +def verify_result(real_result, golden): + real_result = np.fromfile(real_result, dtype=np.int32) # 从bin文件读取实际运算结果 + golden = np.fromfile(golden, dtype=np.int32) # 从bin文件读取预期运算结果 + result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 + deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 + result_atol = np.less_equal(result, loss) # 计算绝对误差 + result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 + if not result_rtol.all() and not result_atol.all(): + if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 + print("[ERROR] result error") + print(real_result) + print(golden) + return False + print("test pass") + return True + +if __name__ == '__main__': + verify_result(sys.argv[1],sys.argv[2]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..ee73e930f --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,68 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include + ${BUILT_PKG_PATH}/include/aclnnop +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..47a39cd5f --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp @@ -0,0 +1,79 @@ +/** +* @file common.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" + +#include +#include +#include +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void) close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..7536fdb6e --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp @@ -0,0 +1,171 @@ +/** +* @file main.cpp +* +* Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "op_runner.h" + +#include "common.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + int64_t b = 5; + int64_t n = 160; + int64_t m = 256; + std::vector xyz1_shape {b,n,3}; + std::vector xyz2_shape {b,m,3}; + std::vector out_shape {b,n,3}; + aclDataType dataType = ACL_FLOAT; + aclDataType indicesType = ACL_INT32; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, xyz1_shape.size(), xyz1_shape.data(), format); + opDesc.AddInputTensorDesc(dataType, xyz2_shape.size(), xyz2_shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, out_shape.size(), out_shape.data(), format); + opDesc.AddOutputTensorDesc(indicesType, out_shape.size(), out_shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_xyz1.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_xyz2.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_dist.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + WriteFile("../output/output_idx.bin", runner.GetOutputBuffer(1), runner.GetOutputSize(1)); + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } + else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..16bbc21d0 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,486 @@ +/** +* @file op_runner.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "op_runner.h" +#include "aclnn_three_nn.h" +#include +#include +#include "acl/acl_op_compiler.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto ret = aclDestroyTensor(inputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free InputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(inputBuffers_[i]); + + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free inputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devInputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostInputs_[i]); + } else { + ret = aclrtFreeHost(hostInputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto ret = aclDestroyTensor(outputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(outputBuffers_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devOutputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostOutputs_[i]); + } else { + ret = aclrtFreeHost(hostOutputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), + nullptr, 0, GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), + nullptr, 0, GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + + auto ret = aclnnThreeNNGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], outputTensor_[1], + &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute GetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + ret = aclnnThreeNN(workspace, workspaceSize, handle, stream); + + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute Operator success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + + +template +void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..a7a1ed3bd --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,56 @@ +/** +* @file operator_desc.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" +#include "operator_desc.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } + +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt new file mode 100644 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json new file mode 100644 index 000000000..8b7181000 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json new file mode 100644 index 000000000..f35459443 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json @@ -0,0 +1,50 @@ +[ + { + "op": "ThreeNN", + "language": "cpp", + "input_desc": [ + { + "name": "xyz1", + "param_type": "required", + "format": [ + "ND" + ], + "type": [ + "float" + ] + }, + { + "name": "xyz2", + "param_type": "required", + "format": [ + "ND" + ], + "type": [ + "float" + ] + } + ], + "output_desc": [ + { + "name": "dist", + "param_type": "required", + "format": [ + "ND" + ], + "type": [ + "float" + ] + }, + { + "name": "indices", + "param_type": "required", + "format": [ + "ND" + ], + "type": [ + "int32" + ] + } + ] + } +] \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh new file mode 100644 index 000000000..27d1968d1 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh @@ -0,0 +1,80 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) +ENABLE_CROSS="-DENABLE_CROSS_COMPILE=True" +ENABLE_BINARY="-DENABLE_BINARY_PACKAGE=True" +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') + +cmake_run_package() +{ + target=$1 + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then exit 1; fi + + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + cmake --build . --target $target -j16 + fi + fi +} + +if [[ $opts =~ $ENABLE_CROSS ]] && [[ $opts =~ $ENABLE_BINARY ]] +then + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DENABLE_CROSS_COMPILE=0 + else + cmake .. --preset=default -DENABLE_CROSS_COMPILE=0 + fi + cmake_run_package $target + cp -r kernel ../ + rm -rf * + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts + else + cmake .. --preset=default + fi + + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + fi + fi + rm -rf ../kernel + +else + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts + else + cmake .. --preset=default + fi + cmake_run_package $target +fi + + +# for debug +# cd build_out +# make +# cpack +# verbose append -v \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake new file mode 100644 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake new file mode 100644 index 000000000..4179dfd25 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake @@ -0,0 +1,192 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} && echo $(MAKE) + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + +endfunction() + +function(add_cross_compile_target) + cmake_parse_arguments(CROSSMP "" "TARGET;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + add_custom_target(${CROSSMP_TARGET} ALL + DEPENDS ${CROSSMP_OUT_DIR} + ) + install(DIRECTORY ${CROSSMP_OUT_DIR} + DESTINATION ${CROSSMP_INSTALL_DIR} + ) +endfunction() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake new file mode 100644 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake new file mode 100644 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp new file mode 100644 index 000000000..113e0d5cd --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp @@ -0,0 +1,127 @@ +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelFastGeluGrad { +public: + __aicore__ inline KernelFastGeluGrad() {} + __aicore__ inline void Init(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, + uint32_t CoreDataNum, uint32_t finalTileNum, uint32_t tileDataNum, uint32_t TailDataNum, + uint32_t xyz2DataNum) { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + + this->coreDataNum = CoreDataNum; + this->tileNum = finalTileNum; + this->tileDataNum = tileDataNum; + this->tailDataNum = TailDataNum; + this->xyz2DataNum = xyz2DataNum; + + xyz1Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ1*)xyz1, this->coreDataNum); + xyz2Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ2*)xyz2, this->xyz2DataNum); + distGm.SetGlobalBuffer((__gm__ DTYPE_DIST*)dist, this->coreDataNum); + indicesGm.SetGlobalBuffer((__gm__ DTYPE_INDICES*)indices, this->coreDataNum); + + // pipe.InitBuffer(inQueueXYZ1, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_XYZ1)); + // pipe.InitBuffer(inQueueXYZ2, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_XYZ2)); + // pipe.InitBuffer(outQueueDIST, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_DIST)); + // pipe.InitBuffer(outQueueINDICES, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_INDICES)); + + } + __aicore__ inline void Process() { + float ux,uy,uz; + float x,y,z; + float best1, best2, best3; + int besti1, besti2, besti3; + for(uint32_t i=0; icoreDataNum; i+=3) + { + ux = xyz1Gm.GetValue(i); + uy = xyz1Gm.GetValue(i+1); + uz = xyz1Gm.GetValue(i+2); + for(j=0; jxyz2DataNum; j+=3) + { + x = xyz2Gm.GetValue(j); + y = xyz2Gm.GetValue(j+1); + z = xyz2Gm.GetValue(j+2); + } + } + // SetValue(count*M*N + i*N+j, xGm.GetValue(count*M*N + i*N+j)); + } +// __aicore__ inline void Process() { + +// int32_t loopCount = this->tileNum; +// this->processDataNum = this->tileDataNum; +// for (int32_t i = 0; i < loopCount; i++) { +// if (i == this->tileNum - 1) { +// this->processDataNum = this->tailDataNum; +// } +// CopyIn(i); +// Compute(i); +// CopyOut(i); +// } +// } + +// private: +// __aicore__ inline void CopyIn(int32_t progress) +// { +// LocalTensor xyz1Local = inQueueXYZ1.AllocTensor(); +// LocalTensor xyz2Local = inQueueXYZ2.AllocTensor(); + +// DataCopy(xyz1Local, xyz1Gm[progress * this->tileDataNum], this->processDataNum); +// DataCopy(xyz2Local, xyz2Gm[progress * this->tileDataNum], this->processDataNum); + +// inQueueXYZ1.EnQue(xyz1Local); +// inQueueXYZ2.EnQue(xyz2Local); +// } +// __aicore__ inline void Compute(int32_t progress) +// { +// LocalTensor xyz1Local = inQueueXYZ1.DeQue(); +// LocalTensor xyz2Local = inQueueXYZ2.DeQue(); +// LocalTensor distLocal = outQueueDIST.AllocTensor(); +// LocalTensor indicesLocal = outQueueINDICES.AllocTensor(); + + +// outQueueDIST.EnQue(distLocal); +// outQueueINDICES.EnQue(indicesLocal); +// inQueueXYZ1.FreeTensor(xyz2Local); +// inQueueXYZ2.FreeTensor(xyz2Local); +// } +// __aicore__ inline void CopyOut(int32_t progress) +// { +// LocalTensor distLocal = outQueueDIST.DeQue(); +// LocalTensor indicesLocal = outQueueINDICES.DeQue(); +// DataCopy(distGm[progress * this->tileDataNum], distLocal, this->processDataNum); +// DataCopy(indicesGm[progress * this->tileDataNum], indicesLocal, this->processDataNum); +// outQueueDIST.FreeTensor(distLocal); +// outQueueINDICES.FreeTensor(indicesLocal); +// } + +private: + TPipe pipe; + TBuf tmpBuffer, signbitBuffer; + TQue inQueueXYZ1, inQueueXYZ2; + TQue outQueueDIST, outQueueINDICES; + GlobalTensor xyz1Gm; + GlobalTensor xyz2Gm; + GlobalTensor distGm; + GlobalTensor indicesGm; + uint32_t coreDataNum; + uint32_t tileNum; + uint32_t tileDataNum; + uint32_t tailDataNum; + uint32_t processDataNum; + uint32_t xyz2DataNum; + // LocalTensor signbit; +}; + +extern "C" __global__ __aicore__ void three_nn(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); + // TODO: user kernel impl + KernelFastGeluGrad op; + op.Init(xyz1, xyz2, dist, indices, + tiling_data.CoreDataNum, tiling_data.finalTileNum, tiling_data.tileDataNum, tiling_data.TailDataNum, tiling_data.xyz2DataNum); + op.Process(); +} +// extern "C" __global__ __aicore__ void three_nn(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling) { +// GET_TILING_DATA(tiling_data, tiling); +// // TODO: user kernel impl +// } \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py new file mode 100644 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py new file mode 100644 index 000000000..63b15c900 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx][:-5] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx][:-5] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = self.soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py new file mode 100644 index 000000000..647801c07 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int4b_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]}} + +def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): + options = [] + x_fmt = x.get("format") + x_dtype = x.get("dtype") + x_n_in_kernel = x_n + '_REF' if is_ref else x_n + options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) + return options + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + input_names = {} + output_names = {} + unique_param_name_set = set() + for idx, x in enumerate(__inputs__): + if x is None: + continue + x_n = input_names[idx].upper() + unique_param_name_set.add(x_n) + options += add_dtype_fmt_option_single(x, x_n) + + for idx, x in enumerate(__outputs__): + if x is None: + continue + x_n = output_names[idx].upper() + if x_n in unique_param_name_set: + options += add_dtype_fmt_option_single(x, x_n, True) + else: + options += add_dtype_fmt_option_single(x, x_n) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +def get_kernel_source(src_file, dir_snake, dir_ex): + src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, src_file) + if os.path.exists(src): + return src + return src_ex + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}", trans_bool_to_s8=False) +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + bisheng = shutil.which("bisheng") + if bisheng != None: + bisheng_path = os.path.dirname(bisheng) + tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir_ex = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is " + kernel_name) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}], param_type_dynamic = {},\\ + mc2_ctx = None, param_type_list = {}) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'list_int': '[]', + 'list_float': '[]', 'list_bool': '[]', 'list_list_int': '[[]]', 'str': ''} + + +def _get_snake_str(s, i): + if s[i - 1] != '_': + if not s[i - 1].isupper(): + return "_" + elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): + return "_" + return "" + return "" + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +def optype_snake_ex(s): + snake_case = "" + for i, c in enumerate(s): + if i == 0: + snake_case += c.lower() + elif c.isupper(): + snake_case += _get_snake_str(s, i) + snake_case += c.lower() + else: + snake_case += c + return snake_case + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD.format(self.input_ori_name, self.output_ori_name)) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake_ex(self.op_type), optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), False, self.input_type + self.output_type, self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py new file mode 100644 index 000000000..b138047af --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, task_ration, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + if core_type == 0 and task_ration == "tilingKey": + bin_list.append({'coreType': core_type, 'simplifiedKey': key, \ + 'multiKernelType':1, 'binPath': objfile}) + else: + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {'MIX': 0, 'AiCore': 1, 'VectorCore': 2, 'MIX_AICORE':3, 'MIX_VECTOR_CORE':4} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + task_ration = contents.get('taskRation') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, task_ration, os.path.join(soc, op_dir, bin_file_name), \ + config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py new file mode 100644 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp new file mode 100644 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py new file mode 100644 index 000000000..d4e6a540c --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b", "ascend910c"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for bisheng cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py new file mode 100644 index 000000000..3e71f5ab8 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1', 'ascend910c': 'Ascend910C1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh new file mode 100644 index 000000000..492034e49 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,20 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh new file mode 100644 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh new file mode 100644 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py new file mode 100644 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py new file mode 100644 index 000000000..7e8dc2c55 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import re +import glob +import json +import argparse + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py new file mode 100644 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp new file mode 100644 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING new file mode 100644 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md new file mode 100644 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION new file mode 100644 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh new file mode 100644 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh new file mode 100644 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 new file mode 100644 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm new file mode 100644 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh new file mode 100644 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh new file mode 100644 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh new file mode 100644 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py new file mode 100644 index 000000000..0d3221602 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b2c": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend910c1": "ascend910c", + "ascend910c2": "ascend910c", + "ascend910c3": "ascend910c", + "ascend910c4": "ascend910c", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} +CONFLICT_KEYWORDS = { + "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", + "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "False", + "None", "True", "nonlocal", "arg", "__inputs__", "__outputs__", "options", "bisheng", + "bisheng_path", "tikcpp_path", "impl_mode", "custom_compile_options", + "custom_all_compile_options", "soc_version", "soc_short", "custom_compile_options_soc", + "custom_all_compile_options_soc", "origin_func_name", "ascendc_src_dir_ex", + "ascendc_src_dir", "ascendc_src_file", "src", "op_type", "code_channel", "op_info", + "compile_op", "get_code_channel", "result", "__attrs__", "isinstance", "attr", + "get_current_build_config", "_build_args", "get_dtype_fmt_options", "shutil", "os", + "get_kernel_source" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_ori_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_ori_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_ori_name.append(self._parse_str(conf)) + self.input_name.append(self.input_ori_name[-1] + '_in__') + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_ori_name.append(self._parse_str(conf)) + self.output_name.append(self.output_ori_name[-1] + '_out_') + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS + if intersection_element: + raise Exception(f'The attribute name: {intersection_element} in op: {{\'{self.op_type}\'}} conflicts with\ + the built-in variable name. Use a complex name or prefix the operator name.') + + @staticmethod + def _camel_to_snake(camel_case_str: str): + snake_case_str = '' + for i, c in enumerate(camel_case_str): + if i == 0: + snake_case_str += c.lower() + elif c.isupper(): + snake_case_str += '_' + c.lower() + else: + snake_case_str += c + return snake_case_str + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._camel_to_snake(self._parse_str(conf)) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return + with open(file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py new file mode 100644 index 000000000..ac931e5d3 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport", "enableVectorCore"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # The owner have all rights，group only have read rights + os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py new file mode 100644 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py new file mode 100644 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp new file mode 100644 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py new file mode 100644 index 000000000..4c5b099b7 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' uint64_t *src = (uint64_t *)tiling;\n' + tiling_source += ' uint64_t *dst = (uint64_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt new file mode 100644 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc new file mode 100644 index 000000000..1809e029a --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("ThreeNN") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("ThreeNN") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt new file mode 100644 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp new file mode 100644 index 000000000..027ef0850 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp @@ -0,0 +1,109 @@ + +#include "three_nn_tiling.h" +#include "register/op_def_registry.h" +#include "tiling/platform/platform_ascendc.h" +#include + +namespace optiling { +const uint32_t BLOCK_SIZE = 32*3*4; +static ge::graphStatus TilingFunc(gert::TilingContext* context) +{ + ThreeNNTilingData tiling; + + uint64_t ubSize; + auto ascendcPlatform = platform_ascendc::PlatformAscendC(context->GetPlatformInfo()); + // auto socVersion = ascendcPlatform.GetSocVersion(); + ascendcPlatform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ubSize); //获取硬件平台存储空间 UB 的内存大小 + uint32_t shapeB = context->GetInputTensor(0)->GetOriginShape().GetDim(0); + uint32_t shapeN = context->GetInputTensor(0)->GetOriginShape().GetDim(1); + uint32_t shapeM = context->GetInputTensor(1)->GetOriginShape().GetDim(1); + + //获取输入shape信息 + uint32_t inputNum = context->GetInputShape(1)->GetStorageShape().GetShapeSize()/shapeB; //输入数量 xyz2 + uint32_t inputBytes = GetSizeByDataType(context->GetInputDesc(1)->GetDataType()); //输入类型 + uint32_t inputLength = inputBytes * inputNum; //输入长度 + + //可使用的ub空间 输入3输出1，手动考虑双缓存 + uint32_t ubDataNumber = 4;//(inputBytes == 2) ? 10 : 6; + + // The number of 32B data blocks that can be used for each data. DOUBLE BUFFER is already counted here + uint32_t tileBlockNum = (ubSize / BLOCK_SIZE) / ubDataNumber; //每个ub段可用的空间块数 + uint32_t tileDataNum = 1440; + + // Input data for 32B alignment + uint32_t inputLengthAlgin32 = (((inputLength + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE); //输入长度 对齐处理 + uint32_t everyCoreInputBlockNum = inputLengthAlgin32 / BLOCK_SIZE;// 输入数据需要多少空间块 + + uint32_t TileNum = inputNum / tileDataNum; + uint32_t finalTileNum = (inputNum % tileDataNum) == 0 ? TileNum : TileNum + 1; //需要循环处理几次 + // Tail block calculation for chunks of data + uint32_t TailDataNum = inputNum - (tileDataNum * TileNum);//CoreDataNum - (tileDataNum * TileNum); + TailDataNum = TailDataNum == 0 ? tileDataNum : TailDataNum; //最后一次需要处理的数据量 + + tiling.set_CoreDataNum((inputNum*shapeB+31)/32*32); //对齐空间后的输入数量 + tiling.set_finalTileNum(finalTileNum);//需要循环处理几次 + tiling.set_tileDataNum(tileDataNum); //每个b处理的数据量 + tiling.set_TailDataNum(TailDataNum); //最后一次需要处理的数据量 + + tiling.set_shapeB(shapeB); + tiling.set_shapeN(shapeN); + tiling.set_shapeM(shapeM); + + context->SetBlockDim(1); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} + + +namespace ge { +static ge::graphStatus InferShape(gert::InferShapeContext* context) +{ + const gert::Shape* x1_shape = context->GetInputShape(0); + gert::Shape* y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} +} + + +namespace ops { +class ThreeNN : public OpDef { +public: + explicit ThreeNN(const char* name) : OpDef(name) + { + this->Input("xyz1") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Input("xyz2") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("dist") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("indices") + .ParamType(REQUIRED) + .DataType({ge::DT_INT32}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape); + + this->AICore() + .SetTiling(optiling::TilingFunc); + this->AICore().AddConfig("ascend310b"); + + } +}; + +OP_ADD(ThreeNN); +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h new file mode 100644 index 000000000..5da548fed --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h @@ -0,0 +1,16 @@ + +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(ThreeNNTilingData) + TILING_DATA_FIELD_DEF(uint32_t, CoreDataNum); + TILING_DATA_FIELD_DEF(uint32_t, finalTileNum); + TILING_DATA_FIELD_DEF(uint32_t, tileDataNum); + TILING_DATA_FIELD_DEF(uint32_t, TailDataNum); + + TILING_DATA_FIELD_DEF(uint32_t, shapeB); + TILING_DATA_FIELD_DEF(uint32_t, shapeN); + TILING_DATA_FIELD_DEF(uint32_t, shapeM); +END_TILING_DATA_DEF; +REGISTER_TILING_DATA_CLASS(ThreeNN, ThreeNNTilingData) +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt new file mode 100644 index 000000000..8c94a952d --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt @@ -0,0 +1,68 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE} AND NOT ${ENABLE_CROSS_COMPILE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + + if (${ENABLE_CROSS_COMPILE} AND ${ENABLE_BINARY_PACKAGE}) + add_cross_compile_target( + TARGET bin_${compute_unit} + OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../kernel + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/ + ) + endif() +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp new file mode 100644 index 000000000..9d60d095b --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp @@ -0,0 +1,166 @@ +#define K_MAX_SHAPE_DIM 0 +#include "kernel_operator.h" +#include +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelFastGeluGrad { +public: + __aicore__ inline KernelFastGeluGrad() {} + __aicore__ inline void Init(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, + uint32_t CoreDataNum, uint32_t finalTileNum, uint32_t tileDataNum, uint32_t TailDataNum, + uint32_t shapeB, uint32_t shapeN, uint32_t shapeM) { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + + this->coreDataNum = CoreDataNum; + this->tileNum = finalTileNum; + this->tileDataNum = tileDataNum; + this->tailDataNum = TailDataNum; + + this->shapeB = shapeB; + this->shapeN = shapeN; + this->shapeM = shapeM; + + xyz1Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ1*)xyz1, this->coreDataNum); + xyz2Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ2*)xyz2, this->coreDataNum); + distGm.SetGlobalBuffer((__gm__ DTYPE_DIST*)dist, this->coreDataNum); + indicesGm.SetGlobalBuffer((__gm__ DTYPE_INDICES*)indices, this->coreDataNum); + + pipe.InitBuffer(inQueueXYZ2, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + pipe.InitBuffer(QueueTmp1, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + pipe.InitBuffer(QueueTmp2, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + pipe.InitBuffer(QueueTmp3, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + + pipe.InitBuffer(QueueTmp10, 32 * sizeof(DTYPE_XYZ2)); + + pipe.InitBuffer(QueueTmpX, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + pipe.InitBuffer(QueueTmpY, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + pipe.InitBuffer(QueueTmpZ, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + + pipe.InitBuffer(QueueMask, sizeof(uint32_t)*(128+36)); + + } + __aicore__ inline void Process() { + int32_t loopCount; + int32_t b=0; + + LocalTensor mask_scl = QueueMask.Get(); + auto tmpx = QueueTmpX.Get(); + auto tmpy = QueueTmpY.Get(); + auto tmpz = QueueTmpZ.Get(); + + uint64_t mask1[2] = {10540996613548315209, 5270498306774157604}; + uint64_t mask2[2] = {2635249153387078802, 10540996613548315209}; + uint64_t mask3[2] = {5270498306774157604, 2635249153387078802}; + uint32_t scalar = 1227133513; + Duplicate(mask_scl, scalar, mask1, 2, 1, 8 ); + + scalar = 2454267026; + Duplicate(mask_scl, scalar, mask2, 2, 1, 8 ); + + scalar = 613566756; + Duplicate(mask_scl, scalar, mask3, 2, 1, 8 ); + + this->processDataNum = this->tailDataNum; + for(int32_t b=0; bshapeB; b++) + { + CopyIn(b, 0); + Compute(b, 0, mask_scl, tmpx, tmpy, tmpz); + } + } + +private: + __aicore__ inline void CopyIn(int32_t b, int32_t progress) + { + LocalTensor xyz2Local = inQueueXYZ2.AllocTensor(); + DataCopy(xyz2Local, xyz2Gm[3*b*this->shapeM], (3*this->shapeM+31)/32*32); + inQueueXYZ2.EnQue(xyz2Local); + } + __aicore__ inline void Compute(int32_t b, int32_t progress, LocalTensor mask_scl, LocalTensor tmpx, LocalTensor tmpy, LocalTensor tmpz) + { + LocalTensor xyz2Local = inQueueXYZ2.DeQue(); + auto tmp1 = QueueTmp1.Get(); + auto tmp2 = QueueTmp2.Get(); + auto tmp3 = QueueTmp3.Get(); + auto tmp10 = QueueTmp10.Get(); + + float ux,uy,uz; + int besti1, besti2, besti3; + uint32_t calculate_num = this->shapeM; + uint32_t mask = this->shapeM*3; + uint64_t rsvdCnt = 0; + GatherMask (tmpx, xyz2Local, mask_scl, true, mask, { 1, 1, 8, 8 }, rsvdCnt); + GatherMask (tmpy, xyz2Local, mask_scl[16], true, mask, { 1, 1, 8, 8 }, rsvdCnt); + GatherMask (tmpz, xyz2Local, mask_scl[8], true, mask, { 1, 1, 8, 8 }, rsvdCnt); + + uint32_t b_xyz1_index = 3*(b*this->shapeN); + for(uint32_t i=0; i < this->shapeN; i++) + { + ux = -xyz1Gm.GetValue(b_xyz1_index); + uy = -xyz1Gm.GetValue(b_xyz1_index+1); + uz = -xyz1Gm.GetValue(b_xyz1_index+2); + + Adds(tmp3, tmpx, ux, calculate_num); + Mul(tmp2, tmp3, tmp3, calculate_num); + + Adds(tmp3, tmpy, uy, calculate_num); + Mul(tmp1, tmp3, tmp3, calculate_num); + + Add(tmp3, tmp2, tmp1, calculate_num); + + Adds(tmp2, tmpz, uz, calculate_num); + Mul(tmp1, tmp2, tmp2, calculate_num); + + Add(tmp3, tmp3, tmp1, calculate_num); + + ReduceMin(tmp2, tmp3, tmp1, calculate_num, true); + besti1 = (tmp2.ReinterpretCast()).GetValue(1); + tmp3.SetValue(besti1, 3.402823466e+38F); + + ReduceMin(tmp2[8], tmp3, tmp1, calculate_num, true); + besti2 = (tmp2.ReinterpretCast()).GetValue(9); + tmp3.SetValue(besti2, 3.402823466e+38F); + + ReduceMin(tmp2[16], tmp3, tmp1, calculate_num, true); + besti3 = (tmp2.ReinterpretCast()).GetValue(17); + + indicesGm.SetValue(b_xyz1_index, besti1); + indicesGm.SetValue(b_xyz1_index+1, besti2); + indicesGm.SetValue(b_xyz1_index+2, besti3); + + distGm.SetValue(b_xyz1_index, tmp2.GetValue(0)); + distGm.SetValue(b_xyz1_index+1, tmp2.GetValue(8)); + distGm.SetValue(b_xyz1_index+2, tmp2.GetValue(16)); + b_xyz1_index+=3; + } + inQueueXYZ2.FreeTensor(xyz2Local); + } + +private: + TPipe pipe; + TQue inQueueXYZ2; + TBuf QueueTmp1, QueueTmp2, QueueTmp3, QueueTmpX, QueueTmpY, QueueTmpZ, QueueMask, QueueTmp10; + GlobalTensor xyz1Gm; + GlobalTensor xyz2Gm; + GlobalTensor distGm; + GlobalTensor indicesGm; + uint32_t coreDataNum; + uint32_t tileNum; + uint32_t tileDataNum; + uint32_t tailDataNum; + uint32_t processDataNum; + uint32_t xyz2DataNum; + uint32_t shapeB; + uint32_t shapeN; + uint32_t shapeM; +}; + +extern "C" __global__ __aicore__ void three_nn(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); + // TODO: user kernel impl + KernelFastGeluGrad op; + op.Init(xyz1, xyz2, dist, indices, + tiling_data.CoreDataNum, tiling_data.finalTileNum, tiling_data.tileDataNum, tiling_data.TailDataNum, + tiling_data.shapeB, tiling_data.shapeN, tiling_data.shapeM); + op.Process(); +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh new file mode 100644 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh new file mode 100644 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/ThreeNNSample/README.en.md b/operator_contrib/ThreeNNSample/README.en.md new file mode 100644 index 000000000..19000a924 --- /dev/null +++ b/operator_contrib/ThreeNNSample/README.en.md @@ -0,0 +1,160 @@ +## ThreeNN Custom Operator Sample Description + +This sample implements the ThreeNN operator using the Ascend C programming language and provides corresponding end-to-end implementations based on different operator invocation methods. + +- [FrameworkLaunch](./FrameworkLaunch): Using a framework to call the ThreeNN custom operator. + +The development process follows the workflow of project creation -> operator implementation -> compilation and deployment -> operator invocation. The entire process relies on the operator project: the operator kernel function and tiling are developed based on the project code framework, the operator is compiled and deployed via project compilation scripts, enabling the use of single operators or operators within third-party frameworks. + +The following invocation methods are included in this sample: + +| Invocation Method | Directory | Description | +| ----------------- | --------- | ----------- | +| [FrameworkLaunch](./FrameworkLaunch) | [AclNNInvocation](./FrameworkLaunch/AclNNInvocation) | Calling the ThreeNN operator through aclnn. | + +## Operator Description + +For each point in the target point set, the ThreeNN operator finds the distances and coordinates of the three closest points from a reference set. The returned points are sorted in ascending order of squared distance. + +## Operator Specification + + + + + + + + + + + +
Operator TypeThreeNN
Operator inputnameshapedata typeformatdefault
xyz1(b, n, 3)float32ND\
xyz2(b, m, 3)float32ND\
Operator output
dist(b, n, 3)float32ND\
indices(b, n, 3)int32ND\
kernel nameThreeNN
+ +### 1. Operator Project Overview + +The operator project directory ThreeNN contains template files, compilation scripts, etc., as follows: + +``` +├── AclNNInvocation // Directory for invoking the ThreeNN operator via aclnn +│ ├── inc // Header file directory +│ │ ├── common.h // Declaration of common methods, used for reading binary files +│ │ ├── op_runner.h // Operator description declaration, including inputs/outputs, operator type, input and output descriptions +│ │ └── operator_desc.h // Operator runtime information declaration file +│ ├── input // Directory for input data generated by scripts +│ ├── output // Directory for storing operator output data and ground truth +│ ├── scripts +│ │ ├── acl.json // ACL configuration file +│ │ ├── gen_data.py // Script for generating input and ground truth data +│ │ └── verify_result.py // File for verifying results +│ ├── src +│ │ ├── CMakeLists.txt // Compilation rules +│ │ ├── common.cpp // Common functions, implementation for reading binary files +│ │ ├── main.cpp // Entry point for single operator invocation +│ │ ├── op_runner.cpp // Main process for single operator invocation +│ │ └── operator_desc.cpp // Construct operator input and output descriptions +│ ├── run.sh // Script for running the test sample +├── ThreeNN // Custom operator project +│ ├── cmake +│ ├── framework // Directory for implementing operator plugins, single operator model generation does not depend on plugins, not needed +│ ├── op_host // Host-side implementation +│ ├── op_kernel // Kernel-side implementation +│ ├── scripts // Directory for packaging custom operator project-related scripts +│ ├── build.sh // Compilation entry script +│ ├── CMakeLists.txt // CMakeLists.txt for the operator project +│ └── CMakePresets.json // Compilation configuration file +│ └── ThreeNN.json // Configuration file for generating the custom operator using msopgen +``` + +The CANN software package provides the project creation tool msopgen. The ThreeNN operator project can be automatically created using ThreeNN.json. For details, please refer to [Ascend C Operator Development](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC) > Operator Development > Operator Project Development > Creating an Operator Project. + +### 2. Compile the Operator Project + +Compile the custom operator project to generate the custom operator package. + +- Run the following commands to navigate to the ReduceCustom operator project directory. + + Use the command line to download the sample code, using the master branch as an example. + + ```bash + cd ${git_clone_path}/samples/operator_contrib/ThreeNNSample/ThreeNN + ``` + +- Modify `CMakePresets.json` to set `ASCEND_CANN_PACKAGE_PATH` to the actual path where the CANN software package is installed. + + ```json + { + …… + "configurePresets": [ + { + …… + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + // Replace with the actual installation path of the CANN software package, e.g., /home/HwHiAiUser/Ascend/ascend-toolkit/latest + }, + …… + } + ] + } + ``` + +- Run the following commands in the ReduceCustom operator project directory to compile the operator project. + + ```bash + ./build.sh + ``` + +After a successful compilation, a `build_out` directory is created in the current directory, which contains the custom operator installation package `custom_opp__.run`, for example, `custom_opp_ubuntu_x86_64.run`. + +Note: To use the dump debug feature, remove the Atlas training series product configuration from the op_host and CMakeLists.txt for Atlas 200/500 A2 inference products. + +### 3. Deploy the Operator Package + +Run the following commands in the directory containing the custom operator installation package to install the custom operator package. + +```bash +cd build_out +./custom_opp__.run +``` + +After successful execution, the relevant files in the custom operator package will be deployed to the `vendors/customize` directory in the OPP operator library of the current environment. + +### 4. Configure Environment Variables + +Please select the appropriate command to configure the environment variables according to the [installation method](https://hiascend.com/document/redirect/CannCommunityInstSoftware) of the CANN development kit package in the current environment. + +- Default path, root user installed CANN software package + + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` + +- Default path, non-root user installed CANN software package + + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` + +- Specify path `install_path`, installed CANN software package + + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ``` + +### 5. Run Operator Sample + +- [aclnn calls the ThreeNN operator project (test sample)](./ThreeNNCase) + +```bash +cd ./AclNNInvocation +./run.sh +``` + +The sample will automatically generate test data, then compile and run the aclnn sample, and finally verify the results. For details, see the `run.sh` script. + +## Update Notes + +| Date | Update Content | +| ---------- | --------------------- | +| 2024/10/24 | Uploaded ThreeNN code | + diff --git a/operator_contrib/ThreeNNSample/README.md b/operator_contrib/ThreeNNSample/README.md new file mode 100644 index 000000000..39f6afce4 --- /dev/null +++ b/operator_contrib/ThreeNNSample/README.md @@ -0,0 +1,145 @@ +## ThreeNN自定义算子样例说明 +本样例通过Ascend C编程语言实现了ThreeNN算子，并按照不同的算子调用方式分别给出了对应的端到端实现。 +- [FrameworkLaunch](./FrameworkLaunch)：使用框架调用ThreeNN自定义算子。 +按照工程创建->算子实现->编译部署>算子调用的流程完成算子开发。整个过程都依赖于算子工程：基于工程代码框架完成算子核函数的开发和Tiling实现，通过工程编译脚本完成算子的编译部署，继而实现单算子调用或第三方框架中的算子调用。 + +本样例中包含如下调用方式： + + + + + + +
调用方式目录描述
FrameworkLaunch AclNNInvocation通过aclnn调用的方式调用ThreeNN算子。
+ +## 算子描述 +对于目标点集中每个点，找到比较点集中距离其最近的3个点的距离、坐标，返回的三个点会按照距离平方升序。 + +## 算子规格描述 + + + + + + + + + + +
算子类型(OpType)ThreeNN
算子输入nameshapedata typeformatdefault
xyz1(b, n, 3)float32ND\
xyz2(b, m, 3)float32ND\
算子输出
dist(b, n, 3)float32ND\
indices(b, n, 3)int32ND\
核函数名ThreeNN
+ +### 1.算子工程介绍 +其中，算子工程目录ThreeNN包含算子实现的模板文件、编译脚本等，如下所示: +``` +├── AclNNInvocation // 通过aclnn调用的方式调用ThreeNN算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ └── operator_desc.h // 算子运行相关信息声明文件，包含算子 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真值数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ └── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ └── operator_desc.cpp // 构造算子的输入与输出描述 +│ ├── run.sh // 测试样例启动脚本 +├── ThreeNN //自定义算子工程 +│ ├── cmake +│ ├── framework // 算子插件实现文件目录，单算子模型文件的生成不依赖算子适配插件，无需关注 +│ ├── op_host // host侧实现文件 +│ ├── op_kernel // kernel侧实现文件 +│ ├── scripts // 自定义算子工程打包相关脚本所在目录 +│ ├── build.sh // 编译入口脚本 +│ ├── CMakeLists.txt // 算子工程的CMakeLists.txt +│ └── CMakePresets.json // 编译配置项 +│ └── ThreeNN.json // msopgen自定义算子生成配置文件 +``` +CANN软件包中提供了工程创建工具msopgen，ThreeNN算子工程可通过ThreeNN.json自动创建，具体请参考[Ascend C算子开发](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC)>算子开发>算子开发工程>基于自定义算子工程的算子开发>创建算子工程 章节。 + +### 2. 编译算子工程 + +编译自定义算子工程，构建生成自定义算子包。 + +- 执行如下命令，切换到算子工程ReduceCustom目录。 + + 以命令行方式下载样例代码，master分支为例。 + + ```bash + cd ${git_clone_path}/samples/operator_contrib/ThreeNNSample/ThreeNN + ``` +- 修改CMakePresets.json中ASCEND_CANN_PACKAGE_PATH为CANN软件包安装后的实际路径。 + + ```json + { + …… + "configurePresets": [ + { + …… + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + //请替换为CANN软件包安装后的实际路径。eg:/home/HwHiAiUser/Ascend/ascend-toolkit/latest + }, + …… + } + ] + } + ``` +- 在算子工程ReduceCustom目录下执行如下命令，进行算子工程编译。 + + ```bash + ./build.sh + ``` + +编译成功后，会在当前目录下创建build_out目录，并在build_out目录下生成自定义算子安装包custom_opp_\_\.run，例如“custom_opp_ubuntu_x86_64.run”。 + +备注：如果要使用dump调试功能，需要移除op_host内和CMakeLists.txt内的Atlas 训练系列产品、Atlas 200/500 A2 推理产品的配置。 + +### 3. 部署算子包 + +执行如下命令，在自定义算子安装包所在路径下，安装自定义算子包。 + +```bash +cd build_out +./custom_opp__.run +``` + +命令执行成功后，自定义算子包中的相关文件将部署至当前环境的OPP算子库的vendors/customize目录中。 + +### 4. 配置环境变量 + +请根据当前环境上CANN开发套件包的[安装方式](https://hiascend.com/document/redirect/CannCommunityInstSoftware)，选择对应配置环境变量的命令。 + +- 默认路径，root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` +- 默认路径，非root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` +- 指定路径install_path，安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ``` + +### 5. 调用执行算子样例 + +- [aclnn调用ThreeNN算子工程(测试样例)](./ThreeNNCase) + +```bash +cd ./AclNNInvocation +./run.sh +``` +样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + +## 更新说明 + | 时间 | 更新事项 | +|----|------| +| 2024/10/24 | 上传ThreeNN代码 | -- Gitee From 44f8a6a09b070c74f77fb50cb75440b5eb290d56 Mon Sep 17 00:00:00 2001 From: appleinsky <2164757809@qq.com> Date: Thu, 24 Oct 2024 01:09:52 +0800 Subject: [PATCH 02/12] test --- .../AclNNInvocation/inc/common.h | 45 - .../AclNNInvocation/inc/op_runner.h | 182 ---- .../AclNNInvocation/inc/operator_desc.h | 57 -- .../FrameworkLaunch/AclNNInvocation/run.sh | 120 --- .../AclNNInvocation/scripts/acl.json | 1 - .../AclNNInvocation/scripts/gen_data.py | 72 -- .../AclNNInvocation/scripts/verify_result.py | 26 - .../scripts/verify_result_int.py | 26 - .../AclNNInvocation/src/CMakeLists.txt | 68 -- .../AclNNInvocation/src/common.cpp | 79 -- .../AclNNInvocation/src/main.cpp | 171 ---- .../AclNNInvocation/src/op_runner.cpp | 486 ----------- .../AclNNInvocation/src/operator_desc.cpp | 56 -- .../FrameworkLaunch/ThreeNN/CMakeLists.txt | 69 -- .../FrameworkLaunch/ThreeNN/CMakePresets.json | 63 -- .../FrameworkLaunch/ThreeNN/ThreeNN.json | 50 -- .../FrameworkLaunch/ThreeNN/build.sh | 80 -- .../ThreeNN/cmake/config.cmake | 25 - .../FrameworkLaunch/ThreeNN/cmake/func.cmake | 192 ---- .../FrameworkLaunch/ThreeNN/cmake/intf.cmake | 26 - .../ThreeNN/cmake/makeself.cmake | 17 - .../ThreeNN/cmake/three_nn.cpp | 127 --- .../ThreeNN/cmake/util/__init__.py | 8 - .../cmake/util/ascendc_bin_param_build.py | 127 --- .../ThreeNN/cmake/util/ascendc_impl_build.py | 501 ----------- .../ThreeNN/cmake/util/ascendc_ops_config.py | 120 --- .../cmake/util/ascendc_replay_build.py | 65 -- .../ThreeNN/cmake/util/batch_replay_impl.temp | 117 --- .../ThreeNN/cmake/util/code_channel_infer.py | 58 -- .../ThreeNN/cmake/util/const_var.py | 33 - .../cmake/util/gen_impl_and_mrege_json.sh | 20 - .../ThreeNN/cmake/util/gen_ops_filter.sh | 62 -- .../ThreeNN/cmake/util/gen_version_info.sh | 6 - .../ThreeNN/cmake/util/insert_op_info.py | 36 - .../cmake/util/insert_simplified_keys.py | 242 ------ .../ThreeNN/cmake/util/kernel_entry.py | 115 --- .../ThreeNN/cmake/util/kernel_impl.temp | 10 - .../ThreeNN/cmake/util/makeself/COPYING | 339 -------- .../ThreeNN/cmake/util/makeself/README.md | 246 ------ .../ThreeNN/cmake/util/makeself/VERSION | 1 - .../cmake/util/makeself/make-release.sh | 9 - .../cmake/util/makeself/makeself-header.sh | 660 -------------- .../ThreeNN/cmake/util/makeself/makeself.1 | 110 --- .../ThreeNN/cmake/util/makeself/makeself.lsm | 16 - .../ThreeNN/cmake/util/makeself/makeself.sh | 822 ------------------ .../ThreeNN/cmake/util/makeself/run-tests.sh | 8 - .../cmake/util/merge_aicpu_info_json.sh | 31 - .../ThreeNN/cmake/util/opdesc_parser.py | 298 ------- .../ThreeNN/cmake/util/parse_ini_to_json.py | 338 ------- .../ThreeNN/cmake/util/preset_parse.py | 23 - .../ThreeNN/cmake/util/replay_codegen.py | 105 --- .../ThreeNN/cmake/util/replay_impl.temp | 120 --- .../cmake/util/tiling_data_def_build.py | 87 -- .../ThreeNN/framework/CMakeLists.txt | 11 - .../framework/tf_plugin/CMakeLists.txt | 14 - .../tf_plugin/tensorflow_three_nn_plugin.cc | 23 - .../ThreeNN/op_host/CMakeLists.txt | 82 -- .../ThreeNN/op_host/three_nn.cpp | 109 --- .../ThreeNN/op_host/three_nn_tiling.h | 16 - .../ThreeNN/op_kernel/CMakeLists.txt | 68 -- .../ThreeNN/op_kernel/three_nn.cpp | 166 ---- .../ThreeNN/scripts/install.sh | 318 ------- .../ThreeNN/scripts/upgrade.sh | 151 ---- operator_contrib/ThreeNNSample/README.en.md | 160 ---- operator_contrib/ThreeNNSample/README.md | 145 --- 65 files changed, 8034 deletions(-) delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh delete mode 100644 operator_contrib/ThreeNNSample/README.en.md delete mode 100644 operator_contrib/ThreeNNSample/README.md diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h deleted file mode 100644 index 7036813a9..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h +++ /dev/null @@ -1,45 +0,0 @@ -/** -* @file common.h -* -* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#ifndef COMMON_H -#define COMMON_H - -#include -#include -#include -#include -#include - -#include "acl/acl.h" - -#define SUCCESS 0 -#define FAILED 1 - -#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) -#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) -#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) - -/** - * @brief Read data from file - * @param [in] filePath: file path - * @param [out] fileSize: file size - * @return read result - */ -bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); - -/** - * @brief Write data to file - * @param [in] filePath: file path - * @param [in] buffer: data to write to file - * @param [in] size: size to write - * @return write result - */ -bool WriteFile(const std::string &filePath, const void *buffer, size_t size); - -#endif // COMMON_H diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h deleted file mode 100644 index bf2a9ef43..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h +++ /dev/null @@ -1,182 +0,0 @@ -/** -* @file op_runner.h -* -* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#ifndef OP_RUNNER_H -#define OP_RUNNER_H - -#include "aclnn/acl_meta.h" -#include "acl/acl.h" -#include "common.h" -#include "operator_desc.h" - -/** - * Op Runner - */ -class OpRunner { -public: - /** - * @brief Constructor - * @param [in] opDesc: op description - */ - explicit OpRunner(OperatorDesc *opDesc); - - /** - * @brief Destructor - */ - virtual ~OpRunner(); - - /** - * @brief Init op runner - */ - bool Init(); - - /** - * @brief Get number of inputs - * @return number of inputs - */ - const size_t NumInputs(); - - /** - * @brief Get number of outputs - * @return number of outputs - */ - const size_t NumOutputs(); - - /** - * @brief Get input size by index - * @param [in] index: input index - * @return size of the input - */ - const size_t GetInputSize(size_t index) const; - const size_t GetInputNumDims(size_t index) const; - aclDataType GetInputDataType(size_t index) const; - aclFormat GetInputFormat(size_t index) const; - - /** - * @brief Get output size by index - * @param [in] index: output index - * @return size of the output - */ - size_t GetOutputSize(size_t index) const; - const size_t GetOutputNumDims(size_t index) const; - aclDataType GetOutputDataType(size_t index) const; - aclFormat GetOutputFormat(size_t index) const; - - /** - * @brief Get input element count by index - * @param i[in] ndex: input index - * @return element count of the input - */ - size_t GetInputElementCount(size_t index) const; - - /** - * @brief Get output element count by index - * @param [in] index: output index - * @return element count of the output - */ - size_t GetOutputElementCount(size_t index) const; - - /** - * @brief Get input shape by index - * @param [in] index: input index - * @return shape of the output - */ - std::vector GetInputShape(size_t index) const; - - /** - * @brief Get output shape by index - * @param [in] index: output index - * @return shape of the output - */ - std::vector GetOutputShape(size_t index) const; - - /** - * @brief Get input buffer(host memory) by index - * @tparam T: data type - * @param [in] index: input index - * @return host address of the input - */ - template - T *GetInputBuffer(size_t index) - { - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return nullptr; - } - return reinterpret_cast(hostInputs_[index]); - } - - /** - * @brief Get output buffer(host memory) by index - * @tparam T: data type - * @param [in] index: output index - * @return host address of the output - */ - template - const T *GetOutputBuffer(size_t index) - { - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return nullptr; - } - - return reinterpret_cast(hostOutputs_[index]); - } - - /** - * @brief Print readable input by index - * @param [in] index: input index - * @param [in] elementsPerRow: number of elements per row - */ - void PrintInput(size_t index, size_t elementsPerRow = 16); - - /** - * @brief Print readable output by index - * @param [in] index: output index - * @param [in] elementsPerRow: number of elements per row - */ - void PrintOutput(size_t index, size_t elementsPerRow = 16); - - /** - * @brief Compile static op - * @return compile result - */ - bool CompileStaticOp(); - - /** - * @brief Compile dynamic op - * @return compile result - */ - bool CompileDynamicOp(); - - /** - * @brief Run op - * @return run result - */ - bool RunOp(); - -private: - size_t numInputs_; - size_t numOutputs_; - - std::vector inputBuffers_; - std::vector outputBuffers_; - - std::vector devInputs_; - std::vector devOutputs_; - - std::vector hostInputs_; - std::vector hostOutputs_; - - std::vector inputTensor_; - std::vector outputTensor_; - OperatorDesc *opDesc_; -}; - -#endif // OP_RUNNER_H diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h deleted file mode 100644 index 4cbdf07e0..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h +++ /dev/null @@ -1,57 +0,0 @@ -/** -* @file operator_desc.h -* -* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#ifndef OPERATOR_DESC_H -#define OPERATOR_DESC_H - -#include -#include - -#include "acl/acl.h" - -/** - * Op description - */ -struct OperatorDesc { - /** - * Constructor - */ - explicit OperatorDesc(); - - /** - * Destructor - */ - virtual ~OperatorDesc(); - - /** - * Add an input tensor description - * @param [in] dataType: data type - * @param [in] numDims: number of dims - * @param [in] dims: dims - * @param [in] format: format - * @return OperatorDesc - */ - OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); - - /** - * Add an output tensor description - * @param [in] dataType: data type - * @param [in] numDims: number of dims - * @param [in] dims: dims - * @param [in] format: format - * @return OperatorDesc - */ - OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); - - std::string opType; - std::vector inputDesc; - std::vector outputDesc; -}; - -#endif // OPERATOR_DESC_H diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh deleted file mode 100644 index 96ca9c3b1..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/bash -export ASCEND_SLOG_PRINT_TO_STDOUT=0 -export ASCEND_GLOBAL_LOG_LEVEL=0 - -CURRENT_DIR=$( - cd $(dirname ${BASH_SOURCE:-$0}) - pwd -) -cd $CURRENT_DIR - -# 导出环境变量 -SHORT=v:, -LONG=dtype:, -OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") -eval set -- "$OPTS" -while : -do - case "$1" in - # float16, float, int32 - (-v | --dtype) - DTYPE="$2" - shift 2;; - (--) - shift; - break;; - (*) - echo "[ERROR] Unexpected option: $1"; - break;; - esac -done - -if [ ! $ASCEND_HOME_DIR ]; then - if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then - export ASCEND_HOME_DIR=$HOME/Ascend/ascend-toolkit/latest - else - export ASCEND_HOME_DIR=/usr/local/Ascend/ascend-toolkit/latest - fi -fi -source $ASCEND_HOME_DIR/bin/setenv.bash - -export DDK_PATH=$ASCEND_HOME_DIR -arch=$(uname -m) -export NPU_HOST_LIB=$ASCEND_HOME_DIR/${arch}-linux/lib64 - -function main { - # 1. 清除算子输出和日志文件 - - rm ./output/execute_op - rm ./input/*.bin - rm ./output/*.bin - # rm -rf ./output/output*.bin > /dev/null - - # 2. 生成或复用输入数据和真值数据 - if [ -d "./input" ]; then - if [ "$(ls -A "./input")" ]; then - echo "已存在测试数据" - else - echo "生成测试数据" - cd $CURRENT_DIR - python3 scripts/gen_data.py - fi - else - echo "生成测试数据" - cd $CURRENT_DIR - python3 scripts/gen_data.py - fi - - if [ $? -ne 0 ]; then - echo "ERROR: generate input data failed!" - return 1 - fi - echo "INFO: generate input data success!" - - # 3. 编译或复用acl可执行文件 - if [ -e "./output/execute_op" ]; then - echo "可执行存在" - else - echo "可执行不存在" - cd $CURRENT_DIR; rm -rf build; mkdir -p build; cd build - cmake ../src - if [ $? -ne 0 ]; then - echo "ERROR: cmake failed!" - return 1 - fi - echo "INFO: cmake success!" - make - if [ $? -ne 0 ]; then - echo "ERROR: make failed!" - return 1 - fi - echo "INFO: make success!" - fi - - # 4. 运行可执行文件 - cd $CURRENT_DIR/output - echo "INFO: execute op!" - timeout 30 ./execute_op - - if [ $? -ne 0 ]; then - echo "ERROR: acl executable run failed! please check your project!" - return 1 - fi - echo "INFO: acl executable run success!" - - # 5. 比较真值文件 - cd $CURRENT_DIR - ret_dist=`python3 scripts/verify_result.py output/output_dist.bin output/golden_dist.bin` - ret_idx=`python3 scripts/verify_result_int.py output/output_idx.bin output/golden_idx.bin` - echo "verify dist $ret_dist" - echo "verify idx $ret_idx" - if [ "x$ret_dist" == "xtest pass" ] && [ "x$ret_idx" == "xtest pass" ]; then - echo "" - echo "#####################################" - echo "INFO: you have passed the Precision!" - echo "#####################################" - echo "" - fi -} - -main diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json deleted file mode 100644 index 9e26dfeeb..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py deleted file mode 100644 index c723df735..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/python3 -# -*- coding:utf-8 -*- -# Copyright 2022-2023 Huawei Technologies Co., Ltd -import numpy as np -import os -import torch - -def run_three_nn(b, n, m, xyz1, xyz2): - dist = np.zeros_like(xyz1, dtype=np.float32) - idx = np.zeros_like(xyz1, dtype=np.int32) - - for ii in range(b): - for jj in range(n): - x1 = xyz1[ii, jj, 0] - y1 = xyz1[ii, jj, 1] - z1 = xyz1[ii, jj, 2] - best1 = 1e40 - best2 = 1e40 - best3 = 1e40 - besti1 = 0 - besti2 = 0 - besti3 = 0 - for kk in range(m): - x2 = xyz2[ii, kk, 0] - y2 = xyz2[ii, kk, 1] - z2 = xyz2[ii, kk, 2] - d = (x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1) + (z2 - z1) * (z2 - z1) - if (d < best1): - best3 = best2 - besti3 = besti2 - best2 = best1 - besti2 = besti1 - best1 = d - besti1 = kk - elif (d < best2): - best3 = best2 - besti3 = besti2 - best2 = d - besti2 = kk - elif (d < best3): - best3 = d - besti3 = kk - dist[ii, jj, 0] = best1 - idx[ii, jj, 0] = besti1 - dist[ii, jj, 1] = best2 - idx[ii, jj, 1] = besti2 - dist[ii, jj, 2] = best3 - idx[ii, jj, 2] = besti3 - - return dist, idx - -def gen_golden_data_simple(): - b = 5 - n = 160 - m = 256 - xyz1_shape = [b,n,3] - xyz2_shape = [b,m,3] - input_xyz1 = np.random.uniform(-10, 10, xyz1_shape).astype(np.float32) - input_xyz2 = np.random.uniform(-10, 10, xyz2_shape).astype(np.float32) - dist, idx = run_three_nn(b, n, m, input_xyz1, input_xyz2) - os.system("mkdir -p input") - os.system("mkdir -p output") - print("dist = ",dist) - print("idx = ",idx) - input_xyz1.tofile("./input/input_xyz1.bin") - input_xyz2.tofile("./input/input_xyz2.bin") - dist.tofile("./output/golden_dist.bin") - idx.tofile("./output/golden_idx.bin") - - -if __name__ == "__main__": - gen_golden_data_simple() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py deleted file mode 100644 index d917ab413..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import sys -import numpy as np -np.set_printoptions(suppress=True) - -loss = 1e-4 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 -minimum = 10e-10 - -def verify_result(real_result, golden): - real_result = np.fromfile(real_result, dtype=np.float32) # 从bin文件读取实际运算结果 - golden = np.fromfile(golden, dtype=np.float32) # 从bin文件读取预期运算结果 - result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 - deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 - result_atol = np.less_equal(result, loss) # 计算绝对误差 - result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 - if not result_rtol.all() and not result_atol.all(): - if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 - print("[ERROR] result error") - print(real_result) - print(golden) - return False - print("test pass") - return True - -if __name__ == '__main__': - verify_result(sys.argv[1],sys.argv[2]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py deleted file mode 100644 index 9cbc3482b..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import sys -import numpy as np -np.set_printoptions(suppress=True) - -loss = 1e-4 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 -minimum = 10e-10 - -def verify_result(real_result, golden): - real_result = np.fromfile(real_result, dtype=np.int32) # 从bin文件读取实际运算结果 - golden = np.fromfile(golden, dtype=np.int32) # 从bin文件读取预期运算结果 - result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 - deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 - result_atol = np.less_equal(result, loss) # 计算绝对误差 - result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 - if not result_rtol.all() and not result_atol.all(): - if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 - print("[ERROR] result error") - print(real_result) - print(golden) - return False - print("test pass") - return True - -if __name__ == '__main__': - verify_result(sys.argv[1],sys.argv[2]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt deleted file mode 100644 index ee73e930f..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. - -# CMake lowest version requirement -cmake_minimum_required(VERSION 3.5.1) - -# project information -project(acl_execute_add) - -# Compile options -add_compile_options(-std=c++11) - -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") - -set(INC_PATH $ENV{DDK_PATH}) - -if (NOT DEFINED ENV{DDK_PATH}) - set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") - message(STATUS "set default INC_PATH: ${INC_PATH}") -else () - message(STATUS "env INC_PATH: ${INC_PATH}") -endif() - -set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") - -set(LIB_PATH $ENV{NPU_HOST_LIB}) - -# Dynamic libraries in the stub directory can only be used for compilation -if (NOT DEFINED ENV{NPU_HOST_LIB}) - set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") - set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") - message(STATUS "set default LIB_PATH: ${LIB_PATH}") -else () - message(STATUS "env LIB_PATH: ${LIB_PATH}") -endif() - -# Header path -include_directories( - ${INC_PATH}/runtime/include - ${INC_PATH}/atc/include - ../inc - ${CUST_PKG_PATH}/include - ${BUILT_PKG_PATH}/include/aclnnop -) - -# add host lib path -link_directories( - ${LIB_PATH} - ${LIB_PATH1} - ${CUST_PKG_PATH}/lib -) - -add_executable(execute_op - operator_desc.cpp - op_runner.cpp - main.cpp - common.cpp -) - -target_link_libraries(execute_op - ascendcl - cust_opapi - acl_op_compiler - nnopbase - stdc++ -) - -install(TARGETS execute_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp deleted file mode 100644 index 47a39cd5f..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/** -* @file common.cpp -* -* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#include "common.h" - -#include -#include -#include -#include - -extern bool g_isDevice; - -bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) -{ - struct stat sBuf; - int fileStatus = stat(filePath.data(), &sBuf); - if (fileStatus == -1) { - ERROR_LOG("failed to get file %s", filePath.c_str()); - return false; - } - if (S_ISREG(sBuf.st_mode) == 0) { - ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); - return false; - } - - std::ifstream file; - file.open(filePath, std::ios::binary); - if (!file.is_open()) { - ERROR_LOG("Open file failed. path = %s", filePath.c_str()); - return false; - } - - std::filebuf *buf = file.rdbuf(); - size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); - if (size == 0) { - ERROR_LOG("file size is 0"); - file.close(); - return false; - } - if (size > bufferSize) { - ERROR_LOG("file size is larger than buffer size"); - file.close(); - return false; - } - buf->pubseekpos(0, std::ios::in); - buf->sgetn(static_cast(buffer), size); - fileSize = size; - file.close(); - return true; -} - -bool WriteFile(const std::string &filePath, const void *buffer, size_t size) -{ - if (buffer == nullptr) { - ERROR_LOG("Write file failed. buffer is nullptr"); - return false; - } - - int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); - if (fd < 0) { - ERROR_LOG("Open file failed. path = %s", filePath.c_str()); - return false; - } - - auto writeSize = write(fd, buffer, size); - (void) close(fd); - if (writeSize != size) { - ERROR_LOG("Write file Failed."); - return false; - } - - return true; -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp deleted file mode 100644 index 7536fdb6e..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp +++ /dev/null @@ -1,171 +0,0 @@ -/** -* @file main.cpp -* -* Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#include -#include -#include -#include -#include - -#include "acl/acl.h" -#include "op_runner.h" - -#include "common.h" - -bool g_isDevice = false; -int deviceId = 0; - -OperatorDesc CreateOpDesc() -{ - // define operator - int64_t b = 5; - int64_t n = 160; - int64_t m = 256; - std::vector xyz1_shape {b,n,3}; - std::vector xyz2_shape {b,m,3}; - std::vector out_shape {b,n,3}; - aclDataType dataType = ACL_FLOAT; - aclDataType indicesType = ACL_INT32; - aclFormat format = ACL_FORMAT_ND; - OperatorDesc opDesc; - opDesc.AddInputTensorDesc(dataType, xyz1_shape.size(), xyz1_shape.data(), format); - opDesc.AddInputTensorDesc(dataType, xyz2_shape.size(), xyz2_shape.data(), format); - opDesc.AddOutputTensorDesc(dataType, out_shape.size(), out_shape.data(), format); - opDesc.AddOutputTensorDesc(indicesType, out_shape.size(), out_shape.data(), format); - return opDesc; -} - -bool SetInputData(OpRunner &runner) -{ - size_t fileSize = 0; - ReadFile("../input/input_xyz1.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); - ReadFile("../input/input_xyz2.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); - INFO_LOG("Set input success"); - return true; -} - -bool ProcessOutputData(OpRunner &runner) -{ - WriteFile("../output/output_dist.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); - WriteFile("../output/output_idx.bin", runner.GetOutputBuffer(1), runner.GetOutputSize(1)); - INFO_LOG("Write output success"); - return true; -} - -void DestoryResource() -{ - bool flag = false; - if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { - ERROR_LOG("Reset device %d failed", deviceId); - flag = true; - } - INFO_LOG("Reset Device success"); - if (aclFinalize() != ACL_SUCCESS) { - ERROR_LOG("Finalize acl failed"); - flag = true; - } - if (flag) { - ERROR_LOG("Destory resource failed"); - } else { - INFO_LOG("Destory resource success"); - } -} - -bool InitResource() -{ - std::string output = "../output"; - if (access(output.c_str(), 0) == -1) { - int ret = mkdir(output.c_str(), 0700); - if (ret == 0) { - INFO_LOG("Make output directory successfully"); - } - else { - ERROR_LOG("Make output directory fail"); - return false; - } - } - - // acl.json is dump or profiling config file - if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { - ERROR_LOG("acl init failed"); - return false; - } - - if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { - ERROR_LOG("Set device failed. deviceId is %d", deviceId); - (void)aclFinalize(); - return false; - } - INFO_LOG("Set device[%d] success", deviceId); - - // runMode is ACL_HOST which represents app is running in host - // runMode is ACL_DEVICE which represents app is running in device - aclrtRunMode runMode; - if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { - ERROR_LOG("Get run mode failed"); - DestoryResource(); - return false; - } - g_isDevice = (runMode == ACL_DEVICE); - INFO_LOG("Get RunMode[%d] success", runMode); - - return true; -} - -bool RunOp() -{ - // create op desc - OperatorDesc opDesc = CreateOpDesc(); - - // create Runner - OpRunner opRunner(&opDesc); - if (!opRunner.Init()) { - ERROR_LOG("Init OpRunner failed"); - return false; - } - - // Load inputs - if (!SetInputData(opRunner)) { - ERROR_LOG("Set input data failed"); - return false; - } - - // Run op - if (!opRunner.RunOp()) { - ERROR_LOG("Run op failed"); - return false; - } - - // process output data - if (!ProcessOutputData(opRunner)) { - ERROR_LOG("Process output data failed"); - return false; - } - - INFO_LOG("Run op success"); - return true; -} - -int main(int argc, char **argv) -{ - if (!InitResource()) { - ERROR_LOG("Init resource failed"); - return FAILED; - } - INFO_LOG("Init resource success"); - - if (!RunOp()) { - DestoryResource(); - return FAILED; - } - - DestoryResource(); - - return SUCCESS; -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp deleted file mode 100644 index 16bbc21d0..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp +++ /dev/null @@ -1,486 +0,0 @@ -/** -* @file op_runner.cpp -* -* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#include "op_runner.h" -#include "aclnn_three_nn.h" -#include -#include -#include "acl/acl_op_compiler.h" -#include "common.h" - -using namespace std; - -extern bool g_isDevice; - -OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) -{ - numInputs_ = opDesc->inputDesc.size(); - numOutputs_ = opDesc->outputDesc.size(); -} - -OpRunner::~OpRunner() -{ - for (size_t i = 0; i < numInputs_; ++i) { - auto ret = aclDestroyTensor(inputTensor_[i]); - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free InputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - ret = aclDestroyDataBuffer(inputBuffers_[i]); - - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free inputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - ret = aclrtFree(devInputs_[i]); - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free devInputs[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - if (g_isDevice) { - ret = aclrtFree(hostInputs_[i]); - } else { - ret = aclrtFreeHost(hostInputs_[i]); - } - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free hostInputs[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - } - - for (size_t i = 0; i < numOutputs_; ++i) { - auto ret = aclDestroyTensor(outputTensor_[i]); - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free outputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - ret = aclDestroyDataBuffer(outputBuffers_[i]); - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free outputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - ret = aclrtFree(devOutputs_[i]); - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free devOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - if (g_isDevice) { - ret = aclrtFree(hostOutputs_[i]); - } else { - ret = aclrtFreeHost(hostOutputs_[i]); - } - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free hostOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - } -} - -bool OpRunner::Init() -{ - for (size_t i = 0; i < numInputs_; ++i) { - auto size = GetInputSize(i); - void *devMem = nullptr; - if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory for input[%zu] failed", i); - return false; - } - devInputs_.emplace_back(devMem); - inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); - - void *hostInput = nullptr; - if (g_isDevice) { - if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory for input[%zu] failed", i); - return false; - } - } else { - if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory for input[%zu] failed", i); - return false; - } - } - if (hostInput == nullptr) { - ERROR_LOG("Malloc memory for input[%zu] failed", i); - return false; - } - hostInputs_.emplace_back(hostInput); - - aclTensor *inputTensor = aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), - nullptr, 0, GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); - if (inputTensor == nullptr) { - ERROR_LOG("Create Tensor for input[%zu] failed", i); - return false; - } - inputTensor_.emplace_back(inputTensor); - } - - for (size_t i = 0; i < numOutputs_; ++i) { - auto size = GetOutputSize(i); - void *devMem = nullptr; - if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory for output[%zu] failed", i); - return false; - } - devOutputs_.emplace_back(devMem); - outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); - - void *hostOutput = nullptr; - if (g_isDevice) { - if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory for output[%zu] failed", i); - return false; - } - } else { - if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory for output[%zu] failed", i); - return false; - } - } - if (hostOutput == nullptr) { - ERROR_LOG("Malloc host memory for output[%zu] failed", i); - return false; - } - hostOutputs_.emplace_back(hostOutput); - - aclTensor *outputTensor = aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), - nullptr, 0, GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); - if (outputTensor == nullptr) { - ERROR_LOG("Create Tensor for output[%zu] failed", i); - return false; - } - outputTensor_.emplace_back(outputTensor); - } - - return true; -} - -const size_t OpRunner::NumInputs() -{ - return numInputs_; -} - -const size_t OpRunner::NumOutputs() -{ - return numOutputs_; -} - -const size_t OpRunner::GetInputSize(size_t index) const -{ - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return 0; - } - - return aclGetTensorDescSize(opDesc_->inputDesc[index]); -} - -const size_t OpRunner::GetInputNumDims(size_t index) const -{ - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return 0; - } - - return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); -} - -aclDataType OpRunner::GetInputDataType(size_t index) const -{ - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return ACL_DT_UNDEFINED; - } - - return aclGetTensorDescType(opDesc_->inputDesc[index]); -} - -aclFormat OpRunner::GetInputFormat(size_t index) const -{ - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return ACL_FORMAT_UNDEFINED; - } - - return aclGetTensorDescFormat(opDesc_->inputDesc[index]); -} - -std::vector OpRunner::GetInputShape(size_t index) const -{ - std::vector ret; - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return ret; - } - - auto desc = opDesc_->inputDesc[index]; - for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { - int64_t dimSize; - if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { - ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); - ret.clear(); - return ret; - } - ret.emplace_back(dimSize); - } - - return ret; -} - -size_t OpRunner::GetOutputSize(size_t index) const -{ - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return 0; - } - - return aclGetTensorDescSize(opDesc_->outputDesc[index]); -} - -const size_t OpRunner::GetOutputNumDims(size_t index) const -{ - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return 0; - } - - return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); -} - -aclDataType OpRunner::GetOutputDataType(size_t index) const -{ - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return ACL_DT_UNDEFINED; - } - - return aclGetTensorDescType(opDesc_->outputDesc[index]); -} - - -aclFormat OpRunner::GetOutputFormat(size_t index) const -{ - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return ACL_FORMAT_UNDEFINED; - } - - return aclGetTensorDescFormat(opDesc_->outputDesc[index]); -} - -std::vector OpRunner::GetOutputShape(size_t index) const -{ - std::vector ret; - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return ret; - } - - auto desc = opDesc_->outputDesc[index]; - for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { - int64_t dimSize; - if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { - ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); - ret.clear(); - return ret; - } - ret.emplace_back(dimSize); - } - return ret; -} - -size_t OpRunner::GetInputElementCount(size_t index) const -{ - if (index >= opDesc_->inputDesc.size()) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return 0; - } - - return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); -} - -size_t OpRunner::GetOutputElementCount(size_t index) const -{ - if (index >= opDesc_->outputDesc.size()) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return 0; - } - - return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); -} - -bool OpRunner::RunOp() -{ - for (size_t i = 0; i < numInputs_; ++i) { - auto size = GetInputSize(i); - aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; - if (g_isDevice) { - kind = ACL_MEMCPY_DEVICE_TO_DEVICE; - } - if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { - ERROR_LOG("Copy input[%zu] failed", i); - return false; - } - INFO_LOG("Copy input[%zu] success", i); - } - - aclrtStream stream = nullptr; - if (aclrtCreateStream(&stream) != ACL_SUCCESS) { - ERROR_LOG("Create stream failed"); - return false; - } - INFO_LOG("Create stream success"); - - size_t workspaceSize = 0; - aclOpExecutor *handle = nullptr; - - auto ret = aclnnThreeNNGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], outputTensor_[1], - &workspaceSize, &handle); - if (ret != ACL_SUCCESS) { - (void)aclrtDestroyStream(stream); - ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); - return false; - } - INFO_LOG("Execute GetWorkspaceSize success, workspace size %lu", workspaceSize); - - void *workspace = nullptr; - if (workspaceSize != 0) { - if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory failed"); - } - } - ret = aclnnThreeNN(workspace, workspaceSize, handle, stream); - - if (ret != ACL_SUCCESS) { - (void)aclrtDestroyStream(stream); - ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); - return false; - } - INFO_LOG("Execute Operator success"); - - ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); - if (ret != SUCCESS) { - ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); - (void)aclrtDestroyStream(stream); - return false; - } - INFO_LOG("Synchronize stream success"); - - for (size_t i = 0; i < numOutputs_; ++i) { - auto size = GetOutputSize(i); - aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; - if (g_isDevice) { - kind = ACL_MEMCPY_DEVICE_TO_DEVICE; - } - if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { - INFO_LOG("Copy output[%zu] success", i); - (void)aclrtDestroyStream(stream); - return false; - } - INFO_LOG("Copy output[%zu] success", i); - } - - (void)aclrtDestroyStream(stream); - return true; -} - - -template -void DoPrintData(const T *data, size_t count, size_t elementsPerRow) -{ - assert(elementsPerRow != 0); - for (size_t i = 0; i < count; ++i) { - std::cout << std::setw(10) << data[i]; - if (i % elementsPerRow == elementsPerRow - 1) { - std::cout << std::endl; - } - } -} - -void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) -{ - assert(elementsPerRow != 0); - for (size_t i = 0; i < count; ++i) { - std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); - if (i % elementsPerRow == elementsPerRow - 1) { - std::cout << std::endl; - } - } -} - -void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) -{ - if (data == nullptr) { - ERROR_LOG("Print data failed. data is nullptr"); - return; - } - - switch (dataType) { - case ACL_BOOL: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_INT8: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_UINT8: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_INT16: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_UINT16: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_INT32: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_UINT32: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_INT64: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_UINT64: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_FLOAT16: - DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_FLOAT: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_DOUBLE: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - default: - ERROR_LOG("Unsupported type: %d", dataType); - } -} - -void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) -{ - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); - return; - } - - auto desc = opDesc_->inputDesc[index]; - PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); -} - -void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) -{ - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return; - } - - auto desc = opDesc_->outputDesc[index]; - PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp deleted file mode 100644 index a7a1ed3bd..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/** -* @file operator_desc.cpp -* -* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#include "common.h" -#include "operator_desc.h" - -using namespace std; - -OperatorDesc::OperatorDesc() {} - -OperatorDesc::~OperatorDesc() -{ - for (auto *desc : inputDesc) { - aclDestroyTensorDesc(desc); - } - - for (auto *desc : outputDesc) { - aclDestroyTensorDesc(desc); - } - -} - -OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, - int numDims, - const int64_t *dims, - aclFormat format) -{ - aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); - if (desc == nullptr) { - ERROR_LOG("create tensor failed"); - return *this; - } - inputDesc.emplace_back(desc); - return *this; -} - -OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, - int numDims, - const int64_t *dims, - aclFormat format) -{ - aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); - if (desc == nullptr) { - ERROR_LOG("create tensor failed"); - return *this; - } - - outputDesc.emplace_back(desc); - return *this; -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt deleted file mode 100644 index 584132d80..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt +++ /dev/null @@ -1,69 +0,0 @@ -cmake_minimum_required(VERSION 3.16.0) -project(opp) -if(ENABLE_CROSS_COMPILE) - if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) - set(CROSS_COMPILE_PLATFORM aarch64) - else() - set(CROSS_COMPILE_PLATFORM x86_64) - endif() - set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) - set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) - set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) - set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) - set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) - set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) -else() - set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) -endif() - -include(cmake/config.cmake) -include(cmake/func.cmake) -include(cmake/intf.cmake) - -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) - add_subdirectory(framework) -endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) - add_subdirectory(op_host) -endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) - add_subdirectory(op_kernel) -endif() -if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) - add_subdirectory(testcases) -endif() - -# modify vendor_name in install.sh and upgrade.sh -add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh - COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts - COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ - COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* -) -add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) -install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) - -install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) - -get_system_info(SYSTEM_INFO) - -# gen version.info -add_custom_target(gen_version_info ALL - COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} -) - -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info - DESTINATION packages/vendors/${vendor_name}/) - -# CPack config -set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) -set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) -set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") -set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") -set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) -set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") -set(CPACK_GENERATOR External) -set(CPACK_CMAKE_GENERATOR "Unix Makefiles") -set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) -set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) -set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) -include(CPack) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json deleted file mode 100644 index 8b7181000..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "version": 1, - "cmakeMinimumRequired": { - "major": 3, - "minor": 19, - "patch": 0 - }, - "configurePresets": [ - { - "name": "default", - "displayName": "Default Config", - "description": "Default build using Unix Makefiles generator", - "generator": "Unix Makefiles", - "binaryDir": "${sourceDir}/build_out", - "cacheVariables": { - "CMAKE_BUILD_TYPE": { - "type": "STRING", - "value": "Release" - }, - "ENABLE_SOURCE_PACKAGE": { - "type": "BOOL", - "value": "True" - }, - "ENABLE_BINARY_PACKAGE": { - "type": "BOOL", - "value": "True" - }, - "ASCEND_COMPUTE_UNIT": { - "type": "STRING", - "value": "ascend310b" - }, - "ENABLE_TEST": { - "type": "BOOL", - "value": "True" - }, - "vendor_name": { - "type": "STRING", - "value": "customize" - }, - "ASCEND_CANN_PACKAGE_PATH": { - "type": "PATH", - "value": "/usr/local/Ascend/ascend-toolkit/latest" - }, - "ASCEND_PYTHON_EXECUTABLE": { - "type": "STRING", - "value": "python3" - }, - "CMAKE_INSTALL_PREFIX": { - "type": "PATH", - "value": "${sourceDir}/build_out" - }, - "ENABLE_CROSS_COMPILE": { - "type": "BOOL", - "value": "False" - }, - "CMAKE_CROSS_PLATFORM_COMPILER": { - "type": "PATH", - "value": "/usr/bin/aarch64-linux-gnu-g++" - } - } - } - ] -} \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json deleted file mode 100644 index f35459443..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json +++ /dev/null @@ -1,50 +0,0 @@ -[ - { - "op": "ThreeNN", - "language": "cpp", - "input_desc": [ - { - "name": "xyz1", - "param_type": "required", - "format": [ - "ND" - ], - "type": [ - "float" - ] - }, - { - "name": "xyz2", - "param_type": "required", - "format": [ - "ND" - ], - "type": [ - "float" - ] - } - ], - "output_desc": [ - { - "name": "dist", - "param_type": "required", - "format": [ - "ND" - ], - "type": [ - "float" - ] - }, - { - "name": "indices", - "param_type": "required", - "format": [ - "ND" - ], - "type": [ - "int32" - ] - } - ] - } -] \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh deleted file mode 100644 index 27d1968d1..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -script_path=$(realpath $(dirname $0)) - - -mkdir -p build_out -rm -rf build_out/* -cd build_out - -opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) -ENABLE_CROSS="-DENABLE_CROSS_COMPILE=True" -ENABLE_BINARY="-DENABLE_BINARY_PACKAGE=True" -cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') - -cmake_run_package() -{ - target=$1 - cmake --build . --target $target -j16 - if [ $? -ne 0 ]; then exit 1; fi - - if [ $target = "package" ]; then - if test -d ./op_kernel/binary ; then - ./cust*.run - if [ $? -ne 0 ]; then exit 1; fi - cmake --build . --target binary -j16 - if [ $? -ne 0 ]; then - echo "[ERROR] Kernel compile failed, the run package will not be generated." - rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; - fi - cmake --build . --target $target -j16 - fi - fi -} - -if [[ $opts =~ $ENABLE_CROSS ]] && [[ $opts =~ $ENABLE_BINARY ]] -then - target=package - if [ "$1"x != ""x ]; then target=$1; fi - if [ "$cmake_version" \< "3.19.0" ] ; then - cmake .. $opts -DENABLE_CROSS_COMPILE=0 - else - cmake .. --preset=default -DENABLE_CROSS_COMPILE=0 - fi - cmake_run_package $target - cp -r kernel ../ - rm -rf * - if [ "$cmake_version" \< "3.19.0" ] ; then - cmake .. $opts - else - cmake .. --preset=default - fi - - cmake --build . --target $target -j16 - if [ $? -ne 0 ]; then - echo "[ERROR] Kernel compile failed, the run package will not be generated." - rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; - fi - if [ $target = "package" ]; then - if test -d ./op_kernel/binary ; then - ./cust*.run - fi - fi - rm -rf ../kernel - -else - target=package - if [ "$1"x != ""x ]; then target=$1; fi - if [ "$cmake_version" \< "3.19.0" ] ; then - cmake .. $opts - else - cmake .. --preset=default - fi - cmake_run_package $target -fi - - -# for debug -# cd build_out -# make -# cpack -# verbose append -v \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake deleted file mode 100644 index 886119daa..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake +++ /dev/null @@ -1,25 +0,0 @@ - -set(CMAKE_CXX_FLAGS_DEBUG "") -set(CMAKE_CXX_FLAGS_RELEASE "") - -if (NOT DEFINED vendor_name) - set(vendor_name customize CACHE STRING "") -endif() -if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) - set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") -endif() -if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) - set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") -endif() -if (NOT DEFINED ASCEND_COMPUTE_UNIT) - message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! -") -endif() -set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) -set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) -set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) -set(ASCEND_FRAMEWORK_TYPE tensorflow) -file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) -set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") -execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} - COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake deleted file mode 100644 index 4179dfd25..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake +++ /dev/null @@ -1,192 +0,0 @@ - -function(get_system_info SYSTEM_INFO) - if (UNIX) - execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) - string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) - set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) - elseif (WIN32) - message(STATUS "System is Windows. Only for pre-build.") - else () - message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") - endif () -endfunction() - -function(opbuild) - message(STATUS "Opbuild generating sources") - cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) - execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 - -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api - -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so - RESULT_VARIABLE EXEC_RESULT - OUTPUT_VARIABLE EXEC_INFO - ERROR_VARIABLE EXEC_ERROR - ) - if (${EXEC_RESULT}) - message("build ops lib info: ${EXEC_INFO}") - message("build ops lib error: ${EXEC_ERROR}") - message(FATAL_ERROR "opbuild run failed!") - endif() - set(proj_env "") - set(prefix_env "") - if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") - set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") - endif() - if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") - set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") - endif() - execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build - ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} - RESULT_VARIABLE EXEC_RESULT - OUTPUT_VARIABLE EXEC_INFO - ERROR_VARIABLE EXEC_ERROR - ) - if (${EXEC_RESULT}) - message("opbuild ops info: ${EXEC_INFO}") - message("opbuild ops error: ${EXEC_ERROR}") - endif() - message(STATUS "Opbuild generating sources - done") -endfunction() - -function(add_ops_info_target) - cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) - get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) - add_custom_command(OUTPUT ${OPINFO_OUTPUT} - COMMAND mkdir -p ${opinfo_file_path} - COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py - ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} - ) - add_custom_target(${OPINFO_TARGET} ALL - DEPENDS ${OPINFO_OUTPUT} - ) - install(FILES ${OPINFO_OUTPUT} - DESTINATION ${OPINFO_INSTALL_DIR} - ) -endfunction() - -function(add_ops_compile_options OP_TYPE) - cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) - file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} - "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") -endfunction() - -function(add_ops_impl_target) - cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) - add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp - COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic - COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py - ${OPIMPL_OPS_INFO} - \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" - ${OPIMPL_IMPL_DIR} - ${OPIMPL_OUT_DIR}/dynamic - ${ASCEND_AUTOGEN_PATH} - - COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp - COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp - DEPENDS ${OPIMPL_OPS_INFO} - ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py - ) - add_custom_target(${OPIMPL_TARGET} ALL - DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) - if (${ENABLE_SOURCE_PACKAGE}) - install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic - DESTINATION ${OPIMPL_INSTALL_DIR} - ) - endif() -endfunction() - -function(add_npu_support_target) - cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) - get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) - add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json - COMMAND mkdir -p ${NPUSUP_OUT_DIR} - COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh - ${NPUSUP_OPS_INFO_DIR} - ${NPUSUP_OUT_DIR} - ) - add_custom_target(npu_supported_ops ALL - DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json - ) - install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json - DESTINATION ${NPUSUP_INSTALL_DIR} - ) -endfunction() - -function(add_bin_compile_target) - cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) - file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) - file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) - file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) - execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py - ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} - RESULT_VARIABLE EXEC_RESULT - OUTPUT_VARIABLE EXEC_INFO - ERROR_VARIABLE EXEC_ERROR - ) - if (${EXEC_RESULT}) - message("ops binary compile scripts gen info: ${EXEC_INFO}") - message("ops binary compile scripts gen error: ${EXEC_ERROR}") - message(FATAL_ERROR "ops binary compile scripts gen failed!") - endif() - if (NOT TARGET binary) - add_custom_target(binary) - endif() - add_custom_target(${BINCMP_TARGET} - COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src - ) - add_custom_target(${BINCMP_TARGET}_gen_ops_config - COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin - COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin - -s ${BINCMP_COMPUTE_UNIT} - ) - add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) - file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) - foreach(bin_script ${bin_scripts}) - get_filename_component(bin_file ${bin_script} NAME_WE) - string(REPLACE "-" ";" bin_sep ${bin_file}) - list(GET bin_sep 0 op_type) - list(GET bin_sep 1 op_file) - list(GET bin_sep 2 op_index) - if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) - file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) - add_custom_target(${BINCMP_TARGET}_${op_file}_copy - COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py - ) - install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} - DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL - ) - install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json - DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL - ) - endif() - add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} - COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} && echo $(MAKE) - WORKING_DIRECTORY ${BINCMP_OUT_DIR} - ) - add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) - add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) - endforeach() - install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json - DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL - ) - - install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} - DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/${BINCMP_COMPUTE_UNIT} OPTIONAL - ) - install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json - DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL - ) - install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json - DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL - ) - -endfunction() - -function(add_cross_compile_target) - cmake_parse_arguments(CROSSMP "" "TARGET;OUT_DIR;INSTALL_DIR" "" ${ARGN}) - add_custom_target(${CROSSMP_TARGET} ALL - DEPENDS ${CROSSMP_OUT_DIR} - ) - install(DIRECTORY ${CROSSMP_OUT_DIR} - DESTINATION ${CROSSMP_INSTALL_DIR} - ) -endfunction() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake deleted file mode 100644 index 2f362c396..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake +++ /dev/null @@ -1,26 +0,0 @@ - -add_library(intf_pub INTERFACE) -target_compile_options(intf_pub INTERFACE - -fPIC - -fvisibility=hidden - -fvisibility-inlines-hidden - $<$:-O2> - $<$:-O0 -g> - $<$:-std=c++11> - $<$,$>:-ftrapv -fstack-check> - $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> - $,-fstack-protector-strong,-fstack-protector-all> -) -target_compile_definitions(intf_pub INTERFACE - _GLIBCXX_USE_CXX11_ABI=0 - $<$:_FORTIFY_SOURCE=2> -) -target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) -target_link_options(intf_pub INTERFACE - $<$,EXECUTABLE>:-pie> - $<$:-s> - -Wl,-z,relro - -Wl,-z,now - -Wl,-z,noexecstack -) -target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake deleted file mode 100644 index 48c565bfb..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake +++ /dev/null @@ -1,17 +0,0 @@ -execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) -execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh - --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh - --help-header ./help.info - --gzip --complevel 4 --nomd5 --sha256 - ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh - WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} - RESULT_VARIABLE EXEC_RESULT - ERROR_VARIABLE EXEC_ERROR -) -if (NOT "${EXEC_RESULT}x" STREQUAL "0x") - message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") -endif() -execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ - COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" - WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} -) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp deleted file mode 100644 index 113e0d5cd..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp +++ /dev/null @@ -1,127 +0,0 @@ -#include "kernel_operator.h" -using namespace AscendC; -constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue - -class KernelFastGeluGrad { -public: - __aicore__ inline KernelFastGeluGrad() {} - __aicore__ inline void Init(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, - uint32_t CoreDataNum, uint32_t finalTileNum, uint32_t tileDataNum, uint32_t TailDataNum, - uint32_t xyz2DataNum) { - ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); - - this->coreDataNum = CoreDataNum; - this->tileNum = finalTileNum; - this->tileDataNum = tileDataNum; - this->tailDataNum = TailDataNum; - this->xyz2DataNum = xyz2DataNum; - - xyz1Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ1*)xyz1, this->coreDataNum); - xyz2Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ2*)xyz2, this->xyz2DataNum); - distGm.SetGlobalBuffer((__gm__ DTYPE_DIST*)dist, this->coreDataNum); - indicesGm.SetGlobalBuffer((__gm__ DTYPE_INDICES*)indices, this->coreDataNum); - - // pipe.InitBuffer(inQueueXYZ1, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_XYZ1)); - // pipe.InitBuffer(inQueueXYZ2, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_XYZ2)); - // pipe.InitBuffer(outQueueDIST, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_DIST)); - // pipe.InitBuffer(outQueueINDICES, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_INDICES)); - - } - __aicore__ inline void Process() { - float ux,uy,uz; - float x,y,z; - float best1, best2, best3; - int besti1, besti2, besti3; - for(uint32_t i=0; icoreDataNum; i+=3) - { - ux = xyz1Gm.GetValue(i); - uy = xyz1Gm.GetValue(i+1); - uz = xyz1Gm.GetValue(i+2); - for(j=0; jxyz2DataNum; j+=3) - { - x = xyz2Gm.GetValue(j); - y = xyz2Gm.GetValue(j+1); - z = xyz2Gm.GetValue(j+2); - } - } - // SetValue(count*M*N + i*N+j, xGm.GetValue(count*M*N + i*N+j)); - } -// __aicore__ inline void Process() { - -// int32_t loopCount = this->tileNum; -// this->processDataNum = this->tileDataNum; -// for (int32_t i = 0; i < loopCount; i++) { -// if (i == this->tileNum - 1) { -// this->processDataNum = this->tailDataNum; -// } -// CopyIn(i); -// Compute(i); -// CopyOut(i); -// } -// } - -// private: -// __aicore__ inline void CopyIn(int32_t progress) -// { -// LocalTensor xyz1Local = inQueueXYZ1.AllocTensor(); -// LocalTensor xyz2Local = inQueueXYZ2.AllocTensor(); - -// DataCopy(xyz1Local, xyz1Gm[progress * this->tileDataNum], this->processDataNum); -// DataCopy(xyz2Local, xyz2Gm[progress * this->tileDataNum], this->processDataNum); - -// inQueueXYZ1.EnQue(xyz1Local); -// inQueueXYZ2.EnQue(xyz2Local); -// } -// __aicore__ inline void Compute(int32_t progress) -// { -// LocalTensor xyz1Local = inQueueXYZ1.DeQue(); -// LocalTensor xyz2Local = inQueueXYZ2.DeQue(); -// LocalTensor distLocal = outQueueDIST.AllocTensor(); -// LocalTensor indicesLocal = outQueueINDICES.AllocTensor(); - - -// outQueueDIST.EnQue(distLocal); -// outQueueINDICES.EnQue(indicesLocal); -// inQueueXYZ1.FreeTensor(xyz2Local); -// inQueueXYZ2.FreeTensor(xyz2Local); -// } -// __aicore__ inline void CopyOut(int32_t progress) -// { -// LocalTensor distLocal = outQueueDIST.DeQue(); -// LocalTensor indicesLocal = outQueueINDICES.DeQue(); -// DataCopy(distGm[progress * this->tileDataNum], distLocal, this->processDataNum); -// DataCopy(indicesGm[progress * this->tileDataNum], indicesLocal, this->processDataNum); -// outQueueDIST.FreeTensor(distLocal); -// outQueueINDICES.FreeTensor(indicesLocal); -// } - -private: - TPipe pipe; - TBuf tmpBuffer, signbitBuffer; - TQue inQueueXYZ1, inQueueXYZ2; - TQue outQueueDIST, outQueueINDICES; - GlobalTensor xyz1Gm; - GlobalTensor xyz2Gm; - GlobalTensor distGm; - GlobalTensor indicesGm; - uint32_t coreDataNum; - uint32_t tileNum; - uint32_t tileDataNum; - uint32_t tailDataNum; - uint32_t processDataNum; - uint32_t xyz2DataNum; - // LocalTensor signbit; -}; - -extern "C" __global__ __aicore__ void three_nn(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling) { - GET_TILING_DATA(tiling_data, tiling); - // TODO: user kernel impl - KernelFastGeluGrad op; - op.Init(xyz1, xyz2, dist, indices, - tiling_data.CoreDataNum, tiling_data.finalTileNum, tiling_data.tileDataNum, tiling_data.TailDataNum, tiling_data.xyz2DataNum); - op.Process(); -} -// extern "C" __global__ __aicore__ void three_nn(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling) { -// GET_TILING_DATA(tiling_data, tiling); -// // TODO: user kernel impl -// } \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py deleted file mode 100644 index c4ddc893a..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import sys -import os - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(PYF_PATH) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py deleted file mode 100644 index 63b15c900..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import sys -import os -import json -import hashlib -import const_var -import opdesc_parser - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - - -class BinParamBuilder(opdesc_parser.OpDesc): - def __init__(self: any, op_type: str): - super().__init__(op_type) - self.soc = '' - self.out_path = '' - - def set_soc_version(self: any, soc: str): - self.soc = soc - - def set_out_path(self: any, out_path: str): - self.out_path = out_path - - def gen_input_json(self: any): - key_map = {} - count = len(self.input_dtype[0].split(',')) - for i in range(0, count): - inputs = [] - outputs = [] - attrs = [] - op_node = {} - for idx in range(0, len(self.input_name)): - idtypes = self.input_dtype[idx].split(',') - ifmts = self.input_fmt[idx].split(',') - itype = self.input_type[idx] - para = {} - para['name'] = self.input_name[idx][:-5] - para['index'] = idx - para['dtype'] = idtypes[i] - para['format'] = ifmts[i] - para['paramType'] = itype - para['shape'] = [-2] - if itype == 'dynamic': - inputs.append([para]) - else: - inputs.append(para) - for idx in range(0, len(self.output_name)): - odtypes = self.output_dtype[idx].split(',') - ofmts = self.output_fmt[idx].split(',') - otype = self.output_type[idx] - para = {} - para['name'] = self.output_name[idx][:-5] - para['index'] = idx - para['dtype'] = odtypes[i] - para['format'] = ofmts[i] - para['paramType'] = otype - para['shape'] = [-2] - if otype == 'dynamic': - outputs.append([para]) - else: - outputs.append(para) - for attr in self.attr_list: - att = {} - att['name'] = attr - atype = self.attr_val.get(attr).get('type').lower() - att['dtype'] = atype - att['value'] = const_var.ATTR_DEF_VAL.get(atype) - attrs.append(att) - op_node['bin_filename'] = '' - op_node['inputs'] = inputs - op_node['outputs'] = outputs - if len(attrs) > 0: - op_node['attrs'] = attrs - param = {} - param['op_type'] = self.op_type - param['op_list'] = [op_node] - objstr = json.dumps(param, indent=' ') - md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() - while key_map.get(md5sum) is not None: - objstr += '1' - md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() - key_map[md5sum] = md5sum - bin_file = self.op_type + '_' + md5sum - op_node['bin_filename'] = bin_file - param_file = os.path.join(self.out_path, bin_file + '_param.json') - param_file = os.path.realpath(param_file) - with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: - json.dump(param, fd, indent=' ') - self._write_buld_cmd(param_file, bin_file, i) - - def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): - hard_soc = const_var.SOC_MAP_EXT.get(self.soc) - if not hard_soc: - hard_soc = self.soc.capitalize() - name_com = [self.op_type, self.op_file, str(index)] - compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') - compile_file = os.path.realpath(compile_file) - with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: - fd.write('#!/bin/bash\n') - fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) - cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') - fd.write(cmd) - chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') - fd.write(chk) - chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') - fd.write(chk) - fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) - - -def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): - op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) - for op_desc in op_descs: - op_desc.set_soc_version(soc) - op_desc.set_out_path(out_dir) - op_desc.gen_input_json() - - -if __name__ == '__main__': - if len(sys.argv) <= 3: - raise RuntimeError('arguments must greater than 3') - gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py deleted file mode 100644 index 647801c07..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py +++ /dev/null @@ -1,501 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import sys -import os -import re -import opdesc_parser -import const_var - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - -IMPL_HEAD = ''' -import os, sys -import ctypes -import json -import shutil -from tbe.common.platform import get_soc_spec -from tbe.common.utils import para_check -from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo -from tbe.common.buildcfg import get_default_build_config -from impl.util.platform_adapter import tbe_register -from tbe.common.buildcfg import get_current_build_config -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - -DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], - "float16": ["DT_FLOAT16", "half"], - "int8": ["DT_INT8", "int8_t"], - "int16": ["DT_INT16", "int16_t"], - "int32": ["DT_INT32", "int32_t"], - "int64": ["DT_INT64", "int64_t"], - "uint1": ["DT_UINT1", "uint8_t"], - "uint8": ["DT_UINT8", "uint8_t"], - "uint16": ["DT_UINT16", "uint16_t"], - "uint32": ["DT_UINT32", "uint32_t"], - "uint64": ["DT_UINT64", "uint64_t"], - "bool": ["DT_BOOL", "bool"], - "double": ["DT_DOUBLE", "double"], - "dual": ["DT_DUAL", "unknown"], - "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], - "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], - "string": ["DT_STRING", "unknown"], - "complex64": ["DT_COMPLEX64", "unknown"], - "complex128": ["DT_COMPLEX128", "unknown"], - "qint8": ["DT_QINT8", "unknown"], - "qint16": ["DT_QINT16", "unknown"], - "qint32": ["DT_QINT32", "unknown"], - "quint8": ["DT_QUINT8", "unknown"], - "quint16": ["DT_QUINT16", "unknown"], - "resource": ["DT_RESOURCE", "unknown"], - "string_ref": ["DT_STRING_REF", "unknown"], - "int4": ["DT_INT4", "int4b_t"], - "bfloat16": ["DT_BF16", "bfloat16_t"]}} - -def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): - options = [] - x_fmt = x.get("format") - x_dtype = x.get("dtype") - x_n_in_kernel = x_n + '_REF' if is_ref else x_n - options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) - options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) - options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) - return options - -def get_dtype_fmt_options(__inputs__, __outputs__): - options = [] - input_names = {} - output_names = {} - unique_param_name_set = set() - for idx, x in enumerate(__inputs__): - if x is None: - continue - x_n = input_names[idx].upper() - unique_param_name_set.add(x_n) - options += add_dtype_fmt_option_single(x, x_n) - - for idx, x in enumerate(__outputs__): - if x is None: - continue - x_n = output_names[idx].upper() - if x_n in unique_param_name_set: - options += add_dtype_fmt_option_single(x, x_n, True) - else: - options += add_dtype_fmt_option_single(x, x_n) - return options - -def load_dso(so_path): - try: - ctypes.CDLL(so_path) - except OSError as error : - print(error) - raise RuntimeError("cannot open %s" %(so_path)) - else: - print("load so succ ", so_path) - -def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): - compile_options = [] - if shortsoc in compile_option_list: - compile_options = compile_option_list[shortsoc] - elif '__ALLSOC__' in compile_option_list: - compile_options = compile_option_list['__ALLSOC__'] - return compile_options - -def get_kernel_source(src_file, dir_snake, dir_ex): - src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) - if os.path.exists(src_ex): - return src_ex - src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) - if os.path.exists(src): - return src - src = os.path.join(PYF_PATH, src_file) - if os.path.exists(src): - return src - return src_ex - -''' - -IMPL_API = ''' -@tbe_register.register_operator("{}", trans_bool_to_s8=False) -@para_check.check_op_params({}) -def {}({}, kernel_name="{}", impl_mode=""): - if get_current_build_config("enable_op_prebuild"): - return - __inputs__, __outputs__, __attrs__ = _build_args({}) - options = get_dtype_fmt_options(__inputs__, __outputs__) - options += ["-x", "cce"] - bisheng = shutil.which("bisheng") - if bisheng != None: - bisheng_path = os.path.dirname(bisheng) - tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) - else: - tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") - options.append("-I" + tikcpp_path) - options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) - options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) - options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) - options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) - if impl_mode == "high_performance": - options.append("-DHIGH_PERFORMANCE=1") - elif impl_mode == "high_precision": - options.append("-DHIGH_PRECISION=1") - if get_default_build_config("enable_deterministic_mode") == 1: - options.append("-DDETEMINISTIC_MODE=1") - - custom_compile_options = {}, - custom_all_compile_options = {}, - soc_version = get_soc_spec("SOC_VERSION") - soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() - custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) - custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) - options += custom_all_compile_options_soc - options += custom_compile_options_soc - - origin_func_name = "{}" - ascendc_src_dir_ex = "{}" - ascendc_src_dir = "{}" - ascendc_src_file = "{}" - src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) -''' - -REPLAY_OP_API = ''' - print("start replay Acend C Operator {}, kernel name is {}") - tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" - tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version - print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") - codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" - replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" - if PYF_PATH.endswith("dynamic"): - op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") - else: - op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") - replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") - load_dso(codegen_so_path) - load_dso(replaystub_so_path) - load_dso(replayapi_so_path) - op_type = "{}" - entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") - code_channel = get_code_channel(src, kernel_name, op_type, options) - op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ - attrs = __attrs__, impl_mode = impl_mode) - res, msg = replay_op(op_info, entry_obj, code_channel, src, options) - if not res: - print("call replay op failed for %s and get into call compile op" %(msg)) - compile_op(src, origin_func_name, op_info, options, code_channel, '{}') -''' - -COMPILE_OP_API = ''' - print("start compile Ascend C operator {}. kernel name is " + kernel_name) - op_type = "{}" - code_channel = get_code_channel(src, kernel_name, op_type, options) - op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ - attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}], param_type_dynamic = {},\\ - mc2_ctx = None, param_type_list = {}) - compile_op(src, origin_func_name, op_info, options, code_channel, '{}') -''' - -SUP_API = ''' -def {}({}, impl_mode=""): - __inputs__, __outputs__, __attrs__ = _build_args({}) - ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) - ret_dict = json.loads(ret_str) - err_code = ret_dict.get("ret_code") - sup = "Unknown" - reason = "Unknown reason" - if err_code is not None: - if err_code is 0: - sup = "True" - reason = "" - elif err_code is 1: - sup = "False" - reason = ret_dict.get("reason") - else: - sup = "Unknown" - reason = ret_dict.get("reason") - return sup, reason -''' -CAP_API = ''' -def {}({}, impl_mode=""): - __inputs__, __outputs__, __attrs__ = _build_args({}) - result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) - return result.decode("utf-8") -''' -GLZ_API = ''' -@tbe_register.register_param_generalization("{}") -def {}_generalization({}, generalize_config=None): - __inputs__, __outputs__, __attrs__ = _build_args({}) - ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) - return [json.loads(ret_str)] -''' - -ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'list_int': '[]', - 'list_float': '[]', 'list_bool': '[]', 'list_list_int': '[[]]', 'str': ''} - - -def _get_snake_str(s, i): - if s[i - 1] != '_': - if not s[i - 1].isupper(): - return "_" - elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): - return "_" - return "" - return "" - - -def optype_snake(origin_str): - temp_str = origin_str[0].lower() + origin_str[1:] - new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() - return new_str - - -def optype_snake_ex(s): - snake_case = "" - for i, c in enumerate(s): - if i == 0: - snake_case += c.lower() - elif c.isupper(): - snake_case += _get_snake_str(s, i) - snake_case += c.lower() - else: - snake_case += c - return snake_case - - -class AdpBuilder(opdesc_parser.OpDesc): - def __init__(self: any, op_type: str): - self.argsdefv = [] - self.op_compile_option:str = '{}' - super().__init__(op_type) - - - def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): - self._build_paradefault() - if impl_path != "": - src_file = os.path.join(impl_path, self.op_file + '.cpp') - if not os.path.exists(src_file): - return - out_path = os.path.abspath(path) - if self.dynamic_shape and not out_path.endswith('dynamic'): - out_path = os.path.join(path, 'dynamic') - os.makedirs(out_path, mode=0o700, exist_ok=True) - adpfile = os.path.join(out_path, self.op_file + '.py') - self._gen_op_compile_option(op_compile_option_all) - with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: - self._write_head(fd) - self._write_argparse(fd) - self._write_impl(fd) - if self.op_chk_support: - self._write_cap('check_supported', fd) - self._write_cap('get_op_support_info', fd) - if self.op_fmt_sel: - self._write_cap('op_select_format', fd) - self._write_cap('get_op_specific_info', fd) - if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': - self._write_glz(fd) - - - def _gen_op_compile_option(self:any, op_compile_option_all:list =None): - if op_compile_option_all is not None: - if self.op_type in op_compile_option_all: - self.op_compile_option = op_compile_option_all[self.op_type] - elif "__all__" in op_compile_option_all: - self.op_compile_option = op_compile_option_all["__all__"] - - - def _ip_argpack(self: any, default: bool = True) -> list: - args = [] - for i in range(len(self.input_name)): - arg = self.input_name[i] - if default and self.argsdefv[i] is not None: - arg += '=' + self.argsdefv[i] - args.append(arg) - return args - - def _op_argpack(self: any, default: bool = True) -> list: - args = [] - argidx = len(self.input_name) - for i in range(len(self.output_name)): - arg = self.output_name[i] - if default and self.argsdefv[i + argidx] is not None: - arg += '=' + self.argsdefv[i + argidx] - args.append(arg) - return args - - def _attr_argpack(self: any, default: bool = True) -> list: - args = [] - argidx = len(self.input_name) + len(self.output_name) - for i in range(len(self.attr_list)): - att = self.attr_list[i] - arg = att - if default and self.argsdefv[i + argidx] is not None: - if self.attr_val.get(att).get('type') == 'str': - arg += '="' + self.argsdefv[i + argidx] + '"' - elif self.attr_val.get(att).get('type') == 'bool': - arg += '=' + self.argsdefv[i + argidx].capitalize() - else: - arg += '=' + self.argsdefv[i + argidx] - args.append(arg) - return args - - def _build_paralist(self: any, default: bool = True) -> str: - args = [] - args.extend(self._ip_argpack(default)) - args.extend(self._op_argpack(default)) - args.extend(self._attr_argpack(default)) - return ', '.join(args) - - def _io_parachk(self: any, types: list, type_name: str) -> list: - chk = [] - for iot in types: - if iot == 'optional': - ptype = 'OPTION' - else: - ptype = iot.upper() - chk.append('para_check.{}_{}'.format(ptype, type_name)) - return chk - - def _attr_parachk(self: any) -> list: - chk = [] - for att in self.attr_list: - if self.attr_val.get(att).get('paramType') == 'optional': - pt = 'OPTION' - else: - pt = self.attr_val.get(att).get('paramType').upper() - att_type = self.attr_val.get(att).get('type').upper() - chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) - return chk - - def _build_parachk(self: any) -> str: - chk = [] - chk.extend(self._io_parachk(self.input_type, 'INPUT')) - chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) - chk.extend(self._attr_parachk()) - chk.append('para_check.KERNEL_NAME') - return ', '.join(chk) - - def _build_paradefault(self: any): - optional = False - argtypes = [] - argtypes.extend(self.input_type) - argtypes.extend(self.output_type) - for atype in argtypes: - if atype == 'optional': - optional = True - if optional: - self.argsdefv.append('None') - else: - self.argsdefv.append(None) - for attr in self.attr_list: - atype = self.attr_val.get(attr).get('paramType') - if atype == 'optional': - optional = True - attrval = self.attr_val.get(attr).get('defaultValue') - if attrval is not None: - optional = True - if type == "bool": - attrval = attrval.capitalize() - elif type == "str": - attrval = "\"" + attrval + "\"" - self.argsdefv.append(attrval) - continue - if optional: - self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) - else: - self.argsdefv.append(None) - - def _write_head(self: any, fd: object): - fd.write(IMPL_HEAD.format(self.input_ori_name, self.output_ori_name)) - - def _write_argparse(self: any, fd: object): - args = self._build_paralist(False) - fd.write('def _build_args({}):\n'.format(args)) - fd.write(' __inputs__ = []\n') - fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) - fd.write(' if arg != None:\n') - fd.write(' if isinstance(arg, (list, tuple)):\n') - fd.write(' if len(arg) == 0:\n') - fd.write(' continue\n') - fd.write(' __inputs__.append(arg[0])\n') - fd.write(' else:\n') - fd.write(' __inputs__.append(arg)\n') - fd.write(' else:\n') - fd.write(' __inputs__.append(arg)\n') - fd.write(' __outputs__ = []\n') - fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) - fd.write(' if arg != None:\n') - fd.write(' if isinstance(arg, (list, tuple)):\n') - fd.write(' if len(arg) == 0:\n') - fd.write(' continue\n') - fd.write(' __outputs__.append(arg[0])\n') - fd.write(' else:\n') - fd.write(' __outputs__.append(arg)\n') - fd.write(' else:\n') - fd.write(' __outputs__.append(arg)\n') - fd.write(' __attrs__ = []\n') - for attr in self.attr_list: - fd.write(' if {} != None:\n'.format(attr)) - fd.write(' attr = {}\n') - fd.write(' attr["name"] = "{}"\n'.format(attr)) - fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) - fd.write(' attr["value"] = {}\n'.format(attr)) - fd.write(' __attrs__.append(attr)\n') - fd.write(' return __inputs__, __outputs__, __attrs__\n') - - def _write_impl(self: any, fd: object): - argsdef = self._build_paralist() - argsval = self._build_paralist(False) - pchk = self._build_parachk() - if len(self.kern_name) > 0: - kern_name = self.kern_name - else: - kern_name = self.op_intf - src = self.op_file + '.cpp' - fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ - self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ - optype_snake_ex(self.op_type), optype_snake(self.op_type), src)) - if self.op_replay_flag: - fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ - self.op_compile_option)) - else: - fd.write(COMPILE_OP_API.format(self.op_type, self.op_type, ', '.join(self.input_name),\ - ', '.join(self.output_name), False, self.input_type + self.output_type, self.op_compile_option)) - - def _write_cap(self: any, cap_name: str, fd: object): - argsdef = self._build_paralist() - argsval = self._build_paralist(False) - if cap_name == 'check_supported': - fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) - else: - fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) - - def _write_glz(self: any, fd: object): - argsdef = self._build_paralist() - argsval = self._build_paralist(False) - fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) - - -def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): - batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') - iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') - file_map = {} - op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ - ops, dirs.get(const_var.AUTO_GEN_DIR)) - for op_desc in op_descs: - op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) - file_map[op_desc.op_type] = op_desc.op_file - return file_map - -if __name__ == '__main__': - if len(sys.argv) <= 6: - raise RuntimeError('arguments must greater equal than 6') - rep_cfg = {} - rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] - rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] - cfg_dir = {} - cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] - cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] - cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] - write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py deleted file mode 100644 index b138047af..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import sys -import os -import glob -import json -import argparse -import const_var - - -def load_json(json_file: str): - with open(json_file, encoding='utf-8') as file: - json_content = json.load(file) - return json_content - - -def get_specified_suffix_file(root_dir, suffix): - specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) - all_suffix_files = glob.glob(specified_suffix, recursive=True) - return all_suffix_files - - -def add_simplified_config(op_type, key, core_type, task_ration, objfile, config): - simple_cfg = config.get('binary_info_config.json') - op_cfg = simple_cfg.get(op_type) - if not op_cfg: - op_cfg = {} - op_cfg['dynamicRankSupport'] = True - op_cfg['simplifiedKeyMode'] = 0 - op_cfg['binaryList'] = [] - simple_cfg[op_type] = op_cfg - bin_list = op_cfg.get('binaryList') - if core_type == 0 and task_ration == "tilingKey": - bin_list.append({'coreType': core_type, 'simplifiedKey': key, \ - 'multiKernelType':1, 'binPath': objfile}) - else: - bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) - - -def add_op_config(op_file, bin_info, config): - op_cfg = config.get(op_file) - if not op_cfg: - op_cfg = {} - op_cfg['binList'] = [] - config[op_file] = op_cfg - op_cfg.get('binList').append(bin_info) - - -def gen_ops_config(json_file, soc, config): - core_type_map = {'MIX': 0, 'AiCore': 1, 'VectorCore': 2, 'MIX_AICORE':3, 'MIX_VECTOR_CORE':4} - contents = load_json(json_file) - if ('binFileName' not in contents) or ('supportInfo' not in contents): - return - json_base_name = os.path.basename(json_file) - op_dir = os.path.basename(os.path.dirname(json_file)) - support_info = contents.get('supportInfo') - bin_name = contents.get('binFileName') - bin_suffix = contents.get('binFileSuffix') - task_ration = contents.get('taskRation') - core_type = core_type_map.get(contents.get("coreType")) - bin_file_name = bin_name + bin_suffix - op_type = bin_name.split('_')[0] - op_file = op_dir + '.json' - bin_info = {} - keys = support_info.get('simplifiedKey') - if keys: - bin_info['simplifiedKey'] = keys - for key in keys: - add_simplified_config(op_type, key, core_type, task_ration, os.path.join(soc, op_dir, bin_file_name), \ - config) - bin_info['staticKey'] = support_info.get('staticKey') - bin_info['int64Mode'] = support_info.get('int64Mode') - bin_info['inputs'] = support_info.get('inputs') - bin_info['outputs'] = support_info.get('outputs') - if support_info.get('attrs'): - bin_info['attrs'] = support_info.get('attrs') - bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} - add_op_config(op_file, bin_info, config) - - -def gen_all_config(root_dir, soc): - suffix = 'json' - config = {} - config['binary_info_config.json'] = {} - all_json_files = get_specified_suffix_file(root_dir, suffix) - for _json in all_json_files: - gen_ops_config(_json, soc, config) - for cfg_key in config.keys(): - cfg_file = os.path.join(root_dir, cfg_key) - with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: - json.dump(config.get(cfg_key), fd, indent=' ') - - -def args_prase(): - parser = argparse.ArgumentParser() - parser.add_argument('-p', - '--path', - nargs='?', - required=True, - help='Parse the path of the json file.') - parser.add_argument('-s', - '--soc', - nargs='?', - required=True, - help='Parse the soc_version of ops.') - return parser.parse_args() - - -def main(): - args = args_prase() - gen_all_config(args.path, args.soc) - - -if __name__ == '__main__': - main() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py deleted file mode 100644 index 1cac7d911..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import sys -import os -import opdesc_parser -import replay_codegen -import const_var -from replay_codegen import ReplayCodeGenParams - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - - -class ReplayBuilder(opdesc_parser.OpDesc): - def __init__(self: any, op_type: str): - super().__init__(op_type) - - def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): - if not self.op_replay_flag: - print('{} replay not enabled'.format(self.op_type)) - return - argn = len(self.input_name) + len(self.output_name) + 1 - if self.op_replay_batch: - print('{} replay in batch mode'.format(self.op_type)) - else: - print('{} replay in normal mode'.format(self.op_type)) - if impl_path.endswith('op_kernel'): - implf = os.path.join(impl_path, self.op_file + '.cpp') - tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') - else: - if self.dynamic_shape: - dyn_path = 'dynamic' - else: - dyn_path = '' - implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') - tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') - rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ - self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) - rep_conf.set_batch(self.op_replay_batch) - rep_conf.set_outdir(out_path) - rep_conf.gen_replay(ops_product) - - -def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): - batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') - iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') - op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) - for op_desc in op_descs: - op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) - - -if __name__ == '__main__': - if len(sys.argv) <= 6: - raise RuntimeError('arguments must greater than 6') - rep_cfg = {} - rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] - rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] - rep_dir = {} - rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] - rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] - gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp deleted file mode 100644 index 0e8834664..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp +++ /dev/null @@ -1,117 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include "replay_def.h" -#include "code_gen.h" -#include "replay_fun.h" -#include "register/op_check.h" -#define __ASCENDC_REPLAY_CODE__ -#include - -using namespace std; -using namespace optiling; -using namespace AscendCReplay; - -extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); -extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], - int atlen, const char* kernelname[]); - -#define KERNEL_N 1 -#define ARG_N (__ARG_NUM__) -#define MAX_L (1024 * 1024 * 100) -#define MAX_E (1024 * 1024) - -int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) -{ - // gen type 1 : direct call codes 0: load .o file - if (param.gentype < 0 || param.gentype > 1) { - printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); - return 0; - } else if (param.gentype == 1 && param.objptr == nullptr) { - printf("Error: call replay with direct call mode, but code obj addr is null\n"); - return 0; - } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { - printf("Error: call replay with object file mode, but object file path is null\n"); - return 0; - } - // core_type 0:MIX 1:CUBE 2:VEC - if (core_type < 0 || core_type > 2) { - printf("Error: call replay core type is %d !\n", core_type); - return 0; - } - g_coreType = __CORE_TYPE__; - g_taskRation = param.task_ration; - g_tilingKey = param.tiling_key; - - unsigned char *buf, *jit; - char *kernel[KERNEL_N]; - int len[KERNEL_N]; - block_idx = 0; - block_num = param.block_dim; - g_ubBase = block_num; - uint8_t *code = (uint8_t *)malloc(MAX_L); - uint8_t *pos = code; - struct timespec tp1, tp2; - - clock_gettime(CLOCK_MONOTONIC, &tp1); - if (block_num > 32) { - printf("Error: block_num > 32\n"); - return 0; - } - //__OP_FOPEN__ - for (int i = 0; i < KERNEL_N; i++) { - //__OP_SET_KERNEL__ - for (int j = 0; j < ARG_N; j++) - AddArg(j, ARG_STEP * (j + 1)); -#ifdef FP_CEILING - SetCtrlFloatEnable(); -#else - SetCtrlFloatDisable(); -#endif - CodeInit(pos, true); - __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); - CodeEnd(); - kernel[i] = (char *)pos; - len[i] = CodeLen(); - pos += len[i]; - } - //__OP_FCLOSE__ - clock_gettime(CLOCK_MONOTONIC, &tp2); - buf = (unsigned char *)malloc(MAX_E); - int fd = open(param.entry_file, O_RDONLY); - if (fd < 0) { - printf("[error]: cannot find entry.o : %s\n", param.entry_file); - return 0; - } - uint32_t bufSize = read(fd, buf, MAX_E); - if (bufSize <= 0) { - printf("[error]: entry.o : %s is too small ! \n", param.entry_file); - } - close(fd); - jit = (unsigned char *)malloc(MAX_L); - printf("total code generated %ld\n", pos - code); - int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); - if (tp1.tv_sec != tp2.tv_sec) { - printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); - } else { - printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); - } - printf("new elf size %d\n", sz); - if (param.gentype == 0) { - fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); - (void)write(fd, jit, sz); - close(fd); - free(jit); - } else if (param.gentype == 1) { - *param.objptr = (char*)jit; - } - free(buf); - free(code); - return sz; -} - -REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py deleted file mode 100644 index d4e6a540c..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" -import os -import stat -import ctypes -import collections -import shutil -import subprocess -import copy - -"""CODE_* is used to cube/vector api is called in operator code -CODE_MIX means both cube and vector api is called -CODE_CUBE means only cube api is called -CODE_VEC means only vector api is called -""" -CODE_MIX = 0 -CODE_CUBE = 1 -CODE_VEC = 2 - - -def _is_v220(op_product: str): - """return if current soc version is V220 - - Returns: - res: True means V220 - """ - if op_product in ["ascend910b", "ascend910c"]: - return True - return False - - -InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ -['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) - - -def infer_code_channel(params: InfoCodeChanelParams): - """get code channel for v220, return CODE_MIX if soc version is not V220 - - Args: - src_file (str): AscendC operator code file - src_file (str): AscendC operator tiling header file - kernel_name (str): kernel function name - optype (str): operator type - compile_options (list): compile options for bisheng cmd - - Raises: - Exception: if not exist L1/L0/UB if code, it's not a aicore code - - Returns: - res (int): CODE_MIX/CODE_CUBE/CODE_VEC - """ - if not _is_v220(params.op_product): - return CODE_MIX - return CODE_VEC diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py deleted file mode 100644 index 3e71f5ab8..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py +++ /dev/null @@ -1,33 +0,0 @@ - -#!/usr/bin/env python -# coding=utf-8 -""" -Function: -The replay funtion entry -Copyright Information: -Huawei Technologies Co., Ltd. All Rights Reserved © 2020 -""" - -import os -import stat - - -REPLAY_BATCH = 'batch' -REPLAY_ITERATE = 'iterate' -CFG_IMPL_DIR = 'impl_dir' -CFG_OUT_DIR = 'out_dir' -AUTO_GEN_DIR = 'auto_gen_dir' -WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC -WMODES = stat.S_IWUSR | stat.S_IRUSR -SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', - 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1', 'ascend910c': 'Ascend910C1'} -BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ ---output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' -CHK_CMD = ''' -if ! test -f $2/{res_file} ; then - echo "$2/{res_file} not generated!" - exit 1 -fi -''' -ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], - 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh deleted file mode 100644 index 492034e49..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/bash - -project_path=$1 -build_path=$2 -vendor_name=customize -if [[ ! -d "$project_path" ]]; then - echo "[ERROR] No projcet path is provided" - exit 1 -fi - -if [[ ! -d "$build_path" ]]; then - echo "[ERROR] No build path is provided" - exit 1 -fi - -# copy aicpu kernel so operators -if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then - cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl - rm -rf ${project_path}/cpukernel/aicpu_kernel_lib -fi diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh deleted file mode 100644 index d4c27d17f..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -# Description: Generate npu_supported_ops.json -# ============================================================================== - -if [[ -z "$1" ]]; then - echo -e "[ERROR] No source dir provided" - exit 1 -fi - -if [[ -z "$2" ]]; then - echo -e "[ERROR] No destination dir provided" - exit 1 -fi - -src=$1 -dest_file=$2/npu_supported_ops.json - -if [ -f "$dest_file" ];then - chmod u+w $dest_file -fi - -echo $* - -add_ops() { - name=$1 - isHeavy=$2 - file=$3 - grep -w "\"$name\"" ${file} >/dev/null - if [ $? == 0 ];then - return - fi - echo " \"${name}\": {" >> ${file} - echo " \"isGray\": false," >> ${file} - echo " \"isHeavy\": ${isHeavy}" >> ${file} - echo " }," >> ${file} -} - -echo "{" > ${dest_file} -ini_files=$(find ${src} -name "*.ini") -for file in ${ini_files} ; do - name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') - grep 'heavyOp.flag' ${file} >/dev/null - if [ $? == 0 ];then - isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') - else - isHeavy="false" - fi - for op in ${name} ; do - add_ops ${op} "false" ${dest_file} - done -done -echo "}" >> ${dest_file} -file_count=$(cat ${dest_file} | wc -l) -line=$(($file_count-1)) -sed -i "${line}{s/,//g}" ${dest_file} - -chmod 640 "${dest_file}" -echo -e "[INFO] Succed generated ${dest_file}" - -exit 0 - diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh deleted file mode 100644 index a06cfc78d..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh +++ /dev/null @@ -1,6 +0,0 @@ -ascend_install_dir=$1 -gen_file_dir=$2 - -# create version.info -compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') -echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py deleted file mode 100644 index 28ba08757..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" -import json -import os -import sys -import stat -import const_var - - -if __name__ == '__main__': - if len(sys.argv) != 3: - print(sys.argv) - print('argv error, inert_op_info.py your_op_file lib_op_file') - sys.exit(2) - - with open(sys.argv[1], 'r') as load_f: - insert_operator = json.load(load_f) - - all_operators = {} - if os.path.exists(sys.argv[2]): - if os.path.getsize(sys.argv[2]) != 0: - with open(sys.argv[2], 'r') as load_f: - all_operators = json.load(load_f) - - for k in insert_operator.keys(): - if k in all_operators.keys(): - print('replace op:[', k, '] success') - else: - print('insert op:[', k, '] success') - all_operators[k] = insert_operator[k] - - with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: - json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py deleted file mode 100644 index 7e8dc2c55..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import os -import re -import glob -import json -import argparse - - -DATA_TPYE_DICT = { - 'float32': 0, - 'float16': 1, - 'int8': 2, - 'int16': 6, - 'uint16': 7, - 'uint8': 4, - 'int32': 3, - 'int64': 9, - 'uint32': 8, - 'uint64': 10, - 'bool': 12, - 'double': 11, - 'complex64': 16, - 'complex128': 17, - 'qint8': 18, - 'qint16': 19, - 'qint32': 20, - 'quint8': 21, - 'quint16': 22, - 'resource': 23, - 'string': 24, - 'dual': 25, - 'variant': 26, - 'bf16': 27, - 'bfloat16': 27, - 'undefined': 28, - 'int4': 29, - 'uint1': 30, - 'int2': 31 -} - -FORMAT_DICT = { - 'NCHW': 0, - 'NHWC': 1, - 'ND': 2, - 'NC1HWC0': 3, - 'FRACTAL_Z': 4, - 'NC1C0HWPAD': 5, - 'NHWC1C0': 6, - 'FSR_NCHW': 7, - 'FRACTAL_DECONV': 8, - 'C1HWNC0': 9, - 'FRACTAL_DECONV_TRANSPOSE': 10, - 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, - 'NC1HWC0_C04': 12, - 'FRACTAL_Z_C04': 13, - 'CHWN': 14, - 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, - 'HWCN': 16, - 'NC1KHKWHWC0': 17, - 'BN_WEIGHT': 18, - 'FILTER_HWCK': 19, - 'HASHTABLE_LOOKUP_LOOKUPS': 20, - 'HASHTABLE_LOOKUP_KEYS': 21, - 'HASHTABLE_LOOKUP_VALUE': 22, - 'HASHTABLE_LOOKUP_OUTPUT': 23, - 'HASHTABLE_LOOKUP_HITS': 24, - 'C1HWNCoC0': 25, - 'MD': 26, - 'NDHWC': 27, - 'FRACTAL_ZZ': 28, - 'FRACTAL_NZ': 29, - 'NCDHW': 30, - 'DHWCN': 31, - 'NDC1HWC0': 32, - 'FRACTAL_Z_3D': 33, - 'CN': 34, - 'NC': 35, - 'DHWNC': 36, - 'FRACTAL_Z_3D_TRANSPOSE': 37, - 'FRACTAL_ZN_LSTM': 38, - 'FRACTAL_Z_G': 39, - 'RESERVED': 40, - 'ALL': 41, - 'NULL': 42, - 'ND_RNN_BIAS': 43, - 'FRACTAL_ZN_RNN': 44, - 'NYUV': 45, - 'NYUV_A': 46 -} - - -def load_json(json_file: str): - with open(json_file, encoding='utf-8') as file: - json_content = json.load(file) - return json_content - - -def get_specified_suffix_file(root_dir, suffix): - specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) - all_suffix_files = glob.glob(specified_suffix, recursive=True) - return all_suffix_files - - -def get_deterministic_value(support_info): - deterministic_key = 'deterministic' - if deterministic_key not in support_info: - return 0 - deterministic_value = support_info.get(deterministic_key) - if deterministic_value == 'true': - return 1 - else: - return 0 - - -def get_precision_value(support_info): - precision_key = 'implMode' - precision_value = support_info.get(precision_key) - if precision_value == 'high_performance': - _value = 1 - elif precision_value == 'high_precision': - _value = 2 - else: - _value = 0 - return _value - - -def get_overflow_value(support_info): - return 0 - - -def get_parameters(info): - if info: - if 'dtype' in info: - data_type = info['dtype'] - data_type_value = DATA_TPYE_DICT.get(data_type) - else: - data_type_value = 0 - if 'format' in info: - _format = info['format'] - _format_value = FORMAT_DICT.get(_format) - else: - _format_value = 0 - else: - data_type_value = 0 - _format_value = 0 - return str(data_type_value), str(_format_value) - - -def get_dynamic_parameters(info): - # 动态输入时只需获取第一个参数 - return get_parameters(info[0]) - - -def get_all_parameters(support_info, _type): - result_list = list() - info_lists = support_info.get(_type) - if info_lists: - for _info in info_lists: - # 输入为列表时是动态输入 - if isinstance(_info, (list, tuple)): - data_type_value, _format_value = get_dynamic_parameters(_info) - else: - data_type_value, _format_value = get_parameters(_info) - result_list.append("{},{}".format(data_type_value, _format_value)) - return result_list - - -def get_all_input_parameters(support_info): - result = get_all_parameters(support_info, 'inputs') - return '/'.join(result) - - -def insert_content_into_file(input_file, content): - with open(input_file, 'r+') as file: - lines = file.readlines() - for index, line in enumerate(lines): - match_result = re.search(r'"staticKey":', line) - if match_result: - count = len(line) - len(line.lstrip()) - new_content = "{}{}".format(' ' * count, content) - # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 - lines.insert(index, new_content) - break - file.seek(0) - file.write(''.join(lines)) - - -def insert_simplified_keys(json_file): - contents = load_json(json_file) - # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 - if ('binFileName' not in contents) or ('supportInfo' not in contents): - return - support_info = contents.get('supportInfo') - bin_file_name = contents.get('binFileName') - # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 - if 'simplifiedKey' in support_info: - return - op_type = bin_file_name.split('_')[0] - deterministic = str(get_deterministic_value(support_info)) - precision = str(get_precision_value(support_info)) - overflow = str(get_overflow_value(support_info)) - input_parameters = get_all_input_parameters(support_info) - key = '{}/d={},p={},o={}/{}/'.format( - op_type, - deterministic, - precision, - overflow, - input_parameters) - result = '"simplifiedKey": "' + key + '",\n' - insert_content_into_file(json_file, result) - - -def insert_all_simplified_keys(root_dir): - suffix = 'json' - all_json_files = get_specified_suffix_file(root_dir, suffix) - for _json in all_json_files: - insert_simplified_keys(_json) - - -def args_prase(): - parser = argparse.ArgumentParser() - parser.add_argument('-p', - '--path', - nargs='?', - required=True, - help='Parse the path of the json file.') - return parser.parse_args() - - -def main(): - args = args_prase() - insert_all_simplified_keys(args.path) - - -if __name__ == '__main__': - main() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py deleted file mode 100644 index 2b77c970d..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - - -def gen_fun_def(title, kernel, argn, arg_type, arg_name): - entry = [] - entry.append(title) - entry.append(kernel) - entry.append('(') - args = [] - for i in range(0, argn): - args.append(arg_type + ' ' + arg_name + str(i)) - entry.append(', '.join(args)) - entry.append(')') - return ' '.join(entry) - - -def gen_batch_kernel_body(fname, argn, arg_name): - body = [] - body.append('{') - fun = [] - fun.append(fname) - fun.append('(') - args = [] - for i in range(0, argn): - args.append(arg_name + str(i)) - fun.append(', '.join(args)) - fun.append(');') - body.append(' '.join(fun)) - body.append('}') - return '\n'.join(body) - - -def gen_mc_kernel_body(kn, argn, arg_name, blknum): - body = [] - body.append('{') - body.append(' switch(block_idx) {') - for blk in range(0, blknum): - fun = [] - fun.append('{}_blk{:02d}'.format(kn, blk)) - fun.append('(') - args = [] - for i in range(0, argn): - args.append(arg_name + str(i)) - fun.append(', '.join(args)) - fun.append(')') - body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) - body.append(' default: break;') - body.append(' }') - body.append('}') - return '\n'.join(body) - - -def gen_proc_body(argn, arg_name): - body = [] - body.append('{') - args = [] - for i in range(0, argn): - args.append(arg_name + str(i)) - body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') - body.append('__asm__ ("NOP");') - body.append('__asm__ ("NOP");') - body.append('__asm__ ("NOP");') - body.append('}') - return '\n'.join(body) - - -def batch_code_gen(kn, argn, argt): - codes = [] - kernel_name = kn - proc_name = kernel_name + '_percore' - arg_num = int(argn) - data_type = argt - arg_type = '__gm__ ' + data_type + '* __restrict__' - arg_name = 'arg' - kernel_title = 'extern \"C\" __global__ __aicore__ void' - proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' - codes.append('#ifndef __aicore__') - codes.append('#define __aicore__ [aicore]') - codes.append('#endif') - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') - codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) - codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) - codes.append(gen_proc_body(arg_num, arg_name)) - return '\n'.join(codes) + '\n' - - -def mc_code_gen(kn, argn, argt, blknum): - codes = [] - kernel_name = kn - core_num = int(blknum) - arg_num = int(argn) - data_type = argt - arg_type = '__gm__ ' + data_type + '* __restrict__' - arg_name = 'arg' - kernel_title = 'extern \"C\" __global__ __aicore__ void' - proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' - codes.append('#ifndef __aicore__') - codes.append('#define __aicore__ [aicore]') - codes.append('#endif') - for i in range(0, core_num): - proc_name = '{}_blk{:02d}'.format(kernel_name, i) - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') - codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) - codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) - for i in range(0, core_num): - proc_name = '{}_blk{:02d}'.format(kernel_name, i) - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) - codes.append(gen_proc_body(arg_num, arg_name)) - return '\n'.join(codes) + '\n' diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp deleted file mode 100644 index 5079a1043..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp +++ /dev/null @@ -1,10 +0,0 @@ -#include -#include -#include -#include -#include -#include "replay_def.h" -#include "code_gen.h" -#include "replay_fun.h" -#define __ASCENDC_REPLAY_CODE__ -#include "__CCE_FILE__" diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING deleted file mode 100644 index d159169d1..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md deleted file mode 100644 index b41f01682..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md +++ /dev/null @@ -1,246 +0,0 @@ -[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) -![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) - -# makeself - Make self-extractable archives on Unix - -[makeself.sh][1] is a small shell script that generates a self-extractable -compressed tar archive from a directory. The resulting file appears as a shell script -(many of those have a **.run** suffix), and can be launched as is. The archive -will then uncompress itself to a temporary directory and an optional arbitrary -command will be executed (for example an installation script). This is pretty -similar to archives generated with WinZip Self-Extractor in the Windows world. -Makeself archives also include checksums for integrity self-validation (CRC -and/or MD5/SHA256 checksums). - -The makeself.sh script itself is used only to create the archives from a -directory of files. The resultant archive is actually a compressed (using -gzip, bzip2, or compress) TAR archive, with a small shell script stub at the -beginning. This small stub performs all the steps of extracting the files, -running the embedded command, and removing the temporary files when done. -All the user has to do to install the software contained in such an -archive is to "run" the archive, i.e **sh nice-software.run**. I recommend -using the ".run" (which was introduced by some Makeself archives released by -Loki Software) or ".sh" suffix for such archives not to confuse the users, -so that they will know they are actually shell scripts (with quite a lot of binary data -attached to them though!). - -I am trying to keep the code of this script as portable as possible, i.e it is -not relying on any bash-specific features and only calls commands that are -installed on any functioning UNIX-compatible system. This script as well as -the archives it generates should run on any Unix flavor, with any compatible -Bourne shell, provided of course that the compression programs are available. - -As of version 2.1, Makeself has been rewritten and tested on the following -platforms : - - * Linux (all distributions) - * Sun Solaris (8 and above) - * HP-UX (tested on 11.0 and 11i on HPPA RISC) - * SCO OpenUnix and OpenServer - * IBM AIX 5.1L - * macOS (Darwin) - * SGI IRIX 6.5 - * FreeBSD - * UnicOS / Cray - * Cygwin (Windows) - -If you successfully run Makeself and/or archives created with it on another -system, then please [let me know][2]! - -Examples of publicly available archives made using makeself are : - - * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; - * All game patches released by [Loki Software][4] for the Linux version of popular games ; - * The [nVidia drivers][5] for Linux - * The installer for the Linux version of [Google Earth][6] - * The [VirtualBox][7] installers for Linux - * The [Makeself][1] distribution itself ;-) - * and countless others... - -**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : - -`AddType application/x-makeself .run` - -**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : - -`export _POSIX2_VERSION=199209` - -## Usage - -The syntax of makeself is the following: - -``` -makeself.sh [args] archive_dir file_name label startup_script [script_args] -``` - - * _args_ are optional options for Makeself. The available ones are : - - * **`--version`** : Prints the version number on stdout, then exits immediately - * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) - * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. - * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. - * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. - * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. - * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. - * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. - * **`--pigz`** : Use pigz for compression. - * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). - * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. - * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. - * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. - * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. - * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) - * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. - * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). - * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. - * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. - * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. - * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. - * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. - * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. - * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. - * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. - * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. - * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. - * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. - * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). - * **`--tar-extra opt`** : Append more options to the tar command line. - - For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` - - * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. - * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. - * **`--license`** : Append a license file. - * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. - * **`--help-header file`** : Add a header to the archive's `--help` output. - * `archive_dir` is the name of the directory that contains the files to be archived - * `file_name` is the name of the archive to be created - * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. - * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. - -Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named -**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : - -`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup -` - -Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : - -`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` - -Archives generated with Makeself can be passed the following arguments: - - * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. - * **`--verbose`** : Will prompt the user before executing the embedded command - * **`--target dir`** : Allows to extract the archive in an arbitrary place. - * **`--nox11`** : Do not spawn a X11 terminal. - * **`--confirm`** : Prompt the user for confirmation before running the embedded command. - * **`--info`** : Print out general information about the archive (does not extract). - * **`--lsm`** : Print out the LSM entry, if it is present. - * **`--list`** : List the files in the archive. - * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. - * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. - * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. - * **`--noexec`** : Do not run the embedded script after extraction. - * **`--noexec-cleanup`** : Do not run the embedded cleanup script. - * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. - * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. - -Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. - -## Startup Script - -The startup script must be a regular Shell script. - -Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. - -`my-self-extracting-script.sh --fooBarFileParameter foo.bar` - -## Building and Testing - -Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. - -* To make a release: `make` -* To run all tests: `make test` - -## Maven Usage - -Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. - -## License - -Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. - -## Contributing - -I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: - - * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. - * Please explain clearly what the purpose of the patch is, and how you achieved it. - -## Download - -Get the latest official distribution [here][9] (version 2.4.2). - -The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. - -## Version history - - * **v1.0:** Initial public release - * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan - * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. - * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. - * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. - * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. - * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. - * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. - * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. - * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. - * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. - * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. - * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. - * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). - * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) - * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. - * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. - * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. - * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. - * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. - * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. - * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. - * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) - -## Links - - * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. - * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] - -## Contact - -This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. - -Contributions were included from John C. Quillan, Bjarni R. Einarsson, -Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot -your name, don't hesitate to contact me. - -This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. - -* * * - -[Stephane Peter][2] - - [1]: http://makeself.io/ - [2]: mailto:megastep@megastep.org - [3]: http://www.idsoftware.com/ - [4]: http://www.lokigames.com/products/myth2/updates.php3 - [5]: http://www.nvidia.com/ - [6]: http://earth.google.com/ - [7]: http://www.virtualbox.org/ - [8]: http://www.gnu.org/copyleft/gpl.html - [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run - [10]: https://github.com/megastep/makeself - [11]: https://github.com/megastep/loki_setup/ - [12]: http://www.unrealtournament2003.com/ - [13]: http://www.icculus.org/ - [14]: http://bre.klaki.net/programs/setup.sh/ - [15]: https://stephanepeter.com/ diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION deleted file mode 100644 index 59aa62c1f..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION +++ /dev/null @@ -1 +0,0 @@ -2.4.5 diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh deleted file mode 100644 index b5692d490..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -# -# Create a distributable archive of the current version of Makeself - -VER=`cat VERSION` -mkdir -p /tmp/makeself-$VER release -cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ -./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" - diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh deleted file mode 100644 index 940903148..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh +++ /dev/null @@ -1,660 +0,0 @@ -cat << EOF > "$archname" -#!/bin/bash -# This script was generated using Makeself $MS_VERSION -# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) -# 2022.3.19-Modified the MS_Help function and some options -# Huawei Technologies Co., Ltd. - -ORIG_UMASK=\`umask\` - -CRCsum="$CRCsum" -MD5="$MD5sum" -SHA="$SHAsum" -SIGNATURE="$Signature" -TMPROOT=\${TMPDIR:="\$HOME"} -if ! test -d "\$TMPROOT"; then - TMPROOT="\$PWD" -fi -export TMPDIR="\$TMPROOT" -USER_PWD="\$PWD" -if ! test -d "\$USER_PWD"; then - exit 1 -fi -export USER_PWD -ARCHIVE_DIR=\`dirname "\$0"\` -export ARCHIVE_DIR - -name_of_file="\$0 " -pwd_of_file="\$PWD" -label="$LABEL" -script="$SCRIPT" -scriptargs="$SCRIPTARGS" -cleanup_script="${CLEANUP_SCRIPT}" -licensetxt="$LICENSE" -helpheader='$HELPHEADER' -targetdir="$archdirname" -filesizes="$filesizes" -totalsize="$totalsize" -keep="$KEEP" -nooverwrite="$NOOVERWRITE" -quiet="n" -accept="n" -nodiskspace="n" -export_conf="$EXPORT_CONF" -decrypt_cmd="$DECRYPT_CMD" -skip="$SKIP" - -print_cmd_arg="" -if type printf > /dev/null; then - print_cmd="printf" -elif test -x /usr/ucb/echo; then - print_cmd="/usr/ucb/echo" -else - print_cmd="echo" -fi - -if test -d /usr/xpg4/bin; then - PATH=/usr/xpg4/bin:\$PATH - export PATH -fi - -if test -d /usr/sfw/bin; then - PATH=\$PATH:/usr/sfw/bin - export PATH -fi - -unset CDPATH - -MS_Printf() -{ - \$print_cmd \$print_cmd_arg "\$1" -} - -MS_PrintLicense() -{ - PAGER=\${PAGER:=more} - if test x"\$licensetxt" != x; then - PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` - if test -x "\$PAGER_PATH"; then - echo "\$licensetxt" | \$PAGER - else - echo "\$licensetxt" - fi - if test x"\$accept" != xy; then - while true - do - MS_Printf "Please type y to accept, n otherwise: " - read yn - if test x"\$yn" = xn; then - keep=n - eval \$finish; exit 1 - break; - elif test x"\$yn" = xy; then - break; - fi - done - fi - fi -} - -MS_diskspace() -{ - ( - df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' - ) -} - -MS_dd() -{ - blocks=\`expr \$3 / 1024\` - bytes=\`expr \$3 % 1024\` - # Test for ibs, obs and conv feature - if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then - dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ - { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ - test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null - else - dd if="\$1" bs=\$2 skip=1 2> /dev/null - fi -} - -MS_dd_Progress() -{ - if test x"\$noprogress" = xy; then - MS_dd "\$@" - return \$? - fi - file="\$1" - offset=\$2 - length=\$3 - pos=0 - bsize=4194304 - while test \$bsize -gt \$length; do - bsize=\`expr \$bsize / 4\` - done - blocks=\`expr \$length / \$bsize\` - bytes=\`expr \$length % \$bsize\` - ( - dd ibs=\$offset skip=1 2>/dev/null - pos=\`expr \$pos \+ \$bsize\` - MS_Printf " 0%% " 1>&2 - if test \$blocks -gt 0; then - while test \$pos -le \$length; do - dd bs=\$bsize count=1 2>/dev/null - pcent=\`expr \$length / 100\` - pcent=\`expr \$pos / \$pcent\` - if test \$pcent -lt 100; then - MS_Printf "\b\b\b\b\b\b\b" 1>&2 - if test \$pcent -lt 10; then - MS_Printf " \$pcent%% " 1>&2 - else - MS_Printf " \$pcent%% " 1>&2 - fi - fi - pos=\`expr \$pos \+ \$bsize\` - done - fi - if test \$bytes -gt 0; then - dd bs=\$bytes count=1 2>/dev/null - fi - MS_Printf "\b\b\b\b\b\b\b" 1>&2 - MS_Printf " 100%% " 1>&2 - ) < "\$file" -} - -MS_Help() -{ - cat << EOH >&2 -Usage: \$0 [options] -Options: - --help | -h Print this message - --info Print embedded info : title, default target directory, embedded script ... - --list Print the list of files in the archive - --check Checks integrity and version dependency of the archive - --quiet Quiet install mode, skip human-computer interactions - --nox11 Do not spawn an xterm - --noexec Do not run embedded script - --extract= Extract directly to a target directory (absolute or relative) - Usually used with --noexec to just extract files without running - --tar arg1 [arg2 ...] Access the contents of the archive through the tar command -\${helpheader} -EOH -} - -MS_Verify_Sig() -{ - GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` - MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` - test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` - test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` - offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` - temp_sig=\`mktemp -t XXXXX\` - echo \$SIGNATURE | base64 --decode > "\$temp_sig" - gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` - gpg_res=\$? - rm -f "\$temp_sig" - if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then - if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then - test x"\$quiet" = xn && echo "GPG signature is good" >&2 - else - echo "GPG Signature key does not match" >&2 - exit 2 - fi - else - test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 - exit 2 - fi -} - -MS_Check() -{ - OLD_PATH="\$PATH" - PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} - MD5_ARG="" - MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` - test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` - test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` - PATH="\$OLD_PATH" - - SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` - test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` - - if test x"\$quiet" = xn; then - MS_Printf "Verifying archive integrity..." - fi - offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` - fsize=\`cat "\$1" | wc -c | tr -d " "\` - if test \$totalsize -ne \`expr \$fsize - \$offset\`; then - echo " Unexpected archive size." >&2 - exit 2 - fi - verb=\$2 - i=1 - for s in \$filesizes - do - crc=\`echo \$CRCsum | cut -d" " -f\$i\` - if test -x "\$SHA_PATH"; then - if test x"\`basename \$SHA_PATH\`" = xshasum; then - SHA_ARG="-a 256" - fi - sha=\`echo \$SHA | cut -d" " -f\$i\` - if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then - test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 - else - shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; - if test x"\$shasum" != x"\$sha"; then - echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 - exit 2 - elif test x"\$quiet" = xn; then - MS_Printf " SHA256 checksums are OK." >&2 - fi - crc="0000000000"; - fi - fi - if test -x "\$MD5_PATH"; then - if test x"\`basename \$MD5_PATH\`" = xdigest; then - MD5_ARG="-a md5" - fi - md5=\`echo \$MD5 | cut -d" " -f\$i\` - if test x"\$md5" = x00000000000000000000000000000000; then - test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 - else - md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; - if test x"\$md5sum" != x"\$md5"; then - echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 - exit 2 - elif test x"\$quiet" = xn; then - MS_Printf " MD5 checksums are OK." >&2 - fi - crc="0000000000"; verb=n - fi - fi - if test x"\$crc" = x0000000000; then - test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 - else - sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` - if test x"\$sum1" != x"\$crc"; then - echo "Error in checksums: \$sum1 is different from \$crc" >&2 - exit 2 - elif test x"\$quiet" = xn; then - MS_Printf " CRC checksums are OK." >&2 - fi - fi - i=\`expr \$i + 1\` - offset=\`expr \$offset + \$s\` - done - if test x"\$quiet" = xn; then - echo " All good." - fi -} - -MS_Decompress() -{ - if test x"\$decrypt_cmd" != x""; then - { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" - else - eval "$GUNZIP_CMD" - fi - - if test \$? -ne 0; then - echo " ... Decompression failed." >&2 - fi -} - -UnTAR() -{ - if test x"\$quiet" = xn; then - tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } - else - tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } - fi -} - -MS_exec_cleanup() { - if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then - cleanup=n - cd "\$tmpdir" - eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" - fi -} - -MS_cleanup() -{ - echo 'Signal caught, cleaning up' >&2 - MS_exec_cleanup - cd "\$TMPROOT" - rm -rf "\$tmpdir" - eval \$finish; exit 15 -} - -Script_Args_Check() -{ - script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) - arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) - - for arg in \${script_supported_args}; - do - if test x"\$arg_to_test" = x"\$arg" ;then - return - fi - done - - MS_Help - exit 1 -} - -finish=true -xterm_loop= -noprogress=$NOPROGRESS -nox11=$NOX11 -copy=$COPY -ownership=$OWNERSHIP -verbose=n -cleanup=y -cleanupargs= -sig_key= - -initargs="\$@" - -while [ -n "\$*" ] -do - case "\$1" in - -h | --help) - MS_Help - exit 0 - ;; - -q | --quiet) - quiet=y - noprogress=y - shift - ;; - --info) - echo Identification: "\$label" - echo Target directory: "\$targetdir" - echo Uncompressed size: $USIZE KB - echo Compression: $COMPRESS - if test x"$ENCRYPT" != x""; then - echo Encryption: $ENCRYPT - fi - echo Date of packaging: $DATE - echo Built with Makeself version $MS_VERSION - echo Build command was: "$MS_COMMAND" - if test x"\$script" != x; then - echo Script run after extraction: - echo " " \$script \$scriptargs - fi - if test x"$copy" = xcopy; then - echo "Archive will copy itself to a temporary location" - fi - if test x"$NEED_ROOT" = xy; then - echo "Root permissions required for extraction" - fi - if test x"$KEEP" = xy; then - echo "directory \$targetdir is permanent" - else - echo "\$targetdir will be removed after extraction" - fi - exit 0 - ;; - --list) - echo Target directory: \$targetdir - offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` - for s in \$filesizes - do - MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t - offset=\`expr \$offset + \$s\` - done - exit 0 - ;; - --tar) - offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` - arg1="\$2" - shift 2 || { MS_Help; exit 1; } - for s in \$filesizes - do - MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" - offset=\`expr \$offset + \$s\` - done - exit 0 - ;; - --check) - MS_Check "\$0" y - scriptargs="\$scriptargs \$1" - shift - ;; - --noexec) - script="" - cleanup_script="" - shift - ;; - --extract=*) - keep=y - targetdir=\`echo \$1 | cut -d"=" -f2 \` - if ! shift; then MS_Help; exit 1; fi - ;; - --nox11) - nox11=y - shift - ;; - --xwin) - if test "$NOWAIT" = n; then - finish="echo Press Return to close this window...; read junk" - fi - xterm_loop=1 - shift - ;; - --phase2) - copy=phase2 - shift - ;; - --repack | --repack-path=*) - Script_Args_Check \$1 - scriptargs="\$scriptargs '\$1'" - shift - if [[ ! "\$1" =~ ^-.* ]]; then - scriptargs="\$scriptargs '\$1'" - shift - fi - ;; - *) - Script_Args_Check \$1 - scriptargs="\$scriptargs '\$1'" - shift - ;; - esac -done - -quiet_para="" -if test x"\$quiet" = xy; then - quiet_para="--quiet " -fi -scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" - -if test x"\$quiet" = xy -a x"\$verbose" = xy; then - echo Cannot be verbose and quiet at the same time. >&2 - exit 1 -fi - -if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then - echo "Administrative privileges required for this archive (use su or sudo)" >&2 - exit 1 -fi - -if test x"\$copy" \!= xphase2; then - MS_PrintLicense -fi - -case "\$copy" in -copy) - tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ - mkdir "\$tmpdir" || { - echo "Could not create temporary directory \$tmpdir" >&2 - exit 1 - } - SCRIPT_COPY="\$tmpdir/makeself" - echo "Copying to a temporary location..." >&2 - cp "\$0" "\$SCRIPT_COPY" - chmod +x "\$SCRIPT_COPY" - cd "\$TMPROOT" - exec "\$SCRIPT_COPY" --phase2 -- \$initargs - ;; -phase2) - finish="\$finish ; rm -rf \`dirname \$0\`" - ;; -esac - -if test x"\$nox11" = xn; then - if tty -s; then # Do we have a terminal? - : - else - if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? - if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable - GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" - for a in \$GUESS_XTERMS; do - if type \$a >/dev/null 2>&1; then - XTERM=\$a - break - fi - done - chmod a+x \$0 || echo Please add execution rights on \$0 - if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! - exec \$XTERM -e "\$0 --xwin \$initargs" - else - exec \$XTERM -e "./\$0 --xwin \$initargs" - fi - fi - fi - fi -fi - -if test x"\$targetdir" = x.; then - tmpdir="." -else - if test x"\$keep" = xy; then - if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then - echo "Target directory \$targetdir already exists, aborting." >&2 - exit 1 - fi - if test x"\$quiet" = xn; then - echo "Creating directory \$targetdir" >&2 - fi - tmpdir="\$targetdir" - dashp="-p" - else - tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" - dashp="" - fi - mkdir \$dashp "\$tmpdir" || { - echo 'Cannot create target directory' \$tmpdir >&2 - echo 'You should try option --extract=' >&2 - eval \$finish - exit 1 - } -fi - -location="\`pwd\`" -if test x"\$SETUP_NOCHECK" != x1; then - MS_Check "\$0" -fi -offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` - -if test x"\$verbose" = xy; then - MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " - read yn - if test x"\$yn" = xn; then - eval \$finish; exit 1 - fi -fi - -if test x"\$quiet" = xn; then - # Decrypting with openssl will ask for password, - # the prompt needs to start on new line - if test x"$ENCRYPT" = x"openssl"; then - echo "Decrypting and uncompressing \$label..." - else - MS_Printf "Uncompressing \$label" - fi -fi -res=3 -if test x"\$keep" = xn; then - trap MS_cleanup 1 2 3 15 -fi - -if test x"\$nodiskspace" = xn; then - leftspace=\`MS_diskspace "\$tmpdir"\` - if test -n "\$leftspace"; then - if test "\$leftspace" -lt $USIZE; then - echo - echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 - if test x"\$keep" = xn; then - echo "Consider setting TMPDIR to a directory with more free space." - fi - eval \$finish; exit 1 - fi - fi -fi - -for s in \$filesizes -do - if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then - if test x"\$ownership" = xy; then - (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) - fi - else - echo >&2 - echo "Unable to decompress \$0" >&2 - eval \$finish; exit 1 - fi - offset=\`expr \$offset + \$s\` -done -if test x"\$quiet" = xn; then - echo -fi - -cd "\$tmpdir" -res=0 -if test x"\$script" != x; then - if test x"\$export_conf" = x"y"; then - MS_BUNDLE="\$0" - MS_LABEL="\$label" - MS_SCRIPT="\$script" - MS_SCRIPTARGS="\$scriptargs" - MS_ARCHDIRNAME="\$archdirname" - MS_KEEP="\$KEEP" - MS_NOOVERWRITE="\$NOOVERWRITE" - MS_COMPRESS="\$COMPRESS" - MS_CLEANUP="\$cleanup" - export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS - export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS - fi - - if test x"\$verbose" = x"y"; then - yn="x" - while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN - do - MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " - read yn - if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then - eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; - elif test x"\$yn" = xn -o x"\$yn" = xN; then - echo "Unable to decompress \$script ,because of aborting! ";res=\$? - else - echo "Input value is unacceptable,please try again." - fi - done - else - eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? - fi - if test "\$res" -ne 0; then - test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 - fi -fi - -MS_exec_cleanup - -if test x"\$keep" = xn; then - cd "\$TMPROOT" - rm -rf "\$tmpdir" -fi -eval \$finish; exit \$res -EOF diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 deleted file mode 100644 index 81bf6e4ff..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 +++ /dev/null @@ -1,110 +0,0 @@ -.TH "MAKESELF" "1" "2.4.5" -.SH "NAME" -makeself \- An utility to generate self-extractable archives. -.SH "SYNTAX" -.B makeself [\fIoptions\fP] archive_dir file_name label -.B [\fIstartup_script\fP] [\fIargs\fP] -.SH "DESCRIPTION" -This program is a free (GPL) utility designed to create self-extractable -archives from a directory. -.SH "OPTIONS" -The following options are supported. -.TP 15 -.B -v, --version -Prints out the makeself version number and exits. -.TP -.B -h, --help -Print out help information. -.TP -.B --tar-quietly -Suppress verbose output from the tar command -.TP -.B --quiet -Do not print any messages other than errors -.TP -.B --gzip -Compress using gzip (default if detected). -.TP -.B --bzip2 -Compress using bzip2. -.TP -.B --pbzip2 -Compress using pbzip2. -.TP -.B --xz -Compress using xz. -.TP -.B --lzo -Compress using lzop. -.TP -.B --lz4 -Compress using lz4. -.TP -.B --compress -Compress using the UNIX 'compress' command. -.TP -.B --nocomp -Do not compress the data. -.TP -.B --complevel lvl -Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 -.TP -.B --notemp -The archive will create archive_dir in the current directory and -uncompress in ./archive_dir. -.TP -.B --copy -Upon extraction, the archive will first copy itself to a temporary directory. -.TP -.B --append -Append more files to an existing makeself archive. The label and startup scripts will then be ignored. -.TP -.B --current -Files will be extracted to the current directory. Both --current and --target dir imply --notemp. -.TP -.B --target dir -Extract directly to a target directory. Directory path can be either absolute or relative. -.TP -.B --header file -Specify location of the header script. -.TP -.B --cleanup file -Specify a cleanup script that executes on interrupt and when finished successfully. -.TP -.B --follow -Follow the symlinks in the archive. -.TP -.B --noprogress -Do not show the progress during the decompression. -.TP -.B --nox11 -Disable automatic spawn of an xterm if running in X11. -.TP -.B --nowait -Do not wait for user input after executing embedded program from an xterm. -.TP -.B --nomd5 -Do not create a MD5 checksum for the archive. -.TP -.B --nocrc -Do not create a CRC32 checksum for the archive. -.TP -.B --lsm file -LSM file describing the package. -.B --packaging-date date -Use provided string as the packaging date instead of the current date. -.SH "EXAMPLES" -Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, -and he wants to generate a self-extracting package named mysoft.sh, which will launch -the "setup" script initially stored in /home/joe/mysoft: -.TP -makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup -.TP -Here is also how I created the makeself.run archive which contains the Makeself distribution: -.TP -makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" -.SH "AUTHORS" -Makeself has been written by Stéphane Peter . -.BR -This man page was originally written by Bartosz Fenski for the -Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm deleted file mode 100644 index 3c4cea8c1..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm +++ /dev/null @@ -1,16 +0,0 @@ -Begin3 -Title: makeself.sh -Version: 2.4.5 -Description: makeself.sh is a shell script that generates a self-extractable - tar.gz archive from a directory. The resulting file appears as a shell - script, and can be launched as is. The archive will then uncompress - itself to a temporary directory and an arbitrary command will be - executed (for example an installation script). This is pretty similar - to archives generated with WinZip Self-Extractor in the Windows world. -Keywords: Installation archive tar winzip -Author: Stephane Peter (megastep@megastep.org) -Maintained-by: Stephane Peter (megastep@megastep.org) -Original-site: https://makeself.io/ -Platform: Unix -Copying-policy: GPL -End diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh deleted file mode 100644 index c8ea56597..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh +++ /dev/null @@ -1,822 +0,0 @@ -#!/bin/sh -# -# Makeself version 2.4.x -# by Stephane Peter -# -# Utility to create self-extracting tar.gz archives. -# The resulting archive is a file holding the tar.gz archive with -# a small Shell script stub that uncompresses the archive to a temporary -# directory and then executes a given script from withing that directory. -# -# Makeself home page: https://makeself.io/ -# -# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. -# -# Version history : -# - 1.0 : Initial public release -# - 1.1 : The archive can be passed parameters that will be passed on to -# the embedded script, thanks to John C. Quillan -# - 1.2 : Package distribution, bzip2 compression, more command line options, -# support for non-temporary archives. Ideas thanks to Francois Petitjean -# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: -# Support for no compression (--nocomp), script is no longer mandatory, -# automatic launch in an xterm, optional verbose output, and -target -# archive option to indicate where to extract the files. -# - 1.4 : Improved UNIX compatibility (Francois Petitjean) -# Automatic integrity checking, support of LSM files (Francois Petitjean) -# - 1.5 : Many bugfixes. Optionally disable xterm spawning. -# - 1.5.1 : More bugfixes, added archive options -list and -check. -# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big -# archives (Quake III demo) -# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. -# More verbosity in xterms and check for embedded command's return value. -# Bugfix for Debian 2.0 systems that have a different "print" command. -# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. -# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to -# bypass checksum verification of archives. -# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) -# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. -# - 2.0.1 : Added --copy -# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. -# Added --nochown for archives -# Stopped doing redundant checksums when not necesary -# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command -# Cleaned up the code to handle error codes from compress. Simplified the extraction code. -# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. -# - 2.1.3 : Bug fixes with command line when spawning terminals. -# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. -# Added --noexec to prevent execution of embedded scripts. -# Added --nomd5 and --nocrc to avoid creating checksums in archives. -# Added command used to create the archive in --info output. -# Run the embedded script through eval. -# - 2.1.4 : Fixed --info output. -# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) -# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) -# Avoid some race conditions (Ludwig Nussel) -# Unset the $CDPATH variable to avoid problems if it is set. (Debian) -# Better handling of dot files in the archive directory. -# - 2.1.5 : Made the md5sum detection consistent with the header code. -# Check for the presence of the archive directory -# Added --encrypt for symmetric encryption through gpg (Eric Windisch) -# Added support for the digest command on Solaris 10 for MD5 checksums -# Check for available disk space before extracting to the target directory (Andreas Schweitzer) -# Allow extraction to run asynchronously (patch by Peter Hatch) -# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) -# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) -# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) -# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) -# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. -# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) -# - 2.4.0 : Optional support for SHA256 checksums in archives. -# - 2.4.2 : Add support for threads for several compressors. (M. Limber) -# Added zstd support. -# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. -# - 2.4.5 : Added --tar-format to override ustar tar archive format -# -# (C) 1998-2021 by Stephane Peter -# -# This software is released under the terms of the GNU GPL version 2 and above -# Please read the license at http://www.gnu.org/copyleft/gpl.html -# Self-extracting archives created with this script are explictly NOT released under the term of the GPL -# - -MS_VERSION=2.4.5 -MS_COMMAND="$0" -unset CDPATH - -for f in ${1+"$@"}; do - MS_COMMAND="$MS_COMMAND \\\\ - \\\"$f\\\"" -done - -# For Solaris systems -if test -d /usr/xpg4/bin; then - PATH=/usr/xpg4/bin:$PATH - export PATH -fi - -# Procedures - -MS_Usage() -{ - echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" - echo "args can be one or more of the following :" - echo " --version | -v : Print out Makeself version number and exit" - echo " --help | -h : Print out this help message" - echo " --tar-quietly : Suppress verbose output from the tar command" - echo " --quiet | -q : Do not print any messages other than errors." - echo " --gzip : Compress using gzip (default if detected)" - echo " --pigz : Compress with pigz" - echo " --zstd : Compress with zstd" - echo " --bzip2 : Compress using bzip2 instead of gzip" - echo " --pbzip2 : Compress using pbzip2 instead of gzip" - echo " --xz : Compress using xz instead of gzip" - echo " --lzo : Compress using lzop instead of gzip" - echo " --lz4 : Compress using lz4 instead of gzip" - echo " --compress : Compress using the UNIX 'compress' command" - echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" - echo " --threads thds : Number of threads to be used by compressors that support parallelization." - echo " Omit to use compressor's default. Most useful (and required) for opting" - echo " into xz's threading, usually with '--threads=0' for all available cores." - echo " pbzip2 and pigz are parallel by default, and setting this value allows" - echo " limiting the number of threads they use." - echo " --base64 : Instead of compressing, encode the data using base64" - echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" - echo " --gpg-asymmetric-encrypt-sign" - echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" - echo " --gpg-extra opt : Append more options to the gpg command line" - echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" - echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" - echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" - echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." - echo " If this option is not supplied, the user will be asked to enter" - echo " encryption password on the current terminal." - echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." - echo " --nochown : Do not give the target folder to the current user (default)" - echo " --chown : Give the target folder to the current user recursively" - echo " --nocomp : Do not compress the data" - echo " --notemp : The archive will create archive_dir in the" - echo " current directory and uncompress in ./archive_dir" - echo " --needroot : Check that the root user is extracting the archive before proceeding" - echo " --copy : Upon extraction, the archive will first copy itself to" - echo " a temporary directory" - echo " --append : Append more files to an existing Makeself archive" - echo " The label and startup scripts will then be ignored" - echo " --target dir : Extract directly to a target directory" - echo " directory path can be either absolute or relative" - echo " --nooverwrite : Do not extract the archive if the specified target directory exists" - echo " --current : Files will be extracted to the current directory" - echo " Both --current and --target imply --notemp" - echo " --tar-format opt : Specify a tar archive format (default is ustar)" - echo " --tar-extra opt : Append more options to the tar command line" - echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" - echo " --nomd5 : Don't calculate an MD5 for archive" - echo " --nocrc : Don't calculate a CRC for archive" - echo " --sha256 : Compute a SHA256 checksum for the archive" - echo " --header file : Specify location of the header script" - echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." - echo " --follow : Follow the symlinks in the archive" - echo " --noprogress : Do not show the progress during the decompression" - echo " --nox11 : Disable automatic spawn of a xterm" - echo " --nowait : Do not wait for user input after executing embedded" - echo " program from an xterm" - echo " --sign passphrase : Signature private key to sign the package with" - echo " --lsm file : LSM file describing the package" - echo " --license file : Append a license file" - echo " --help-header file : Add a header to the archive's --help output" - echo " --packaging-date date" - echo " : Use provided string as the packaging date" - echo " instead of the current date." - echo - echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." - echo " --export-conf : Export configuration variables to startup_script" - echo - echo "Do not forget to give a fully qualified startup script name" - echo "(i.e. with a ./ prefix if inside the archive)." - exit 1 -} - -# Default settings -if type gzip >/dev/null 2>&1; then - COMPRESS=gzip -elif type compress >/dev/null 2>&1; then - COMPRESS=compress -else - echo "ERROR: missing commands: gzip, compress" >&2 - MS_Usage -fi -ENCRYPT=n -PASSWD="" -PASSWD_SRC="" -OPENSSL_NO_MD=n -COMPRESS_LEVEL=9 -DEFAULT_THREADS=123456 # Sentinel value -THREADS=$DEFAULT_THREADS -KEEP=n -CURRENT=n -NOX11=n -NOWAIT=n -APPEND=n -TAR_QUIETLY=n -KEEP_UMASK=n -QUIET=n -NOPROGRESS=n -COPY=none -NEED_ROOT=n -TAR_ARGS=rvf -TAR_FORMAT=ustar -TAR_EXTRA="" -GPG_EXTRA="" -DU_ARGS=-ks -HEADER=`dirname "$0"`/makeself-header.sh -SIGNATURE="" -TARGETDIR="" -NOOVERWRITE=n -DATE=`LC_ALL=C date` -EXPORT_CONF=n -SHA256=n -OWNERSHIP=n -SIGN=n -GPG_PASSPHRASE="" - -# LSM file stuff -LSM_CMD="echo No LSM. >> \"\$archname\"" - -while true -do - case "$1" in - --version | -v) - echo Makeself version $MS_VERSION - exit 0 - ;; - --pbzip2) - COMPRESS=pbzip2 - shift - ;; - --bzip2) - COMPRESS=bzip2 - shift - ;; - --gzip) - COMPRESS=gzip - shift - ;; - --pigz) - COMPRESS=pigz - shift - ;; - --zstd) - COMPRESS=zstd - shift - ;; - --xz) - COMPRESS=xz - shift - ;; - --lzo) - COMPRESS=lzo - shift - ;; - --lz4) - COMPRESS=lz4 - shift - ;; - --compress) - COMPRESS=compress - shift - ;; - --base64) - COMPRESS=base64 - shift - ;; - --gpg-encrypt) - COMPRESS=gpg - shift - ;; - --gpg-asymmetric-encrypt-sign) - COMPRESS=gpg-asymmetric - shift - ;; - --gpg-extra) - GPG_EXTRA="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --ssl-encrypt) - ENCRYPT=openssl - shift - ;; - --ssl-passwd) - PASSWD=$2 - shift 2 || { MS_Usage; exit 1; } - ;; - --ssl-pass-src) - PASSWD_SRC=$2 - shift 2 || { MS_Usage; exit 1; } - ;; - --ssl-no-md) - OPENSSL_NO_MD=y - shift - ;; - --nocomp) - COMPRESS=none - shift - ;; - --complevel) - COMPRESS_LEVEL="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --threads) - THREADS="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --nochown) - OWNERSHIP=n - shift - ;; - --chown) - OWNERSHIP=y - shift - ;; - --notemp) - KEEP=y - shift - ;; - --copy) - COPY=copy - shift - ;; - --current) - CURRENT=y - KEEP=y - shift - ;; - --tar-format) - TAR_FORMAT="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --tar-extra) - TAR_EXTRA="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --untar-extra) - UNTAR_EXTRA="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --target) - TARGETDIR="$2" - KEEP=y - shift 2 || { MS_Usage; exit 1; } - ;; - --sign) - SIGN=y - GPG_PASSPHRASE="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --nooverwrite) - NOOVERWRITE=y - shift - ;; - --needroot) - NEED_ROOT=y - shift - ;; - --header) - HEADER="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --cleanup) - CLEANUP_SCRIPT="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --license) - # We need to escape all characters having a special meaning in double quotes - LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") - shift 2 || { MS_Usage; exit 1; } - ;; - --follow) - TAR_ARGS=rvhf - DU_ARGS=-ksL - shift - ;; - --noprogress) - NOPROGRESS=y - shift - ;; - --nox11) - NOX11=y - shift - ;; - --nowait) - NOWAIT=y - shift - ;; - --nomd5) - NOMD5=y - shift - ;; - --sha256) - SHA256=y - shift - ;; - --nocrc) - NOCRC=y - shift - ;; - --append) - APPEND=y - shift - ;; - --lsm) - LSM_CMD="cat \"$2\" >> \"\$archname\"" - shift 2 || { MS_Usage; exit 1; } - ;; - --packaging-date) - DATE="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --help-header) - HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` - shift 2 || { MS_Usage; exit 1; } - [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER -" - ;; - --tar-quietly) - TAR_QUIETLY=y - shift - ;; - --keep-umask) - KEEP_UMASK=y - shift - ;; - --export-conf) - EXPORT_CONF=y - shift - ;; - -q | --quiet) - QUIET=y - shift - ;; - -h | --help) - MS_Usage - ;; - -*) - echo Unrecognized flag : "$1" - MS_Usage - ;; - *) - break - ;; - esac -done - -if test $# -lt 1; then - MS_Usage -else - if test -d "$1"; then - archdir="$1" - else - echo "Directory $1 does not exist." >&2 - exit 1 - fi -fi -archname="$2" - -if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then - if test "$TAR_ARGS" = "rvf"; then - TAR_ARGS="rf" - elif test "$TAR_ARGS" = "rvhf"; then - TAR_ARGS="rhf" - fi -fi - -if test "$APPEND" = y; then - if test $# -lt 2; then - MS_Usage - fi - - # Gather the info from the original archive - OLDENV=`sh "$archname" --dumpconf` - if test $? -ne 0; then - echo "Unable to update archive: $archname" >&2 - exit 1 - else - eval "$OLDENV" - OLDSKIP=`expr $SKIP + 1` - fi -else - if test "$KEEP" = n -a $# = 3; then - echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 - echo >&2 - MS_Usage - fi - # We don't want to create an absolute directory unless a target directory is defined - if test "$CURRENT" = y; then - archdirname="." - elif test x"$TARGETDIR" != x; then - archdirname="$TARGETDIR" - else - archdirname=`basename "$1"` - fi - - if test $# -lt 3; then - MS_Usage - fi - - LABEL="$3" - SCRIPT="$4" - test "x$SCRIPT" = x || shift 1 - shift 3 - SCRIPTARGS="$*" -fi - -if test "$KEEP" = n -a "$CURRENT" = y; then - echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 - exit 1 -fi - -case $COMPRESS in -gzip) - GZIP_CMD="gzip -c$COMPRESS_LEVEL" - GUNZIP_CMD="gzip -cd" - ;; -pigz) - GZIP_CMD="pigz -$COMPRESS_LEVEL" - if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated - GZIP_CMD="$GZIP_CMD --processes $THREADS" - fi - GUNZIP_CMD="gzip -cd" - ;; -zstd) - GZIP_CMD="zstd -$COMPRESS_LEVEL" - if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated - GZIP_CMD="$GZIP_CMD --threads=$THREADS" - fi - GUNZIP_CMD="zstd -cd" - ;; -pbzip2) - GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" - if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated - GZIP_CMD="$GZIP_CMD -p$THREADS" - fi - GUNZIP_CMD="bzip2 -d" - ;; -bzip2) - GZIP_CMD="bzip2 -$COMPRESS_LEVEL" - GUNZIP_CMD="bzip2 -d" - ;; -xz) - GZIP_CMD="xz -c$COMPRESS_LEVEL" - # Must opt-in by specifying a value since not all versions of xz support threads - if test $THREADS -ne $DEFAULT_THREADS; then - GZIP_CMD="$GZIP_CMD --threads=$THREADS" - fi - GUNZIP_CMD="xz -d" - ;; -lzo) - GZIP_CMD="lzop -c$COMPRESS_LEVEL" - GUNZIP_CMD="lzop -d" - ;; -lz4) - GZIP_CMD="lz4 -c$COMPRESS_LEVEL" - GUNZIP_CMD="lz4 -d" - ;; -base64) - GZIP_CMD="base64" - GUNZIP_CMD="base64 --decode -i -" - ;; -gpg) - GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" - GUNZIP_CMD="gpg -d" - ENCRYPT="gpg" - ;; -gpg-asymmetric) - GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" - GUNZIP_CMD="gpg --yes -d" - ENCRYPT="gpg" - ;; -compress) - GZIP_CMD="compress -fc" - GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" - ;; -none) - GZIP_CMD="cat" - GUNZIP_CMD="cat" - ;; -esac - -if test x"$ENCRYPT" = x"openssl"; then - if test x"$APPEND" = x"y"; then - echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 - fi - - ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" - DECRYPT_CMD="openssl enc -aes-256-cbc -d" - - if test x"$OPENSSL_NO_MD" != x"y"; then - ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" - DECRYPT_CMD="$DECRYPT_CMD -md sha256" - fi - - if test -n "$PASSWD_SRC"; then - ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" - elif test -n "$PASSWD"; then - ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" - fi -fi - -tmpfile="${TMPDIR:-/tmp}/mkself$$" - -if test -f "$HEADER"; then - oldarchname="$archname" - archname="$tmpfile" - # Generate a fake header to count its lines - SKIP=0 - . "$HEADER" - SKIP=`cat "$tmpfile" |wc -l` - # Get rid of any spaces - SKIP=`expr $SKIP` - rm -f "$tmpfile" - if test "$QUIET" = "n"; then - echo "Header is $SKIP lines long" >&2 - fi - archname="$oldarchname" -else - echo "Unable to open header file: $HEADER" >&2 - exit 1 -fi - -if test "$QUIET" = "n"; then - echo -fi - -if test "$APPEND" = n; then - if test -f "$archname"; then - echo "WARNING: Overwriting existing file: $archname" >&2 - fi -fi - -USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` - -if test "." = "$archdirname"; then - if test "$KEEP" = n; then - archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" - fi -fi - -test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } -if test "$QUIET" = "n"; then - echo "About to compress $USIZE KB of data..." - echo "Adding files to archive named \"$archname\"..." -fi - -# See if we have GNU tar -TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` -test -x "$TAR" || TAR=tar - -tmparch="${TMPDIR:-/tmp}/mkself$$.tar" -( - if test "$APPEND" = "y"; then - tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" - fi - cd "$archdir" - # "Determining if a directory is empty" - # https://www.etalabs.net/sh_tricks.html - find . \ - \( \ - ! -type d \ - -o \ - \( -links 2 -exec sh -c ' - is_empty () ( - cd "$1" - set -- .[!.]* ; test -f "$1" && return 1 - set -- ..?* ; test -f "$1" && return 1 - set -- * ; test -f "$1" && return 1 - return 0 - ) - is_empty "$0"' {} \; \ - \) \ - \) -print \ - | LC_ALL=C sort \ - | sed 's/./\\&/g' \ - | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" -) || { - echo "ERROR: failed to create temporary archive: $tmparch" - rm -f "$tmparch" "$tmpfile" - exit 1 -} - -USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` - -eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { - echo "ERROR: failed to create temporary file: $tmpfile" - rm -f "$tmparch" "$tmpfile" - exit 1 -} -rm -f "$tmparch" - -if test x"$ENCRYPT" = x"openssl"; then - echo "About to encrypt archive \"$archname\"..." - { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ - { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } -fi - -fsize=`cat "$tmpfile" | wc -c | tr -d " "` - -# Compute the checksums - -shasum=0000000000000000000000000000000000000000000000000000000000000000 -md5sum=00000000000000000000000000000000 -crcsum=0000000000 - -if test "$NOCRC" = y; then - if test "$QUIET" = "n"; then - echo "skipping crc at user request" - fi -else - crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` - if test "$QUIET" = "n"; then - echo "CRC: $crcsum" - fi -fi - -if test "$SHA256" = y; then - SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` - if test -x "$SHA_PATH"; then - shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` - else - SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` - shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` - fi - if test "$QUIET" = "n"; then - if test -x "$SHA_PATH"; then - echo "SHA256: $shasum" - else - echo "SHA256: none, SHA command not found" - fi - fi -fi -if test "$NOMD5" = y; then - if test "$QUIET" = "n"; then - echo "Skipping md5sum at user request" - fi -else - # Try to locate a MD5 binary - OLD_PATH=$PATH - PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} - MD5_ARG="" - MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` - test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` - test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` - PATH=$OLD_PATH - if test -x "$MD5_PATH"; then - if test `basename ${MD5_PATH}`x = digestx; then - MD5_ARG="-a md5" - fi - md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` - if test "$QUIET" = "n"; then - echo "MD5: $md5sum" - fi - else - if test "$QUIET" = "n"; then - echo "MD5: none, MD5 command not found" - fi - fi -fi -if test "$SIGN" = y; then - GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` - if test -x "$GPG_PATH"; then - SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` - if test "$QUIET" = "n"; then - echo "Signature: $SIGNATURE" - fi - else - echo "Missing gpg command" >&2 - fi -fi - -totalsize=0 -for size in $fsize; -do - totalsize=`expr $totalsize + $size` -done - -if test "$APPEND" = y; then - mv "$archname" "$archname".bak || exit - - # Prepare entry for new archive - filesizes="$fsize" - CRCsum="$crcsum" - MD5sum="$md5sum" - SHAsum="$shasum" - Signature="$SIGNATURE" - # Generate the header - . "$HEADER" - # Append the new data - cat "$tmpfile" >> "$archname" - - chmod +x "$archname" - rm -f "$archname".bak - if test "$QUIET" = "n"; then - echo "Self-extractable archive \"$archname\" successfully updated." - fi -else - filesizes="$fsize" - CRCsum="$crcsum" - MD5sum="$md5sum" - SHAsum="$shasum" - Signature="$SIGNATURE" - - # Generate the header - . "$HEADER" - - # Append the compressed tar data after the stub - if test "$QUIET" = "n"; then - echo - fi - cat "$tmpfile" >> "$archname" - chmod +x "$archname" - if test "$QUIET" = "n"; then - echo Self-extractable archive \"$archname\" successfully created. - fi -fi -rm -f "$tmpfile" diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh deleted file mode 100644 index 31ee16511..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# Run every available test - Bash needed -cd test -for test in *test; -do - echo "Running test $test ..." - bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } -done diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh deleted file mode 100644 index a977bd51d..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -project_path=$1 -build_path=$2 -vendor_name=customize -echo $@ -if [[ ! -d "$project_path" ]]; then - echo "[ERROR] No projcet path is provided" - exit 1 -fi - -if [[ ! -d "$build_path" ]]; then - echo "[ERROR] No build path is provided" - exit 1 -fi - -if [[ ! -d "$ASCEND_OPP_PATH" ]]; then - echo "[ERROR] No opp install path is provided" - exit 1 -fi -custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json -custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json -temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json - -if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then - cp -f $custom_exist_info_json $temp_info_json - chmod +w $temp_info_json - python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} - cp -f $temp_info_json $custom_new_info_json - rm -f $temp_info_json -fi diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py deleted file mode 100644 index 0d3221602..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import sys -import os - - -OP_ALL = '__ALLOP__' -SOC_ALL = '__ALLSOC__' -SOC_TO_SHORT_SOC_MAP = { - "ascend910a": "ascend910", - "ascend910proa": "ascend910", - "ascend910b": "ascend910", - "ascend910prob": "ascend910", - "ascend910premiuma": "ascend910", - "ascend910b1": "ascend910b", - "ascend910b2": "ascend910b", - "ascend910b2c": "ascend910b", - "ascend910b3": "ascend910b", - "ascend910b4": "ascend910b", - "ascend910c1": "ascend910c", - "ascend910c2": "ascend910c", - "ascend910c3": "ascend910c", - "ascend910c4": "ascend910c", - "ascend310p1": "ascend310p", - "ascend310p3": "ascend310p", - "ascend310p3vir01": "ascend310p", - "ascend310p3vir02": "ascend310p", - "ascend310p3vir04": "ascend310p", - "ascend310p3vir08": "ascend310p", - "ascend310b1": "ascend310b", - "bs9sx1aa": "bs9sx1a" -} -CONFLICT_KEYWORDS = { - "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", - "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", - "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "False", - "None", "True", "nonlocal", "arg", "__inputs__", "__outputs__", "options", "bisheng", - "bisheng_path", "tikcpp_path", "impl_mode", "custom_compile_options", - "custom_all_compile_options", "soc_version", "soc_short", "custom_compile_options_soc", - "custom_all_compile_options_soc", "origin_func_name", "ascendc_src_dir_ex", - "ascendc_src_dir", "ascendc_src_file", "src", "op_type", "code_channel", "op_info", - "compile_op", "get_code_channel", "result", "__attrs__", "isinstance", "attr", - "get_current_build_config", "_build_args", "get_dtype_fmt_options", "shutil", "os", - "get_kernel_source" -} - - -class OpDesc: - def __init__(self: any, op_type: str): - self.op_type = op_type - self.attr_list = [] - self.attr_val = {} - self.input_name = [] - self.input_ori_name = [] - self.input_type = [] - self.input_dtype = [] - self.input_fmt = [] - self.output_name = [] - self.output_ori_name = [] - self.output_type = [] - self.output_dtype = [] - self.output_fmt = [] - self.op_fmt_sel = False - self.op_chk_support = False - self.op_intf = '' - self.kern_name = '' - self.op_file = '' - self.op_replay_flag = False - self.op_replay_batch = False - self.input_idx = -1 - self.output_idx = -1 - self.max_block_dim = 32 - self.max_shape_size = 268435456 - self.dynamic_shape = False - self.op_range_limit = '' - self.custom_compile_options = {} - self.custom_all_compile_options = {} - - @staticmethod - def _parse_digit(conf: str) -> int: - return int(conf.split('=')[1]) - - @staticmethod - def _parse_flag(conf: str) -> bool: - if 'true' == conf.split('=')[1]: - return True - return False - - @staticmethod - def _parse_str(conf: str) -> str: - return conf.split('=')[1] - - @staticmethod - def _parse_list(conf: str) -> list: - return conf.split('=')[1].split(',') - - def parse_input(self: any, conf: str): - if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): - self.input_idx += 1 - self.input_ori_name.append(self._parse_str(conf)) - self.input_name.append(self.input_ori_name[-1] + '_in__') - elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): - self.input_type.append(self._parse_str(conf)) - elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): - self.input_dtype.append(self._parse_str(conf)) - elif conf.startswith('input{}.format'.format(int(self.input_idx))): - self.input_fmt.append(self._parse_str(conf)) - else: - return - - def parse_output(self: any, conf: str): - if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): - self.output_idx += 1 - self.output_ori_name.append(self._parse_str(conf)) - self.output_name.append(self.output_ori_name[-1] + '_out_') - elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): - self.output_type.append(self._parse_str(conf)) - elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): - self.output_dtype.append(self._parse_str(conf)) - elif conf.startswith('output{}.format'.format(int(self.output_idx))): - self.output_fmt.append(self._parse_str(conf)) - else: - return - - def parse_op_format(self: any, conf: str): - self.op_fmt_sel = self._parse_flag(conf) - - def parse_check_support(self: any, conf: str): - self.op_chk_support = self._parse_flag(conf) - - def parse_range_limit(self: any, conf: str): - self.op_range_limit = self._parse_str(conf) - - def parse_kern_name(self: any, conf: str): - self.kern_name = self._parse_str(conf) - - def parse_op_intf(self: any, conf: str): - self.op_intf = self._parse_str(conf) - - def parse_op_file(self: any, conf: str): - self.op_file = self._parse_str(conf) - - def parse_dynamic_shape(self: any, conf: str): - self.dynamic_shape = self._parse_flag(conf) - - def parse_attr_list(self: any, conf: str): - self.attr_list = self._parse_list(conf) - intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS - if intersection_element: - raise Exception(f'The attribute name: {intersection_element} in op: {{\'{self.op_type}\'}} conflicts with\ - the built-in variable name. Use a complex name or prefix the operator name.') - - @staticmethod - def _camel_to_snake(camel_case_str: str): - snake_case_str = '' - for i, c in enumerate(camel_case_str): - if i == 0: - snake_case_str += c.lower() - elif c.isupper(): - snake_case_str += '_' + c.lower() - else: - snake_case_str += c - return snake_case_str - - def parse_attr_val(self: any, conf: str): - for attr in self.attr_list: - if self.attr_val.get(attr) is None: - self.attr_val[attr] = {} - if conf.startswith('attr_{}.type'.format(attr)): - self.attr_val.get(attr)['type'] = self._camel_to_snake(self._parse_str(conf)) - elif conf.startswith('attr_{}.paramType'.format(attr)): - self.attr_val.get(attr)['paramType'] = self._parse_str(conf) - elif conf.startswith('attr_{}.defaultValue'.format(attr)): - self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) - - def parse_replay_val(self: any, batch_list: list, iterator_list: list): - if self.op_type in batch_list: - self.op_replay_flag = True - self.op_replay_batch = True - elif self.op_type in iterator_list: - self.op_replay_flag = True - self.op_replay_batch = False - - -def _is_op_type_in_opdesc(op_descs: list, op_type: str): - for op in op_descs: - if op_type == op.op_type: - return True - return False - - -def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): - for op in op_descs: - op.custom_all_compile_options = soc_ver_compile_options - - -def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): - for op in op_descs: - if op.op_type != op_type: - continue - op.custom_compile_options = soc_ver_compile_options - - -def _trans_soc_ver_to_short(soc_ver: str): - low_soc_ver = soc_ver.lower() - if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: - print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' - f'use ascend910b1~4 instead') - return SOC_TO_SHORT_SOC_MAP[low_soc_ver] - - -def _get_op_custom_options(op_descs: list, auto_gen_dir: str): - if auto_gen_dir is None: - return - file = os.path.join(auto_gen_dir, "custom_compile_options.ini") - if not os.path.exists(file): - print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') - return - with open(file, 'r') as fd: - lines = fd.readlines() - for line in lines: - param_list = str.split(line.rstrip('\n'), ',') - if len(param_list) != 3: - raise Exception(f'ERROR: custom compile option {param_list} len is not 3') - op_type = param_list[0] - if op_type.upper() == 'ALL': - op_type = OP_ALL - if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: - print(f'WARNING: op: {op_type} are not exists in this project') - continue - soc_ver_compile_options = {} - soc_ver = param_list[1] - options_str = param_list[2] - options = str.split(options_str, ';') - if soc_ver == '': - soc_ver_compile_options[SOC_ALL] = options - else: - soc_ver_list = str.split(soc_ver, ';') - for ver in soc_ver_list: - short_ver = _trans_soc_ver_to_short(ver) - soc_ver_compile_options[short_ver] = options - if op_type == OP_ALL: - _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) - else: - _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) - - -def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, - op_type: list, auto_gen_dir: str = None) -> list: - op_descs = [] - op_match = False - with open (file, 'r') as fd: - lines = fd.readlines() - for line in lines: - line = line.strip() - if line.startswith('['): - name = line[1:-1] - if op_type is None or name in op_type: - op_match = True - op_desc = builder(name) - op_desc.parse_replay_val(batch_list, iterator_list) - op_descs.append(op_desc) - else: - op_match = False - if op_type is not None and len(op_descs) == len(op_type): - return op_descs - continue - if not op_match: - continue - if line.startswith('input'): - op_desc.parse_input(line) - elif line.startswith('output'): - op_desc.parse_output(line) - elif line.startswith('dynamicFormat.flag'): - op_desc.parse_op_format(line) - elif line.startswith('needCheckSupport.flag'): - op_desc.parse_check_support(line) - elif line.startswith('rangeLimit.value'): - op_desc.parse_range_limit(line) - elif line.startswith('opInterface.value'): - op_desc.parse_op_intf(line) - elif line.startswith('kernel.name'): - op_desc.parse_kern_name(line) - elif line.startswith('opFile.value'): - op_desc.parse_op_file(line) - elif line.startswith('dynamicShapeSupport.flag'): - op_desc.parse_dynamic_shape(line) - elif line.startswith('attr.list'): - op_desc.parse_attr_list(line) - elif line.startswith('attr_'): - op_desc.parse_attr_val(line) - _get_op_custom_options(op_descs, auto_gen_dir) - return op_descs diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py deleted file mode 100644 index ac931e5d3..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright 2020-2021 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -parser ini to json -""" - -import json -import os -import stat -import sys - - -ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", - "type", "listType", "tensor", "listTensor"] -ATTR_PARAMTYPE_LIST = ["optional", "required"] -BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", - "needCheckSupport", "enableVectorCore"] -BOOL_LIST = ["true", "false"] -DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", - "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", - "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", - "int4", "bfloat16", "uint1"] -FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", - "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", - "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", - "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", - "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", - "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", - "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] - - -def parse_ini_files(ini_files): - """ - parse ini files to json - Parameters: - ---------------- - ini_files:input file list - return:ops_info - ---------------- - """ - tbe_ops_info = {} - for ini_file in ini_files: - check_file_size(ini_file) - parse_ini_to_obj(ini_file, tbe_ops_info) - return tbe_ops_info - - -def check_file_size(input_file): - try: - file_size = os.path.getsize(input_file) - except OSError as os_error: - print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) - raise OSError from os_error - if file_size > 10*1024*1024: - print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) - - -def parse_ini_to_obj(ini_file, tbe_ops_info): - """ - parse ini file to json obj - Parameters: - ---------------- - ini_file:ini file path - tbe_ops_info:ops_info - ---------------- - """ - with open(ini_file) as ini_file: - lines = ini_file.readlines() - op_dict = {} - op_name = "" - find_op_type = False - for line in lines: - line = line.rstrip() - if line == "": - continue - if line.startswith("["): - if line.endswith("]"): - op_name = line[1:-1] - op_dict = {} - tbe_ops_info[op_name] = op_dict - find_op_type = True - elif "=" in line: - key1 = line[:line.index("=")] - key2 = line[line.index("=")+1:] - key1_0, key1_1 = key1.split(".") - if key1_0 not in op_dict: - op_dict[key1_0] = {} - if key1_1 in op_dict.get(key1_0): - raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + - key1_1 + " is repeated!") - dic_key = op_dict.get(key1_0) - dic_key[key1_1] = key2 - else: - continue - if not find_op_type: - raise RuntimeError("Not find OpType in .ini file.") - - -def check_output_exist(op_dict, is_valid): - """ - Function Description: - Check output is exist - Parameter: op_dict - Parameter: is_valid - """ - if "output0" in op_dict: - output0_dict = op_dict.get("output0") - if output0_dict.get("name", None) is None: - is_valid = False - print("output0.name is required in .ini file!") - else: - is_valid = False - print("output0 is required in .ini file!") - return is_valid - - -def check_attr_dict(attr_dict, is_valid, attr): - """ - Function Description: - Check attr_dict - Parameter: attr_dict - Parameter: is_valid - Parameter: attr - """ - attr_type = attr_dict.get("type") - value = attr_dict.get("value") - param_type = attr_dict.get("paramType") - if attr_type is None or value is None: - is_valid = False - print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) - if param_type and param_type not in ATTR_PARAMTYPE_LIST: - is_valid = False - print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) - if attr_type and attr_type not in ATTR_TYPE_LIST: - is_valid = False - print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) - return is_valid - - -def check_attr(op_dict, is_valid): - """ - Function Description: - Check attr - Parameter: op_dict - Parameter: is_valid - """ - if "attr" in op_dict: - attr_dict = op_dict.get("attr") - attr_list_str = attr_dict.get("list", None) - if attr_list_str is None: - is_valid = False - print("attr.list is required in .ini file!") - else: - attr_list = attr_list_str.split(",") - for attr_name in attr_list: - attr = "attr_" + attr_name.strip() - attr_dict = op_dict.get(attr) - if attr_dict: - is_valid = check_attr_dict(attr_dict, is_valid, attr) - else: - is_valid = False - print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) - return is_valid - - -def check_bool_flag(op_dict, is_valid): - """ - Function Description: - check_bool_flag - Parameter: op_dict - Parameter: is_valid - """ - for key in BOOL_FLAG_KEY: - if key in op_dict: - op_bool_key = op_dict.get(key) - if op_bool_key.get("flag").strip() not in BOOL_LIST: - is_valid = False - print("{0}.flag only support {1}.".format(key, BOOL_LIST)) - return is_valid - - -def check_type_format(op_info, is_valid, op_info_key): - """ - Function Description: - Check type and format - Parameter: op_info - Parameter: is_valid - Parameter: op_info_key - """ - op_info_dtype_str = op_info.get("dtype") - op_info_dtype_num = 0 - op_info_format_num = 0 - if op_info_dtype_str: - op_info_dtype = op_info_dtype_str.split(",") - op_info_dtype_num = len(op_info_dtype) - for dtype in op_info_dtype: - if dtype.strip() not in DTYPE_LIST: - is_valid = False - print("{0}.dtype not support {1}.".format(op_info_key, dtype)) - op_info_format_str = op_info.get("format") - if op_info_format_str: - op_info_format = op_info_format_str.split(",") - op_info_format_num = len(op_info_format) - for op_format in op_info_format: - if op_format.strip() not in FORMAT_LIST: - is_valid = False - print("{0}.format not support {1}.".format(op_info_key, op_format)) - if op_info_dtype_num > 0 and op_info_format_num > 0: - if op_info_dtype_num != op_info_format_num: - is_valid = False - print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) - return is_valid - - -def check_op_info(tbe_ops): - """ - Function Description: - Check info. - Parameter: tbe_ops - Return Value: is_valid - """ - print("\n\n==============check valid for ops info start==============") - required_op_input_info_keys = ["paramType", "name"] - required_op_output_info_keys = ["paramType", "name"] - param_type_valid_value = ["dynamic", "optional", "required"] - is_valid = True - for op_key in tbe_ops: - op_dict = tbe_ops[op_key] - for op_info_key in op_dict: - if op_info_key.startswith("input"): - op_input_info = op_dict[op_info_key] - missing_keys = [] - for required_op_input_info_key in required_op_input_info_keys: - if required_op_input_info_key not in op_input_info: - missing_keys.append(required_op_input_info_key) - if len(missing_keys) > 0: - print("op: " + op_key + " " + op_info_key + " missing: " + - ",".join(missing_keys)) - is_valid = False - else: - if not op_input_info["paramType"] in param_type_valid_value: - print("op: " + op_key + " " + op_info_key + \ - " paramType not valid, valid key:[dynamic, " - "optional, required]") - is_valid = False - is_valid = check_type_format(op_input_info, is_valid, op_info_key) - if op_info_key.startswith("output"): - op_input_info = op_dict[op_info_key] - missing_keys = [] - for required_op_input_info_key in required_op_output_info_keys: - if required_op_input_info_key not in op_input_info: - missing_keys.append(required_op_input_info_key) - if len(missing_keys) > 0: - print("op: " + op_key + " " + op_info_key + " missing: " + - ",".join(missing_keys)) - is_valid = False - else: - if not op_input_info["paramType"] in param_type_valid_value: - print("op: " + op_key + " " + op_info_key + - " paramType not valid, valid key:[dynamic, " - "optional, required]") - is_valid = False - is_valid = check_type_format(op_input_info, is_valid, op_info_key) - is_valid = check_attr(op_dict, is_valid) - is_valid = check_bool_flag(op_dict, is_valid) - print("==============check valid for ops info end================\n\n") - return is_valid - - -def write_json_file(tbe_ops_info, json_file_path): - """ - Save info to json file - Parameters: - ---------------- - tbe_ops_info: ops_info - json_file_path: json file path - ---------------- - """ - json_file_real_path = os.path.realpath(json_file_path) - wr_flag = os.O_WRONLY | os.O_CREAT - wr_mode = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: - # The owner have all rights，group only have read rights - os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP - + stat.S_IRUSR) - json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, - separators=(',', ':')) - print("Compile op info cfg successfully.") - - -def parse_ini_to_json(ini_file_paths, outfile_path): - """ - parse ini files to json file - Parameters: - ---------------- - ini_file_paths: list of ini file path - outfile_path: output file path - ---------------- - """ - tbe_ops_info = parse_ini_files(ini_file_paths) - if not check_op_info(tbe_ops_info): - print("Compile op info cfg failed.") - return False - write_json_file(tbe_ops_info, outfile_path) - return True - - -if __name__ == '__main__': - args = sys.argv - - OUTPUT_FILE_PATH = "tbe_ops_info.json" - ini_file_path_list = [] - - for arg in args: - if arg.endswith("ini"): - ini_file_path_list.append(arg) - OUTPUT_FILE_PATH = arg.replace(".ini", ".json") - if arg.endswith("json"): - OUTPUT_FILE_PATH = arg - - if len(ini_file_path_list) == 0: - ini_file_path_list.append("tbe_ops_info.ini") - - if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): - sys.exit(1) - sys.exit(0) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py deleted file mode 100644 index 8f1124b1d..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py +++ /dev/null @@ -1,23 +0,0 @@ -import json -import sys -import os - - -def get_config_opts(file): - src_dir = os.path.abspath(os.path.dirname(file)) - opts = '' - with open(file, 'r') as fd: - config = json.load(fd) - for conf in config: - if conf == 'configurePresets': - for node in config[conf]: - macros = node.get('cacheVariables') - if macros is not None: - for key in macros: - opts += '-D{}={} '.format(key, macros[key]['value']) - opts = opts.replace('${sourceDir}', src_dir) - print(opts) - - -if __name__ == "__main__": - get_config_opts(sys.argv[1]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py deleted file mode 100644 index 1baa364ef..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import os -import stat -import collections -import kernel_entry as keb -from tiling_data_def_build import gen_tiling -import code_channel_infer -import const_var - -PYF_PATH = os.path.dirname(__file__) - -ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ -['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) - - -class ReplayCodeGen: - def __init__(self, replayCodeGenParams): - self.op_type = replayCodeGenParams.op_type - self.impl = replayCodeGenParams.impl - self.tiling_file = replayCodeGenParams.tiling_file - self.tiling_data_file = '' - self.kernel = replayCodeGenParams.kernel - self.entry = replayCodeGenParams.entry - self.argn = replayCodeGenParams.argn - self.batch = False - self.outdir = '' - self.data_type = 'uint8_t' - self.blknum = 32 - self.op_replay_batch = replayCodeGenParams.op_replay_batch - self.max_block_dim = replayCodeGenParams.max_block_dim - self.max_shape_size = replayCodeGenParams.max_shape_size - - def set_batch(self, is_batch): - self.batch = is_batch - - def set_outdir(self, outdir): - self.outdir = outdir - - def gen_replay(self, ops_product: str): - kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') - kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') - replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') - if self.batch: - reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') - else: - reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') - kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') - self._gen_kentry(kerentry) - self._gen_kimpl_code(kerimpl, kertmp) - self._gen_tiling_data_header() - self._gen_replay_code(replayimpl, reptmp, ops_product) - - def _gen_tiling_data_header(self): - self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') - gen_tiling(self.tiling_file, self.tiling_data_file) - - def _gen_kimpl_code(self, src, tmpfile): - with open(tmpfile, 'r') as fd: - temp = fd.read() - temp = temp.replace('__CCE_FILE__', self.impl) - with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: - ofd.write(temp) - - def _gen_replay_code(self, src, tmpfile, ops_product: str): - with open(tmpfile, 'r') as fd: - temp = fd.read() - temp = temp.replace('__ARG_NUM__', str(self.argn)) - argdef = [] - kargs = [] - for i in range(0, self.argn): - argdef.append('{} *'.format(self.data_type)) - kargs.append('({} *)GetArg({})'.format(self.data_type, i)) - temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) - temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) - temp = temp.replace('__KERNEL_FUN__', self.entry) - core_type_infer = 'core_type' - code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ - self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) - if code_channel == code_channel_infer.CODE_VEC: - core_type_infer = '0' - elif code_channel == code_channel_infer.CODE_CUBE: - core_type_infer = '1' - temp = temp.replace('__CORE_TYPE__', core_type_infer) - # regist function - temp = temp.replace('__OPS_PRODUCT__', ops_product) - temp = temp.replace('__OPTYPE__', self.op_type) - with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: - ofd.write(temp) - - def _gen_kentry(self, src): - kf = '' - pre_alloc_str = 'A' * 256 - if self.batch: - kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) - else: - kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ - self.argn, self.data_type, self.blknum) - with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: - ofd.write(kf) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp deleted file mode 100644 index 1d30dd865..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp +++ /dev/null @@ -1,120 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include "replay_def.h" -#include "code_gen.h" -#include "replay_fun.h" -#include "register/op_check.h" -#define __ASCENDC_REPLAY_CODE__ -using namespace std; -using namespace optiling; -using namespace AscendCReplay; - -extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); -extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], - int alen[], int atlen, const char* kernelname[]); - -#define KERNEL_N 1 -#define ARG_N (__ARG_NUM__) -#define MAX_L (1024 * 1024 * 100) -#define MAX_E (1024 * 1024) - -int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) -{ - // gen type 1 : direct call codes 0: load .o file - if (param.gentype < 0 || param.gentype > 1) { - printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); - return 0; - } else if (param.gentype == 1 && param.objptr == nullptr) { - printf("Error: call replay with direct call mode, but code obj addr is null\n"); - return 0; - } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { - printf("Error: call replay with object file mode, but object file path is null\n"); - return 0; - } - // core_type 0:MIX 1:CUBE 2:VEC - if (core_type < 0 || core_type > 2) { - printf("Error: call replay core type is %d !\n", core_type); - return 0; - } - g_coreType = __CORE_TYPE__; - g_taskRation = param.task_ration; - g_tilingKey = param.tiling_key; - - unsigned char *buf, *jit; - char *kernel[KERNEL_N * 32]; - int len[KERNEL_N * 32]; - int blknum[KERNEL_N]; - int max; - block_num = param.block_dim; - g_ubBase = block_num; - uint8_t *code = (uint8_t *)malloc(MAX_L); - uint8_t *pos = code; - struct timespec tp1, tp2; - - clock_gettime(CLOCK_MONOTONIC, &tp1); - if (block_num > 32) { - printf("Error: block_num > 32\n"); - return 0; - } - //__OP_FOPEN__ - for (int i = 0; i < KERNEL_N; i++) { - for (int j = 0; j < ARG_N; j++) - AddArg(j, ARG_STEP * (j + 1)); - for (block_idx = 0; block_idx < block_num; block_idx++) { - //__OP_SET_KERNEL__ - int code_idx = i * block_num + block_idx; -#ifdef FP_CEILING - SetCtrlFloatEnable(); -#else - SetCtrlFloatDisable(); -#endif - CodeInit(pos, false); - __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); - CodeEnd(); - kernel[code_idx] = (char *)pos; - len[code_idx] = CodeLen(); - pos += len[code_idx]; - printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); - } - blknum[i] = block_num; - } - //__OP_FCLOSE__ - clock_gettime(CLOCK_MONOTONIC, &tp2); - buf = (unsigned char *)malloc(MAX_E); - int fd = open(param.entry_file, O_RDONLY); - if (fd < 0) { - printf("[error]: cannot find entry.o : %s\n", param.entry_file); - return 0; - } - uint32_t bufSize = read(fd, buf, MAX_E); - if (bufSize <= 0) { - printf("[error]: entry.o : %s is too small ! \n", param.entry_file); - } - close(fd); - jit = (unsigned char *)malloc(MAX_L); - printf("total code generated %ld\n", pos - code); - int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); - if (tp1.tv_sec != tp2.tv_sec) { - printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); - } else { - printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); - } - printf("new elf size %d\n", sz); - if (param.gentype == 0) { - fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); - (void)write(fd, jit, sz); - close(fd); - free(jit); - } else if (param.gentype == 1) { - *param.objptr = (char*)jit; - } - free(buf); - free(code); - return sz; -} - -REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py deleted file mode 100644 index 4c5b099b7..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -""" -Function: -The replay funtion entry -Copyright Information: -Huawei Technologies Co., Ltd. All Rights Reserved © 2020 -""" - -import sys -import os -import stat -import re -import const_var - - -def gen_tiling(tiling_header_file: str, tiling_file_out: str): - if not os.path.exists(tiling_header_file): - print("warning: no userdef tiling header file: ", tiling_header_file) - return - print("generate tiling def header file: ", tiling_file_out) - tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() - tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) - tiling_source += '#define __{}_H__\n\n'.format(tmp_name) - tiling_source += '#include \n' - tiling_source += '#include \n\n' - tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' - end_source = "" - pattern = re.compile(r'[(](.*)[)]', re.S) - with open(tiling_header_file, 'r') as fd: - lines = fd.readlines() - for line in lines: - line = line.strip() - if (line.startswith('BEGIN_TILING_DATA_DEF')): - tiling_source += '#pragma pack(1)\n' - tiling_source += 'struct ' - struct_def = re.findall(pattern, line)[0] - tiling_source += struct_def + ' {\n' - elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): - field_params = re.findall(pattern, line)[0] - fds = field_params.split(',') - tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) - elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): - field_params = re.findall(pattern, line)[0] - fds = field_params.split(',') - tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) - elif (line.startswith('TILING_DATA_FIELD_DEF')): - field_params = re.findall(pattern, line)[0] - fds = field_params.split(',') - tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) - elif (line.startswith('END_TILING_DATA_DEF')): - tiling_source += '};\n' - tiling_source += '#pragma pack()\n\n' - tiling_source += '#ifdef __NPU_TILING__\n' - tiling_source += \ - 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ - .format(stru=struct_def) - tiling_source += '{\n' - tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' - tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' - tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ - .format(struct_def) - tiling_source += '}\n' - tiling_source += '#else\n' - tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) - tiling_source += '{\n' - tiling_source += ' uint64_t *src = (uint64_t *)tiling;\n' - tiling_source += ' uint64_t *dst = (uint64_t *)const_data;\n' - tiling_source += ' for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n'\ - .format(struct_def) - tiling_source += '}\n' - tiling_source += '#endif\n\n' - end_source = ''' -#define GET_TILING_DATA(tiling_data, tiling_arg) \\ -{stru} tiling_data; \\ -Init{stru}(tiling_arg, &tiling_data)\n -'''.format(stru=struct_def) - tiling_source += end_source - tiling_source += '#endif' - with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: - ofd.write(tiling_source) - - -if __name__ == '__main__': - if len(sys.argv) <= 2: - raise RuntimeError('arguments must greater than 2') - gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt deleted file mode 100644 index b6be9b492..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") - if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") - add_subdirectory(caffe_plugin) - endif() - if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") - add_subdirectory(tf_plugin) - endif() - if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") - add_subdirectory(onnx_plugin) - endif() -endif() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt deleted file mode 100644 index a6aba5c20..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ - -aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) -add_library(cust_tf_parsers SHARED ${plugin_srcs}) -target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) -if(ENABLE_CROSS_COMPILE) - target_link_directories(cust_tf_parsers PRIVATE - ${CMAKE_COMPILE_COMPILER_LIBRARY} - ${CMAKE_COMPILE_RUNTIME_LIBRARY} - ) -endif() -target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) -install(TARGETS cust_tf_parsers - LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow -) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc deleted file mode 100644 index 1809e029a..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc +++ /dev/null @@ -1,23 +0,0 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All -rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the Apache License Version 2.0. - * You may not use this file except in compliance with the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Apache License for more details at - * http://www.apache.org/licenses/LICENSE-2.0 - */ - -#include "register/register.h" - -namespace domi { -// register op info to GE -REGISTER_CUSTOM_OP("ThreeNN") - .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW - .OriginOpType("ThreeNN") // name in tf module - .ParseParamsByOperatorFn(AutoMappingByOpFn); -} // namespace domi diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt deleted file mode 100644 index 40dd51cfa..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt +++ /dev/null @@ -1,82 +0,0 @@ - -aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) - -opbuild(OPS_SRC ${ops_srcs} - OUT_DIR ${ASCEND_AUTOGEN_PATH} -) - -add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) -target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) -target_compile_options(cust_op_proto PRIVATE - -fvisibility=hidden -) -if(ENABLE_CROSS_COMPILE) - target_link_directories(cust_op_proto PRIVATE - ${CMAKE_COMPILE_COMPILER_LIBRARY} - ${CMAKE_COMPILE_RUNTIME_LIBRARY} - ) -endif() -target_link_libraries(cust_op_proto PRIVATE - intf_pub - exe_graph - register - tiling_api - -Wl,--whole-archive - rt2_registry - -Wl,--no-whole-archive -) -set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME - cust_opsproto_rt2.0 -) -add_library(cust_optiling SHARED ${ops_srcs}) -target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) -target_compile_options(cust_optiling PRIVATE - -fvisibility=hidden -) -if(ENABLE_CROSS_COMPILE) - target_link_directories(cust_optiling PRIVATE - ${CMAKE_COMPILE_COMPILER_LIBRARY} - ${CMAKE_COMPILE_RUNTIME_LIBRARY} - ) -endif() -target_link_libraries(cust_optiling PRIVATE - intf_pub - exe_graph - register - tiling_api - -Wl,--whole-archive - rt2_registry - -Wl,--no-whole-archive -) -set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME - cust_opmaster_rt2.0 -) - -file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) -file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) -add_library(cust_opapi SHARED ${aclnn_src}) -if(ENABLE_CROSS_COMPILE) - target_link_directories(cust_opapi PRIVATE - ${CMAKE_COMPILE_COMPILER_LIBRARY} - ${CMAKE_COMPILE_RUNTIME_LIBRARY} - ) -endif() -target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) - -add_custom_target(optiling_compat ALL - COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ - ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so -) - -install(TARGETS cust_op_proto - LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) -install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h - DESTINATION packages/vendors/${vendor_name}/op_proto/inc) -install(TARGETS cust_optiling - LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so - DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) -install(TARGETS cust_opapi - LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) -install(FILES ${aclnn_inc} - DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp deleted file mode 100644 index 027ef0850..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp +++ /dev/null @@ -1,109 +0,0 @@ - -#include "three_nn_tiling.h" -#include "register/op_def_registry.h" -#include "tiling/platform/platform_ascendc.h" -#include - -namespace optiling { -const uint32_t BLOCK_SIZE = 32*3*4; -static ge::graphStatus TilingFunc(gert::TilingContext* context) -{ - ThreeNNTilingData tiling; - - uint64_t ubSize; - auto ascendcPlatform = platform_ascendc::PlatformAscendC(context->GetPlatformInfo()); - // auto socVersion = ascendcPlatform.GetSocVersion(); - ascendcPlatform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ubSize); //获取硬件平台存储空间 UB 的内存大小 - uint32_t shapeB = context->GetInputTensor(0)->GetOriginShape().GetDim(0); - uint32_t shapeN = context->GetInputTensor(0)->GetOriginShape().GetDim(1); - uint32_t shapeM = context->GetInputTensor(1)->GetOriginShape().GetDim(1); - - //获取输入shape信息 - uint32_t inputNum = context->GetInputShape(1)->GetStorageShape().GetShapeSize()/shapeB; //输入数量 xyz2 - uint32_t inputBytes = GetSizeByDataType(context->GetInputDesc(1)->GetDataType()); //输入类型 - uint32_t inputLength = inputBytes * inputNum; //输入长度 - - //可使用的ub空间 输入3输出1，手动考虑双缓存 - uint32_t ubDataNumber = 4;//(inputBytes == 2) ? 10 : 6; - - // The number of 32B data blocks that can be used for each data. DOUBLE BUFFER is already counted here - uint32_t tileBlockNum = (ubSize / BLOCK_SIZE) / ubDataNumber; //每个ub段可用的空间块数 - uint32_t tileDataNum = 1440; - - // Input data for 32B alignment - uint32_t inputLengthAlgin32 = (((inputLength + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE); //输入长度 对齐处理 - uint32_t everyCoreInputBlockNum = inputLengthAlgin32 / BLOCK_SIZE;// 输入数据需要多少空间块 - - uint32_t TileNum = inputNum / tileDataNum; - uint32_t finalTileNum = (inputNum % tileDataNum) == 0 ? TileNum : TileNum + 1; //需要循环处理几次 - // Tail block calculation for chunks of data - uint32_t TailDataNum = inputNum - (tileDataNum * TileNum);//CoreDataNum - (tileDataNum * TileNum); - TailDataNum = TailDataNum == 0 ? tileDataNum : TailDataNum; //最后一次需要处理的数据量 - - tiling.set_CoreDataNum((inputNum*shapeB+31)/32*32); //对齐空间后的输入数量 - tiling.set_finalTileNum(finalTileNum);//需要循环处理几次 - tiling.set_tileDataNum(tileDataNum); //每个b处理的数据量 - tiling.set_TailDataNum(TailDataNum); //最后一次需要处理的数据量 - - tiling.set_shapeB(shapeB); - tiling.set_shapeN(shapeN); - tiling.set_shapeM(shapeM); - - context->SetBlockDim(1); - tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); - context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); - size_t *currentWorkspace = context->GetWorkspaceSizes(1); - currentWorkspace[0] = 0; - return ge::GRAPH_SUCCESS; -} -} - - -namespace ge { -static ge::graphStatus InferShape(gert::InferShapeContext* context) -{ - const gert::Shape* x1_shape = context->GetInputShape(0); - gert::Shape* y_shape = context->GetOutputShape(0); - *y_shape = *x1_shape; - return GRAPH_SUCCESS; -} -} - - -namespace ops { -class ThreeNN : public OpDef { -public: - explicit ThreeNN(const char* name) : OpDef(name) - { - this->Input("xyz1") - .ParamType(REQUIRED) - .DataType({ge::DT_FLOAT}) - .Format({ge::FORMAT_ND}) - .UnknownShapeFormat({ge::FORMAT_ND}); - this->Input("xyz2") - .ParamType(REQUIRED) - .DataType({ge::DT_FLOAT}) - .Format({ge::FORMAT_ND}) - .UnknownShapeFormat({ge::FORMAT_ND}); - this->Output("dist") - .ParamType(REQUIRED) - .DataType({ge::DT_FLOAT}) - .Format({ge::FORMAT_ND}) - .UnknownShapeFormat({ge::FORMAT_ND}); - this->Output("indices") - .ParamType(REQUIRED) - .DataType({ge::DT_INT32}) - .Format({ge::FORMAT_ND}) - .UnknownShapeFormat({ge::FORMAT_ND}); - - this->SetInferShape(ge::InferShape); - - this->AICore() - .SetTiling(optiling::TilingFunc); - this->AICore().AddConfig("ascend310b"); - - } -}; - -OP_ADD(ThreeNN); -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h deleted file mode 100644 index 5da548fed..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h +++ /dev/null @@ -1,16 +0,0 @@ - -#include "register/tilingdata_base.h" - -namespace optiling { -BEGIN_TILING_DATA_DEF(ThreeNNTilingData) - TILING_DATA_FIELD_DEF(uint32_t, CoreDataNum); - TILING_DATA_FIELD_DEF(uint32_t, finalTileNum); - TILING_DATA_FIELD_DEF(uint32_t, tileDataNum); - TILING_DATA_FIELD_DEF(uint32_t, TailDataNum); - - TILING_DATA_FIELD_DEF(uint32_t, shapeB); - TILING_DATA_FIELD_DEF(uint32_t, shapeN); - TILING_DATA_FIELD_DEF(uint32_t, shapeM); -END_TILING_DATA_DEF; -REGISTER_TILING_DATA_CLASS(ThreeNN, ThreeNNTilingData) -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt deleted file mode 100644 index 8c94a952d..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt +++ /dev/null @@ -1,68 +0,0 @@ -# set custom compile options -if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") - add_ops_compile_options(ALL OPTIONS -g -O0) -endif() - -foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) - - # generate aic-${compute_unit}-ops-info.json - add_ops_info_target(TARGET ops_info_gen_${compute_unit} - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json - OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini - INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} - ) - - # generate ascendc impl py once - if (NOT TARGET ascendc_impl_gen) - add_ops_impl_target(TARGET ascendc_impl_gen - OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini - IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} - OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe - INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl - ) - endif() - - # dynamic shape binary compile - if (${ENABLE_BINARY_PACKAGE} AND NOT ${ENABLE_CROSS_COMPILE}) - add_bin_compile_target(TARGET ascendc_bin_${compute_unit} - OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini - IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} - ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic - OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} - INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel - COMPUTE_UNIT ${compute_unit} - ) - add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) - endif() - - if (${ENABLE_CROSS_COMPILE} AND ${ENABLE_BINARY_PACKAGE}) - add_cross_compile_target( - TARGET bin_${compute_unit} - OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../kernel - INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/ - ) - endif() -endforeach() - -# generate npu_supported_ops.json -add_npu_support_target(TARGET npu_supported_ops - OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} - OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core - INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} -) - -if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) - add_subdirectory(testcases) -endif() - -# install kernel file -if (${ENABLE_SOURCE_PACKAGE}) - file(GLOB KERNEL_FILES - ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/*.h - ${CMAKE_CURRENT_SOURCE_DIR}/*.py - ) - install(FILES ${KERNEL_FILES} - DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic - ) -endif() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp deleted file mode 100644 index 9d60d095b..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp +++ /dev/null @@ -1,166 +0,0 @@ -#define K_MAX_SHAPE_DIM 0 -#include "kernel_operator.h" -#include -using namespace AscendC; -constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue - -class KernelFastGeluGrad { -public: - __aicore__ inline KernelFastGeluGrad() {} - __aicore__ inline void Init(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, - uint32_t CoreDataNum, uint32_t finalTileNum, uint32_t tileDataNum, uint32_t TailDataNum, - uint32_t shapeB, uint32_t shapeN, uint32_t shapeM) { - ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); - - this->coreDataNum = CoreDataNum; - this->tileNum = finalTileNum; - this->tileDataNum = tileDataNum; - this->tailDataNum = TailDataNum; - - this->shapeB = shapeB; - this->shapeN = shapeN; - this->shapeM = shapeM; - - xyz1Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ1*)xyz1, this->coreDataNum); - xyz2Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ2*)xyz2, this->coreDataNum); - distGm.SetGlobalBuffer((__gm__ DTYPE_DIST*)dist, this->coreDataNum); - indicesGm.SetGlobalBuffer((__gm__ DTYPE_INDICES*)indices, this->coreDataNum); - - pipe.InitBuffer(inQueueXYZ2, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - pipe.InitBuffer(QueueTmp1, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - pipe.InitBuffer(QueueTmp2, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - pipe.InitBuffer(QueueTmp3, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - - pipe.InitBuffer(QueueTmp10, 32 * sizeof(DTYPE_XYZ2)); - - pipe.InitBuffer(QueueTmpX, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - pipe.InitBuffer(QueueTmpY, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - pipe.InitBuffer(QueueTmpZ, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - - pipe.InitBuffer(QueueMask, sizeof(uint32_t)*(128+36)); - - } - __aicore__ inline void Process() { - int32_t loopCount; - int32_t b=0; - - LocalTensor mask_scl = QueueMask.Get(); - auto tmpx = QueueTmpX.Get(); - auto tmpy = QueueTmpY.Get(); - auto tmpz = QueueTmpZ.Get(); - - uint64_t mask1[2] = {10540996613548315209, 5270498306774157604}; - uint64_t mask2[2] = {2635249153387078802, 10540996613548315209}; - uint64_t mask3[2] = {5270498306774157604, 2635249153387078802}; - uint32_t scalar = 1227133513; - Duplicate(mask_scl, scalar, mask1, 2, 1, 8 ); - - scalar = 2454267026; - Duplicate(mask_scl, scalar, mask2, 2, 1, 8 ); - - scalar = 613566756; - Duplicate(mask_scl, scalar, mask3, 2, 1, 8 ); - - this->processDataNum = this->tailDataNum; - for(int32_t b=0; bshapeB; b++) - { - CopyIn(b, 0); - Compute(b, 0, mask_scl, tmpx, tmpy, tmpz); - } - } - -private: - __aicore__ inline void CopyIn(int32_t b, int32_t progress) - { - LocalTensor xyz2Local = inQueueXYZ2.AllocTensor(); - DataCopy(xyz2Local, xyz2Gm[3*b*this->shapeM], (3*this->shapeM+31)/32*32); - inQueueXYZ2.EnQue(xyz2Local); - } - __aicore__ inline void Compute(int32_t b, int32_t progress, LocalTensor mask_scl, LocalTensor tmpx, LocalTensor tmpy, LocalTensor tmpz) - { - LocalTensor xyz2Local = inQueueXYZ2.DeQue(); - auto tmp1 = QueueTmp1.Get(); - auto tmp2 = QueueTmp2.Get(); - auto tmp3 = QueueTmp3.Get(); - auto tmp10 = QueueTmp10.Get(); - - float ux,uy,uz; - int besti1, besti2, besti3; - uint32_t calculate_num = this->shapeM; - uint32_t mask = this->shapeM*3; - uint64_t rsvdCnt = 0; - GatherMask (tmpx, xyz2Local, mask_scl, true, mask, { 1, 1, 8, 8 }, rsvdCnt); - GatherMask (tmpy, xyz2Local, mask_scl[16], true, mask, { 1, 1, 8, 8 }, rsvdCnt); - GatherMask (tmpz, xyz2Local, mask_scl[8], true, mask, { 1, 1, 8, 8 }, rsvdCnt); - - uint32_t b_xyz1_index = 3*(b*this->shapeN); - for(uint32_t i=0; i < this->shapeN; i++) - { - ux = -xyz1Gm.GetValue(b_xyz1_index); - uy = -xyz1Gm.GetValue(b_xyz1_index+1); - uz = -xyz1Gm.GetValue(b_xyz1_index+2); - - Adds(tmp3, tmpx, ux, calculate_num); - Mul(tmp2, tmp3, tmp3, calculate_num); - - Adds(tmp3, tmpy, uy, calculate_num); - Mul(tmp1, tmp3, tmp3, calculate_num); - - Add(tmp3, tmp2, tmp1, calculate_num); - - Adds(tmp2, tmpz, uz, calculate_num); - Mul(tmp1, tmp2, tmp2, calculate_num); - - Add(tmp3, tmp3, tmp1, calculate_num); - - ReduceMin(tmp2, tmp3, tmp1, calculate_num, true); - besti1 = (tmp2.ReinterpretCast()).GetValue(1); - tmp3.SetValue(besti1, 3.402823466e+38F); - - ReduceMin(tmp2[8], tmp3, tmp1, calculate_num, true); - besti2 = (tmp2.ReinterpretCast()).GetValue(9); - tmp3.SetValue(besti2, 3.402823466e+38F); - - ReduceMin(tmp2[16], tmp3, tmp1, calculate_num, true); - besti3 = (tmp2.ReinterpretCast()).GetValue(17); - - indicesGm.SetValue(b_xyz1_index, besti1); - indicesGm.SetValue(b_xyz1_index+1, besti2); - indicesGm.SetValue(b_xyz1_index+2, besti3); - - distGm.SetValue(b_xyz1_index, tmp2.GetValue(0)); - distGm.SetValue(b_xyz1_index+1, tmp2.GetValue(8)); - distGm.SetValue(b_xyz1_index+2, tmp2.GetValue(16)); - b_xyz1_index+=3; - } - inQueueXYZ2.FreeTensor(xyz2Local); - } - -private: - TPipe pipe; - TQue inQueueXYZ2; - TBuf QueueTmp1, QueueTmp2, QueueTmp3, QueueTmpX, QueueTmpY, QueueTmpZ, QueueMask, QueueTmp10; - GlobalTensor xyz1Gm; - GlobalTensor xyz2Gm; - GlobalTensor distGm; - GlobalTensor indicesGm; - uint32_t coreDataNum; - uint32_t tileNum; - uint32_t tileDataNum; - uint32_t tailDataNum; - uint32_t processDataNum; - uint32_t xyz2DataNum; - uint32_t shapeB; - uint32_t shapeN; - uint32_t shapeM; -}; - -extern "C" __global__ __aicore__ void three_nn(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling) { - GET_TILING_DATA(tiling_data, tiling); - // TODO: user kernel impl - KernelFastGeluGrad op; - op.Init(xyz1, xyz2, dist, indices, - tiling_data.CoreDataNum, tiling_data.finalTileNum, tiling_data.tileDataNum, tiling_data.TailDataNum, - tiling_data.shapeB, tiling_data.shapeN, tiling_data.shapeM); - op.Process(); -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh deleted file mode 100644 index 8468c5a25..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh +++ /dev/null @@ -1,318 +0,0 @@ -#!/bin/bash -vendor_name=customize -targetdir=/usr/local/Ascend/opp -target_custom=0 - -sourcedir=$PWD/packages -vendordir=vendors/$vendor_name - -QUIET="y" - -while true -do - case $1 in - --quiet) - QUIET="y" - shift - ;; - --install-path=*) - INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) - INSTALL_PATH=${INSTALL_PATH%*/} - shift - ;; - --*) - shift - ;; - *) - break - ;; - esac -done - -log() { - cur_date=`date +"%Y-%m-%d %H:%M:%S"` - echo "[runtime] [$cur_date] "$1 -} - -if [ -n "${INSTALL_PATH}" ]; then - if [[ ! "${INSTALL_PATH}" = /* ]]; then - log "[ERROR] use absolute path for --install-path argument" - exit 1 - fi - if [ ! -d ${INSTALL_PATH} ]; then - mkdir ${INSTALL_PATH} >> /dev/null 2>&1 - if [ $? -ne 0 ]; then - log "[ERROR] create ${INSTALL_PATH} failed" - exit 1 - fi - fi - targetdir=${INSTALL_PATH} -elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then - if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then - mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 - if [ $? -ne 0 ]; then - log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" - fi - fi - targetdir=${ASCEND_CUSTOM_OPP_PATH} -else - if [ "x${ASCEND_OPP_PATH}" == "x" ]; then - log "[ERROR] env ASCEND_OPP_PATH no exist" - exit 1 - fi - targetdir="${ASCEND_OPP_PATH}" -fi - -if [ ! -d $targetdir ];then - log "[ERROR] $targetdir no exist" - exit 1 -fi - -upgrade() -{ - if [ ! -d ${sourcedir}/$vendordir/$1 ]; then - log "[INFO] no need to upgrade ops $1 files" - return 0 - fi - - if [ ! -d ${targetdir}/$vendordir/$1 ];then - log "[INFO] create ${targetdir}/$vendordir/$1." - mkdir -p ${targetdir}/$vendordir/$1 - if [ $? -ne 0 ];then - log "[ERROR] create ${targetdir}/$vendordir/$1 failed" - return 1 - fi - else - has_same_file=-1 - for file_a in ${sourcedir}/$vendordir/$1/*; do - file_b=${file_a##*/}; - if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then - log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" - return 1 - fi - grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; - if [[ $? -eq 0 ]]; then - echo -n "${file_b} " - has_same_file=0 - fi - done - if [ 0 -eq $has_same_file ]; then - if test $QUIET = "n"; then - echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ - you want to Overlay Installation , please enter:[o]; \ - or replace directory installation , please enter: [r]; \ - or not install , please enter:[n]." - - while true - do - read orn - if [ "$orn" = n ]; then - return 0 - elif [ "$orn" = m ]; then - break; - elif [ "$0rn" = r ]; then - [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* - break; - else - echo "[ERROR] input error, please input again!" - fi - done - fi - fi - log "[INFO] replace or merge old ops $1 files .g....." - fi - - log "copy new ops $1 files ......" - if [ -d ${targetdir}/$vendordir/$1/ ]; then - chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 - fi - cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ - if [ $? -ne 0 ];then - log "[ERROR] copy new $1 files failed" - return 1 - fi - - return 0 -} -upgrade_proto() -{ - if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then - log "[INFO] no need to upgrade custom.proto files" - return 0 - fi - if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then - log "[INFO] create ${targetdir}/$vendordir/framework/caffe." - mkdir -p ${targetdir}/$vendordir/framework/caffe - if [ $? -ne 0 ];then - log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" - return 1 - fi - else - if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then - # 有老版本,判断是否要覆盖式安装 - if test $QUIET = "n"; then - echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ - "custom.proto file. Do you want to replace? [y/n] " - - while true - do - read yn - if [ "$yn" = n ]; then - return 0 - elif [ "$yn" = y ]; then - break; - else - echo "[ERROR] input error, please input again!" - fi - done - fi - fi - log "[INFO] replace old caffe.proto files ......" - fi - chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 - cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ - if [ $? -ne 0 ];then - log "[ERROR] copy new custom.proto failed" - return 1 - fi - log "[INFO] copy custom.proto success" - - return 0 -} - -upgrade_file() -{ - if [ ! -e ${sourcedir}/$vendordir/$1 ]; then - log "[INFO] no need to upgrade ops $1 file" - return 0 - fi - - log "copy new $1 files ......" - cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 - if [ $? -ne 0 ];then - log "[ERROR] copy new $1 file failed" - return 1 - fi - - return 0 -} - -delete_optiling_file() -{ - if [ ! -d ${targetdir}/vendors ];then - log "[INFO] $1 not exist, no need to uninstall" - return 0 - fi - sys_info=$(uname -m) - if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then - rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so - fi - return 0 -} - -log "[INFO] copy uninstall sh success" - -if [ ! -d ${targetdir}/vendors ];then - log "[INFO] create ${targetdir}/vendors." - mkdir -p ${targetdir}/vendors - if [ $? -ne 0 ];then - log "[ERROR] create ${targetdir}/vendors failed" - return 1 - fi -fi -chmod u+w ${targetdir}/vendors - -echo "[ops_custom]upgrade framework" -upgrade framework -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade op proto" -upgrade op_proto -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade version.info" -upgrade_file version.info -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade op impl" -delete_optiling_file op_impl -upgrade op_impl -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade op api" -upgrade op_api -if [ $? -ne 0 ];then - exit 1 -fi - -upgrade_proto -if [ $? -ne 0 ];then - exit 1 -fi - -# set the set_env.bash -if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then - _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} - bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" - set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" - if [ ! -d ${bin_path} ]; then - mkdir -p ${bin_path} >> /dev/null 2>&1 - if [ $? -ne 0 ]; then - log "[ERROR] create ${bin_path} failed" - exit 1 - fi - fi - echo -e ${set_env_variable} > ${bin_path}/set_env.bash - if [ $? -ne 0 ]; then - log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" - exit 1 - else - log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ - execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" - fi -else - config_file=${targetdir}/vendors/config.ini - if [ ! -f ${config_file} ]; then - touch ${config_file} - chmod 640 ${config_file} - echo "load_priority=$vendor_name" > ${config_file} - if [ $? -ne 0 ];then - echo "echo load_priority failed" - exit 1 - fi - else - found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" - found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') - vendor=$(echo $found_vendor | tr -s ' ' ',') - if [ "$vendor" != "" ]; then - sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" - fi - fi -fi - -chmod u-w ${targetdir}/vendors - -if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then - chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 -fi -if [ -f ${targetdir}/ascend_install.info ]; then - chmod -R 440 ${targetdir}/ascend_install.info -fi -if [ -f ${targetdir}/scene.info ]; then - chmod -R 440 ${targetdir}/scene.info -fi -if [ -f ${targetdir}/version.info ]; then - chmod -R 440 ${targetdir}/version.info -fi - -echo "SUCCESS" -exit 0 - diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh deleted file mode 100644 index e09173485..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash -vendor_name=customize -targetdir=/usr/local/Ascend/opp -target_custom=0 - -sourcedir=$PWD/packages -vendordir=vendors/$vendor_name - -log() { - cur_date=`date +"%Y-%m-%d %H:%M:%S"` - echo "[runtime] [$cur_date] "$1 -} - -if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then - log "[ERROR] env ASCEND_OPP_PATH no exist" - exit 1 -fi - -targetdir=${ASCEND_OPP_PATH} - -if [ ! -d $targetdir ];then - log "[ERROR] $targetdir no exist" - exit 1 -fi - -upgrade() -{ - if [ ! -d ${sourcedir}/$vendordir/$1 ]; then - log "[INFO] no need to upgrade ops $1 files" - return 0 - fi - - if [ ! -d ${targetdir}/$vendordir/$1 ];then - log "[INFO] create ${targetdir}/$vendordir/$1." - mkdir -p ${targetdir}/$vendordir/$1 - if [ $? -ne 0 ];then - log "[ERROR] create ${targetdir}/$vendordir/$1 failed" - return 1 - fi - else - vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) - for i in $vendor_installed_dir;do - vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) - if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then - echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ - you want to Overlay Installation , please enter:[o]; \ - or replace directory installation , please enter: [r]; \ - or not install , please enter:[n]." - fi - while true - do - read mrn - if [ "$mrn" = m ]; then - break - elif [ "$mrn" = r ]; then - [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" - break - elif [ "$mrn" = n ]; then - return 0 - else - echo "[WARNING]: Input error, please input m or r or n to choose!" - fi - done - done - log "[INFO] replace old ops $1 files ......" - fi - - log "copy new ops $1 files ......" - cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ - if [ $? -ne 0 ];then - log "[ERROR] copy new $1 files failed" - return 1 - fi - - return 0 -} - -upgrade_file() -{ - if [ ! -e ${sourcedir}/$vendordir/$1 ]; then - log "[INFO] no need to upgrade ops $1 file" - return 0 - fi - - log "copy new $1 files ......" - cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 - if [ $? -ne 0 ];then - log "[ERROR] copy new $1 file failed" - return 1 - fi - - return 0 -} - -log "[INFO] copy uninstall sh success" - -echo "[ops_custom]upgrade framework" -upgrade framework -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade op proto" -upgrade op_proto -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade op impl" -upgrade op_impl -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade op api" -upgrade op_api -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade version.info" -upgrade_file version.info -if [ $? -ne 0 ];then - exit 1 -fi - -config_file=${targetdir}/vendors/config.ini -found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" -found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') -vendor=$(echo $found_vendor | tr -s ' ' ',') -if [ "$vendor" != "" ]; then - sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" -fi - -changemode() -{ - if [ -d ${targetdir} ];then - chmod -R 550 ${targetdir}>/dev/null 2>&1 - fi - - return 0 -} -echo "[ops_custom]changemode..." -#changemode -if [ $? -ne 0 ];then - exit 1 -fi - -echo "SUCCESS" -exit 0 - diff --git a/operator_contrib/ThreeNNSample/README.en.md b/operator_contrib/ThreeNNSample/README.en.md deleted file mode 100644 index 19000a924..000000000 --- a/operator_contrib/ThreeNNSample/README.en.md +++ /dev/null @@ -1,160 +0,0 @@ -## ThreeNN Custom Operator Sample Description - -This sample implements the ThreeNN operator using the Ascend C programming language and provides corresponding end-to-end implementations based on different operator invocation methods. - -- [FrameworkLaunch](./FrameworkLaunch): Using a framework to call the ThreeNN custom operator. - -The development process follows the workflow of project creation -> operator implementation -> compilation and deployment -> operator invocation. The entire process relies on the operator project: the operator kernel function and tiling are developed based on the project code framework, the operator is compiled and deployed via project compilation scripts, enabling the use of single operators or operators within third-party frameworks. - -The following invocation methods are included in this sample: - -| Invocation Method | Directory | Description | -| ----------------- | --------- | ----------- | -| [FrameworkLaunch](./FrameworkLaunch) | [AclNNInvocation](./FrameworkLaunch/AclNNInvocation) | Calling the ThreeNN operator through aclnn. | - -## Operator Description - -For each point in the target point set, the ThreeNN operator finds the distances and coordinates of the three closest points from a reference set. The returned points are sorted in ascending order of squared distance. - -## Operator Specification - - - - - - - - - - - -
Operator TypeThreeNN
Operator inputnameshapedata typeformatdefault
xyz1(b, n, 3)float32ND\
xyz2(b, m, 3)float32ND\
Operator output
dist(b, n, 3)float32ND\
indices(b, n, 3)int32ND\
kernel nameThreeNN
- -### 1. Operator Project Overview - -The operator project directory ThreeNN contains template files, compilation scripts, etc., as follows: - -``` -├── AclNNInvocation // Directory for invoking the ThreeNN operator via aclnn -│ ├── inc // Header file directory -│ │ ├── common.h // Declaration of common methods, used for reading binary files -│ │ ├── op_runner.h // Operator description declaration, including inputs/outputs, operator type, input and output descriptions -│ │ └── operator_desc.h // Operator runtime information declaration file -│ ├── input // Directory for input data generated by scripts -│ ├── output // Directory for storing operator output data and ground truth -│ ├── scripts -│ │ ├── acl.json // ACL configuration file -│ │ ├── gen_data.py // Script for generating input and ground truth data -│ │ └── verify_result.py // File for verifying results -│ ├── src -│ │ ├── CMakeLists.txt // Compilation rules -│ │ ├── common.cpp // Common functions, implementation for reading binary files -│ │ ├── main.cpp // Entry point for single operator invocation -│ │ ├── op_runner.cpp // Main process for single operator invocation -│ │ └── operator_desc.cpp // Construct operator input and output descriptions -│ ├── run.sh // Script for running the test sample -├── ThreeNN // Custom operator project -│ ├── cmake -│ ├── framework // Directory for implementing operator plugins, single operator model generation does not depend on plugins, not needed -│ ├── op_host // Host-side implementation -│ ├── op_kernel // Kernel-side implementation -│ ├── scripts // Directory for packaging custom operator project-related scripts -│ ├── build.sh // Compilation entry script -│ ├── CMakeLists.txt // CMakeLists.txt for the operator project -│ └── CMakePresets.json // Compilation configuration file -│ └── ThreeNN.json // Configuration file for generating the custom operator using msopgen -``` - -The CANN software package provides the project creation tool msopgen. The ThreeNN operator project can be automatically created using ThreeNN.json. For details, please refer to [Ascend C Operator Development](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC) > Operator Development > Operator Project Development > Creating an Operator Project. - -### 2. Compile the Operator Project - -Compile the custom operator project to generate the custom operator package. - -- Run the following commands to navigate to the ReduceCustom operator project directory. - - Use the command line to download the sample code, using the master branch as an example. - - ```bash - cd ${git_clone_path}/samples/operator_contrib/ThreeNNSample/ThreeNN - ``` - -- Modify `CMakePresets.json` to set `ASCEND_CANN_PACKAGE_PATH` to the actual path where the CANN software package is installed. - - ```json - { - …… - "configurePresets": [ - { - …… - "ASCEND_CANN_PACKAGE_PATH": { - "type": "PATH", - "value": "/usr/local/Ascend/ascend-toolkit/latest" - // Replace with the actual installation path of the CANN software package, e.g., /home/HwHiAiUser/Ascend/ascend-toolkit/latest - }, - …… - } - ] - } - ``` - -- Run the following commands in the ReduceCustom operator project directory to compile the operator project. - - ```bash - ./build.sh - ``` - -After a successful compilation, a `build_out` directory is created in the current directory, which contains the custom operator installation package `custom_opp__.run`, for example, `custom_opp_ubuntu_x86_64.run`. - -Note: To use the dump debug feature, remove the Atlas training series product configuration from the op_host and CMakeLists.txt for Atlas 200/500 A2 inference products. - -### 3. Deploy the Operator Package - -Run the following commands in the directory containing the custom operator installation package to install the custom operator package. - -```bash -cd build_out -./custom_opp__.run -``` - -After successful execution, the relevant files in the custom operator package will be deployed to the `vendors/customize` directory in the OPP operator library of the current environment. - -### 4. Configure Environment Variables - -Please select the appropriate command to configure the environment variables according to the [installation method](https://hiascend.com/document/redirect/CannCommunityInstSoftware) of the CANN development kit package in the current environment. - -- Default path, root user installed CANN software package - - ```bash - export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest - ``` - -- Default path, non-root user installed CANN software package - - ```bash - export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest - ``` - -- Specify path `install_path`, installed CANN software package - - ```bash - export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest - ``` - -### 5. Run Operator Sample - -- [aclnn calls the ThreeNN operator project (test sample)](./ThreeNNCase) - -```bash -cd ./AclNNInvocation -./run.sh -``` - -The sample will automatically generate test data, then compile and run the aclnn sample, and finally verify the results. For details, see the `run.sh` script. - -## Update Notes - -| Date | Update Content | -| ---------- | --------------------- | -| 2024/10/24 | Uploaded ThreeNN code | - diff --git a/operator_contrib/ThreeNNSample/README.md b/operator_contrib/ThreeNNSample/README.md deleted file mode 100644 index 39f6afce4..000000000 --- a/operator_contrib/ThreeNNSample/README.md +++ /dev/null @@ -1,145 +0,0 @@ -## ThreeNN自定义算子样例说明 -本样例通过Ascend C编程语言实现了ThreeNN算子，并按照不同的算子调用方式分别给出了对应的端到端实现。 -- [FrameworkLaunch](./FrameworkLaunch)：使用框架调用ThreeNN自定义算子。 -按照工程创建->算子实现->编译部署>算子调用的流程完成算子开发。整个过程都依赖于算子工程：基于工程代码框架完成算子核函数的开发和Tiling实现，通过工程编译脚本完成算子的编译部署，继而实现单算子调用或第三方框架中的算子调用。 - -本样例中包含如下调用方式： - - - - - - -
调用方式目录描述
FrameworkLaunch AclNNInvocation通过aclnn调用的方式调用ThreeNN算子。
- -## 算子描述 -对于目标点集中每个点，找到比较点集中距离其最近的3个点的距离、坐标，返回的三个点会按照距离平方升序。 - -## 算子规格描述 - - - - - - - - - - -
算子类型(OpType)ThreeNN
算子输入nameshapedata typeformatdefault
xyz1(b, n, 3)float32ND\
xyz2(b, m, 3)float32ND\
算子输出
dist(b, n, 3)float32ND\
indices(b, n, 3)int32ND\
核函数名ThreeNN
- -### 1.算子工程介绍 -其中，算子工程目录ThreeNN包含算子实现的模板文件、编译脚本等，如下所示: -``` -├── AclNNInvocation // 通过aclnn调用的方式调用ThreeNN算子 -│ ├── inc // 头文件目录 -│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 -│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 -│ │ └── operator_desc.h // 算子运行相关信息声明文件，包含算子 -│ ├── input // 存放脚本生成的输入数据目录 -│ ├── output // 存放算子运行输出数据和真值数据的目录 -│ ├── scripts -│ │ ├── acl.json // acl配置文件 -│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 -│ │ └── verify_result.py // 真值对比文件 -│ ├── src -│ │ ├── CMakeLists.txt // 编译规则文件 -│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 -│ │ ├── main.cpp // 单算子调用应用的入口 -│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 -│ │ └── operator_desc.cpp // 构造算子的输入与输出描述 -│ ├── run.sh // 测试样例启动脚本 -├── ThreeNN //自定义算子工程 -│ ├── cmake -│ ├── framework // 算子插件实现文件目录，单算子模型文件的生成不依赖算子适配插件，无需关注 -│ ├── op_host // host侧实现文件 -│ ├── op_kernel // kernel侧实现文件 -│ ├── scripts // 自定义算子工程打包相关脚本所在目录 -│ ├── build.sh // 编译入口脚本 -│ ├── CMakeLists.txt // 算子工程的CMakeLists.txt -│ └── CMakePresets.json // 编译配置项 -│ └── ThreeNN.json // msopgen自定义算子生成配置文件 -``` -CANN软件包中提供了工程创建工具msopgen，ThreeNN算子工程可通过ThreeNN.json自动创建，具体请参考[Ascend C算子开发](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC)>算子开发>算子开发工程>基于自定义算子工程的算子开发>创建算子工程 章节。 - -### 2. 编译算子工程 - -编译自定义算子工程，构建生成自定义算子包。 - -- 执行如下命令，切换到算子工程ReduceCustom目录。 - - 以命令行方式下载样例代码，master分支为例。 - - ```bash - cd ${git_clone_path}/samples/operator_contrib/ThreeNNSample/ThreeNN - ``` -- 修改CMakePresets.json中ASCEND_CANN_PACKAGE_PATH为CANN软件包安装后的实际路径。 - - ```json - { - …… - "configurePresets": [ - { - …… - "ASCEND_CANN_PACKAGE_PATH": { - "type": "PATH", - "value": "/usr/local/Ascend/ascend-toolkit/latest" - //请替换为CANN软件包安装后的实际路径。eg:/home/HwHiAiUser/Ascend/ascend-toolkit/latest - }, - …… - } - ] - } - ``` -- 在算子工程ReduceCustom目录下执行如下命令，进行算子工程编译。 - - ```bash - ./build.sh - ``` - -编译成功后，会在当前目录下创建build_out目录，并在build_out目录下生成自定义算子安装包custom_opp_\_\.run，例如“custom_opp_ubuntu_x86_64.run”。 - -备注：如果要使用dump调试功能，需要移除op_host内和CMakeLists.txt内的Atlas 训练系列产品、Atlas 200/500 A2 推理产品的配置。 - -### 3. 部署算子包 - -执行如下命令，在自定义算子安装包所在路径下，安装自定义算子包。 - -```bash -cd build_out -./custom_opp__.run -``` - -命令执行成功后，自定义算子包中的相关文件将部署至当前环境的OPP算子库的vendors/customize目录中。 - -### 4. 配置环境变量 - -请根据当前环境上CANN开发套件包的[安装方式](https://hiascend.com/document/redirect/CannCommunityInstSoftware)，选择对应配置环境变量的命令。 - -- 默认路径，root用户安装CANN软件包 - ```bash - export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest - ``` -- 默认路径，非root用户安装CANN软件包 - ```bash - export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest - ``` -- 指定路径install_path，安装CANN软件包 - ```bash - export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest - ``` - -### 5. 调用执行算子样例 - -- [aclnn调用ThreeNN算子工程(测试样例)](./ThreeNNCase) - -```bash -cd ./AclNNInvocation -./run.sh -``` -样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 - -## 更新说明 - | 时间 | 更新事项 | -|----|------| -| 2024/10/24 | 上传ThreeNN代码 | -- Gitee From 1a24595b92f49753d5b5e82397aa0f0db0bdd7d0 Mon Sep 17 00:00:00 2001 From: appleinsky <2164757809@qq.com> Date: Thu, 24 Oct 2024 01:11:35 +0800 Subject: [PATCH 03/12] =?UTF-8?q?=E3=80=90S2=E7=AE=97=E5=AD=90=E6=8F=90?= =?UTF-8?q?=E4=BA=A4=E3=80=91ThreeNN=E7=AE=97=E5=AD=90=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 182 ++++ .../AclNNInvocation/inc/operator_desc.h | 57 ++ .../FrameworkLaunch/AclNNInvocation/run.sh | 120 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 72 ++ .../AclNNInvocation/scripts/verify_result.py | 26 + .../scripts/verify_result_int.py | 26 + .../AclNNInvocation/src/CMakeLists.txt | 68 ++ .../AclNNInvocation/src/common.cpp | 79 ++ .../AclNNInvocation/src/main.cpp | 171 ++++ .../AclNNInvocation/src/op_runner.cpp | 486 +++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 56 ++ .../FrameworkLaunch/ThreeNN/CMakeLists.txt | 69 ++ .../FrameworkLaunch/ThreeNN/CMakePresets.json | 63 ++ .../FrameworkLaunch/ThreeNN/ThreeNN.json | 50 ++ .../FrameworkLaunch/ThreeNN/build.sh | 80 ++ .../ThreeNN/cmake/config.cmake | 25 + .../FrameworkLaunch/ThreeNN/cmake/func.cmake | 192 ++++ .../FrameworkLaunch/ThreeNN/cmake/intf.cmake | 26 + .../ThreeNN/cmake/makeself.cmake | 17 + .../ThreeNN/cmake/three_nn.cpp | 127 +++ .../ThreeNN/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 127 +++ .../ThreeNN/cmake/util/ascendc_impl_build.py | 501 +++++++++++ .../ThreeNN/cmake/util/ascendc_ops_config.py | 120 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../ThreeNN/cmake/util/batch_replay_impl.temp | 117 +++ .../ThreeNN/cmake/util/code_channel_infer.py | 58 ++ .../ThreeNN/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 20 + .../ThreeNN/cmake/util/gen_ops_filter.sh | 62 ++ .../ThreeNN/cmake/util/gen_version_info.sh | 6 + .../ThreeNN/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 242 ++++++ .../ThreeNN/cmake/util/kernel_entry.py | 115 +++ .../ThreeNN/cmake/util/kernel_impl.temp | 10 + .../ThreeNN/cmake/util/makeself/COPYING | 339 ++++++++ .../ThreeNN/cmake/util/makeself/README.md | 246 ++++++ .../ThreeNN/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../ThreeNN/cmake/util/makeself/makeself.1 | 110 +++ .../ThreeNN/cmake/util/makeself/makeself.lsm | 16 + .../ThreeNN/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../ThreeNN/cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../ThreeNN/cmake/util/opdesc_parser.py | 298 +++++++ .../ThreeNN/cmake/util/parse_ini_to_json.py | 338 +++++++ .../ThreeNN/cmake/util/preset_parse.py | 23 + .../ThreeNN/cmake/util/replay_codegen.py | 105 +++ .../ThreeNN/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 87 ++ .../ThreeNN/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_three_nn_plugin.cc | 23 + .../ThreeNN/op_host/CMakeLists.txt | 82 ++ .../ThreeNN/op_host/three_nn.cpp | 109 +++ .../ThreeNN/op_host/three_nn_tiling.h | 16 + .../ThreeNN/op_kernel/CMakeLists.txt | 68 ++ .../ThreeNN/op_kernel/three_nn.cpp | 166 ++++ .../ThreeNN/scripts/install.sh | 318 +++++++ .../ThreeNN/scripts/upgrade.sh | 151 ++++ operator_contrib/ThreeNNSample/README.en.md | 160 ++++ operator_contrib/ThreeNNSample/README.md | 145 +++ 65 files changed, 8034 insertions(+) create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh create mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh create mode 100644 operator_contrib/ThreeNNSample/README.en.md create mode 100644 operator_contrib/ThreeNNSample/README.md diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..7036813a9 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** +* @file common.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..bf2a9ef43 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,182 @@ +/** +* @file op_runner.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "aclnn/acl_meta.h" +#include "acl/acl.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template + T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template + const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..4cbdf07e0 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,57 @@ +/** +* @file operator_desc.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh new file mode 100644 index 000000000..96ca9c3b1 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh @@ -0,0 +1,120 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while : +do + case "$1" in + # float16, float, int32 + (-v | --dtype) + DTYPE="$2" + shift 2;; + (--) + shift; + break;; + (*) + echo "[ERROR] Unexpected option: $1"; + break;; + esac +done + +if [ ! $ASCEND_HOME_DIR ]; then + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + export ASCEND_HOME_DIR=$HOME/Ascend/ascend-toolkit/latest + else + export ASCEND_HOME_DIR=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $ASCEND_HOME_DIR/bin/setenv.bash + +export DDK_PATH=$ASCEND_HOME_DIR +arch=$(uname -m) +export NPU_HOST_LIB=$ASCEND_HOME_DIR/${arch}-linux/lib64 + +function main { + # 1. 清除算子输出和日志文件 + + rm ./output/execute_op + rm ./input/*.bin + rm ./output/*.bin + # rm -rf ./output/output*.bin > /dev/null + + # 2. 生成或复用输入数据和真值数据 + if [ -d "./input" ]; then + if [ "$(ls -A "./input")" ]; then + echo "已存在测试数据" + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译或复用acl可执行文件 + if [ -e "./output/execute_op" ]; then + echo "可执行存在" + else + echo "可执行不存在" + cd $CURRENT_DIR; rm -rf build; mkdir -p build; cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + fi + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + timeout 30 ./execute_op + + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + ret_dist=`python3 scripts/verify_result.py output/output_dist.bin output/golden_dist.bin` + ret_idx=`python3 scripts/verify_result_int.py output/output_idx.bin output/golden_idx.bin` + echo "verify dist $ret_dist" + echo "verify idx $ret_idx" + if [ "x$ret_dist" == "xtest pass" ] && [ "x$ret_idx" == "xtest pass" ]; then + echo "" + echo "#####################################" + echo "INFO: you have passed the Precision!" + echo "#####################################" + echo "" + fi +} + +main diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..c723df735 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,72 @@ +#!/usr/bin/python3 +# -*- coding:utf-8 -*- +# Copyright 2022-2023 Huawei Technologies Co., Ltd +import numpy as np +import os +import torch + +def run_three_nn(b, n, m, xyz1, xyz2): + dist = np.zeros_like(xyz1, dtype=np.float32) + idx = np.zeros_like(xyz1, dtype=np.int32) + + for ii in range(b): + for jj in range(n): + x1 = xyz1[ii, jj, 0] + y1 = xyz1[ii, jj, 1] + z1 = xyz1[ii, jj, 2] + best1 = 1e40 + best2 = 1e40 + best3 = 1e40 + besti1 = 0 + besti2 = 0 + besti3 = 0 + for kk in range(m): + x2 = xyz2[ii, kk, 0] + y2 = xyz2[ii, kk, 1] + z2 = xyz2[ii, kk, 2] + d = (x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1) + (z2 - z1) * (z2 - z1) + if (d < best1): + best3 = best2 + besti3 = besti2 + best2 = best1 + besti2 = besti1 + best1 = d + besti1 = kk + elif (d < best2): + best3 = best2 + besti3 = besti2 + best2 = d + besti2 = kk + elif (d < best3): + best3 = d + besti3 = kk + dist[ii, jj, 0] = best1 + idx[ii, jj, 0] = besti1 + dist[ii, jj, 1] = best2 + idx[ii, jj, 1] = besti2 + dist[ii, jj, 2] = best3 + idx[ii, jj, 2] = besti3 + + return dist, idx + +def gen_golden_data_simple(): + b = 5 + n = 160 + m = 256 + xyz1_shape = [b,n,3] + xyz2_shape = [b,m,3] + input_xyz1 = np.random.uniform(-10, 10, xyz1_shape).astype(np.float32) + input_xyz2 = np.random.uniform(-10, 10, xyz2_shape).astype(np.float32) + dist, idx = run_three_nn(b, n, m, input_xyz1, input_xyz2) + os.system("mkdir -p input") + os.system("mkdir -p output") + print("dist = ",dist) + print("idx = ",idx) + input_xyz1.tofile("./input/input_xyz1.bin") + input_xyz2.tofile("./input/input_xyz2.bin") + dist.tofile("./output/golden_dist.bin") + idx.tofile("./output/golden_idx.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..d917ab413 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,26 @@ +import os +import sys +import numpy as np +np.set_printoptions(suppress=True) + +loss = 1e-4 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 +minimum = 10e-10 + +def verify_result(real_result, golden): + real_result = np.fromfile(real_result, dtype=np.float32) # 从bin文件读取实际运算结果 + golden = np.fromfile(golden, dtype=np.float32) # 从bin文件读取预期运算结果 + result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 + deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 + result_atol = np.less_equal(result, loss) # 计算绝对误差 + result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 + if not result_rtol.all() and not result_atol.all(): + if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 + print("[ERROR] result error") + print(real_result) + print(golden) + return False + print("test pass") + return True + +if __name__ == '__main__': + verify_result(sys.argv[1],sys.argv[2]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py new file mode 100644 index 000000000..9cbc3482b --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py @@ -0,0 +1,26 @@ +import os +import sys +import numpy as np +np.set_printoptions(suppress=True) + +loss = 1e-4 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 +minimum = 10e-10 + +def verify_result(real_result, golden): + real_result = np.fromfile(real_result, dtype=np.int32) # 从bin文件读取实际运算结果 + golden = np.fromfile(golden, dtype=np.int32) # 从bin文件读取预期运算结果 + result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 + deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 + result_atol = np.less_equal(result, loss) # 计算绝对误差 + result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 + if not result_rtol.all() and not result_atol.all(): + if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 + print("[ERROR] result error") + print(real_result) + print(golden) + return False + print("test pass") + return True + +if __name__ == '__main__': + verify_result(sys.argv[1],sys.argv[2]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..ee73e930f --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,68 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include + ${BUILT_PKG_PATH}/include/aclnnop +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..47a39cd5f --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp @@ -0,0 +1,79 @@ +/** +* @file common.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" + +#include +#include +#include +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void) close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..7536fdb6e --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp @@ -0,0 +1,171 @@ +/** +* @file main.cpp +* +* Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "op_runner.h" + +#include "common.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + int64_t b = 5; + int64_t n = 160; + int64_t m = 256; + std::vector xyz1_shape {b,n,3}; + std::vector xyz2_shape {b,m,3}; + std::vector out_shape {b,n,3}; + aclDataType dataType = ACL_FLOAT; + aclDataType indicesType = ACL_INT32; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.AddInputTensorDesc(dataType, xyz1_shape.size(), xyz1_shape.data(), format); + opDesc.AddInputTensorDesc(dataType, xyz2_shape.size(), xyz2_shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, out_shape.size(), out_shape.data(), format); + opDesc.AddOutputTensorDesc(indicesType, out_shape.size(), out_shape.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_xyz1.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + ReadFile("../input/input_xyz2.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output_dist.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + WriteFile("../output/output_idx.bin", runner.GetOutputBuffer(1), runner.GetOutputSize(1)); + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } + else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..16bbc21d0 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,486 @@ +/** +* @file op_runner.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "op_runner.h" +#include "aclnn_three_nn.h" +#include +#include +#include "acl/acl_op_compiler.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto ret = aclDestroyTensor(inputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free InputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(inputBuffers_[i]); + + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free inputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devInputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostInputs_[i]); + } else { + ret = aclrtFreeHost(hostInputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto ret = aclDestroyTensor(outputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(outputBuffers_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devOutputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostOutputs_[i]); + } else { + ret = aclrtFreeHost(hostOutputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), + nullptr, 0, GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), + nullptr, 0, GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + + auto ret = aclnnThreeNNGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], outputTensor_[1], + &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute GetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + ret = aclnnThreeNN(workspace, workspaceSize, handle, stream); + + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute Operator success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + + +template +void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..a7a1ed3bd --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,56 @@ +/** +* @file operator_desc.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" +#include "operator_desc.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } + +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt new file mode 100644 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json new file mode 100644 index 000000000..8b7181000 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json new file mode 100644 index 000000000..f35459443 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json @@ -0,0 +1,50 @@ +[ + { + "op": "ThreeNN", + "language": "cpp", + "input_desc": [ + { + "name": "xyz1", + "param_type": "required", + "format": [ + "ND" + ], + "type": [ + "float" + ] + }, + { + "name": "xyz2", + "param_type": "required", + "format": [ + "ND" + ], + "type": [ + "float" + ] + } + ], + "output_desc": [ + { + "name": "dist", + "param_type": "required", + "format": [ + "ND" + ], + "type": [ + "float" + ] + }, + { + "name": "indices", + "param_type": "required", + "format": [ + "ND" + ], + "type": [ + "int32" + ] + } + ] + } +] \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh new file mode 100644 index 000000000..27d1968d1 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh @@ -0,0 +1,80 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) +ENABLE_CROSS="-DENABLE_CROSS_COMPILE=True" +ENABLE_BINARY="-DENABLE_BINARY_PACKAGE=True" +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') + +cmake_run_package() +{ + target=$1 + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then exit 1; fi + + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + cmake --build . --target $target -j16 + fi + fi +} + +if [[ $opts =~ $ENABLE_CROSS ]] && [[ $opts =~ $ENABLE_BINARY ]] +then + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DENABLE_CROSS_COMPILE=0 + else + cmake .. --preset=default -DENABLE_CROSS_COMPILE=0 + fi + cmake_run_package $target + cp -r kernel ../ + rm -rf * + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts + else + cmake .. --preset=default + fi + + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + fi + fi + rm -rf ../kernel + +else + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts + else + cmake .. --preset=default + fi + cmake_run_package $target +fi + + +# for debug +# cd build_out +# make +# cpack +# verbose append -v \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake new file mode 100644 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake new file mode 100644 index 000000000..4179dfd25 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake @@ -0,0 +1,192 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} && echo $(MAKE) + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + +endfunction() + +function(add_cross_compile_target) + cmake_parse_arguments(CROSSMP "" "TARGET;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + add_custom_target(${CROSSMP_TARGET} ALL + DEPENDS ${CROSSMP_OUT_DIR} + ) + install(DIRECTORY ${CROSSMP_OUT_DIR} + DESTINATION ${CROSSMP_INSTALL_DIR} + ) +endfunction() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake new file mode 100644 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake new file mode 100644 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp new file mode 100644 index 000000000..113e0d5cd --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp @@ -0,0 +1,127 @@ +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelFastGeluGrad { +public: + __aicore__ inline KernelFastGeluGrad() {} + __aicore__ inline void Init(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, + uint32_t CoreDataNum, uint32_t finalTileNum, uint32_t tileDataNum, uint32_t TailDataNum, + uint32_t xyz2DataNum) { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + + this->coreDataNum = CoreDataNum; + this->tileNum = finalTileNum; + this->tileDataNum = tileDataNum; + this->tailDataNum = TailDataNum; + this->xyz2DataNum = xyz2DataNum; + + xyz1Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ1*)xyz1, this->coreDataNum); + xyz2Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ2*)xyz2, this->xyz2DataNum); + distGm.SetGlobalBuffer((__gm__ DTYPE_DIST*)dist, this->coreDataNum); + indicesGm.SetGlobalBuffer((__gm__ DTYPE_INDICES*)indices, this->coreDataNum); + + // pipe.InitBuffer(inQueueXYZ1, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_XYZ1)); + // pipe.InitBuffer(inQueueXYZ2, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_XYZ2)); + // pipe.InitBuffer(outQueueDIST, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_DIST)); + // pipe.InitBuffer(outQueueINDICES, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_INDICES)); + + } + __aicore__ inline void Process() { + float ux,uy,uz; + float x,y,z; + float best1, best2, best3; + int besti1, besti2, besti3; + for(uint32_t i=0; icoreDataNum; i+=3) + { + ux = xyz1Gm.GetValue(i); + uy = xyz1Gm.GetValue(i+1); + uz = xyz1Gm.GetValue(i+2); + for(j=0; jxyz2DataNum; j+=3) + { + x = xyz2Gm.GetValue(j); + y = xyz2Gm.GetValue(j+1); + z = xyz2Gm.GetValue(j+2); + } + } + // SetValue(count*M*N + i*N+j, xGm.GetValue(count*M*N + i*N+j)); + } +// __aicore__ inline void Process() { + +// int32_t loopCount = this->tileNum; +// this->processDataNum = this->tileDataNum; +// for (int32_t i = 0; i < loopCount; i++) { +// if (i == this->tileNum - 1) { +// this->processDataNum = this->tailDataNum; +// } +// CopyIn(i); +// Compute(i); +// CopyOut(i); +// } +// } + +// private: +// __aicore__ inline void CopyIn(int32_t progress) +// { +// LocalTensor xyz1Local = inQueueXYZ1.AllocTensor(); +// LocalTensor xyz2Local = inQueueXYZ2.AllocTensor(); + +// DataCopy(xyz1Local, xyz1Gm[progress * this->tileDataNum], this->processDataNum); +// DataCopy(xyz2Local, xyz2Gm[progress * this->tileDataNum], this->processDataNum); + +// inQueueXYZ1.EnQue(xyz1Local); +// inQueueXYZ2.EnQue(xyz2Local); +// } +// __aicore__ inline void Compute(int32_t progress) +// { +// LocalTensor xyz1Local = inQueueXYZ1.DeQue(); +// LocalTensor xyz2Local = inQueueXYZ2.DeQue(); +// LocalTensor distLocal = outQueueDIST.AllocTensor(); +// LocalTensor indicesLocal = outQueueINDICES.AllocTensor(); + + +// outQueueDIST.EnQue(distLocal); +// outQueueINDICES.EnQue(indicesLocal); +// inQueueXYZ1.FreeTensor(xyz2Local); +// inQueueXYZ2.FreeTensor(xyz2Local); +// } +// __aicore__ inline void CopyOut(int32_t progress) +// { +// LocalTensor distLocal = outQueueDIST.DeQue(); +// LocalTensor indicesLocal = outQueueINDICES.DeQue(); +// DataCopy(distGm[progress * this->tileDataNum], distLocal, this->processDataNum); +// DataCopy(indicesGm[progress * this->tileDataNum], indicesLocal, this->processDataNum); +// outQueueDIST.FreeTensor(distLocal); +// outQueueINDICES.FreeTensor(indicesLocal); +// } + +private: + TPipe pipe; + TBuf tmpBuffer, signbitBuffer; + TQue inQueueXYZ1, inQueueXYZ2; + TQue outQueueDIST, outQueueINDICES; + GlobalTensor xyz1Gm; + GlobalTensor xyz2Gm; + GlobalTensor distGm; + GlobalTensor indicesGm; + uint32_t coreDataNum; + uint32_t tileNum; + uint32_t tileDataNum; + uint32_t tailDataNum; + uint32_t processDataNum; + uint32_t xyz2DataNum; + // LocalTensor signbit; +}; + +extern "C" __global__ __aicore__ void three_nn(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); + // TODO: user kernel impl + KernelFastGeluGrad op; + op.Init(xyz1, xyz2, dist, indices, + tiling_data.CoreDataNum, tiling_data.finalTileNum, tiling_data.tileDataNum, tiling_data.TailDataNum, tiling_data.xyz2DataNum); + op.Process(); +} +// extern "C" __global__ __aicore__ void three_nn(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling) { +// GET_TILING_DATA(tiling_data, tiling); +// // TODO: user kernel impl +// } \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py new file mode 100644 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py new file mode 100644 index 000000000..63b15c900 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx][:-5] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx][:-5] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = self.soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py new file mode 100644 index 000000000..647801c07 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int4b_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]}} + +def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): + options = [] + x_fmt = x.get("format") + x_dtype = x.get("dtype") + x_n_in_kernel = x_n + '_REF' if is_ref else x_n + options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) + return options + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + input_names = {} + output_names = {} + unique_param_name_set = set() + for idx, x in enumerate(__inputs__): + if x is None: + continue + x_n = input_names[idx].upper() + unique_param_name_set.add(x_n) + options += add_dtype_fmt_option_single(x, x_n) + + for idx, x in enumerate(__outputs__): + if x is None: + continue + x_n = output_names[idx].upper() + if x_n in unique_param_name_set: + options += add_dtype_fmt_option_single(x, x_n, True) + else: + options += add_dtype_fmt_option_single(x, x_n) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +def get_kernel_source(src_file, dir_snake, dir_ex): + src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, src_file) + if os.path.exists(src): + return src + return src_ex + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}", trans_bool_to_s8=False) +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + bisheng = shutil.which("bisheng") + if bisheng != None: + bisheng_path = os.path.dirname(bisheng) + tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir_ex = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is " + kernel_name) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}], param_type_dynamic = {},\\ + mc2_ctx = None, param_type_list = {}) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'list_int': '[]', + 'list_float': '[]', 'list_bool': '[]', 'list_list_int': '[[]]', 'str': ''} + + +def _get_snake_str(s, i): + if s[i - 1] != '_': + if not s[i - 1].isupper(): + return "_" + elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): + return "_" + return "" + return "" + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +def optype_snake_ex(s): + snake_case = "" + for i, c in enumerate(s): + if i == 0: + snake_case += c.lower() + elif c.isupper(): + snake_case += _get_snake_str(s, i) + snake_case += c.lower() + else: + snake_case += c + return snake_case + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD.format(self.input_ori_name, self.output_ori_name)) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake_ex(self.op_type), optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), False, self.input_type + self.output_type, self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py new file mode 100644 index 000000000..b138047af --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, task_ration, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + if core_type == 0 and task_ration == "tilingKey": + bin_list.append({'coreType': core_type, 'simplifiedKey': key, \ + 'multiKernelType':1, 'binPath': objfile}) + else: + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {'MIX': 0, 'AiCore': 1, 'VectorCore': 2, 'MIX_AICORE':3, 'MIX_VECTOR_CORE':4} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + task_ration = contents.get('taskRation') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, task_ration, os.path.join(soc, op_dir, bin_file_name), \ + config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py new file mode 100644 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp new file mode 100644 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py new file mode 100644 index 000000000..d4e6a540c --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b", "ascend910c"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for bisheng cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py new file mode 100644 index 000000000..3e71f5ab8 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1', 'ascend910c': 'Ascend910C1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh new file mode 100644 index 000000000..492034e49 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,20 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh new file mode 100644 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh new file mode 100644 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py new file mode 100644 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py new file mode 100644 index 000000000..7e8dc2c55 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import re +import glob +import json +import argparse + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py new file mode 100644 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp new file mode 100644 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING new file mode 100644 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md new file mode 100644 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION new file mode 100644 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh new file mode 100644 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh new file mode 100644 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 new file mode 100644 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm new file mode 100644 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh new file mode 100644 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh new file mode 100644 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh new file mode 100644 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py new file mode 100644 index 000000000..0d3221602 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b2c": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend910c1": "ascend910c", + "ascend910c2": "ascend910c", + "ascend910c3": "ascend910c", + "ascend910c4": "ascend910c", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} +CONFLICT_KEYWORDS = { + "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", + "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "False", + "None", "True", "nonlocal", "arg", "__inputs__", "__outputs__", "options", "bisheng", + "bisheng_path", "tikcpp_path", "impl_mode", "custom_compile_options", + "custom_all_compile_options", "soc_version", "soc_short", "custom_compile_options_soc", + "custom_all_compile_options_soc", "origin_func_name", "ascendc_src_dir_ex", + "ascendc_src_dir", "ascendc_src_file", "src", "op_type", "code_channel", "op_info", + "compile_op", "get_code_channel", "result", "__attrs__", "isinstance", "attr", + "get_current_build_config", "_build_args", "get_dtype_fmt_options", "shutil", "os", + "get_kernel_source" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_ori_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_ori_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_ori_name.append(self._parse_str(conf)) + self.input_name.append(self.input_ori_name[-1] + '_in__') + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_ori_name.append(self._parse_str(conf)) + self.output_name.append(self.output_ori_name[-1] + '_out_') + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS + if intersection_element: + raise Exception(f'The attribute name: {intersection_element} in op: {{\'{self.op_type}\'}} conflicts with\ + the built-in variable name. Use a complex name or prefix the operator name.') + + @staticmethod + def _camel_to_snake(camel_case_str: str): + snake_case_str = '' + for i, c in enumerate(camel_case_str): + if i == 0: + snake_case_str += c.lower() + elif c.isupper(): + snake_case_str += '_' + c.lower() + else: + snake_case_str += c + return snake_case_str + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._camel_to_snake(self._parse_str(conf)) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return + with open(file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py new file mode 100644 index 000000000..ac931e5d3 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport", "enableVectorCore"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # The owner have all rights，group only have read rights + os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py new file mode 100644 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py new file mode 100644 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp new file mode 100644 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py new file mode 100644 index 000000000..4c5b099b7 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' uint64_t *src = (uint64_t *)tiling;\n' + tiling_source += ' uint64_t *dst = (uint64_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt new file mode 100644 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc new file mode 100644 index 000000000..1809e029a --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("ThreeNN") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("ThreeNN") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt new file mode 100644 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp new file mode 100644 index 000000000..027ef0850 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp @@ -0,0 +1,109 @@ + +#include "three_nn_tiling.h" +#include "register/op_def_registry.h" +#include "tiling/platform/platform_ascendc.h" +#include + +namespace optiling { +const uint32_t BLOCK_SIZE = 32*3*4; +static ge::graphStatus TilingFunc(gert::TilingContext* context) +{ + ThreeNNTilingData tiling; + + uint64_t ubSize; + auto ascendcPlatform = platform_ascendc::PlatformAscendC(context->GetPlatformInfo()); + // auto socVersion = ascendcPlatform.GetSocVersion(); + ascendcPlatform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ubSize); //获取硬件平台存储空间 UB 的内存大小 + uint32_t shapeB = context->GetInputTensor(0)->GetOriginShape().GetDim(0); + uint32_t shapeN = context->GetInputTensor(0)->GetOriginShape().GetDim(1); + uint32_t shapeM = context->GetInputTensor(1)->GetOriginShape().GetDim(1); + + //获取输入shape信息 + uint32_t inputNum = context->GetInputShape(1)->GetStorageShape().GetShapeSize()/shapeB; //输入数量 xyz2 + uint32_t inputBytes = GetSizeByDataType(context->GetInputDesc(1)->GetDataType()); //输入类型 + uint32_t inputLength = inputBytes * inputNum; //输入长度 + + //可使用的ub空间 输入3输出1，手动考虑双缓存 + uint32_t ubDataNumber = 4;//(inputBytes == 2) ? 10 : 6; + + // The number of 32B data blocks that can be used for each data. DOUBLE BUFFER is already counted here + uint32_t tileBlockNum = (ubSize / BLOCK_SIZE) / ubDataNumber; //每个ub段可用的空间块数 + uint32_t tileDataNum = 1440; + + // Input data for 32B alignment + uint32_t inputLengthAlgin32 = (((inputLength + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE); //输入长度 对齐处理 + uint32_t everyCoreInputBlockNum = inputLengthAlgin32 / BLOCK_SIZE;// 输入数据需要多少空间块 + + uint32_t TileNum = inputNum / tileDataNum; + uint32_t finalTileNum = (inputNum % tileDataNum) == 0 ? TileNum : TileNum + 1; //需要循环处理几次 + // Tail block calculation for chunks of data + uint32_t TailDataNum = inputNum - (tileDataNum * TileNum);//CoreDataNum - (tileDataNum * TileNum); + TailDataNum = TailDataNum == 0 ? tileDataNum : TailDataNum; //最后一次需要处理的数据量 + + tiling.set_CoreDataNum((inputNum*shapeB+31)/32*32); //对齐空间后的输入数量 + tiling.set_finalTileNum(finalTileNum);//需要循环处理几次 + tiling.set_tileDataNum(tileDataNum); //每个b处理的数据量 + tiling.set_TailDataNum(TailDataNum); //最后一次需要处理的数据量 + + tiling.set_shapeB(shapeB); + tiling.set_shapeN(shapeN); + tiling.set_shapeM(shapeM); + + context->SetBlockDim(1); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} + + +namespace ge { +static ge::graphStatus InferShape(gert::InferShapeContext* context) +{ + const gert::Shape* x1_shape = context->GetInputShape(0); + gert::Shape* y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} +} + + +namespace ops { +class ThreeNN : public OpDef { +public: + explicit ThreeNN(const char* name) : OpDef(name) + { + this->Input("xyz1") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Input("xyz2") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("dist") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + this->Output("indices") + .ParamType(REQUIRED) + .DataType({ge::DT_INT32}) + .Format({ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND}); + + this->SetInferShape(ge::InferShape); + + this->AICore() + .SetTiling(optiling::TilingFunc); + this->AICore().AddConfig("ascend310b"); + + } +}; + +OP_ADD(ThreeNN); +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h new file mode 100644 index 000000000..5da548fed --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h @@ -0,0 +1,16 @@ + +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(ThreeNNTilingData) + TILING_DATA_FIELD_DEF(uint32_t, CoreDataNum); + TILING_DATA_FIELD_DEF(uint32_t, finalTileNum); + TILING_DATA_FIELD_DEF(uint32_t, tileDataNum); + TILING_DATA_FIELD_DEF(uint32_t, TailDataNum); + + TILING_DATA_FIELD_DEF(uint32_t, shapeB); + TILING_DATA_FIELD_DEF(uint32_t, shapeN); + TILING_DATA_FIELD_DEF(uint32_t, shapeM); +END_TILING_DATA_DEF; +REGISTER_TILING_DATA_CLASS(ThreeNN, ThreeNNTilingData) +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt new file mode 100644 index 000000000..8c94a952d --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt @@ -0,0 +1,68 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE} AND NOT ${ENABLE_CROSS_COMPILE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + + if (${ENABLE_CROSS_COMPILE} AND ${ENABLE_BINARY_PACKAGE}) + add_cross_compile_target( + TARGET bin_${compute_unit} + OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../kernel + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/ + ) + endif() +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp new file mode 100644 index 000000000..9d60d095b --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp @@ -0,0 +1,166 @@ +#define K_MAX_SHAPE_DIM 0 +#include "kernel_operator.h" +#include +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +class KernelFastGeluGrad { +public: + __aicore__ inline KernelFastGeluGrad() {} + __aicore__ inline void Init(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, + uint32_t CoreDataNum, uint32_t finalTileNum, uint32_t tileDataNum, uint32_t TailDataNum, + uint32_t shapeB, uint32_t shapeN, uint32_t shapeM) { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + + this->coreDataNum = CoreDataNum; + this->tileNum = finalTileNum; + this->tileDataNum = tileDataNum; + this->tailDataNum = TailDataNum; + + this->shapeB = shapeB; + this->shapeN = shapeN; + this->shapeM = shapeM; + + xyz1Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ1*)xyz1, this->coreDataNum); + xyz2Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ2*)xyz2, this->coreDataNum); + distGm.SetGlobalBuffer((__gm__ DTYPE_DIST*)dist, this->coreDataNum); + indicesGm.SetGlobalBuffer((__gm__ DTYPE_INDICES*)indices, this->coreDataNum); + + pipe.InitBuffer(inQueueXYZ2, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + pipe.InitBuffer(QueueTmp1, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + pipe.InitBuffer(QueueTmp2, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + pipe.InitBuffer(QueueTmp3, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + + pipe.InitBuffer(QueueTmp10, 32 * sizeof(DTYPE_XYZ2)); + + pipe.InitBuffer(QueueTmpX, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + pipe.InitBuffer(QueueTmpY, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + pipe.InitBuffer(QueueTmpZ, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); + + pipe.InitBuffer(QueueMask, sizeof(uint32_t)*(128+36)); + + } + __aicore__ inline void Process() { + int32_t loopCount; + int32_t b=0; + + LocalTensor mask_scl = QueueMask.Get(); + auto tmpx = QueueTmpX.Get(); + auto tmpy = QueueTmpY.Get(); + auto tmpz = QueueTmpZ.Get(); + + uint64_t mask1[2] = {10540996613548315209, 5270498306774157604}; + uint64_t mask2[2] = {2635249153387078802, 10540996613548315209}; + uint64_t mask3[2] = {5270498306774157604, 2635249153387078802}; + uint32_t scalar = 1227133513; + Duplicate(mask_scl, scalar, mask1, 2, 1, 8 ); + + scalar = 2454267026; + Duplicate(mask_scl, scalar, mask2, 2, 1, 8 ); + + scalar = 613566756; + Duplicate(mask_scl, scalar, mask3, 2, 1, 8 ); + + this->processDataNum = this->tailDataNum; + for(int32_t b=0; bshapeB; b++) + { + CopyIn(b, 0); + Compute(b, 0, mask_scl, tmpx, tmpy, tmpz); + } + } + +private: + __aicore__ inline void CopyIn(int32_t b, int32_t progress) + { + LocalTensor xyz2Local = inQueueXYZ2.AllocTensor(); + DataCopy(xyz2Local, xyz2Gm[3*b*this->shapeM], (3*this->shapeM+31)/32*32); + inQueueXYZ2.EnQue(xyz2Local); + } + __aicore__ inline void Compute(int32_t b, int32_t progress, LocalTensor mask_scl, LocalTensor tmpx, LocalTensor tmpy, LocalTensor tmpz) + { + LocalTensor xyz2Local = inQueueXYZ2.DeQue(); + auto tmp1 = QueueTmp1.Get(); + auto tmp2 = QueueTmp2.Get(); + auto tmp3 = QueueTmp3.Get(); + auto tmp10 = QueueTmp10.Get(); + + float ux,uy,uz; + int besti1, besti2, besti3; + uint32_t calculate_num = this->shapeM; + uint32_t mask = this->shapeM*3; + uint64_t rsvdCnt = 0; + GatherMask (tmpx, xyz2Local, mask_scl, true, mask, { 1, 1, 8, 8 }, rsvdCnt); + GatherMask (tmpy, xyz2Local, mask_scl[16], true, mask, { 1, 1, 8, 8 }, rsvdCnt); + GatherMask (tmpz, xyz2Local, mask_scl[8], true, mask, { 1, 1, 8, 8 }, rsvdCnt); + + uint32_t b_xyz1_index = 3*(b*this->shapeN); + for(uint32_t i=0; i < this->shapeN; i++) + { + ux = -xyz1Gm.GetValue(b_xyz1_index); + uy = -xyz1Gm.GetValue(b_xyz1_index+1); + uz = -xyz1Gm.GetValue(b_xyz1_index+2); + + Adds(tmp3, tmpx, ux, calculate_num); + Mul(tmp2, tmp3, tmp3, calculate_num); + + Adds(tmp3, tmpy, uy, calculate_num); + Mul(tmp1, tmp3, tmp3, calculate_num); + + Add(tmp3, tmp2, tmp1, calculate_num); + + Adds(tmp2, tmpz, uz, calculate_num); + Mul(tmp1, tmp2, tmp2, calculate_num); + + Add(tmp3, tmp3, tmp1, calculate_num); + + ReduceMin(tmp2, tmp3, tmp1, calculate_num, true); + besti1 = (tmp2.ReinterpretCast()).GetValue(1); + tmp3.SetValue(besti1, 3.402823466e+38F); + + ReduceMin(tmp2[8], tmp3, tmp1, calculate_num, true); + besti2 = (tmp2.ReinterpretCast()).GetValue(9); + tmp3.SetValue(besti2, 3.402823466e+38F); + + ReduceMin(tmp2[16], tmp3, tmp1, calculate_num, true); + besti3 = (tmp2.ReinterpretCast()).GetValue(17); + + indicesGm.SetValue(b_xyz1_index, besti1); + indicesGm.SetValue(b_xyz1_index+1, besti2); + indicesGm.SetValue(b_xyz1_index+2, besti3); + + distGm.SetValue(b_xyz1_index, tmp2.GetValue(0)); + distGm.SetValue(b_xyz1_index+1, tmp2.GetValue(8)); + distGm.SetValue(b_xyz1_index+2, tmp2.GetValue(16)); + b_xyz1_index+=3; + } + inQueueXYZ2.FreeTensor(xyz2Local); + } + +private: + TPipe pipe; + TQue inQueueXYZ2; + TBuf QueueTmp1, QueueTmp2, QueueTmp3, QueueTmpX, QueueTmpY, QueueTmpZ, QueueMask, QueueTmp10; + GlobalTensor xyz1Gm; + GlobalTensor xyz2Gm; + GlobalTensor distGm; + GlobalTensor indicesGm; + uint32_t coreDataNum; + uint32_t tileNum; + uint32_t tileDataNum; + uint32_t tailDataNum; + uint32_t processDataNum; + uint32_t xyz2DataNum; + uint32_t shapeB; + uint32_t shapeN; + uint32_t shapeM; +}; + +extern "C" __global__ __aicore__ void three_nn(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); + // TODO: user kernel impl + KernelFastGeluGrad op; + op.Init(xyz1, xyz2, dist, indices, + tiling_data.CoreDataNum, tiling_data.finalTileNum, tiling_data.tileDataNum, tiling_data.TailDataNum, + tiling_data.shapeB, tiling_data.shapeN, tiling_data.shapeM); + op.Process(); +} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh new file mode 100644 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh new file mode 100644 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/ThreeNNSample/README.en.md b/operator_contrib/ThreeNNSample/README.en.md new file mode 100644 index 000000000..19000a924 --- /dev/null +++ b/operator_contrib/ThreeNNSample/README.en.md @@ -0,0 +1,160 @@ +## ThreeNN Custom Operator Sample Description + +This sample implements the ThreeNN operator using the Ascend C programming language and provides corresponding end-to-end implementations based on different operator invocation methods. + +- [FrameworkLaunch](./FrameworkLaunch): Using a framework to call the ThreeNN custom operator. + +The development process follows the workflow of project creation -> operator implementation -> compilation and deployment -> operator invocation. The entire process relies on the operator project: the operator kernel function and tiling are developed based on the project code framework, the operator is compiled and deployed via project compilation scripts, enabling the use of single operators or operators within third-party frameworks. + +The following invocation methods are included in this sample: + +| Invocation Method | Directory | Description | +| ----------------- | --------- | ----------- | +| [FrameworkLaunch](./FrameworkLaunch) | [AclNNInvocation](./FrameworkLaunch/AclNNInvocation) | Calling the ThreeNN operator through aclnn. | + +## Operator Description + +For each point in the target point set, the ThreeNN operator finds the distances and coordinates of the three closest points from a reference set. The returned points are sorted in ascending order of squared distance. + +## Operator Specification + + + + + + + + + + + +
Operator TypeThreeNN
Operator inputnameshapedata typeformatdefault
xyz1(b, n, 3)float32ND\
xyz2(b, m, 3)float32ND\
Operator output
dist(b, n, 3)float32ND\
indices(b, n, 3)int32ND\
kernel nameThreeNN
+ +### 1. Operator Project Overview + +The operator project directory ThreeNN contains template files, compilation scripts, etc., as follows: + +``` +├── AclNNInvocation // Directory for invoking the ThreeNN operator via aclnn +│ ├── inc // Header file directory +│ │ ├── common.h // Declaration of common methods, used for reading binary files +│ │ ├── op_runner.h // Operator description declaration, including inputs/outputs, operator type, input and output descriptions +│ │ └── operator_desc.h // Operator runtime information declaration file +│ ├── input // Directory for input data generated by scripts +│ ├── output // Directory for storing operator output data and ground truth +│ ├── scripts +│ │ ├── acl.json // ACL configuration file +│ │ ├── gen_data.py // Script for generating input and ground truth data +│ │ └── verify_result.py // File for verifying results +│ ├── src +│ │ ├── CMakeLists.txt // Compilation rules +│ │ ├── common.cpp // Common functions, implementation for reading binary files +│ │ ├── main.cpp // Entry point for single operator invocation +│ │ ├── op_runner.cpp // Main process for single operator invocation +│ │ └── operator_desc.cpp // Construct operator input and output descriptions +│ ├── run.sh // Script for running the test sample +├── ThreeNN // Custom operator project +│ ├── cmake +│ ├── framework // Directory for implementing operator plugins, single operator model generation does not depend on plugins, not needed +│ ├── op_host // Host-side implementation +│ ├── op_kernel // Kernel-side implementation +│ ├── scripts // Directory for packaging custom operator project-related scripts +│ ├── build.sh // Compilation entry script +│ ├── CMakeLists.txt // CMakeLists.txt for the operator project +│ └── CMakePresets.json // Compilation configuration file +│ └── ThreeNN.json // Configuration file for generating the custom operator using msopgen +``` + +The CANN software package provides the project creation tool msopgen. The ThreeNN operator project can be automatically created using ThreeNN.json. For details, please refer to [Ascend C Operator Development](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC) > Operator Development > Operator Project Development > Creating an Operator Project. + +### 2. Compile the Operator Project + +Compile the custom operator project to generate the custom operator package. + +- Run the following commands to navigate to the ReduceCustom operator project directory. + + Use the command line to download the sample code, using the master branch as an example. + + ```bash + cd ${git_clone_path}/samples/operator_contrib/ThreeNNSample/ThreeNN + ``` + +- Modify `CMakePresets.json` to set `ASCEND_CANN_PACKAGE_PATH` to the actual path where the CANN software package is installed. + + ```json + { + …… + "configurePresets": [ + { + …… + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + // Replace with the actual installation path of the CANN software package, e.g., /home/HwHiAiUser/Ascend/ascend-toolkit/latest + }, + …… + } + ] + } + ``` + +- Run the following commands in the ReduceCustom operator project directory to compile the operator project. + + ```bash + ./build.sh + ``` + +After a successful compilation, a `build_out` directory is created in the current directory, which contains the custom operator installation package `custom_opp__.run`, for example, `custom_opp_ubuntu_x86_64.run`. + +Note: To use the dump debug feature, remove the Atlas training series product configuration from the op_host and CMakeLists.txt for Atlas 200/500 A2 inference products. + +### 3. Deploy the Operator Package + +Run the following commands in the directory containing the custom operator installation package to install the custom operator package. + +```bash +cd build_out +./custom_opp__.run +``` + +After successful execution, the relevant files in the custom operator package will be deployed to the `vendors/customize` directory in the OPP operator library of the current environment. + +### 4. Configure Environment Variables + +Please select the appropriate command to configure the environment variables according to the [installation method](https://hiascend.com/document/redirect/CannCommunityInstSoftware) of the CANN development kit package in the current environment. + +- Default path, root user installed CANN software package + + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` + +- Default path, non-root user installed CANN software package + + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` + +- Specify path `install_path`, installed CANN software package + + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ``` + +### 5. Run Operator Sample + +- [aclnn calls the ThreeNN operator project (test sample)](./ThreeNNCase) + +```bash +cd ./AclNNInvocation +./run.sh +``` + +The sample will automatically generate test data, then compile and run the aclnn sample, and finally verify the results. For details, see the `run.sh` script. + +## Update Notes + +| Date | Update Content | +| ---------- | --------------------- | +| 2024/10/24 | Uploaded ThreeNN code | + diff --git a/operator_contrib/ThreeNNSample/README.md b/operator_contrib/ThreeNNSample/README.md new file mode 100644 index 000000000..39f6afce4 --- /dev/null +++ b/operator_contrib/ThreeNNSample/README.md @@ -0,0 +1,145 @@ +## ThreeNN自定义算子样例说明 +本样例通过Ascend C编程语言实现了ThreeNN算子，并按照不同的算子调用方式分别给出了对应的端到端实现。 +- [FrameworkLaunch](./FrameworkLaunch)：使用框架调用ThreeNN自定义算子。 +按照工程创建->算子实现->编译部署>算子调用的流程完成算子开发。整个过程都依赖于算子工程：基于工程代码框架完成算子核函数的开发和Tiling实现，通过工程编译脚本完成算子的编译部署，继而实现单算子调用或第三方框架中的算子调用。 + +本样例中包含如下调用方式： + + + + + + +
调用方式目录描述
FrameworkLaunch AclNNInvocation通过aclnn调用的方式调用ThreeNN算子。
+ +## 算子描述 +对于目标点集中每个点，找到比较点集中距离其最近的3个点的距离、坐标，返回的三个点会按照距离平方升序。 + +## 算子规格描述 + + + + + + + + + + +
算子类型(OpType)ThreeNN
算子输入nameshapedata typeformatdefault
xyz1(b, n, 3)float32ND\
xyz2(b, m, 3)float32ND\
算子输出
dist(b, n, 3)float32ND\
indices(b, n, 3)int32ND\
核函数名ThreeNN
+ +### 1.算子工程介绍 +其中，算子工程目录ThreeNN包含算子实现的模板文件、编译脚本等，如下所示: +``` +├── AclNNInvocation // 通过aclnn调用的方式调用ThreeNN算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ └── operator_desc.h // 算子运行相关信息声明文件，包含算子 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真值数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ └── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ └── operator_desc.cpp // 构造算子的输入与输出描述 +│ ├── run.sh // 测试样例启动脚本 +├── ThreeNN //自定义算子工程 +│ ├── cmake +│ ├── framework // 算子插件实现文件目录，单算子模型文件的生成不依赖算子适配插件，无需关注 +│ ├── op_host // host侧实现文件 +│ ├── op_kernel // kernel侧实现文件 +│ ├── scripts // 自定义算子工程打包相关脚本所在目录 +│ ├── build.sh // 编译入口脚本 +│ ├── CMakeLists.txt // 算子工程的CMakeLists.txt +│ └── CMakePresets.json // 编译配置项 +│ └── ThreeNN.json // msopgen自定义算子生成配置文件 +``` +CANN软件包中提供了工程创建工具msopgen，ThreeNN算子工程可通过ThreeNN.json自动创建，具体请参考[Ascend C算子开发](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC)>算子开发>算子开发工程>基于自定义算子工程的算子开发>创建算子工程 章节。 + +### 2. 编译算子工程 + +编译自定义算子工程，构建生成自定义算子包。 + +- 执行如下命令，切换到算子工程ReduceCustom目录。 + + 以命令行方式下载样例代码，master分支为例。 + + ```bash + cd ${git_clone_path}/samples/operator_contrib/ThreeNNSample/ThreeNN + ``` +- 修改CMakePresets.json中ASCEND_CANN_PACKAGE_PATH为CANN软件包安装后的实际路径。 + + ```json + { + …… + "configurePresets": [ + { + …… + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + //请替换为CANN软件包安装后的实际路径。eg:/home/HwHiAiUser/Ascend/ascend-toolkit/latest + }, + …… + } + ] + } + ``` +- 在算子工程ReduceCustom目录下执行如下命令，进行算子工程编译。 + + ```bash + ./build.sh + ``` + +编译成功后，会在当前目录下创建build_out目录，并在build_out目录下生成自定义算子安装包custom_opp_\_\.run，例如“custom_opp_ubuntu_x86_64.run”。 + +备注：如果要使用dump调试功能，需要移除op_host内和CMakeLists.txt内的Atlas 训练系列产品、Atlas 200/500 A2 推理产品的配置。 + +### 3. 部署算子包 + +执行如下命令，在自定义算子安装包所在路径下，安装自定义算子包。 + +```bash +cd build_out +./custom_opp__.run +``` + +命令执行成功后，自定义算子包中的相关文件将部署至当前环境的OPP算子库的vendors/customize目录中。 + +### 4. 配置环境变量 + +请根据当前环境上CANN开发套件包的[安装方式](https://hiascend.com/document/redirect/CannCommunityInstSoftware)，选择对应配置环境变量的命令。 + +- 默认路径，root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` +- 默认路径，非root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` +- 指定路径install_path，安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ``` + +### 5. 调用执行算子样例 + +- [aclnn调用ThreeNN算子工程(测试样例)](./ThreeNNCase) + +```bash +cd ./AclNNInvocation +./run.sh +``` +样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + +## 更新说明 + | 时间 | 更新事项 | +|----|------| +| 2024/10/24 | 上传ThreeNN代码 | -- Gitee From cdba10e18d32d208f2d73f7a1419f4d478399cfe Mon Sep 17 00:00:00 2001 From: appleinsky <2164757809@qq.com> Date: Thu, 24 Oct 2024 02:51:28 +0800 Subject: [PATCH 04/12] =?UTF-8?q?=E3=80=90S2=E7=AE=97=E5=AD=90=E6=8F=90?= =?UTF-8?q?=E4=BA=A4=E3=80=91Histogram=E7=AE=97=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AclNNInvocation/inc/common.h | 45 + .../AclNNInvocation/inc/op_runner.h | 182 ++++ .../AclNNInvocation/inc/operator_desc.h | 60 ++ .../FrameworkLaunch/AclNNInvocation/run.sh | 118 +++ .../AclNNInvocation/scripts/acl.json | 1 + .../AclNNInvocation/scripts/gen_data.py | 21 + .../AclNNInvocation/scripts/verify_result.py | 27 + .../AclNNInvocation/src/CMakeLists.txt | 68 ++ .../AclNNInvocation/src/common.cpp | 79 ++ .../AclNNInvocation/src/main.cpp | 166 ++++ .../AclNNInvocation/src/op_runner.cpp | 490 +++++++++++ .../AclNNInvocation/src/operator_desc.cpp | 56 ++ .../FrameworkLaunch/Histogram/CMakeLists.txt | 69 ++ .../Histogram/CMakePresets.json | 63 ++ .../FrameworkLaunch/Histogram/Histogram.json | 58 ++ .../FrameworkLaunch/Histogram/build.sh | 80 ++ .../Histogram/cmake/config.cmake | 25 + .../Histogram/cmake/func.cmake | 192 ++++ .../Histogram/cmake/intf.cmake | 26 + .../Histogram/cmake/makeself.cmake | 17 + .../Histogram/cmake/util/__init__.py | 8 + .../cmake/util/ascendc_bin_param_build.py | 127 +++ .../cmake/util/ascendc_impl_build.py | 501 +++++++++++ .../cmake/util/ascendc_ops_config.py | 120 +++ .../cmake/util/ascendc_replay_build.py | 65 ++ .../cmake/util/batch_replay_impl.temp | 117 +++ .../cmake/util/code_channel_infer.py | 58 ++ .../Histogram/cmake/util/const_var.py | 33 + .../cmake/util/gen_impl_and_mrege_json.sh | 20 + .../Histogram/cmake/util/gen_ops_filter.sh | 62 ++ .../Histogram/cmake/util/gen_version_info.sh | 6 + .../Histogram/cmake/util/insert_op_info.py | 36 + .../cmake/util/insert_simplified_keys.py | 242 ++++++ .../Histogram/cmake/util/kernel_entry.py | 115 +++ .../Histogram/cmake/util/kernel_impl.temp | 10 + .../Histogram/cmake/util/makeself/COPYING | 339 ++++++++ .../Histogram/cmake/util/makeself/README.md | 246 ++++++ .../Histogram/cmake/util/makeself/VERSION | 1 + .../cmake/util/makeself/make-release.sh | 9 + .../cmake/util/makeself/makeself-header.sh | 660 ++++++++++++++ .../Histogram/cmake/util/makeself/makeself.1 | 110 +++ .../cmake/util/makeself/makeself.lsm | 16 + .../Histogram/cmake/util/makeself/makeself.sh | 822 ++++++++++++++++++ .../cmake/util/makeself/run-tests.sh | 8 + .../cmake/util/merge_aicpu_info_json.sh | 31 + .../Histogram/cmake/util/opdesc_parser.py | 298 +++++++ .../Histogram/cmake/util/parse_ini_to_json.py | 338 +++++++ .../Histogram/cmake/util/preset_parse.py | 23 + .../Histogram/cmake/util/replay_codegen.py | 105 +++ .../Histogram/cmake/util/replay_impl.temp | 120 +++ .../cmake/util/tiling_data_def_build.py | 87 ++ .../Histogram/framework/CMakeLists.txt | 11 + .../framework/tf_plugin/CMakeLists.txt | 14 + .../tf_plugin/tensorflow_histogram_plugin.cc | 23 + .../Histogram/op_host/CMakeLists.txt | 82 ++ .../Histogram/op_host/histogram.cpp | 91 ++ .../Histogram/op_host/histogram_tiling.h | 16 + .../Histogram/op_kernel/CMakeLists.txt | 68 ++ .../Histogram/op_kernel/histogram.cpp | 207 +++++ .../Histogram/scripts/install.sh | 318 +++++++ .../Histogram/scripts/upgrade.sh | 151 ++++ operator_contrib/Histogram/README.en.md | 164 ++++ operator_contrib/Histogram/README.md | 150 ++++ 63 files changed, 7871 insertions(+) create mode 100644 operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/common.h create mode 100644 operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/op_runner.h create mode 100644 operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h create mode 100644 operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/run.sh create mode 100644 operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/acl.json create mode 100644 operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt create mode 100644 operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/common.cpp create mode 100644 operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/main.cpp create mode 100644 operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp create mode 100644 operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/CMakeLists.txt create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/CMakePresets.json create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/Histogram.json create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/build.sh create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/config.cmake create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/func.cmake create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/intf.cmake create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/makeself.cmake create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/__init__.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_bin_param_build.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_impl_build.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_ops_config.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_replay_build.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/batch_replay_impl.temp create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/code_channel_infer.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/const_var.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_impl_and_mrege_json.sh create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_ops_filter.sh create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_version_info.sh create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/insert_op_info.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/insert_simplified_keys.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/kernel_entry.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/kernel_impl.temp create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/COPYING create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/README.md create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/VERSION create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/make-release.sh create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself-header.sh create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.1 create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.lsm create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.sh create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/run-tests.sh create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/merge_aicpu_info_json.sh create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/opdesc_parser.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/parse_ini_to_json.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/preset_parse.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/replay_codegen.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/replay_impl.temp create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/tiling_data_def_build.py create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/CMakeLists.txt create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/tf_plugin/CMakeLists.txt create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/tf_plugin/tensorflow_histogram_plugin.cc create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/CMakeLists.txt create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/histogram.cpp create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/histogram_tiling.h create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/op_kernel/CMakeLists.txt create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/op_kernel/histogram.cpp create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/scripts/install.sh create mode 100644 operator_contrib/Histogram/FrameworkLaunch/Histogram/scripts/upgrade.sh create mode 100644 operator_contrib/Histogram/README.en.md create mode 100644 operator_contrib/Histogram/README.md diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/common.h b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/common.h new file mode 100644 index 000000000..7036813a9 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/common.h @@ -0,0 +1,45 @@ +/** +* @file common.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef COMMON_H +#define COMMON_H + +#include +#include +#include +#include +#include + +#include "acl/acl.h" + +#define SUCCESS 0 +#define FAILED 1 + +#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) +#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) +#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) + +/** + * @brief Read data from file + * @param [in] filePath: file path + * @param [out] fileSize: file size + * @return read result + */ +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); + +/** + * @brief Write data to file + * @param [in] filePath: file path + * @param [in] buffer: data to write to file + * @param [in] size: size to write + * @return write result + */ +bool WriteFile(const std::string &filePath, const void *buffer, size_t size); + +#endif // COMMON_H diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/op_runner.h b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/op_runner.h new file mode 100644 index 000000000..bf2a9ef43 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/op_runner.h @@ -0,0 +1,182 @@ +/** +* @file op_runner.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OP_RUNNER_H +#define OP_RUNNER_H + +#include "aclnn/acl_meta.h" +#include "acl/acl.h" +#include "common.h" +#include "operator_desc.h" + +/** + * Op Runner + */ +class OpRunner { +public: + /** + * @brief Constructor + * @param [in] opDesc: op description + */ + explicit OpRunner(OperatorDesc *opDesc); + + /** + * @brief Destructor + */ + virtual ~OpRunner(); + + /** + * @brief Init op runner + */ + bool Init(); + + /** + * @brief Get number of inputs + * @return number of inputs + */ + const size_t NumInputs(); + + /** + * @brief Get number of outputs + * @return number of outputs + */ + const size_t NumOutputs(); + + /** + * @brief Get input size by index + * @param [in] index: input index + * @return size of the input + */ + const size_t GetInputSize(size_t index) const; + const size_t GetInputNumDims(size_t index) const; + aclDataType GetInputDataType(size_t index) const; + aclFormat GetInputFormat(size_t index) const; + + /** + * @brief Get output size by index + * @param [in] index: output index + * @return size of the output + */ + size_t GetOutputSize(size_t index) const; + const size_t GetOutputNumDims(size_t index) const; + aclDataType GetOutputDataType(size_t index) const; + aclFormat GetOutputFormat(size_t index) const; + + /** + * @brief Get input element count by index + * @param i[in] ndex: input index + * @return element count of the input + */ + size_t GetInputElementCount(size_t index) const; + + /** + * @brief Get output element count by index + * @param [in] index: output index + * @return element count of the output + */ + size_t GetOutputElementCount(size_t index) const; + + /** + * @brief Get input shape by index + * @param [in] index: input index + * @return shape of the output + */ + std::vector GetInputShape(size_t index) const; + + /** + * @brief Get output shape by index + * @param [in] index: output index + * @return shape of the output + */ + std::vector GetOutputShape(size_t index) const; + + /** + * @brief Get input buffer(host memory) by index + * @tparam T: data type + * @param [in] index: input index + * @return host address of the input + */ + template + T *GetInputBuffer(size_t index) + { + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return nullptr; + } + return reinterpret_cast(hostInputs_[index]); + } + + /** + * @brief Get output buffer(host memory) by index + * @tparam T: data type + * @param [in] index: output index + * @return host address of the output + */ + template + const T *GetOutputBuffer(size_t index) + { + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return nullptr; + } + + return reinterpret_cast(hostOutputs_[index]); + } + + /** + * @brief Print readable input by index + * @param [in] index: input index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintInput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Print readable output by index + * @param [in] index: output index + * @param [in] elementsPerRow: number of elements per row + */ + void PrintOutput(size_t index, size_t elementsPerRow = 16); + + /** + * @brief Compile static op + * @return compile result + */ + bool CompileStaticOp(); + + /** + * @brief Compile dynamic op + * @return compile result + */ + bool CompileDynamicOp(); + + /** + * @brief Run op + * @return run result + */ + bool RunOp(); + +private: + size_t numInputs_; + size_t numOutputs_; + + std::vector inputBuffers_; + std::vector outputBuffers_; + + std::vector devInputs_; + std::vector devOutputs_; + + std::vector hostInputs_; + std::vector hostOutputs_; + + std::vector inputTensor_; + std::vector outputTensor_; + OperatorDesc *opDesc_; +}; + +#endif // OP_RUNNER_H diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h new file mode 100644 index 000000000..25ffa7e23 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h @@ -0,0 +1,60 @@ +/** +* @file operator_desc.h +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#ifndef OPERATOR_DESC_H +#define OPERATOR_DESC_H + +#include +#include + +#include "acl/acl.h" + +/** + * Op description + */ +struct OperatorDesc { + /** + * Constructor + */ + explicit OperatorDesc(); + + /** + * Destructor + */ + virtual ~OperatorDesc(); + + /** + * Add an input tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + /** + * Add an output tensor description + * @param [in] dataType: data type + * @param [in] numDims: number of dims + * @param [in] dims: dims + * @param [in] format: format + * @return OperatorDesc + */ + OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); + + int64_t bins; + double min; + double max; + std::string opType; + std::vector inputDesc; + std::vector outputDesc; +}; + +#endif // OPERATOR_DESC_H diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/run.sh b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/run.sh new file mode 100644 index 000000000..70e7c74fc --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/run.sh @@ -0,0 +1,118 @@ +#!/bin/bash +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=0 + +CURRENT_DIR=$( + cd $(dirname ${BASH_SOURCE:-$0}) + pwd +) +cd $CURRENT_DIR + +# 导出环境变量 +SHORT=v:, +LONG=dtype:, +OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") +eval set -- "$OPTS" +while : +do + case "$1" in + # float16, float, int32 + (-v | --dtype) + DTYPE="$2" + shift 2;; + (--) + shift; + break;; + (*) + echo "[ERROR] Unexpected option: $1"; + break;; + esac +done + +if [ ! $ASCEND_HOME_DIR ]; then + if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then + export ASCEND_HOME_DIR=$HOME/Ascend/ascend-toolkit/latest + else + export ASCEND_HOME_DIR=/usr/local/Ascend/ascend-toolkit/latest + fi +fi +source $ASCEND_HOME_DIR/bin/setenv.bash + +export DDK_PATH=$ASCEND_HOME_DIR +arch=$(uname -m) +export NPU_HOST_LIB=$ASCEND_HOME_DIR/${arch}-linux/lib64 + +function main { + # 1. 清除算子输出和日志文件 + + rm ./output/execute_op + rm ./input/*.bin + rm ./output/*.bin + # rm -rf ./output/output*.bin > /dev/null + + # 2. 生成或复用输入数据和真值数据 + if [ -d "./input" ]; then + if [ "$(ls -A "./input")" ]; then + echo "已存在测试数据" + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + else + echo "生成测试数据" + cd $CURRENT_DIR + python3 scripts/gen_data.py + fi + + if [ $? -ne 0 ]; then + echo "ERROR: generate input data failed!" + return 1 + fi + echo "INFO: generate input data success!" + + # 3. 编译或复用acl可执行文件 + if [ -e "./output/execute_op" ]; then + echo "可执行存在" + else + echo "可执行不存在" + cd $CURRENT_DIR; rm -rf build; mkdir -p build; cd build + cmake ../src + if [ $? -ne 0 ]; then + echo "ERROR: cmake failed!" + return 1 + fi + echo "INFO: cmake success!" + make + if [ $? -ne 0 ]; then + echo "ERROR: make failed!" + return 1 + fi + echo "INFO: make success!" + fi + + # 4. 运行可执行文件 + cd $CURRENT_DIR/output + echo "INFO: execute op!" + timeout 30 ./execute_op + + if [ $? -ne 0 ]; then + echo "ERROR: acl executable run failed! please check your project!" + return 1 + fi + echo "INFO: acl executable run success!" + + # 5. 比较真值文件 + cd $CURRENT_DIR + ret=`python3 scripts/verify_result.py output/output.bin output/golden.bin` + echo $ret + if [ "x$ret" == "xtest pass" ]; then + echo "" + echo "#####################################" + echo "INFO: you have passed the Precision!" + echo "#####################################" + echo "" + fi +} + +main diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/acl.json b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/acl.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/acl.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py new file mode 100644 index 000000000..6994f4381 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py @@ -0,0 +1,21 @@ +import torch +import numpy as np +import os + + +def gen_golden_data_simple(): + input_x = np.random.uniform(-10, 10, [2000]).astype(np.int32) + bins = 100 + min = 0 + max = 0 + res = torch.histc(torch.Tensor(input_x), bins=bins, min=min, max=max) + golden = res.numpy().astype(np.int32) + print(golden) + os.system("mkdir -p input") + os.system("mkdir -p output") + input_x.tofile("./input/input_x.bin") + golden.tofile("./output/golden.bin") + + +if __name__ == "__main__": + gen_golden_data_simple() diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py new file mode 100644 index 000000000..f5243eb98 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py @@ -0,0 +1,27 @@ +import os +import sys +import numpy as np + +loss = 1e-3 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 +minimum = 10e-10 + +def verify_result(real_result, golden): + input_x = np.fromfile("./input/input_x.bin", dtype=np.int32) + real_result = np.fromfile(real_result, dtype=np.int32) # 从bin文件读取实际运算结果 + golden = np.fromfile(golden, dtype=np.int32) # 从bin文件读取预期运算结果 + result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 + deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 + result_atol = np.less_equal(result, loss) # 计算绝对误差 + result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 + if not result_rtol.all() and not result_atol.all(): + if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 + print("[ERROR] result error") + print(input_x) + print(real_result) + print(golden) + return False + print("test pass") + return True + +if __name__ == '__main__': + verify_result(sys.argv[1],sys.argv[2]) diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt new file mode 100644 index 000000000..ee73e930f --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt @@ -0,0 +1,68 @@ +# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. + +# CMake lowest version requirement +cmake_minimum_required(VERSION 3.5.1) + +# project information +project(acl_execute_add) + +# Compile options +add_compile_options(-std=c++11) + +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") + +set(INC_PATH $ENV{DDK_PATH}) + +if (NOT DEFINED ENV{DDK_PATH}) + set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") + message(STATUS "set default INC_PATH: ${INC_PATH}") +else () + message(STATUS "env INC_PATH: ${INC_PATH}") +endif() + +set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") + +set(LIB_PATH $ENV{NPU_HOST_LIB}) + +# Dynamic libraries in the stub directory can only be used for compilation +if (NOT DEFINED ENV{NPU_HOST_LIB}) + set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") + set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") + message(STATUS "set default LIB_PATH: ${LIB_PATH}") +else () + message(STATUS "env LIB_PATH: ${LIB_PATH}") +endif() + +# Header path +include_directories( + ${INC_PATH}/runtime/include + ${INC_PATH}/atc/include + ../inc + ${CUST_PKG_PATH}/include + ${BUILT_PKG_PATH}/include/aclnnop +) + +# add host lib path +link_directories( + ${LIB_PATH} + ${LIB_PATH1} + ${CUST_PKG_PATH}/lib +) + +add_executable(execute_op + operator_desc.cpp + op_runner.cpp + main.cpp + common.cpp +) + +target_link_libraries(execute_op + ascendcl + cust_opapi + acl_op_compiler + nnopbase + stdc++ +) + +install(TARGETS execute_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/common.cpp b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/common.cpp new file mode 100644 index 000000000..47a39cd5f --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/common.cpp @@ -0,0 +1,79 @@ +/** +* @file common.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" + +#include +#include +#include +#include + +extern bool g_isDevice; + +bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) +{ + struct stat sBuf; + int fileStatus = stat(filePath.data(), &sBuf); + if (fileStatus == -1) { + ERROR_LOG("failed to get file %s", filePath.c_str()); + return false; + } + if (S_ISREG(sBuf.st_mode) == 0) { + ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); + return false; + } + + std::ifstream file; + file.open(filePath, std::ios::binary); + if (!file.is_open()) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + std::filebuf *buf = file.rdbuf(); + size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); + if (size == 0) { + ERROR_LOG("file size is 0"); + file.close(); + return false; + } + if (size > bufferSize) { + ERROR_LOG("file size is larger than buffer size"); + file.close(); + return false; + } + buf->pubseekpos(0, std::ios::in); + buf->sgetn(static_cast(buffer), size); + fileSize = size; + file.close(); + return true; +} + +bool WriteFile(const std::string &filePath, const void *buffer, size_t size) +{ + if (buffer == nullptr) { + ERROR_LOG("Write file failed. buffer is nullptr"); + return false; + } + + int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); + if (fd < 0) { + ERROR_LOG("Open file failed. path = %s", filePath.c_str()); + return false; + } + + auto writeSize = write(fd, buffer, size); + (void) close(fd); + if (writeSize != size) { + ERROR_LOG("Write file Failed."); + return false; + } + + return true; +} diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/main.cpp b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/main.cpp new file mode 100644 index 000000000..30963b3cb --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/main.cpp @@ -0,0 +1,166 @@ +/** +* @file main.cpp +* +* Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include +#include +#include +#include +#include + +#include "acl/acl.h" +#include "op_runner.h" + +#include "common.h" + +bool g_isDevice = false; +int deviceId = 0; + +OperatorDesc CreateOpDesc() +{ + // define operator + std::vector shape {2000}; + std::vector shape_out {100}; + aclDataType dataType = ACL_INT32; + aclFormat format = ACL_FORMAT_ND; + OperatorDesc opDesc; + opDesc.bins = 100L; + opDesc.min = 0.0; + opDesc.max = 0.0; + opDesc.AddInputTensorDesc(dataType, shape.size(), shape.data(), format); + opDesc.AddOutputTensorDesc(dataType, shape_out.size(), shape_out.data(), format); + return opDesc; +} + +bool SetInputData(OpRunner &runner) +{ + size_t fileSize = 0; + ReadFile("../input/input_x.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); + INFO_LOG("Set input success"); + return true; +} + +bool ProcessOutputData(OpRunner &runner) +{ + WriteFile("../output/output.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); + + INFO_LOG("Write output success"); + return true; +} + +void DestoryResource() +{ + bool flag = false; + if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Reset device %d failed", deviceId); + flag = true; + } + INFO_LOG("Reset Device success"); + if (aclFinalize() != ACL_SUCCESS) { + ERROR_LOG("Finalize acl failed"); + flag = true; + } + if (flag) { + ERROR_LOG("Destory resource failed"); + } else { + INFO_LOG("Destory resource success"); + } +} + +bool InitResource() +{ + std::string output = "../output"; + if (access(output.c_str(), 0) == -1) { + int ret = mkdir(output.c_str(), 0700); + if (ret == 0) { + INFO_LOG("Make output directory successfully"); + } + else { + ERROR_LOG("Make output directory fail"); + return false; + } + } + + // acl.json is dump or profiling config file + if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { + ERROR_LOG("acl init failed"); + return false; + } + + if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { + ERROR_LOG("Set device failed. deviceId is %d", deviceId); + (void)aclFinalize(); + return false; + } + INFO_LOG("Set device[%d] success", deviceId); + + // runMode is ACL_HOST which represents app is running in host + // runMode is ACL_DEVICE which represents app is running in device + aclrtRunMode runMode; + if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { + ERROR_LOG("Get run mode failed"); + DestoryResource(); + return false; + } + g_isDevice = (runMode == ACL_DEVICE); + INFO_LOG("Get RunMode[%d] success", runMode); + + return true; +} + +bool RunOp() +{ + // create op desc + OperatorDesc opDesc = CreateOpDesc(); + + // create Runner + OpRunner opRunner(&opDesc); + if (!opRunner.Init()) { + ERROR_LOG("Init OpRunner failed"); + return false; + } + + // Load inputs + if (!SetInputData(opRunner)) { + ERROR_LOG("Set input data failed"); + return false; + } + + // Run op + if (!opRunner.RunOp()) { + ERROR_LOG("Run op failed"); + return false; + } + + // process output data + if (!ProcessOutputData(opRunner)) { + ERROR_LOG("Process output data failed"); + return false; + } + + INFO_LOG("Run op success"); + return true; +} + +int main(int argc, char **argv) +{ + if (!InitResource()) { + ERROR_LOG("Init resource failed"); + return FAILED; + } + INFO_LOG("Init resource success"); + + if (!RunOp()) { + DestoryResource(); + return FAILED; + } + + DestoryResource(); + + return SUCCESS; +} diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp new file mode 100644 index 000000000..c1d3d321b --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp @@ -0,0 +1,490 @@ +/** +* @file op_runner.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "op_runner.h" +#include "aclnn_histogram.h" +#include +#include +#include "acl/acl_op_compiler.h" +#include "common.h" + +using namespace std; + +extern bool g_isDevice; + +OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) +{ + numInputs_ = opDesc->inputDesc.size(); + numOutputs_ = opDesc->outputDesc.size(); +} + +OpRunner::~OpRunner() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto ret = aclDestroyTensor(inputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free InputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(inputBuffers_[i]); + + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free inputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devInputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostInputs_[i]); + } else { + ret = aclrtFreeHost(hostInputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostInputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto ret = aclDestroyTensor(outputTensor_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclDestroyDataBuffer(outputBuffers_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free outputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + ret = aclrtFree(devOutputs_[i]); + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free devOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + if (g_isDevice) { + ret = aclrtFree(hostOutputs_[i]); + } else { + ret = aclrtFreeHost(hostOutputs_[i]); + } + if (ret != ACL_SUCCESS) { + ERROR_LOG("Free hostOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); + exit(EXIT_FAILURE); + } + } +} + +bool OpRunner::Init() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + devInputs_.emplace_back(devMem); + inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostInput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for input[%zu] failed", i); + return false; + } + } + if (hostInput == nullptr) { + ERROR_LOG("Malloc memory for input[%zu] failed", i); + return false; + } + hostInputs_.emplace_back(hostInput); + + aclTensor *inputTensor = aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), + nullptr, 0, GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); + if (inputTensor == nullptr) { + ERROR_LOG("Create Tensor for input[%zu] failed", i); + return false; + } + inputTensor_.emplace_back(inputTensor); + } + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + void *devMem = nullptr; + if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + devOutputs_.emplace_back(devMem); + outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); + + void *hostOutput = nullptr; + if (g_isDevice) { + if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } else { + if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory for output[%zu] failed", i); + return false; + } + } + if (hostOutput == nullptr) { + ERROR_LOG("Malloc host memory for output[%zu] failed", i); + return false; + } + hostOutputs_.emplace_back(hostOutput); + + aclTensor *outputTensor = aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), + nullptr, 0, GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); + if (outputTensor == nullptr) { + ERROR_LOG("Create Tensor for output[%zu] failed", i); + return false; + } + outputTensor_.emplace_back(outputTensor); + } + + return true; +} + +const size_t OpRunner::NumInputs() +{ + return numInputs_; +} + +const size_t OpRunner::NumOutputs() +{ + return numOutputs_; +} + +const size_t OpRunner::GetInputSize(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->inputDesc[index]); +} + +const size_t OpRunner::GetInputNumDims(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); +} + +aclDataType OpRunner::GetInputDataType(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->inputDesc[index]); +} + +aclFormat OpRunner::GetInputFormat(size_t index) const +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->inputDesc[index]); +} + +std::vector OpRunner::GetInputShape(size_t index) const +{ + std::vector ret; + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return ret; + } + + auto desc = opDesc_->inputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + + return ret; +} + +size_t OpRunner::GetOutputSize(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescSize(opDesc_->outputDesc[index]); +} + +const size_t OpRunner::GetOutputNumDims(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); +} + +aclDataType OpRunner::GetOutputDataType(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_DT_UNDEFINED; + } + + return aclGetTensorDescType(opDesc_->outputDesc[index]); +} + + +aclFormat OpRunner::GetOutputFormat(size_t index) const +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ACL_FORMAT_UNDEFINED; + } + + return aclGetTensorDescFormat(opDesc_->outputDesc[index]); +} + +std::vector OpRunner::GetOutputShape(size_t index) const +{ + std::vector ret; + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return ret; + } + + auto desc = opDesc_->outputDesc[index]; + for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { + int64_t dimSize; + if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { + ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); + ret.clear(); + return ret; + } + ret.emplace_back(dimSize); + } + return ret; +} + +size_t OpRunner::GetInputElementCount(size_t index) const +{ + if (index >= opDesc_->inputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); +} + +size_t OpRunner::GetOutputElementCount(size_t index) const +{ + if (index >= opDesc_->outputDesc.size()) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return 0; + } + + return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); +} + +bool OpRunner::RunOp() +{ + for (size_t i = 0; i < numInputs_; ++i) { + auto size = GetInputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { + ERROR_LOG("Copy input[%zu] failed", i); + return false; + } + INFO_LOG("Copy input[%zu] success", i); + } + + aclrtStream stream = nullptr; + if (aclrtCreateStream(&stream) != ACL_SUCCESS) { + ERROR_LOG("Create stream failed"); + return false; + } + INFO_LOG("Create stream success"); + + size_t workspaceSize = 0; + aclOpExecutor *handle = nullptr; + + auto ret = aclnnHistogramGetWorkspaceSize(inputTensor_[0], + opDesc_->bins, + opDesc_->min, + opDesc_->max, + outputTensor_[0], + &workspaceSize, &handle); + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute GetWorkspaceSize success, workspace size %lu", workspaceSize); + + void *workspace = nullptr; + if (workspaceSize != 0) { + if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { + ERROR_LOG("Malloc device memory failed"); + } + } + ret = aclnnHistogram(workspace, workspaceSize, handle, stream); + + if (ret != ACL_SUCCESS) { + (void)aclrtDestroyStream(stream); + ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); + return false; + } + INFO_LOG("Execute Operator success"); + + ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); + if (ret != SUCCESS) { + ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Synchronize stream success"); + + for (size_t i = 0; i < numOutputs_; ++i) { + auto size = GetOutputSize(i); + aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; + if (g_isDevice) { + kind = ACL_MEMCPY_DEVICE_TO_DEVICE; + } + if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { + INFO_LOG("Copy output[%zu] success", i); + (void)aclrtDestroyStream(stream); + return false; + } + INFO_LOG("Copy output[%zu] success", i); + } + + (void)aclrtDestroyStream(stream); + return true; +} + + +template +void DoPrintData(const T *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << data[i]; + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) +{ + assert(elementsPerRow != 0); + for (size_t i = 0; i < count; ++i) { + std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); + if (i % elementsPerRow == elementsPerRow - 1) { + std::cout << std::endl; + } + } +} + +void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) +{ + if (data == nullptr) { + ERROR_LOG("Print data failed. data is nullptr"); + return; + } + + switch (dataType) { + case ACL_BOOL: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT8: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT16: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT32: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_INT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_UINT64: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT16: + DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_FLOAT: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + case ACL_DOUBLE: + DoPrintData(reinterpret_cast(data), count, elementsPerRow); + break; + default: + ERROR_LOG("Unsupported type: %d", dataType); + } +} + +void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) +{ + if (index >= numInputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); + return; + } + + auto desc = opDesc_->inputDesc[index]; + PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} + +void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) +{ + if (index >= numOutputs_) { + ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); + return; + } + + auto desc = opDesc_->outputDesc[index]; + PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); +} diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp new file mode 100644 index 000000000..a7a1ed3bd --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp @@ -0,0 +1,56 @@ +/** +* @file operator_desc.cpp +* +* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +*/ +#include "common.h" +#include "operator_desc.h" + +using namespace std; + +OperatorDesc::OperatorDesc() {} + +OperatorDesc::~OperatorDesc() +{ + for (auto *desc : inputDesc) { + aclDestroyTensorDesc(desc); + } + + for (auto *desc : outputDesc) { + aclDestroyTensorDesc(desc); + } + +} + +OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + inputDesc.emplace_back(desc); + return *this; +} + +OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, + int numDims, + const int64_t *dims, + aclFormat format) +{ + aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); + if (desc == nullptr) { + ERROR_LOG("create tensor failed"); + return *this; + } + + outputDesc.emplace_back(desc); + return *this; +} diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/CMakeLists.txt b/operator_contrib/Histogram/FrameworkLaunch/Histogram/CMakeLists.txt new file mode 100644 index 000000000..584132d80 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.16.0) +project(opp) +if(ENABLE_CROSS_COMPILE) + if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) + set(CROSS_COMPILE_PLATFORM aarch64) + else() + set(CROSS_COMPILE_PLATFORM x86_64) + endif() + set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) + set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) + set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) + set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) +else() + set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) +endif() + +include(cmake/config.cmake) +include(cmake/func.cmake) +include(cmake/intf.cmake) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) + add_subdirectory(framework) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) + add_subdirectory(op_host) +endif() +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) + add_subdirectory(op_kernel) +endif() +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# modify vendor_name in install.sh and upgrade.sh +add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh + COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts + COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ + COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* +) +add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) +install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) + +install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) + +get_system_info(SYSTEM_INFO) + +# gen version.info +add_custom_target(gen_version_info ALL + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} +) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info + DESTINATION packages/vendors/${vendor_name}/) + +# CPack config +set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) +set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) +set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") +set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") +set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) +set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") +set(CPACK_GENERATOR External) +set(CPACK_CMAKE_GENERATOR "Unix Makefiles") +set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) +set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) +set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) +include(CPack) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/CMakePresets.json b/operator_contrib/Histogram/FrameworkLaunch/Histogram/CMakePresets.json new file mode 100644 index 000000000..8b7181000 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/CMakePresets.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "cmakeMinimumRequired": { + "major": 3, + "minor": 19, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default Config", + "description": "Default build using Unix Makefiles generator", + "generator": "Unix Makefiles", + "binaryDir": "${sourceDir}/build_out", + "cacheVariables": { + "CMAKE_BUILD_TYPE": { + "type": "STRING", + "value": "Release" + }, + "ENABLE_SOURCE_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ENABLE_BINARY_PACKAGE": { + "type": "BOOL", + "value": "True" + }, + "ASCEND_COMPUTE_UNIT": { + "type": "STRING", + "value": "ascend310b" + }, + "ENABLE_TEST": { + "type": "BOOL", + "value": "True" + }, + "vendor_name": { + "type": "STRING", + "value": "customize" + }, + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + }, + "ASCEND_PYTHON_EXECUTABLE": { + "type": "STRING", + "value": "python3" + }, + "CMAKE_INSTALL_PREFIX": { + "type": "PATH", + "value": "${sourceDir}/build_out" + }, + "ENABLE_CROSS_COMPILE": { + "type": "BOOL", + "value": "False" + }, + "CMAKE_CROSS_PLATFORM_COMPILER": { + "type": "PATH", + "value": "/usr/bin/aarch64-linux-gnu-g++" + } + } + } + ] +} \ No newline at end of file diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/Histogram.json b/operator_contrib/Histogram/FrameworkLaunch/Histogram/Histogram.json new file mode 100644 index 000000000..76d818c73 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/Histogram.json @@ -0,0 +1,58 @@ +[ + { + "op": "Histogram", + "language": "cpp", + "input_desc": [ + { + "name": "x", + "param_type": "required", + "format": [ + "ND", + "ND", + "ND" + ], + "type": [ + "float", + "fp16", + "int32" + ] + } + ], + "output_desc": [ + { + "name": "y", + "param_type": "required", + "format": [ + "ND", + "ND", + "ND" + ], + "type": [ + "float", + "fp16", + "int32" + ] + } + ], + "attr": [ + { + "name": "bins", + "param_type": "optional", + "type": "int", + "default_value": 100 + }, + { + "name": "min", + "param_type": "optional", + "type": "float", + "default_value": 0 + }, + { + "name": "max", + "param_type": "optional", + "type": "float", + "default_value": 0 + } + ] + } +] \ No newline at end of file diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/build.sh b/operator_contrib/Histogram/FrameworkLaunch/Histogram/build.sh new file mode 100644 index 000000000..27d1968d1 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/build.sh @@ -0,0 +1,80 @@ +#!/bin/bash +script_path=$(realpath $(dirname $0)) + + +mkdir -p build_out +rm -rf build_out/* +cd build_out + +opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) +ENABLE_CROSS="-DENABLE_CROSS_COMPILE=True" +ENABLE_BINARY="-DENABLE_BINARY_PACKAGE=True" +cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') + +cmake_run_package() +{ + target=$1 + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then exit 1; fi + + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + if [ $? -ne 0 ]; then exit 1; fi + cmake --build . --target binary -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + cmake --build . --target $target -j16 + fi + fi +} + +if [[ $opts =~ $ENABLE_CROSS ]] && [[ $opts =~ $ENABLE_BINARY ]] +then + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts -DENABLE_CROSS_COMPILE=0 + else + cmake .. --preset=default -DENABLE_CROSS_COMPILE=0 + fi + cmake_run_package $target + cp -r kernel ../ + rm -rf * + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts + else + cmake .. --preset=default + fi + + cmake --build . --target $target -j16 + if [ $? -ne 0 ]; then + echo "[ERROR] Kernel compile failed, the run package will not be generated." + rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; + fi + if [ $target = "package" ]; then + if test -d ./op_kernel/binary ; then + ./cust*.run + fi + fi + rm -rf ../kernel + +else + target=package + if [ "$1"x != ""x ]; then target=$1; fi + if [ "$cmake_version" \< "3.19.0" ] ; then + cmake .. $opts + else + cmake .. --preset=default + fi + cmake_run_package $target +fi + + +# for debug +# cd build_out +# make +# cpack +# verbose append -v \ No newline at end of file diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/config.cmake b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/config.cmake new file mode 100644 index 000000000..886119daa --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/config.cmake @@ -0,0 +1,25 @@ + +set(CMAKE_CXX_FLAGS_DEBUG "") +set(CMAKE_CXX_FLAGS_RELEASE "") + +if (NOT DEFINED vendor_name) + set(vendor_name customize CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) + set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") +endif() +if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) + set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") +endif() +if (NOT DEFINED ASCEND_COMPUTE_UNIT) + message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! +") +endif() +set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) +set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) +set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) +set(ASCEND_FRAMEWORK_TYPE tensorflow) +file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) +set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") +execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/func.cmake b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/func.cmake new file mode 100644 index 000000000..4179dfd25 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/func.cmake @@ -0,0 +1,192 @@ + +function(get_system_info SYSTEM_INFO) + if (UNIX) + execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) + string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) + set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) + elseif (WIN32) + message(STATUS "System is Windows. Only for pre-build.") + else () + message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") + endif () +endfunction() + +function(opbuild) + message(STATUS "Opbuild generating sources") + cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) + execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 + -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api + -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("build ops lib info: ${EXEC_INFO}") + message("build ops lib error: ${EXEC_ERROR}") + message(FATAL_ERROR "opbuild run failed!") + endif() + set(proj_env "") + set(prefix_env "") + if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") + set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") + endif() + if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") + set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") + endif() + execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build + ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("opbuild ops info: ${EXEC_INFO}") + message("opbuild ops error: ${EXEC_ERROR}") + endif() + message(STATUS "Opbuild generating sources - done") +endfunction() + +function(add_ops_info_target) + cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) + get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) + add_custom_command(OUTPUT ${OPINFO_OUTPUT} + COMMAND mkdir -p ${opinfo_file_path} + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py + ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} + ) + add_custom_target(${OPINFO_TARGET} ALL + DEPENDS ${OPINFO_OUTPUT} + ) + install(FILES ${OPINFO_OUTPUT} + DESTINATION ${OPINFO_INSTALL_DIR} + ) +endfunction() + +function(add_ops_compile_options OP_TYPE) + cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) + file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} + "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") +endfunction() + +function(add_ops_impl_target) + cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) + add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ${OPIMPL_OPS_INFO} + \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" + ${OPIMPL_IMPL_DIR} + ${OPIMPL_OUT_DIR}/dynamic + ${ASCEND_AUTOGEN_PATH} + + COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp + COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp + DEPENDS ${OPIMPL_OPS_INFO} + ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py + ) + add_custom_target(${OPIMPL_TARGET} ALL + DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) + if (${ENABLE_SOURCE_PACKAGE}) + install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic + DESTINATION ${OPIMPL_INSTALL_DIR} + ) + endif() +endfunction() + +function(add_npu_support_target) + cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) + add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json + COMMAND mkdir -p ${NPUSUP_OUT_DIR} + COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh + ${NPUSUP_OPS_INFO_DIR} + ${NPUSUP_OUT_DIR} + ) + add_custom_target(npu_supported_ops ALL + DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json + ) + install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json + DESTINATION ${NPUSUP_INSTALL_DIR} + ) +endfunction() + +function(add_bin_compile_target) + cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) + execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py + ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} + RESULT_VARIABLE EXEC_RESULT + OUTPUT_VARIABLE EXEC_INFO + ERROR_VARIABLE EXEC_ERROR + ) + if (${EXEC_RESULT}) + message("ops binary compile scripts gen info: ${EXEC_INFO}") + message("ops binary compile scripts gen error: ${EXEC_ERROR}") + message(FATAL_ERROR "ops binary compile scripts gen failed!") + endif() + if (NOT TARGET binary) + add_custom_target(binary) + endif() + add_custom_target(${BINCMP_TARGET} + COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src + ) + add_custom_target(${BINCMP_TARGET}_gen_ops_config + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin + COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin + -s ${BINCMP_COMPUTE_UNIT} + ) + add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) + file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) + foreach(bin_script ${bin_scripts}) + get_filename_component(bin_file ${bin_script} NAME_WE) + string(REPLACE "-" ";" bin_sep ${bin_file}) + list(GET bin_sep 0 op_type) + list(GET bin_sep 1 op_file) + list(GET bin_sep 2 op_index) + if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) + file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) + add_custom_target(${BINCMP_TARGET}_${op_file}_copy + COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py + ) + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL + ) + endif() + add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} + COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} && echo $(MAKE) + WORKING_DIRECTORY ${BINCMP_OUT_DIR} + ) + add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) + add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) + endforeach() + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + + install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL + ) + +endfunction() + +function(add_cross_compile_target) + cmake_parse_arguments(CROSSMP "" "TARGET;OUT_DIR;INSTALL_DIR" "" ${ARGN}) + add_custom_target(${CROSSMP_TARGET} ALL + DEPENDS ${CROSSMP_OUT_DIR} + ) + install(DIRECTORY ${CROSSMP_OUT_DIR} + DESTINATION ${CROSSMP_INSTALL_DIR} + ) +endfunction() diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/intf.cmake b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/intf.cmake new file mode 100644 index 000000000..2f362c396 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/intf.cmake @@ -0,0 +1,26 @@ + +add_library(intf_pub INTERFACE) +target_compile_options(intf_pub INTERFACE + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + $<$:-O2> + $<$:-O0 -g> + $<$:-std=c++11> + $<$,$>:-ftrapv -fstack-check> + $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> + $,-fstack-protector-strong,-fstack-protector-all> +) +target_compile_definitions(intf_pub INTERFACE + _GLIBCXX_USE_CXX11_ABI=0 + $<$:_FORTIFY_SOURCE=2> +) +target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) +target_link_options(intf_pub INTERFACE + $<$,EXECUTABLE>:-pie> + $<$:-s> + -Wl,-z,relro + -Wl,-z,now + -Wl,-z,noexecstack +) +target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/makeself.cmake b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/makeself.cmake new file mode 100644 index 000000000..48c565bfb --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/makeself.cmake @@ -0,0 +1,17 @@ +execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) +execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh + --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh + --help-header ./help.info + --gzip --complevel 4 --nomd5 --sha256 + ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} + RESULT_VARIABLE EXEC_RESULT + ERROR_VARIABLE EXEC_ERROR +) +if (NOT "${EXEC_RESULT}x" STREQUAL "0x") + message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") +endif() +execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ + COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" + WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} +) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/__init__.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/__init__.py new file mode 100644 index 000000000..c4ddc893a --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import sys +import os + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(PYF_PATH) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_bin_param_build.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_bin_param_build.py new file mode 100644 index 000000000..63b15c900 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_bin_param_build.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import json +import hashlib +import const_var +import opdesc_parser + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class BinParamBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + self.soc = '' + self.out_path = '' + + def set_soc_version(self: any, soc: str): + self.soc = soc + + def set_out_path(self: any, out_path: str): + self.out_path = out_path + + def gen_input_json(self: any): + key_map = {} + count = len(self.input_dtype[0].split(',')) + for i in range(0, count): + inputs = [] + outputs = [] + attrs = [] + op_node = {} + for idx in range(0, len(self.input_name)): + idtypes = self.input_dtype[idx].split(',') + ifmts = self.input_fmt[idx].split(',') + itype = self.input_type[idx] + para = {} + para['name'] = self.input_name[idx][:-5] + para['index'] = idx + para['dtype'] = idtypes[i] + para['format'] = ifmts[i] + para['paramType'] = itype + para['shape'] = [-2] + if itype == 'dynamic': + inputs.append([para]) + else: + inputs.append(para) + for idx in range(0, len(self.output_name)): + odtypes = self.output_dtype[idx].split(',') + ofmts = self.output_fmt[idx].split(',') + otype = self.output_type[idx] + para = {} + para['name'] = self.output_name[idx][:-5] + para['index'] = idx + para['dtype'] = odtypes[i] + para['format'] = ofmts[i] + para['paramType'] = otype + para['shape'] = [-2] + if otype == 'dynamic': + outputs.append([para]) + else: + outputs.append(para) + for attr in self.attr_list: + att = {} + att['name'] = attr + atype = self.attr_val.get(attr).get('type').lower() + att['dtype'] = atype + att['value'] = const_var.ATTR_DEF_VAL.get(atype) + attrs.append(att) + op_node['bin_filename'] = '' + op_node['inputs'] = inputs + op_node['outputs'] = outputs + if len(attrs) > 0: + op_node['attrs'] = attrs + param = {} + param['op_type'] = self.op_type + param['op_list'] = [op_node] + objstr = json.dumps(param, indent=' ') + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + while key_map.get(md5sum) is not None: + objstr += '1' + md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() + key_map[md5sum] = md5sum + bin_file = self.op_type + '_' + md5sum + op_node['bin_filename'] = bin_file + param_file = os.path.join(self.out_path, bin_file + '_param.json') + param_file = os.path.realpath(param_file) + with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(param, fd, indent=' ') + self._write_buld_cmd(param_file, bin_file, i) + + def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): + hard_soc = const_var.SOC_MAP_EXT.get(self.soc) + if not hard_soc: + hard_soc = self.soc.capitalize() + name_com = [self.op_type, self.op_file, str(index)] + compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') + compile_file = os.path.realpath(compile_file) + with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + fd.write('#!/bin/bash\n') + fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) + cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') + fd.write(cmd) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') + fd.write(chk) + chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') + fd.write(chk) + fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) + + +def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): + op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) + for op_desc in op_descs: + op_desc.set_soc_version(soc) + op_desc.set_out_path(out_dir) + op_desc.gen_input_json() + + +if __name__ == '__main__': + if len(sys.argv) <= 3: + raise RuntimeError('arguments must greater than 3') + gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_impl_build.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_impl_build.py new file mode 100644 index 000000000..647801c07 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_impl_build.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import re +import opdesc_parser +import const_var + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +IMPL_HEAD = ''' +import os, sys +import ctypes +import json +import shutil +from tbe.common.platform import get_soc_spec +from tbe.common.utils import para_check +from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo +from tbe.common.buildcfg import get_default_build_config +from impl.util.platform_adapter import tbe_register +from tbe.common.buildcfg import get_current_build_config +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + +DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], + "float16": ["DT_FLOAT16", "half"], + "int8": ["DT_INT8", "int8_t"], + "int16": ["DT_INT16", "int16_t"], + "int32": ["DT_INT32", "int32_t"], + "int64": ["DT_INT64", "int64_t"], + "uint1": ["DT_UINT1", "uint8_t"], + "uint8": ["DT_UINT8", "uint8_t"], + "uint16": ["DT_UINT16", "uint16_t"], + "uint32": ["DT_UINT32", "uint32_t"], + "uint64": ["DT_UINT64", "uint64_t"], + "bool": ["DT_BOOL", "bool"], + "double": ["DT_DOUBLE", "double"], + "dual": ["DT_DUAL", "unknown"], + "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], + "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], + "string": ["DT_STRING", "unknown"], + "complex64": ["DT_COMPLEX64", "unknown"], + "complex128": ["DT_COMPLEX128", "unknown"], + "qint8": ["DT_QINT8", "unknown"], + "qint16": ["DT_QINT16", "unknown"], + "qint32": ["DT_QINT32", "unknown"], + "quint8": ["DT_QUINT8", "unknown"], + "quint16": ["DT_QUINT16", "unknown"], + "resource": ["DT_RESOURCE", "unknown"], + "string_ref": ["DT_STRING_REF", "unknown"], + "int4": ["DT_INT4", "int4b_t"], + "bfloat16": ["DT_BF16", "bfloat16_t"]}} + +def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): + options = [] + x_fmt = x.get("format") + x_dtype = x.get("dtype") + x_n_in_kernel = x_n + '_REF' if is_ref else x_n + options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) + options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) + options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) + return options + +def get_dtype_fmt_options(__inputs__, __outputs__): + options = [] + input_names = {} + output_names = {} + unique_param_name_set = set() + for idx, x in enumerate(__inputs__): + if x is None: + continue + x_n = input_names[idx].upper() + unique_param_name_set.add(x_n) + options += add_dtype_fmt_option_single(x, x_n) + + for idx, x in enumerate(__outputs__): + if x is None: + continue + x_n = output_names[idx].upper() + if x_n in unique_param_name_set: + options += add_dtype_fmt_option_single(x, x_n, True) + else: + options += add_dtype_fmt_option_single(x, x_n) + return options + +def load_dso(so_path): + try: + ctypes.CDLL(so_path) + except OSError as error : + print(error) + raise RuntimeError("cannot open %s" %(so_path)) + else: + print("load so succ ", so_path) + +def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): + compile_options = [] + if shortsoc in compile_option_list: + compile_options = compile_option_list[shortsoc] + elif '__ALLSOC__' in compile_option_list: + compile_options = compile_option_list['__ALLSOC__'] + return compile_options + +def get_kernel_source(src_file, dir_snake, dir_ex): + src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) + if os.path.exists(src_ex): + return src_ex + src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) + if os.path.exists(src): + return src + src = os.path.join(PYF_PATH, src_file) + if os.path.exists(src): + return src + return src_ex + +''' + +IMPL_API = ''' +@tbe_register.register_operator("{}", trans_bool_to_s8=False) +@para_check.check_op_params({}) +def {}({}, kernel_name="{}", impl_mode=""): + if get_current_build_config("enable_op_prebuild"): + return + __inputs__, __outputs__, __attrs__ = _build_args({}) + options = get_dtype_fmt_options(__inputs__, __outputs__) + options += ["-x", "cce"] + bisheng = shutil.which("bisheng") + if bisheng != None: + bisheng_path = os.path.dirname(bisheng) + tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) + else: + tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") + options.append("-I" + tikcpp_path) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) + options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) + options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) + if impl_mode == "high_performance": + options.append("-DHIGH_PERFORMANCE=1") + elif impl_mode == "high_precision": + options.append("-DHIGH_PRECISION=1") + if get_default_build_config("enable_deterministic_mode") == 1: + options.append("-DDETEMINISTIC_MODE=1") + + custom_compile_options = {}, + custom_all_compile_options = {}, + soc_version = get_soc_spec("SOC_VERSION") + soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() + custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) + custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) + options += custom_all_compile_options_soc + options += custom_compile_options_soc + + origin_func_name = "{}" + ascendc_src_dir_ex = "{}" + ascendc_src_dir = "{}" + ascendc_src_file = "{}" + src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) +''' + +REPLAY_OP_API = ''' + print("start replay Acend C Operator {}, kernel name is {}") + tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" + tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version + print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") + codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" + replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" + if PYF_PATH.endswith("dynamic"): + op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") + else: + op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") + replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") + load_dso(codegen_so_path) + load_dso(replaystub_so_path) + load_dso(replayapi_so_path) + op_type = "{}" + entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode) + res, msg = replay_op(op_info, entry_obj, code_channel, src, options) + if not res: + print("call replay op failed for %s and get into call compile op" %(msg)) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +COMPILE_OP_API = ''' + print("start compile Ascend C operator {}. kernel name is " + kernel_name) + op_type = "{}" + code_channel = get_code_channel(src, kernel_name, op_type, options) + op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ + attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}], param_type_dynamic = {},\\ + mc2_ctx = None, param_type_list = {}) + compile_op(src, origin_func_name, op_info, options, code_channel, '{}') +''' + +SUP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + ret_dict = json.loads(ret_str) + err_code = ret_dict.get("ret_code") + sup = "Unknown" + reason = "Unknown reason" + if err_code is not None: + if err_code is 0: + sup = "True" + reason = "" + elif err_code is 1: + sup = "False" + reason = ret_dict.get("reason") + else: + sup = "Unknown" + reason = ret_dict.get("reason") + return sup, reason +''' +CAP_API = ''' +def {}({}, impl_mode=""): + __inputs__, __outputs__, __attrs__ = _build_args({}) + result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) + return result.decode("utf-8") +''' +GLZ_API = ''' +@tbe_register.register_param_generalization("{}") +def {}_generalization({}, generalize_config=None): + __inputs__, __outputs__, __attrs__ = _build_args({}) + ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) + return [json.loads(ret_str)] +''' + +ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'list_int': '[]', + 'list_float': '[]', 'list_bool': '[]', 'list_list_int': '[[]]', 'str': ''} + + +def _get_snake_str(s, i): + if s[i - 1] != '_': + if not s[i - 1].isupper(): + return "_" + elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): + return "_" + return "" + return "" + + +def optype_snake(origin_str): + temp_str = origin_str[0].lower() + origin_str[1:] + new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() + return new_str + + +def optype_snake_ex(s): + snake_case = "" + for i, c in enumerate(s): + if i == 0: + snake_case += c.lower() + elif c.isupper(): + snake_case += _get_snake_str(s, i) + snake_case += c.lower() + else: + snake_case += c + return snake_case + + +class AdpBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + self.argsdefv = [] + self.op_compile_option:str = '{}' + super().__init__(op_type) + + + def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): + self._build_paradefault() + if impl_path != "": + src_file = os.path.join(impl_path, self.op_file + '.cpp') + if not os.path.exists(src_file): + return + out_path = os.path.abspath(path) + if self.dynamic_shape and not out_path.endswith('dynamic'): + out_path = os.path.join(path, 'dynamic') + os.makedirs(out_path, mode=0o700, exist_ok=True) + adpfile = os.path.join(out_path, self.op_file + '.py') + self._gen_op_compile_option(op_compile_option_all) + with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + self._write_head(fd) + self._write_argparse(fd) + self._write_impl(fd) + if self.op_chk_support: + self._write_cap('check_supported', fd) + self._write_cap('get_op_support_info', fd) + if self.op_fmt_sel: + self._write_cap('op_select_format', fd) + self._write_cap('get_op_specific_info', fd) + if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': + self._write_glz(fd) + + + def _gen_op_compile_option(self:any, op_compile_option_all:list =None): + if op_compile_option_all is not None: + if self.op_type in op_compile_option_all: + self.op_compile_option = op_compile_option_all[self.op_type] + elif "__all__" in op_compile_option_all: + self.op_compile_option = op_compile_option_all["__all__"] + + + def _ip_argpack(self: any, default: bool = True) -> list: + args = [] + for i in range(len(self.input_name)): + arg = self.input_name[i] + if default and self.argsdefv[i] is not None: + arg += '=' + self.argsdefv[i] + args.append(arg) + return args + + def _op_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + for i in range(len(self.output_name)): + arg = self.output_name[i] + if default and self.argsdefv[i + argidx] is not None: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _attr_argpack(self: any, default: bool = True) -> list: + args = [] + argidx = len(self.input_name) + len(self.output_name) + for i in range(len(self.attr_list)): + att = self.attr_list[i] + arg = att + if default and self.argsdefv[i + argidx] is not None: + if self.attr_val.get(att).get('type') == 'str': + arg += '="' + self.argsdefv[i + argidx] + '"' + elif self.attr_val.get(att).get('type') == 'bool': + arg += '=' + self.argsdefv[i + argidx].capitalize() + else: + arg += '=' + self.argsdefv[i + argidx] + args.append(arg) + return args + + def _build_paralist(self: any, default: bool = True) -> str: + args = [] + args.extend(self._ip_argpack(default)) + args.extend(self._op_argpack(default)) + args.extend(self._attr_argpack(default)) + return ', '.join(args) + + def _io_parachk(self: any, types: list, type_name: str) -> list: + chk = [] + for iot in types: + if iot == 'optional': + ptype = 'OPTION' + else: + ptype = iot.upper() + chk.append('para_check.{}_{}'.format(ptype, type_name)) + return chk + + def _attr_parachk(self: any) -> list: + chk = [] + for att in self.attr_list: + if self.attr_val.get(att).get('paramType') == 'optional': + pt = 'OPTION' + else: + pt = self.attr_val.get(att).get('paramType').upper() + att_type = self.attr_val.get(att).get('type').upper() + chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) + return chk + + def _build_parachk(self: any) -> str: + chk = [] + chk.extend(self._io_parachk(self.input_type, 'INPUT')) + chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) + chk.extend(self._attr_parachk()) + chk.append('para_check.KERNEL_NAME') + return ', '.join(chk) + + def _build_paradefault(self: any): + optional = False + argtypes = [] + argtypes.extend(self.input_type) + argtypes.extend(self.output_type) + for atype in argtypes: + if atype == 'optional': + optional = True + if optional: + self.argsdefv.append('None') + else: + self.argsdefv.append(None) + for attr in self.attr_list: + atype = self.attr_val.get(attr).get('paramType') + if atype == 'optional': + optional = True + attrval = self.attr_val.get(attr).get('defaultValue') + if attrval is not None: + optional = True + if type == "bool": + attrval = attrval.capitalize() + elif type == "str": + attrval = "\"" + attrval + "\"" + self.argsdefv.append(attrval) + continue + if optional: + self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) + else: + self.argsdefv.append(None) + + def _write_head(self: any, fd: object): + fd.write(IMPL_HEAD.format(self.input_ori_name, self.output_ori_name)) + + def _write_argparse(self: any, fd: object): + args = self._build_paralist(False) + fd.write('def _build_args({}):\n'.format(args)) + fd.write(' __inputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __inputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __inputs__.append(arg)\n') + fd.write(' __outputs__ = []\n') + fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) + fd.write(' if arg != None:\n') + fd.write(' if isinstance(arg, (list, tuple)):\n') + fd.write(' if len(arg) == 0:\n') + fd.write(' continue\n') + fd.write(' __outputs__.append(arg[0])\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' else:\n') + fd.write(' __outputs__.append(arg)\n') + fd.write(' __attrs__ = []\n') + for attr in self.attr_list: + fd.write(' if {} != None:\n'.format(attr)) + fd.write(' attr = {}\n') + fd.write(' attr["name"] = "{}"\n'.format(attr)) + fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) + fd.write(' attr["value"] = {}\n'.format(attr)) + fd.write(' __attrs__.append(attr)\n') + fd.write(' return __inputs__, __outputs__, __attrs__\n') + + def _write_impl(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + pchk = self._build_parachk() + if len(self.kern_name) > 0: + kern_name = self.kern_name + else: + kern_name = self.op_intf + src = self.op_file + '.cpp' + fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ + self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ + optype_snake_ex(self.op_type), optype_snake(self.op_type), src)) + if self.op_replay_flag: + fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ + self.op_compile_option)) + else: + fd.write(COMPILE_OP_API.format(self.op_type, self.op_type, ', '.join(self.input_name),\ + ', '.join(self.output_name), False, self.input_type + self.output_type, self.op_compile_option)) + + def _write_cap(self: any, cap_name: str, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + if cap_name == 'check_supported': + fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + else: + fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) + + def _write_glz(self: any, fd: object): + argsdef = self._build_paralist() + argsval = self._build_paralist(False) + fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) + + +def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + file_map = {} + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ + ops, dirs.get(const_var.AUTO_GEN_DIR)) + for op_desc in op_descs: + op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) + file_map[op_desc.op_type] = op_desc.op_file + return file_map + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater equal than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + cfg_dir = {} + cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] + write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_ops_config.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_ops_config.py new file mode 100644 index 000000000..b138047af --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_ops_config.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import glob +import json +import argparse +import const_var + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def add_simplified_config(op_type, key, core_type, task_ration, objfile, config): + simple_cfg = config.get('binary_info_config.json') + op_cfg = simple_cfg.get(op_type) + if not op_cfg: + op_cfg = {} + op_cfg['dynamicRankSupport'] = True + op_cfg['simplifiedKeyMode'] = 0 + op_cfg['binaryList'] = [] + simple_cfg[op_type] = op_cfg + bin_list = op_cfg.get('binaryList') + if core_type == 0 and task_ration == "tilingKey": + bin_list.append({'coreType': core_type, 'simplifiedKey': key, \ + 'multiKernelType':1, 'binPath': objfile}) + else: + bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) + + +def add_op_config(op_file, bin_info, config): + op_cfg = config.get(op_file) + if not op_cfg: + op_cfg = {} + op_cfg['binList'] = [] + config[op_file] = op_cfg + op_cfg.get('binList').append(bin_info) + + +def gen_ops_config(json_file, soc, config): + core_type_map = {'MIX': 0, 'AiCore': 1, 'VectorCore': 2, 'MIX_AICORE':3, 'MIX_VECTOR_CORE':4} + contents = load_json(json_file) + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + json_base_name = os.path.basename(json_file) + op_dir = os.path.basename(os.path.dirname(json_file)) + support_info = contents.get('supportInfo') + bin_name = contents.get('binFileName') + bin_suffix = contents.get('binFileSuffix') + task_ration = contents.get('taskRation') + core_type = core_type_map.get(contents.get("coreType")) + bin_file_name = bin_name + bin_suffix + op_type = bin_name.split('_')[0] + op_file = op_dir + '.json' + bin_info = {} + keys = support_info.get('simplifiedKey') + if keys: + bin_info['simplifiedKey'] = keys + for key in keys: + add_simplified_config(op_type, key, core_type, task_ration, os.path.join(soc, op_dir, bin_file_name), \ + config) + bin_info['staticKey'] = support_info.get('staticKey') + bin_info['int64Mode'] = support_info.get('int64Mode') + bin_info['inputs'] = support_info.get('inputs') + bin_info['outputs'] = support_info.get('outputs') + if support_info.get('attrs'): + bin_info['attrs'] = support_info.get('attrs') + bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} + add_op_config(op_file, bin_info, config) + + +def gen_all_config(root_dir, soc): + suffix = 'json' + config = {} + config['binary_info_config.json'] = {} + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + gen_ops_config(_json, soc, config) + for cfg_key in config.keys(): + cfg_file = os.path.join(root_dir, cfg_key) + with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: + json.dump(config.get(cfg_key), fd, indent=' ') + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + parser.add_argument('-s', + '--soc', + nargs='?', + required=True, + help='Parse the soc_version of ops.') + return parser.parse_args() + + +def main(): + args = args_prase() + gen_all_config(args.path, args.soc) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_replay_build.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_replay_build.py new file mode 100644 index 000000000..1cac7d911 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_replay_build.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os +import opdesc_parser +import replay_codegen +import const_var +from replay_codegen import ReplayCodeGenParams + +PYF_PATH = os.path.dirname(os.path.realpath(__file__)) + + +class ReplayBuilder(opdesc_parser.OpDesc): + def __init__(self: any, op_type: str): + super().__init__(op_type) + + def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): + if not self.op_replay_flag: + print('{} replay not enabled'.format(self.op_type)) + return + argn = len(self.input_name) + len(self.output_name) + 1 + if self.op_replay_batch: + print('{} replay in batch mode'.format(self.op_type)) + else: + print('{} replay in normal mode'.format(self.op_type)) + if impl_path.endswith('op_kernel'): + implf = os.path.join(impl_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') + else: + if self.dynamic_shape: + dyn_path = 'dynamic' + else: + dyn_path = '' + implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') + tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') + rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ + self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) + rep_conf.set_batch(self.op_replay_batch) + rep_conf.set_outdir(out_path) + rep_conf.gen_replay(ops_product) + + +def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): + batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') + iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') + op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) + for op_desc in op_descs: + op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) + + +if __name__ == '__main__': + if len(sys.argv) <= 6: + raise RuntimeError('arguments must greater than 6') + rep_cfg = {} + rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] + rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] + rep_dir = {} + rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] + rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] + gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/batch_replay_impl.temp b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/batch_replay_impl.temp new file mode 100644 index 000000000..0e8834664 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/batch_replay_impl.temp @@ -0,0 +1,117 @@ +#include +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +#include + +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], + int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N]; + int len[KERNEL_N]; + block_idx = 0; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + //__OP_SET_KERNEL__ + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, true); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[i] = (char *)pos; + len[i] = CodeLen(); + pos += len[i]; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/code_channel_infer.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/code_channel_infer.py new file mode 100644 index 000000000..d4e6a540c --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/code_channel_infer.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import os +import stat +import ctypes +import collections +import shutil +import subprocess +import copy + +"""CODE_* is used to cube/vector api is called in operator code +CODE_MIX means both cube and vector api is called +CODE_CUBE means only cube api is called +CODE_VEC means only vector api is called +""" +CODE_MIX = 0 +CODE_CUBE = 1 +CODE_VEC = 2 + + +def _is_v220(op_product: str): + """return if current soc version is V220 + + Returns: + res: True means V220 + """ + if op_product in ["ascend910b", "ascend910c"]: + return True + return False + + +InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ +['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) + + +def infer_code_channel(params: InfoCodeChanelParams): + """get code channel for v220, return CODE_MIX if soc version is not V220 + + Args: + src_file (str): AscendC operator code file + src_file (str): AscendC operator tiling header file + kernel_name (str): kernel function name + optype (str): operator type + compile_options (list): compile options for bisheng cmd + + Raises: + Exception: if not exist L1/L0/UB if code, it's not a aicore code + + Returns: + res (int): CODE_MIX/CODE_CUBE/CODE_VEC + """ + if not _is_v220(params.op_product): + return CODE_MIX + return CODE_VEC diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/const_var.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/const_var.py new file mode 100644 index 000000000..3e71f5ab8 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/const_var.py @@ -0,0 +1,33 @@ + +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import os +import stat + + +REPLAY_BATCH = 'batch' +REPLAY_ITERATE = 'iterate' +CFG_IMPL_DIR = 'impl_dir' +CFG_OUT_DIR = 'out_dir' +AUTO_GEN_DIR = 'auto_gen_dir' +WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC +WMODES = stat.S_IWUSR | stat.S_IRUSR +SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', + 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1', 'ascend910c': 'Ascend910C1'} +BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ +--output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' +CHK_CMD = ''' +if ! test -f $2/{res_file} ; then + echo "$2/{res_file} not generated!" + exit 1 +fi +''' +ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], + 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_impl_and_mrege_json.sh new file mode 100644 index 000000000..492034e49 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_impl_and_mrege_json.sh @@ -0,0 +1,20 @@ +#!/usr/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +# copy aicpu kernel so operators +if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then + cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl + rm -rf ${project_path}/cpukernel/aicpu_kernel_lib +fi diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_ops_filter.sh b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_ops_filter.sh new file mode 100644 index 000000000..d4c27d17f --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_ops_filter.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +# Description: Generate npu_supported_ops.json +# ============================================================================== + +if [[ -z "$1" ]]; then + echo -e "[ERROR] No source dir provided" + exit 1 +fi + +if [[ -z "$2" ]]; then + echo -e "[ERROR] No destination dir provided" + exit 1 +fi + +src=$1 +dest_file=$2/npu_supported_ops.json + +if [ -f "$dest_file" ];then + chmod u+w $dest_file +fi + +echo $* + +add_ops() { + name=$1 + isHeavy=$2 + file=$3 + grep -w "\"$name\"" ${file} >/dev/null + if [ $? == 0 ];then + return + fi + echo " \"${name}\": {" >> ${file} + echo " \"isGray\": false," >> ${file} + echo " \"isHeavy\": ${isHeavy}" >> ${file} + echo " }," >> ${file} +} + +echo "{" > ${dest_file} +ini_files=$(find ${src} -name "*.ini") +for file in ${ini_files} ; do + name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') + grep 'heavyOp.flag' ${file} >/dev/null + if [ $? == 0 ];then + isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') + else + isHeavy="false" + fi + for op in ${name} ; do + add_ops ${op} "false" ${dest_file} + done +done +echo "}" >> ${dest_file} +file_count=$(cat ${dest_file} | wc -l) +line=$(($file_count-1)) +sed -i "${line}{s/,//g}" ${dest_file} + +chmod 640 "${dest_file}" +echo -e "[INFO] Succed generated ${dest_file}" + +exit 0 + diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_version_info.sh b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_version_info.sh new file mode 100644 index 000000000..a06cfc78d --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_version_info.sh @@ -0,0 +1,6 @@ +ascend_install_dir=$1 +gen_file_dir=$2 + +# create version.info +compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') +echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/insert_op_info.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/insert_op_info.py new file mode 100644 index 000000000..28ba08757 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/insert_op_info.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" +import json +import os +import sys +import stat +import const_var + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print(sys.argv) + print('argv error, inert_op_info.py your_op_file lib_op_file') + sys.exit(2) + + with open(sys.argv[1], 'r') as load_f: + insert_operator = json.load(load_f) + + all_operators = {} + if os.path.exists(sys.argv[2]): + if os.path.getsize(sys.argv[2]) != 0: + with open(sys.argv[2], 'r') as load_f: + all_operators = json.load(load_f) + + for k in insert_operator.keys(): + if k in all_operators.keys(): + print('replace op:[', k, '] success') + else: + print('insert op:[', k, '] success') + all_operators[k] = insert_operator[k] + + with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: + json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/insert_simplified_keys.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/insert_simplified_keys.py new file mode 100644 index 000000000..7e8dc2c55 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/insert_simplified_keys.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import re +import glob +import json +import argparse + + +DATA_TPYE_DICT = { + 'float32': 0, + 'float16': 1, + 'int8': 2, + 'int16': 6, + 'uint16': 7, + 'uint8': 4, + 'int32': 3, + 'int64': 9, + 'uint32': 8, + 'uint64': 10, + 'bool': 12, + 'double': 11, + 'complex64': 16, + 'complex128': 17, + 'qint8': 18, + 'qint16': 19, + 'qint32': 20, + 'quint8': 21, + 'quint16': 22, + 'resource': 23, + 'string': 24, + 'dual': 25, + 'variant': 26, + 'bf16': 27, + 'bfloat16': 27, + 'undefined': 28, + 'int4': 29, + 'uint1': 30, + 'int2': 31 +} + +FORMAT_DICT = { + 'NCHW': 0, + 'NHWC': 1, + 'ND': 2, + 'NC1HWC0': 3, + 'FRACTAL_Z': 4, + 'NC1C0HWPAD': 5, + 'NHWC1C0': 6, + 'FSR_NCHW': 7, + 'FRACTAL_DECONV': 8, + 'C1HWNC0': 9, + 'FRACTAL_DECONV_TRANSPOSE': 10, + 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, + 'NC1HWC0_C04': 12, + 'FRACTAL_Z_C04': 13, + 'CHWN': 14, + 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, + 'HWCN': 16, + 'NC1KHKWHWC0': 17, + 'BN_WEIGHT': 18, + 'FILTER_HWCK': 19, + 'HASHTABLE_LOOKUP_LOOKUPS': 20, + 'HASHTABLE_LOOKUP_KEYS': 21, + 'HASHTABLE_LOOKUP_VALUE': 22, + 'HASHTABLE_LOOKUP_OUTPUT': 23, + 'HASHTABLE_LOOKUP_HITS': 24, + 'C1HWNCoC0': 25, + 'MD': 26, + 'NDHWC': 27, + 'FRACTAL_ZZ': 28, + 'FRACTAL_NZ': 29, + 'NCDHW': 30, + 'DHWCN': 31, + 'NDC1HWC0': 32, + 'FRACTAL_Z_3D': 33, + 'CN': 34, + 'NC': 35, + 'DHWNC': 36, + 'FRACTAL_Z_3D_TRANSPOSE': 37, + 'FRACTAL_ZN_LSTM': 38, + 'FRACTAL_Z_G': 39, + 'RESERVED': 40, + 'ALL': 41, + 'NULL': 42, + 'ND_RNN_BIAS': 43, + 'FRACTAL_ZN_RNN': 44, + 'NYUV': 45, + 'NYUV_A': 46 +} + + +def load_json(json_file: str): + with open(json_file, encoding='utf-8') as file: + json_content = json.load(file) + return json_content + + +def get_specified_suffix_file(root_dir, suffix): + specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) + all_suffix_files = glob.glob(specified_suffix, recursive=True) + return all_suffix_files + + +def get_deterministic_value(support_info): + deterministic_key = 'deterministic' + if deterministic_key not in support_info: + return 0 + deterministic_value = support_info.get(deterministic_key) + if deterministic_value == 'true': + return 1 + else: + return 0 + + +def get_precision_value(support_info): + precision_key = 'implMode' + precision_value = support_info.get(precision_key) + if precision_value == 'high_performance': + _value = 1 + elif precision_value == 'high_precision': + _value = 2 + else: + _value = 0 + return _value + + +def get_overflow_value(support_info): + return 0 + + +def get_parameters(info): + if info: + if 'dtype' in info: + data_type = info['dtype'] + data_type_value = DATA_TPYE_DICT.get(data_type) + else: + data_type_value = 0 + if 'format' in info: + _format = info['format'] + _format_value = FORMAT_DICT.get(_format) + else: + _format_value = 0 + else: + data_type_value = 0 + _format_value = 0 + return str(data_type_value), str(_format_value) + + +def get_dynamic_parameters(info): + # 动态输入时只需获取第一个参数 + return get_parameters(info[0]) + + +def get_all_parameters(support_info, _type): + result_list = list() + info_lists = support_info.get(_type) + if info_lists: + for _info in info_lists: + # 输入为列表时是动态输入 + if isinstance(_info, (list, tuple)): + data_type_value, _format_value = get_dynamic_parameters(_info) + else: + data_type_value, _format_value = get_parameters(_info) + result_list.append("{},{}".format(data_type_value, _format_value)) + return result_list + + +def get_all_input_parameters(support_info): + result = get_all_parameters(support_info, 'inputs') + return '/'.join(result) + + +def insert_content_into_file(input_file, content): + with open(input_file, 'r+') as file: + lines = file.readlines() + for index, line in enumerate(lines): + match_result = re.search(r'"staticKey":', line) + if match_result: + count = len(line) - len(line.lstrip()) + new_content = "{}{}".format(' ' * count, content) + # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 + lines.insert(index, new_content) + break + file.seek(0) + file.write(''.join(lines)) + + +def insert_simplified_keys(json_file): + contents = load_json(json_file) + # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 + if ('binFileName' not in contents) or ('supportInfo' not in contents): + return + support_info = contents.get('supportInfo') + bin_file_name = contents.get('binFileName') + # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 + if 'simplifiedKey' in support_info: + return + op_type = bin_file_name.split('_')[0] + deterministic = str(get_deterministic_value(support_info)) + precision = str(get_precision_value(support_info)) + overflow = str(get_overflow_value(support_info)) + input_parameters = get_all_input_parameters(support_info) + key = '{}/d={},p={},o={}/{}/'.format( + op_type, + deterministic, + precision, + overflow, + input_parameters) + result = '"simplifiedKey": "' + key + '",\n' + insert_content_into_file(json_file, result) + + +def insert_all_simplified_keys(root_dir): + suffix = 'json' + all_json_files = get_specified_suffix_file(root_dir, suffix) + for _json in all_json_files: + insert_simplified_keys(_json) + + +def args_prase(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', + '--path', + nargs='?', + required=True, + help='Parse the path of the json file.') + return parser.parse_args() + + +def main(): + args = args_prase() + insert_all_simplified_keys(args.path) + + +if __name__ == '__main__': + main() diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/kernel_entry.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/kernel_entry.py new file mode 100644 index 000000000..2b77c970d --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/kernel_entry.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + + +def gen_fun_def(title, kernel, argn, arg_type, arg_name): + entry = [] + entry.append(title) + entry.append(kernel) + entry.append('(') + args = [] + for i in range(0, argn): + args.append(arg_type + ' ' + arg_name + str(i)) + entry.append(', '.join(args)) + entry.append(')') + return ' '.join(entry) + + +def gen_batch_kernel_body(fname, argn, arg_name): + body = [] + body.append('{') + fun = [] + fun.append(fname) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(');') + body.append(' '.join(fun)) + body.append('}') + return '\n'.join(body) + + +def gen_mc_kernel_body(kn, argn, arg_name, blknum): + body = [] + body.append('{') + body.append(' switch(block_idx) {') + for blk in range(0, blknum): + fun = [] + fun.append('{}_blk{:02d}'.format(kn, blk)) + fun.append('(') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + fun.append(', '.join(args)) + fun.append(')') + body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) + body.append(' default: break;') + body.append(' }') + body.append('}') + return '\n'.join(body) + + +def gen_proc_body(argn, arg_name): + body = [] + body.append('{') + args = [] + for i in range(0, argn): + args.append(arg_name + str(i)) + body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('__asm__ ("NOP");') + body.append('}') + return '\n'.join(body) + + +def batch_code_gen(kn, argn, argt): + codes = [] + kernel_name = kn + proc_name = kernel_name + '_percore' + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' + + +def mc_code_gen(kn, argn, argt, blknum): + codes = [] + kernel_name = kn + core_num = int(blknum) + arg_num = int(argn) + data_type = argt + arg_type = '__gm__ ' + data_type + '* __restrict__' + arg_name = 'arg' + kernel_title = 'extern \"C\" __global__ __aicore__ void' + proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' + codes.append('#ifndef __aicore__') + codes.append('#define __aicore__ [aicore]') + codes.append('#endif') + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') + codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) + codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) + for i in range(0, core_num): + proc_name = '{}_blk{:02d}'.format(kernel_name, i) + codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) + codes.append(gen_proc_body(arg_num, arg_name)) + return '\n'.join(codes) + '\n' diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/kernel_impl.temp b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/kernel_impl.temp new file mode 100644 index 000000000..5079a1043 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/kernel_impl.temp @@ -0,0 +1,10 @@ +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#define __ASCENDC_REPLAY_CODE__ +#include "__CCE_FILE__" diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/COPYING b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/COPYING new file mode 100644 index 000000000..d159169d1 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/README.md b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/README.md new file mode 100644 index 000000000..b41f01682 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/README.md @@ -0,0 +1,246 @@ +[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) + +# makeself - Make self-extractable archives on Unix + +[makeself.sh][1] is a small shell script that generates a self-extractable +compressed tar archive from a directory. The resulting file appears as a shell script +(many of those have a **.run** suffix), and can be launched as is. The archive +will then uncompress itself to a temporary directory and an optional arbitrary +command will be executed (for example an installation script). This is pretty +similar to archives generated with WinZip Self-Extractor in the Windows world. +Makeself archives also include checksums for integrity self-validation (CRC +and/or MD5/SHA256 checksums). + +The makeself.sh script itself is used only to create the archives from a +directory of files. The resultant archive is actually a compressed (using +gzip, bzip2, or compress) TAR archive, with a small shell script stub at the +beginning. This small stub performs all the steps of extracting the files, +running the embedded command, and removing the temporary files when done. +All the user has to do to install the software contained in such an +archive is to "run" the archive, i.e **sh nice-software.run**. I recommend +using the ".run" (which was introduced by some Makeself archives released by +Loki Software) or ".sh" suffix for such archives not to confuse the users, +so that they will know they are actually shell scripts (with quite a lot of binary data +attached to them though!). + +I am trying to keep the code of this script as portable as possible, i.e it is +not relying on any bash-specific features and only calls commands that are +installed on any functioning UNIX-compatible system. This script as well as +the archives it generates should run on any Unix flavor, with any compatible +Bourne shell, provided of course that the compression programs are available. + +As of version 2.1, Makeself has been rewritten and tested on the following +platforms : + + * Linux (all distributions) + * Sun Solaris (8 and above) + * HP-UX (tested on 11.0 and 11i on HPPA RISC) + * SCO OpenUnix and OpenServer + * IBM AIX 5.1L + * macOS (Darwin) + * SGI IRIX 6.5 + * FreeBSD + * UnicOS / Cray + * Cygwin (Windows) + +If you successfully run Makeself and/or archives created with it on another +system, then please [let me know][2]! + +Examples of publicly available archives made using makeself are : + + * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; + * All game patches released by [Loki Software][4] for the Linux version of popular games ; + * The [nVidia drivers][5] for Linux + * The installer for the Linux version of [Google Earth][6] + * The [VirtualBox][7] installers for Linux + * The [Makeself][1] distribution itself ;-) + * and countless others... + +**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : + +`AddType application/x-makeself .run` + +**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : + +`export _POSIX2_VERSION=199209` + +## Usage + +The syntax of makeself is the following: + +``` +makeself.sh [args] archive_dir file_name label startup_script [script_args] +``` + + * _args_ are optional options for Makeself. The available ones are : + + * **`--version`** : Prints the version number on stdout, then exits immediately + * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) + * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. + * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. + * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. + * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. + * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. + * **`--pigz`** : Use pigz for compression. + * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). + * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. + * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. + * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. + * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. + * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) + * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. + * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). + * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. + * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. + * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. + * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. + * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. + * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. + * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. + * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. + * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. + * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. + * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. + * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). + * **`--tar-extra opt`** : Append more options to the tar command line. + + For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` + + * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. + * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. + * **`--license`** : Append a license file. + * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. + * **`--help-header file`** : Add a header to the archive's `--help` output. + * `archive_dir` is the name of the directory that contains the files to be archived + * `file_name` is the name of the archive to be created + * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. + * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. + +Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named +**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : + +`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +` + +Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : + +`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` + +Archives generated with Makeself can be passed the following arguments: + + * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. + * **`--verbose`** : Will prompt the user before executing the embedded command + * **`--target dir`** : Allows to extract the archive in an arbitrary place. + * **`--nox11`** : Do not spawn a X11 terminal. + * **`--confirm`** : Prompt the user for confirmation before running the embedded command. + * **`--info`** : Print out general information about the archive (does not extract). + * **`--lsm`** : Print out the LSM entry, if it is present. + * **`--list`** : List the files in the archive. + * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. + * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. + * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. + * **`--noexec`** : Do not run the embedded script after extraction. + * **`--noexec-cleanup`** : Do not run the embedded cleanup script. + * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. + * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. + +Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. + +## Startup Script + +The startup script must be a regular Shell script. + +Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. + +`my-self-extracting-script.sh --fooBarFileParameter foo.bar` + +## Building and Testing + +Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. + +* To make a release: `make` +* To run all tests: `make test` + +## Maven Usage + +Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. + +## License + +Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. + +## Contributing + +I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: + + * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. + * Please explain clearly what the purpose of the patch is, and how you achieved it. + +## Download + +Get the latest official distribution [here][9] (version 2.4.2). + +The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. + +## Version history + + * **v1.0:** Initial public release + * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan + * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. + * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. + * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. + * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. + * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. + * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. + * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. + * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. + * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. + * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. + * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. + * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). + * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) + * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. + * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. + * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. + * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. + * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. + * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. + * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. + * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) + +## Links + + * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. + * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] + +## Contact + +This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. + +Contributions were included from John C. Quillan, Bjarni R. Einarsson, +Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot +your name, don't hesitate to contact me. + +This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. + +* * * + +[Stephane Peter][2] + + [1]: http://makeself.io/ + [2]: mailto:megastep@megastep.org + [3]: http://www.idsoftware.com/ + [4]: http://www.lokigames.com/products/myth2/updates.php3 + [5]: http://www.nvidia.com/ + [6]: http://earth.google.com/ + [7]: http://www.virtualbox.org/ + [8]: http://www.gnu.org/copyleft/gpl.html + [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run + [10]: https://github.com/megastep/makeself + [11]: https://github.com/megastep/loki_setup/ + [12]: http://www.unrealtournament2003.com/ + [13]: http://www.icculus.org/ + [14]: http://bre.klaki.net/programs/setup.sh/ + [15]: https://stephanepeter.com/ diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/VERSION b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/VERSION new file mode 100644 index 000000000..59aa62c1f --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/VERSION @@ -0,0 +1 @@ +2.4.5 diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/make-release.sh b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/make-release.sh new file mode 100644 index 000000000..b5692d490 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/make-release.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Create a distributable archive of the current version of Makeself + +VER=`cat VERSION` +mkdir -p /tmp/makeself-$VER release +cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ +./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" + diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself-header.sh b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself-header.sh new file mode 100644 index 000000000..940903148 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself-header.sh @@ -0,0 +1,660 @@ +cat << EOF > "$archname" +#!/bin/bash +# This script was generated using Makeself $MS_VERSION +# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) +# 2022.3.19-Modified the MS_Help function and some options +# Huawei Technologies Co., Ltd. + +ORIG_UMASK=\`umask\` + +CRCsum="$CRCsum" +MD5="$MD5sum" +SHA="$SHAsum" +SIGNATURE="$Signature" +TMPROOT=\${TMPDIR:="\$HOME"} +if ! test -d "\$TMPROOT"; then + TMPROOT="\$PWD" +fi +export TMPDIR="\$TMPROOT" +USER_PWD="\$PWD" +if ! test -d "\$USER_PWD"; then + exit 1 +fi +export USER_PWD +ARCHIVE_DIR=\`dirname "\$0"\` +export ARCHIVE_DIR + +name_of_file="\$0 " +pwd_of_file="\$PWD" +label="$LABEL" +script="$SCRIPT" +scriptargs="$SCRIPTARGS" +cleanup_script="${CLEANUP_SCRIPT}" +licensetxt="$LICENSE" +helpheader='$HELPHEADER' +targetdir="$archdirname" +filesizes="$filesizes" +totalsize="$totalsize" +keep="$KEEP" +nooverwrite="$NOOVERWRITE" +quiet="n" +accept="n" +nodiskspace="n" +export_conf="$EXPORT_CONF" +decrypt_cmd="$DECRYPT_CMD" +skip="$SKIP" + +print_cmd_arg="" +if type printf > /dev/null; then + print_cmd="printf" +elif test -x /usr/ucb/echo; then + print_cmd="/usr/ucb/echo" +else + print_cmd="echo" +fi + +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:\$PATH + export PATH +fi + +if test -d /usr/sfw/bin; then + PATH=\$PATH:/usr/sfw/bin + export PATH +fi + +unset CDPATH + +MS_Printf() +{ + \$print_cmd \$print_cmd_arg "\$1" +} + +MS_PrintLicense() +{ + PAGER=\${PAGER:=more} + if test x"\$licensetxt" != x; then + PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` + if test -x "\$PAGER_PATH"; then + echo "\$licensetxt" | \$PAGER + else + echo "\$licensetxt" + fi + if test x"\$accept" != xy; then + while true + do + MS_Printf "Please type y to accept, n otherwise: " + read yn + if test x"\$yn" = xn; then + keep=n + eval \$finish; exit 1 + break; + elif test x"\$yn" = xy; then + break; + fi + done + fi + fi +} + +MS_diskspace() +{ + ( + df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' + ) +} + +MS_dd() +{ + blocks=\`expr \$3 / 1024\` + bytes=\`expr \$3 % 1024\` + # Test for ibs, obs and conv feature + if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then + dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ + { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ + test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null + else + dd if="\$1" bs=\$2 skip=1 2> /dev/null + fi +} + +MS_dd_Progress() +{ + if test x"\$noprogress" = xy; then + MS_dd "\$@" + return \$? + fi + file="\$1" + offset=\$2 + length=\$3 + pos=0 + bsize=4194304 + while test \$bsize -gt \$length; do + bsize=\`expr \$bsize / 4\` + done + blocks=\`expr \$length / \$bsize\` + bytes=\`expr \$length % \$bsize\` + ( + dd ibs=\$offset skip=1 2>/dev/null + pos=\`expr \$pos \+ \$bsize\` + MS_Printf " 0%% " 1>&2 + if test \$blocks -gt 0; then + while test \$pos -le \$length; do + dd bs=\$bsize count=1 2>/dev/null + pcent=\`expr \$length / 100\` + pcent=\`expr \$pos / \$pcent\` + if test \$pcent -lt 100; then + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + if test \$pcent -lt 10; then + MS_Printf " \$pcent%% " 1>&2 + else + MS_Printf " \$pcent%% " 1>&2 + fi + fi + pos=\`expr \$pos \+ \$bsize\` + done + fi + if test \$bytes -gt 0; then + dd bs=\$bytes count=1 2>/dev/null + fi + MS_Printf "\b\b\b\b\b\b\b" 1>&2 + MS_Printf " 100%% " 1>&2 + ) < "\$file" +} + +MS_Help() +{ + cat << EOH >&2 +Usage: \$0 [options] +Options: + --help | -h Print this message + --info Print embedded info : title, default target directory, embedded script ... + --list Print the list of files in the archive + --check Checks integrity and version dependency of the archive + --quiet Quiet install mode, skip human-computer interactions + --nox11 Do not spawn an xterm + --noexec Do not run embedded script + --extract= Extract directly to a target directory (absolute or relative) + Usually used with --noexec to just extract files without running + --tar arg1 [arg2 ...] Access the contents of the archive through the tar command +\${helpheader} +EOH +} + +MS_Verify_Sig() +{ + GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` + test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + temp_sig=\`mktemp -t XXXXX\` + echo \$SIGNATURE | base64 --decode > "\$temp_sig" + gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` + gpg_res=\$? + rm -f "\$temp_sig" + if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then + if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then + test x"\$quiet" = xn && echo "GPG signature is good" >&2 + else + echo "GPG Signature key does not match" >&2 + exit 2 + fi + else + test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 + exit 2 + fi +} + +MS_Check() +{ + OLD_PATH="\$PATH" + PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` + test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` + PATH="\$OLD_PATH" + + SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` + test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` + + if test x"\$quiet" = xn; then + MS_Printf "Verifying archive integrity..." + fi + offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` + fsize=\`cat "\$1" | wc -c | tr -d " "\` + if test \$totalsize -ne \`expr \$fsize - \$offset\`; then + echo " Unexpected archive size." >&2 + exit 2 + fi + verb=\$2 + i=1 + for s in \$filesizes + do + crc=\`echo \$CRCsum | cut -d" " -f\$i\` + if test -x "\$SHA_PATH"; then + if test x"\`basename \$SHA_PATH\`" = xshasum; then + SHA_ARG="-a 256" + fi + sha=\`echo \$SHA | cut -d" " -f\$i\` + if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 + else + shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; + if test x"\$shasum" != x"\$sha"; then + echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " SHA256 checksums are OK." >&2 + fi + crc="0000000000"; + fi + fi + if test -x "\$MD5_PATH"; then + if test x"\`basename \$MD5_PATH\`" = xdigest; then + MD5_ARG="-a md5" + fi + md5=\`echo \$MD5 | cut -d" " -f\$i\` + if test x"\$md5" = x00000000000000000000000000000000; then + test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 + else + md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; + if test x"\$md5sum" != x"\$md5"; then + echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " MD5 checksums are OK." >&2 + fi + crc="0000000000"; verb=n + fi + fi + if test x"\$crc" = x0000000000; then + test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 + else + sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` + if test x"\$sum1" != x"\$crc"; then + echo "Error in checksums: \$sum1 is different from \$crc" >&2 + exit 2 + elif test x"\$quiet" = xn; then + MS_Printf " CRC checksums are OK." >&2 + fi + fi + i=\`expr \$i + 1\` + offset=\`expr \$offset + \$s\` + done + if test x"\$quiet" = xn; then + echo " All good." + fi +} + +MS_Decompress() +{ + if test x"\$decrypt_cmd" != x""; then + { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" + else + eval "$GUNZIP_CMD" + fi + + if test \$? -ne 0; then + echo " ... Decompression failed." >&2 + fi +} + +UnTAR() +{ + if test x"\$quiet" = xn; then + tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } + else + tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } + fi +} + +MS_exec_cleanup() { + if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then + cleanup=n + cd "\$tmpdir" + eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" + fi +} + +MS_cleanup() +{ + echo 'Signal caught, cleaning up' >&2 + MS_exec_cleanup + cd "\$TMPROOT" + rm -rf "\$tmpdir" + eval \$finish; exit 15 +} + +Script_Args_Check() +{ + script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) + arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) + + for arg in \${script_supported_args}; + do + if test x"\$arg_to_test" = x"\$arg" ;then + return + fi + done + + MS_Help + exit 1 +} + +finish=true +xterm_loop= +noprogress=$NOPROGRESS +nox11=$NOX11 +copy=$COPY +ownership=$OWNERSHIP +verbose=n +cleanup=y +cleanupargs= +sig_key= + +initargs="\$@" + +while [ -n "\$*" ] +do + case "\$1" in + -h | --help) + MS_Help + exit 0 + ;; + -q | --quiet) + quiet=y + noprogress=y + shift + ;; + --info) + echo Identification: "\$label" + echo Target directory: "\$targetdir" + echo Uncompressed size: $USIZE KB + echo Compression: $COMPRESS + if test x"$ENCRYPT" != x""; then + echo Encryption: $ENCRYPT + fi + echo Date of packaging: $DATE + echo Built with Makeself version $MS_VERSION + echo Build command was: "$MS_COMMAND" + if test x"\$script" != x; then + echo Script run after extraction: + echo " " \$script \$scriptargs + fi + if test x"$copy" = xcopy; then + echo "Archive will copy itself to a temporary location" + fi + if test x"$NEED_ROOT" = xy; then + echo "Root permissions required for extraction" + fi + if test x"$KEEP" = xy; then + echo "directory \$targetdir is permanent" + else + echo "\$targetdir will be removed after extraction" + fi + exit 0 + ;; + --list) + echo Target directory: \$targetdir + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --tar) + offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + arg1="\$2" + shift 2 || { MS_Help; exit 1; } + for s in \$filesizes + do + MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" + offset=\`expr \$offset + \$s\` + done + exit 0 + ;; + --check) + MS_Check "\$0" y + scriptargs="\$scriptargs \$1" + shift + ;; + --noexec) + script="" + cleanup_script="" + shift + ;; + --extract=*) + keep=y + targetdir=\`echo \$1 | cut -d"=" -f2 \` + if ! shift; then MS_Help; exit 1; fi + ;; + --nox11) + nox11=y + shift + ;; + --xwin) + if test "$NOWAIT" = n; then + finish="echo Press Return to close this window...; read junk" + fi + xterm_loop=1 + shift + ;; + --phase2) + copy=phase2 + shift + ;; + --repack | --repack-path=*) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + if [[ ! "\$1" =~ ^-.* ]]; then + scriptargs="\$scriptargs '\$1'" + shift + fi + ;; + *) + Script_Args_Check \$1 + scriptargs="\$scriptargs '\$1'" + shift + ;; + esac +done + +quiet_para="" +if test x"\$quiet" = xy; then + quiet_para="--quiet " +fi +scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" + +if test x"\$quiet" = xy -a x"\$verbose" = xy; then + echo Cannot be verbose and quiet at the same time. >&2 + exit 1 +fi + +if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then + echo "Administrative privileges required for this archive (use su or sudo)" >&2 + exit 1 +fi + +if test x"\$copy" \!= xphase2; then + MS_PrintLicense +fi + +case "\$copy" in +copy) + tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ + mkdir "\$tmpdir" || { + echo "Could not create temporary directory \$tmpdir" >&2 + exit 1 + } + SCRIPT_COPY="\$tmpdir/makeself" + echo "Copying to a temporary location..." >&2 + cp "\$0" "\$SCRIPT_COPY" + chmod +x "\$SCRIPT_COPY" + cd "\$TMPROOT" + exec "\$SCRIPT_COPY" --phase2 -- \$initargs + ;; +phase2) + finish="\$finish ; rm -rf \`dirname \$0\`" + ;; +esac + +if test x"\$nox11" = xn; then + if tty -s; then # Do we have a terminal? + : + else + if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? + if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable + GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" + for a in \$GUESS_XTERMS; do + if type \$a >/dev/null 2>&1; then + XTERM=\$a + break + fi + done + chmod a+x \$0 || echo Please add execution rights on \$0 + if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! + exec \$XTERM -e "\$0 --xwin \$initargs" + else + exec \$XTERM -e "./\$0 --xwin \$initargs" + fi + fi + fi + fi +fi + +if test x"\$targetdir" = x.; then + tmpdir="." +else + if test x"\$keep" = xy; then + if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then + echo "Target directory \$targetdir already exists, aborting." >&2 + exit 1 + fi + if test x"\$quiet" = xn; then + echo "Creating directory \$targetdir" >&2 + fi + tmpdir="\$targetdir" + dashp="-p" + else + tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" + dashp="" + fi + mkdir \$dashp "\$tmpdir" || { + echo 'Cannot create target directory' \$tmpdir >&2 + echo 'You should try option --extract=' >&2 + eval \$finish + exit 1 + } +fi + +location="\`pwd\`" +if test x"\$SETUP_NOCHECK" != x1; then + MS_Check "\$0" +fi +offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` + +if test x"\$verbose" = xy; then + MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " + read yn + if test x"\$yn" = xn; then + eval \$finish; exit 1 + fi +fi + +if test x"\$quiet" = xn; then + # Decrypting with openssl will ask for password, + # the prompt needs to start on new line + if test x"$ENCRYPT" = x"openssl"; then + echo "Decrypting and uncompressing \$label..." + else + MS_Printf "Uncompressing \$label" + fi +fi +res=3 +if test x"\$keep" = xn; then + trap MS_cleanup 1 2 3 15 +fi + +if test x"\$nodiskspace" = xn; then + leftspace=\`MS_diskspace "\$tmpdir"\` + if test -n "\$leftspace"; then + if test "\$leftspace" -lt $USIZE; then + echo + echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 + if test x"\$keep" = xn; then + echo "Consider setting TMPDIR to a directory with more free space." + fi + eval \$finish; exit 1 + fi + fi +fi + +for s in \$filesizes +do + if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then + if test x"\$ownership" = xy; then + (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) + fi + else + echo >&2 + echo "Unable to decompress \$0" >&2 + eval \$finish; exit 1 + fi + offset=\`expr \$offset + \$s\` +done +if test x"\$quiet" = xn; then + echo +fi + +cd "\$tmpdir" +res=0 +if test x"\$script" != x; then + if test x"\$export_conf" = x"y"; then + MS_BUNDLE="\$0" + MS_LABEL="\$label" + MS_SCRIPT="\$script" + MS_SCRIPTARGS="\$scriptargs" + MS_ARCHDIRNAME="\$archdirname" + MS_KEEP="\$KEEP" + MS_NOOVERWRITE="\$NOOVERWRITE" + MS_COMPRESS="\$COMPRESS" + MS_CLEANUP="\$cleanup" + export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS + export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS + fi + + if test x"\$verbose" = x"y"; then + yn="x" + while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN + do + MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " + read yn + if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; + elif test x"\$yn" = xn -o x"\$yn" = xN; then + echo "Unable to decompress \$script ,because of aborting! ";res=\$? + else + echo "Input value is unacceptable,please try again." + fi + done + else + eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? + fi + if test "\$res" -ne 0; then + test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 + fi +fi + +MS_exec_cleanup + +if test x"\$keep" = xn; then + cd "\$TMPROOT" + rm -rf "\$tmpdir" +fi +eval \$finish; exit \$res +EOF diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.1 b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.1 new file mode 100644 index 000000000..81bf6e4ff --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.1 @@ -0,0 +1,110 @@ +.TH "MAKESELF" "1" "2.4.5" +.SH "NAME" +makeself \- An utility to generate self-extractable archives. +.SH "SYNTAX" +.B makeself [\fIoptions\fP] archive_dir file_name label +.B [\fIstartup_script\fP] [\fIargs\fP] +.SH "DESCRIPTION" +This program is a free (GPL) utility designed to create self-extractable +archives from a directory. +.SH "OPTIONS" +The following options are supported. +.TP 15 +.B -v, --version +Prints out the makeself version number and exits. +.TP +.B -h, --help +Print out help information. +.TP +.B --tar-quietly +Suppress verbose output from the tar command +.TP +.B --quiet +Do not print any messages other than errors +.TP +.B --gzip +Compress using gzip (default if detected). +.TP +.B --bzip2 +Compress using bzip2. +.TP +.B --pbzip2 +Compress using pbzip2. +.TP +.B --xz +Compress using xz. +.TP +.B --lzo +Compress using lzop. +.TP +.B --lz4 +Compress using lz4. +.TP +.B --compress +Compress using the UNIX 'compress' command. +.TP +.B --nocomp +Do not compress the data. +.TP +.B --complevel lvl +Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 +.TP +.B --notemp +The archive will create archive_dir in the current directory and +uncompress in ./archive_dir. +.TP +.B --copy +Upon extraction, the archive will first copy itself to a temporary directory. +.TP +.B --append +Append more files to an existing makeself archive. The label and startup scripts will then be ignored. +.TP +.B --current +Files will be extracted to the current directory. Both --current and --target dir imply --notemp. +.TP +.B --target dir +Extract directly to a target directory. Directory path can be either absolute or relative. +.TP +.B --header file +Specify location of the header script. +.TP +.B --cleanup file +Specify a cleanup script that executes on interrupt and when finished successfully. +.TP +.B --follow +Follow the symlinks in the archive. +.TP +.B --noprogress +Do not show the progress during the decompression. +.TP +.B --nox11 +Disable automatic spawn of an xterm if running in X11. +.TP +.B --nowait +Do not wait for user input after executing embedded program from an xterm. +.TP +.B --nomd5 +Do not create a MD5 checksum for the archive. +.TP +.B --nocrc +Do not create a CRC32 checksum for the archive. +.TP +.B --lsm file +LSM file describing the package. +.B --packaging-date date +Use provided string as the packaging date instead of the current date. +.SH "EXAMPLES" +Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, +and he wants to generate a self-extracting package named mysoft.sh, which will launch +the "setup" script initially stored in /home/joe/mysoft: +.TP +makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup +.TP +Here is also how I created the makeself.run archive which contains the Makeself distribution: +.TP +makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" +.SH "AUTHORS" +Makeself has been written by Stéphane Peter . +.BR +This man page was originally written by Bartosz Fenski for the +Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.lsm b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.lsm new file mode 100644 index 000000000..3c4cea8c1 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.lsm @@ -0,0 +1,16 @@ +Begin3 +Title: makeself.sh +Version: 2.4.5 +Description: makeself.sh is a shell script that generates a self-extractable + tar.gz archive from a directory. The resulting file appears as a shell + script, and can be launched as is. The archive will then uncompress + itself to a temporary directory and an arbitrary command will be + executed (for example an installation script). This is pretty similar + to archives generated with WinZip Self-Extractor in the Windows world. +Keywords: Installation archive tar winzip +Author: Stephane Peter (megastep@megastep.org) +Maintained-by: Stephane Peter (megastep@megastep.org) +Original-site: https://makeself.io/ +Platform: Unix +Copying-policy: GPL +End diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.sh b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.sh new file mode 100644 index 000000000..c8ea56597 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.sh @@ -0,0 +1,822 @@ +#!/bin/sh +# +# Makeself version 2.4.x +# by Stephane Peter +# +# Utility to create self-extracting tar.gz archives. +# The resulting archive is a file holding the tar.gz archive with +# a small Shell script stub that uncompresses the archive to a temporary +# directory and then executes a given script from withing that directory. +# +# Makeself home page: https://makeself.io/ +# +# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. +# +# Version history : +# - 1.0 : Initial public release +# - 1.1 : The archive can be passed parameters that will be passed on to +# the embedded script, thanks to John C. Quillan +# - 1.2 : Package distribution, bzip2 compression, more command line options, +# support for non-temporary archives. Ideas thanks to Francois Petitjean +# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: +# Support for no compression (--nocomp), script is no longer mandatory, +# automatic launch in an xterm, optional verbose output, and -target +# archive option to indicate where to extract the files. +# - 1.4 : Improved UNIX compatibility (Francois Petitjean) +# Automatic integrity checking, support of LSM files (Francois Petitjean) +# - 1.5 : Many bugfixes. Optionally disable xterm spawning. +# - 1.5.1 : More bugfixes, added archive options -list and -check. +# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big +# archives (Quake III demo) +# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. +# More verbosity in xterms and check for embedded command's return value. +# Bugfix for Debian 2.0 systems that have a different "print" command. +# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. +# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to +# bypass checksum verification of archives. +# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) +# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. +# - 2.0.1 : Added --copy +# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. +# Added --nochown for archives +# Stopped doing redundant checksums when not necesary +# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command +# Cleaned up the code to handle error codes from compress. Simplified the extraction code. +# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. +# - 2.1.3 : Bug fixes with command line when spawning terminals. +# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. +# Added --noexec to prevent execution of embedded scripts. +# Added --nomd5 and --nocrc to avoid creating checksums in archives. +# Added command used to create the archive in --info output. +# Run the embedded script through eval. +# - 2.1.4 : Fixed --info output. +# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) +# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) +# Avoid some race conditions (Ludwig Nussel) +# Unset the $CDPATH variable to avoid problems if it is set. (Debian) +# Better handling of dot files in the archive directory. +# - 2.1.5 : Made the md5sum detection consistent with the header code. +# Check for the presence of the archive directory +# Added --encrypt for symmetric encryption through gpg (Eric Windisch) +# Added support for the digest command on Solaris 10 for MD5 checksums +# Check for available disk space before extracting to the target directory (Andreas Schweitzer) +# Allow extraction to run asynchronously (patch by Peter Hatch) +# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) +# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) +# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) +# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) +# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. +# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) +# - 2.4.0 : Optional support for SHA256 checksums in archives. +# - 2.4.2 : Add support for threads for several compressors. (M. Limber) +# Added zstd support. +# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. +# - 2.4.5 : Added --tar-format to override ustar tar archive format +# +# (C) 1998-2021 by Stephane Peter +# +# This software is released under the terms of the GNU GPL version 2 and above +# Please read the license at http://www.gnu.org/copyleft/gpl.html +# Self-extracting archives created with this script are explictly NOT released under the term of the GPL +# + +MS_VERSION=2.4.5 +MS_COMMAND="$0" +unset CDPATH + +for f in ${1+"$@"}; do + MS_COMMAND="$MS_COMMAND \\\\ + \\\"$f\\\"" +done + +# For Solaris systems +if test -d /usr/xpg4/bin; then + PATH=/usr/xpg4/bin:$PATH + export PATH +fi + +# Procedures + +MS_Usage() +{ + echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" + echo "args can be one or more of the following :" + echo " --version | -v : Print out Makeself version number and exit" + echo " --help | -h : Print out this help message" + echo " --tar-quietly : Suppress verbose output from the tar command" + echo " --quiet | -q : Do not print any messages other than errors." + echo " --gzip : Compress using gzip (default if detected)" + echo " --pigz : Compress with pigz" + echo " --zstd : Compress with zstd" + echo " --bzip2 : Compress using bzip2 instead of gzip" + echo " --pbzip2 : Compress using pbzip2 instead of gzip" + echo " --xz : Compress using xz instead of gzip" + echo " --lzo : Compress using lzop instead of gzip" + echo " --lz4 : Compress using lz4 instead of gzip" + echo " --compress : Compress using the UNIX 'compress' command" + echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" + echo " --threads thds : Number of threads to be used by compressors that support parallelization." + echo " Omit to use compressor's default. Most useful (and required) for opting" + echo " into xz's threading, usually with '--threads=0' for all available cores." + echo " pbzip2 and pigz are parallel by default, and setting this value allows" + echo " limiting the number of threads they use." + echo " --base64 : Instead of compressing, encode the data using base64" + echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" + echo " --gpg-asymmetric-encrypt-sign" + echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" + echo " --gpg-extra opt : Append more options to the gpg command line" + echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" + echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" + echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" + echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." + echo " If this option is not supplied, the user will be asked to enter" + echo " encryption password on the current terminal." + echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." + echo " --nochown : Do not give the target folder to the current user (default)" + echo " --chown : Give the target folder to the current user recursively" + echo " --nocomp : Do not compress the data" + echo " --notemp : The archive will create archive_dir in the" + echo " current directory and uncompress in ./archive_dir" + echo " --needroot : Check that the root user is extracting the archive before proceeding" + echo " --copy : Upon extraction, the archive will first copy itself to" + echo " a temporary directory" + echo " --append : Append more files to an existing Makeself archive" + echo " The label and startup scripts will then be ignored" + echo " --target dir : Extract directly to a target directory" + echo " directory path can be either absolute or relative" + echo " --nooverwrite : Do not extract the archive if the specified target directory exists" + echo " --current : Files will be extracted to the current directory" + echo " Both --current and --target imply --notemp" + echo " --tar-format opt : Specify a tar archive format (default is ustar)" + echo " --tar-extra opt : Append more options to the tar command line" + echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" + echo " --nomd5 : Don't calculate an MD5 for archive" + echo " --nocrc : Don't calculate a CRC for archive" + echo " --sha256 : Compute a SHA256 checksum for the archive" + echo " --header file : Specify location of the header script" + echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." + echo " --follow : Follow the symlinks in the archive" + echo " --noprogress : Do not show the progress during the decompression" + echo " --nox11 : Disable automatic spawn of a xterm" + echo " --nowait : Do not wait for user input after executing embedded" + echo " program from an xterm" + echo " --sign passphrase : Signature private key to sign the package with" + echo " --lsm file : LSM file describing the package" + echo " --license file : Append a license file" + echo " --help-header file : Add a header to the archive's --help output" + echo " --packaging-date date" + echo " : Use provided string as the packaging date" + echo " instead of the current date." + echo + echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." + echo " --export-conf : Export configuration variables to startup_script" + echo + echo "Do not forget to give a fully qualified startup script name" + echo "(i.e. with a ./ prefix if inside the archive)." + exit 1 +} + +# Default settings +if type gzip >/dev/null 2>&1; then + COMPRESS=gzip +elif type compress >/dev/null 2>&1; then + COMPRESS=compress +else + echo "ERROR: missing commands: gzip, compress" >&2 + MS_Usage +fi +ENCRYPT=n +PASSWD="" +PASSWD_SRC="" +OPENSSL_NO_MD=n +COMPRESS_LEVEL=9 +DEFAULT_THREADS=123456 # Sentinel value +THREADS=$DEFAULT_THREADS +KEEP=n +CURRENT=n +NOX11=n +NOWAIT=n +APPEND=n +TAR_QUIETLY=n +KEEP_UMASK=n +QUIET=n +NOPROGRESS=n +COPY=none +NEED_ROOT=n +TAR_ARGS=rvf +TAR_FORMAT=ustar +TAR_EXTRA="" +GPG_EXTRA="" +DU_ARGS=-ks +HEADER=`dirname "$0"`/makeself-header.sh +SIGNATURE="" +TARGETDIR="" +NOOVERWRITE=n +DATE=`LC_ALL=C date` +EXPORT_CONF=n +SHA256=n +OWNERSHIP=n +SIGN=n +GPG_PASSPHRASE="" + +# LSM file stuff +LSM_CMD="echo No LSM. >> \"\$archname\"" + +while true +do + case "$1" in + --version | -v) + echo Makeself version $MS_VERSION + exit 0 + ;; + --pbzip2) + COMPRESS=pbzip2 + shift + ;; + --bzip2) + COMPRESS=bzip2 + shift + ;; + --gzip) + COMPRESS=gzip + shift + ;; + --pigz) + COMPRESS=pigz + shift + ;; + --zstd) + COMPRESS=zstd + shift + ;; + --xz) + COMPRESS=xz + shift + ;; + --lzo) + COMPRESS=lzo + shift + ;; + --lz4) + COMPRESS=lz4 + shift + ;; + --compress) + COMPRESS=compress + shift + ;; + --base64) + COMPRESS=base64 + shift + ;; + --gpg-encrypt) + COMPRESS=gpg + shift + ;; + --gpg-asymmetric-encrypt-sign) + COMPRESS=gpg-asymmetric + shift + ;; + --gpg-extra) + GPG_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-encrypt) + ENCRYPT=openssl + shift + ;; + --ssl-passwd) + PASSWD=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-pass-src) + PASSWD_SRC=$2 + shift 2 || { MS_Usage; exit 1; } + ;; + --ssl-no-md) + OPENSSL_NO_MD=y + shift + ;; + --nocomp) + COMPRESS=none + shift + ;; + --complevel) + COMPRESS_LEVEL="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --threads) + THREADS="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nochown) + OWNERSHIP=n + shift + ;; + --chown) + OWNERSHIP=y + shift + ;; + --notemp) + KEEP=y + shift + ;; + --copy) + COPY=copy + shift + ;; + --current) + CURRENT=y + KEEP=y + shift + ;; + --tar-format) + TAR_FORMAT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --tar-extra) + TAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --untar-extra) + UNTAR_EXTRA="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --target) + TARGETDIR="$2" + KEEP=y + shift 2 || { MS_Usage; exit 1; } + ;; + --sign) + SIGN=y + GPG_PASSPHRASE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --nooverwrite) + NOOVERWRITE=y + shift + ;; + --needroot) + NEED_ROOT=y + shift + ;; + --header) + HEADER="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --cleanup) + CLEANUP_SCRIPT="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --license) + # We need to escape all characters having a special meaning in double quotes + LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") + shift 2 || { MS_Usage; exit 1; } + ;; + --follow) + TAR_ARGS=rvhf + DU_ARGS=-ksL + shift + ;; + --noprogress) + NOPROGRESS=y + shift + ;; + --nox11) + NOX11=y + shift + ;; + --nowait) + NOWAIT=y + shift + ;; + --nomd5) + NOMD5=y + shift + ;; + --sha256) + SHA256=y + shift + ;; + --nocrc) + NOCRC=y + shift + ;; + --append) + APPEND=y + shift + ;; + --lsm) + LSM_CMD="cat \"$2\" >> \"\$archname\"" + shift 2 || { MS_Usage; exit 1; } + ;; + --packaging-date) + DATE="$2" + shift 2 || { MS_Usage; exit 1; } + ;; + --help-header) + HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` + shift 2 || { MS_Usage; exit 1; } + [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER +" + ;; + --tar-quietly) + TAR_QUIETLY=y + shift + ;; + --keep-umask) + KEEP_UMASK=y + shift + ;; + --export-conf) + EXPORT_CONF=y + shift + ;; + -q | --quiet) + QUIET=y + shift + ;; + -h | --help) + MS_Usage + ;; + -*) + echo Unrecognized flag : "$1" + MS_Usage + ;; + *) + break + ;; + esac +done + +if test $# -lt 1; then + MS_Usage +else + if test -d "$1"; then + archdir="$1" + else + echo "Directory $1 does not exist." >&2 + exit 1 + fi +fi +archname="$2" + +if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then + if test "$TAR_ARGS" = "rvf"; then + TAR_ARGS="rf" + elif test "$TAR_ARGS" = "rvhf"; then + TAR_ARGS="rhf" + fi +fi + +if test "$APPEND" = y; then + if test $# -lt 2; then + MS_Usage + fi + + # Gather the info from the original archive + OLDENV=`sh "$archname" --dumpconf` + if test $? -ne 0; then + echo "Unable to update archive: $archname" >&2 + exit 1 + else + eval "$OLDENV" + OLDSKIP=`expr $SKIP + 1` + fi +else + if test "$KEEP" = n -a $# = 3; then + echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 + echo >&2 + MS_Usage + fi + # We don't want to create an absolute directory unless a target directory is defined + if test "$CURRENT" = y; then + archdirname="." + elif test x"$TARGETDIR" != x; then + archdirname="$TARGETDIR" + else + archdirname=`basename "$1"` + fi + + if test $# -lt 3; then + MS_Usage + fi + + LABEL="$3" + SCRIPT="$4" + test "x$SCRIPT" = x || shift 1 + shift 3 + SCRIPTARGS="$*" +fi + +if test "$KEEP" = n -a "$CURRENT" = y; then + echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 + exit 1 +fi + +case $COMPRESS in +gzip) + GZIP_CMD="gzip -c$COMPRESS_LEVEL" + GUNZIP_CMD="gzip -cd" + ;; +pigz) + GZIP_CMD="pigz -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --processes $THREADS" + fi + GUNZIP_CMD="gzip -cd" + ;; +zstd) + GZIP_CMD="zstd -$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="zstd -cd" + ;; +pbzip2) + GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" + if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated + GZIP_CMD="$GZIP_CMD -p$THREADS" + fi + GUNZIP_CMD="bzip2 -d" + ;; +bzip2) + GZIP_CMD="bzip2 -$COMPRESS_LEVEL" + GUNZIP_CMD="bzip2 -d" + ;; +xz) + GZIP_CMD="xz -c$COMPRESS_LEVEL" + # Must opt-in by specifying a value since not all versions of xz support threads + if test $THREADS -ne $DEFAULT_THREADS; then + GZIP_CMD="$GZIP_CMD --threads=$THREADS" + fi + GUNZIP_CMD="xz -d" + ;; +lzo) + GZIP_CMD="lzop -c$COMPRESS_LEVEL" + GUNZIP_CMD="lzop -d" + ;; +lz4) + GZIP_CMD="lz4 -c$COMPRESS_LEVEL" + GUNZIP_CMD="lz4 -d" + ;; +base64) + GZIP_CMD="base64" + GUNZIP_CMD="base64 --decode -i -" + ;; +gpg) + GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" + GUNZIP_CMD="gpg -d" + ENCRYPT="gpg" + ;; +gpg-asymmetric) + GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" + GUNZIP_CMD="gpg --yes -d" + ENCRYPT="gpg" + ;; +compress) + GZIP_CMD="compress -fc" + GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" + ;; +none) + GZIP_CMD="cat" + GUNZIP_CMD="cat" + ;; +esac + +if test x"$ENCRYPT" = x"openssl"; then + if test x"$APPEND" = x"y"; then + echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 + fi + + ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" + DECRYPT_CMD="openssl enc -aes-256-cbc -d" + + if test x"$OPENSSL_NO_MD" != x"y"; then + ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" + DECRYPT_CMD="$DECRYPT_CMD -md sha256" + fi + + if test -n "$PASSWD_SRC"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" + elif test -n "$PASSWD"; then + ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" + fi +fi + +tmpfile="${TMPDIR:-/tmp}/mkself$$" + +if test -f "$HEADER"; then + oldarchname="$archname" + archname="$tmpfile" + # Generate a fake header to count its lines + SKIP=0 + . "$HEADER" + SKIP=`cat "$tmpfile" |wc -l` + # Get rid of any spaces + SKIP=`expr $SKIP` + rm -f "$tmpfile" + if test "$QUIET" = "n"; then + echo "Header is $SKIP lines long" >&2 + fi + archname="$oldarchname" +else + echo "Unable to open header file: $HEADER" >&2 + exit 1 +fi + +if test "$QUIET" = "n"; then + echo +fi + +if test "$APPEND" = n; then + if test -f "$archname"; then + echo "WARNING: Overwriting existing file: $archname" >&2 + fi +fi + +USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` + +if test "." = "$archdirname"; then + if test "$KEEP" = n; then + archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" + fi +fi + +test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } +if test "$QUIET" = "n"; then + echo "About to compress $USIZE KB of data..." + echo "Adding files to archive named \"$archname\"..." +fi + +# See if we have GNU tar +TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` +test -x "$TAR" || TAR=tar + +tmparch="${TMPDIR:-/tmp}/mkself$$.tar" +( + if test "$APPEND" = "y"; then + tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" + fi + cd "$archdir" + # "Determining if a directory is empty" + # https://www.etalabs.net/sh_tricks.html + find . \ + \( \ + ! -type d \ + -o \ + \( -links 2 -exec sh -c ' + is_empty () ( + cd "$1" + set -- .[!.]* ; test -f "$1" && return 1 + set -- ..?* ; test -f "$1" && return 1 + set -- * ; test -f "$1" && return 1 + return 0 + ) + is_empty "$0"' {} \; \ + \) \ + \) -print \ + | LC_ALL=C sort \ + | sed 's/./\\&/g' \ + | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" +) || { + echo "ERROR: failed to create temporary archive: $tmparch" + rm -f "$tmparch" "$tmpfile" + exit 1 +} + +USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` + +eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { + echo "ERROR: failed to create temporary file: $tmpfile" + rm -f "$tmparch" "$tmpfile" + exit 1 +} +rm -f "$tmparch" + +if test x"$ENCRYPT" = x"openssl"; then + echo "About to encrypt archive \"$archname\"..." + { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ + { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } +fi + +fsize=`cat "$tmpfile" | wc -c | tr -d " "` + +# Compute the checksums + +shasum=0000000000000000000000000000000000000000000000000000000000000000 +md5sum=00000000000000000000000000000000 +crcsum=0000000000 + +if test "$NOCRC" = y; then + if test "$QUIET" = "n"; then + echo "skipping crc at user request" + fi +else + crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` + if test "$QUIET" = "n"; then + echo "CRC: $crcsum" + fi +fi + +if test "$SHA256" = y; then + SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` + if test -x "$SHA_PATH"; then + shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` + else + SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` + shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` + fi + if test "$QUIET" = "n"; then + if test -x "$SHA_PATH"; then + echo "SHA256: $shasum" + else + echo "SHA256: none, SHA command not found" + fi + fi +fi +if test "$NOMD5" = y; then + if test "$QUIET" = "n"; then + echo "Skipping md5sum at user request" + fi +else + # Try to locate a MD5 binary + OLD_PATH=$PATH + PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} + MD5_ARG="" + MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` + test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` + PATH=$OLD_PATH + if test -x "$MD5_PATH"; then + if test `basename ${MD5_PATH}`x = digestx; then + MD5_ARG="-a md5" + fi + md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` + if test "$QUIET" = "n"; then + echo "MD5: $md5sum" + fi + else + if test "$QUIET" = "n"; then + echo "MD5: none, MD5 command not found" + fi + fi +fi +if test "$SIGN" = y; then + GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` + if test -x "$GPG_PATH"; then + SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` + if test "$QUIET" = "n"; then + echo "Signature: $SIGNATURE" + fi + else + echo "Missing gpg command" >&2 + fi +fi + +totalsize=0 +for size in $fsize; +do + totalsize=`expr $totalsize + $size` +done + +if test "$APPEND" = y; then + mv "$archname" "$archname".bak || exit + + # Prepare entry for new archive + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + # Generate the header + . "$HEADER" + # Append the new data + cat "$tmpfile" >> "$archname" + + chmod +x "$archname" + rm -f "$archname".bak + if test "$QUIET" = "n"; then + echo "Self-extractable archive \"$archname\" successfully updated." + fi +else + filesizes="$fsize" + CRCsum="$crcsum" + MD5sum="$md5sum" + SHAsum="$shasum" + Signature="$SIGNATURE" + + # Generate the header + . "$HEADER" + + # Append the compressed tar data after the stub + if test "$QUIET" = "n"; then + echo + fi + cat "$tmpfile" >> "$archname" + chmod +x "$archname" + if test "$QUIET" = "n"; then + echo Self-extractable archive \"$archname\" successfully created. + fi +fi +rm -f "$tmpfile" diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/run-tests.sh b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/run-tests.sh new file mode 100644 index 000000000..31ee16511 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/run-tests.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Run every available test - Bash needed +cd test +for test in *test; +do + echo "Running test $test ..." + bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } +done diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/merge_aicpu_info_json.sh new file mode 100644 index 000000000..a977bd51d --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/merge_aicpu_info_json.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +project_path=$1 +build_path=$2 +vendor_name=customize +echo $@ +if [[ ! -d "$project_path" ]]; then + echo "[ERROR] No projcet path is provided" + exit 1 +fi + +if [[ ! -d "$build_path" ]]; then + echo "[ERROR] No build path is provided" + exit 1 +fi + +if [[ ! -d "$ASCEND_OPP_PATH" ]]; then + echo "[ERROR] No opp install path is provided" + exit 1 +fi +custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json +temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json + +if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then + cp -f $custom_exist_info_json $temp_info_json + chmod +w $temp_info_json + python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} + cp -f $temp_info_json $custom_new_info_json + rm -f $temp_info_json +fi diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/opdesc_parser.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/opdesc_parser.py new file mode 100644 index 000000000..0d3221602 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/opdesc_parser.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import sys +import os + + +OP_ALL = '__ALLOP__' +SOC_ALL = '__ALLSOC__' +SOC_TO_SHORT_SOC_MAP = { + "ascend910a": "ascend910", + "ascend910proa": "ascend910", + "ascend910b": "ascend910", + "ascend910prob": "ascend910", + "ascend910premiuma": "ascend910", + "ascend910b1": "ascend910b", + "ascend910b2": "ascend910b", + "ascend910b2c": "ascend910b", + "ascend910b3": "ascend910b", + "ascend910b4": "ascend910b", + "ascend910c1": "ascend910c", + "ascend910c2": "ascend910c", + "ascend910c3": "ascend910c", + "ascend910c4": "ascend910c", + "ascend310p1": "ascend310p", + "ascend310p3": "ascend310p", + "ascend310p3vir01": "ascend310p", + "ascend310p3vir02": "ascend310p", + "ascend310p3vir04": "ascend310p", + "ascend310p3vir08": "ascend310p", + "ascend310b1": "ascend310b", + "bs9sx1aa": "bs9sx1a" +} +CONFLICT_KEYWORDS = { + "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", + "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", + "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "False", + "None", "True", "nonlocal", "arg", "__inputs__", "__outputs__", "options", "bisheng", + "bisheng_path", "tikcpp_path", "impl_mode", "custom_compile_options", + "custom_all_compile_options", "soc_version", "soc_short", "custom_compile_options_soc", + "custom_all_compile_options_soc", "origin_func_name", "ascendc_src_dir_ex", + "ascendc_src_dir", "ascendc_src_file", "src", "op_type", "code_channel", "op_info", + "compile_op", "get_code_channel", "result", "__attrs__", "isinstance", "attr", + "get_current_build_config", "_build_args", "get_dtype_fmt_options", "shutil", "os", + "get_kernel_source" +} + + +class OpDesc: + def __init__(self: any, op_type: str): + self.op_type = op_type + self.attr_list = [] + self.attr_val = {} + self.input_name = [] + self.input_ori_name = [] + self.input_type = [] + self.input_dtype = [] + self.input_fmt = [] + self.output_name = [] + self.output_ori_name = [] + self.output_type = [] + self.output_dtype = [] + self.output_fmt = [] + self.op_fmt_sel = False + self.op_chk_support = False + self.op_intf = '' + self.kern_name = '' + self.op_file = '' + self.op_replay_flag = False + self.op_replay_batch = False + self.input_idx = -1 + self.output_idx = -1 + self.max_block_dim = 32 + self.max_shape_size = 268435456 + self.dynamic_shape = False + self.op_range_limit = '' + self.custom_compile_options = {} + self.custom_all_compile_options = {} + + @staticmethod + def _parse_digit(conf: str) -> int: + return int(conf.split('=')[1]) + + @staticmethod + def _parse_flag(conf: str) -> bool: + if 'true' == conf.split('=')[1]: + return True + return False + + @staticmethod + def _parse_str(conf: str) -> str: + return conf.split('=')[1] + + @staticmethod + def _parse_list(conf: str) -> list: + return conf.split('=')[1].split(',') + + def parse_input(self: any, conf: str): + if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): + self.input_idx += 1 + self.input_ori_name.append(self._parse_str(conf)) + self.input_name.append(self.input_ori_name[-1] + '_in__') + elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): + self.input_type.append(self._parse_str(conf)) + elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): + self.input_dtype.append(self._parse_str(conf)) + elif conf.startswith('input{}.format'.format(int(self.input_idx))): + self.input_fmt.append(self._parse_str(conf)) + else: + return + + def parse_output(self: any, conf: str): + if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): + self.output_idx += 1 + self.output_ori_name.append(self._parse_str(conf)) + self.output_name.append(self.output_ori_name[-1] + '_out_') + elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): + self.output_type.append(self._parse_str(conf)) + elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): + self.output_dtype.append(self._parse_str(conf)) + elif conf.startswith('output{}.format'.format(int(self.output_idx))): + self.output_fmt.append(self._parse_str(conf)) + else: + return + + def parse_op_format(self: any, conf: str): + self.op_fmt_sel = self._parse_flag(conf) + + def parse_check_support(self: any, conf: str): + self.op_chk_support = self._parse_flag(conf) + + def parse_range_limit(self: any, conf: str): + self.op_range_limit = self._parse_str(conf) + + def parse_kern_name(self: any, conf: str): + self.kern_name = self._parse_str(conf) + + def parse_op_intf(self: any, conf: str): + self.op_intf = self._parse_str(conf) + + def parse_op_file(self: any, conf: str): + self.op_file = self._parse_str(conf) + + def parse_dynamic_shape(self: any, conf: str): + self.dynamic_shape = self._parse_flag(conf) + + def parse_attr_list(self: any, conf: str): + self.attr_list = self._parse_list(conf) + intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS + if intersection_element: + raise Exception(f'The attribute name: {intersection_element} in op: {{\'{self.op_type}\'}} conflicts with\ + the built-in variable name. Use a complex name or prefix the operator name.') + + @staticmethod + def _camel_to_snake(camel_case_str: str): + snake_case_str = '' + for i, c in enumerate(camel_case_str): + if i == 0: + snake_case_str += c.lower() + elif c.isupper(): + snake_case_str += '_' + c.lower() + else: + snake_case_str += c + return snake_case_str + + def parse_attr_val(self: any, conf: str): + for attr in self.attr_list: + if self.attr_val.get(attr) is None: + self.attr_val[attr] = {} + if conf.startswith('attr_{}.type'.format(attr)): + self.attr_val.get(attr)['type'] = self._camel_to_snake(self._parse_str(conf)) + elif conf.startswith('attr_{}.paramType'.format(attr)): + self.attr_val.get(attr)['paramType'] = self._parse_str(conf) + elif conf.startswith('attr_{}.defaultValue'.format(attr)): + self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) + + def parse_replay_val(self: any, batch_list: list, iterator_list: list): + if self.op_type in batch_list: + self.op_replay_flag = True + self.op_replay_batch = True + elif self.op_type in iterator_list: + self.op_replay_flag = True + self.op_replay_batch = False + + +def _is_op_type_in_opdesc(op_descs: list, op_type: str): + for op in op_descs: + if op_type == op.op_type: + return True + return False + + +def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): + for op in op_descs: + op.custom_all_compile_options = soc_ver_compile_options + + +def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): + for op in op_descs: + if op.op_type != op_type: + continue + op.custom_compile_options = soc_ver_compile_options + + +def _trans_soc_ver_to_short(soc_ver: str): + low_soc_ver = soc_ver.lower() + if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: + print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' + f'use ascend910b1~4 instead') + return SOC_TO_SHORT_SOC_MAP[low_soc_ver] + + +def _get_op_custom_options(op_descs: list, auto_gen_dir: str): + if auto_gen_dir is None: + return + file = os.path.join(auto_gen_dir, "custom_compile_options.ini") + if not os.path.exists(file): + print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') + return + with open(file, 'r') as fd: + lines = fd.readlines() + for line in lines: + param_list = str.split(line.rstrip('\n'), ',') + if len(param_list) != 3: + raise Exception(f'ERROR: custom compile option {param_list} len is not 3') + op_type = param_list[0] + if op_type.upper() == 'ALL': + op_type = OP_ALL + if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: + print(f'WARNING: op: {op_type} are not exists in this project') + continue + soc_ver_compile_options = {} + soc_ver = param_list[1] + options_str = param_list[2] + options = str.split(options_str, ';') + if soc_ver == '': + soc_ver_compile_options[SOC_ALL] = options + else: + soc_ver_list = str.split(soc_ver, ';') + for ver in soc_ver_list: + short_ver = _trans_soc_ver_to_short(ver) + soc_ver_compile_options[short_ver] = options + if op_type == OP_ALL: + _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) + else: + _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) + + +def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, + op_type: list, auto_gen_dir: str = None) -> list: + op_descs = [] + op_match = False + with open (file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if line.startswith('['): + name = line[1:-1] + if op_type is None or name in op_type: + op_match = True + op_desc = builder(name) + op_desc.parse_replay_val(batch_list, iterator_list) + op_descs.append(op_desc) + else: + op_match = False + if op_type is not None and len(op_descs) == len(op_type): + return op_descs + continue + if not op_match: + continue + if line.startswith('input'): + op_desc.parse_input(line) + elif line.startswith('output'): + op_desc.parse_output(line) + elif line.startswith('dynamicFormat.flag'): + op_desc.parse_op_format(line) + elif line.startswith('needCheckSupport.flag'): + op_desc.parse_check_support(line) + elif line.startswith('rangeLimit.value'): + op_desc.parse_range_limit(line) + elif line.startswith('opInterface.value'): + op_desc.parse_op_intf(line) + elif line.startswith('kernel.name'): + op_desc.parse_kern_name(line) + elif line.startswith('opFile.value'): + op_desc.parse_op_file(line) + elif line.startswith('dynamicShapeSupport.flag'): + op_desc.parse_dynamic_shape(line) + elif line.startswith('attr.list'): + op_desc.parse_attr_list(line) + elif line.startswith('attr_'): + op_desc.parse_attr_val(line) + _get_op_custom_options(op_descs, auto_gen_dir) + return op_descs diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/parse_ini_to_json.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/parse_ini_to_json.py new file mode 100644 index 000000000..ac931e5d3 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/parse_ini_to_json.py @@ -0,0 +1,338 @@ +# Copyright 2020-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +parser ini to json +""" + +import json +import os +import stat +import sys + + +ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", + "type", "listType", "tensor", "listTensor"] +ATTR_PARAMTYPE_LIST = ["optional", "required"] +BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", + "needCheckSupport", "enableVectorCore"] +BOOL_LIST = ["true", "false"] +DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", + "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", + "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", + "int4", "bfloat16", "uint1"] +FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", + "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", + "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", + "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", + "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", + "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", + "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] + + +def parse_ini_files(ini_files): + """ + parse ini files to json + Parameters: + ---------------- + ini_files:input file list + return:ops_info + ---------------- + """ + tbe_ops_info = {} + for ini_file in ini_files: + check_file_size(ini_file) + parse_ini_to_obj(ini_file, tbe_ops_info) + return tbe_ops_info + + +def check_file_size(input_file): + try: + file_size = os.path.getsize(input_file) + except OSError as os_error: + print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) + raise OSError from os_error + if file_size > 10*1024*1024: + print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) + + +def parse_ini_to_obj(ini_file, tbe_ops_info): + """ + parse ini file to json obj + Parameters: + ---------------- + ini_file:ini file path + tbe_ops_info:ops_info + ---------------- + """ + with open(ini_file) as ini_file: + lines = ini_file.readlines() + op_dict = {} + op_name = "" + find_op_type = False + for line in lines: + line = line.rstrip() + if line == "": + continue + if line.startswith("["): + if line.endswith("]"): + op_name = line[1:-1] + op_dict = {} + tbe_ops_info[op_name] = op_dict + find_op_type = True + elif "=" in line: + key1 = line[:line.index("=")] + key2 = line[line.index("=")+1:] + key1_0, key1_1 = key1.split(".") + if key1_0 not in op_dict: + op_dict[key1_0] = {} + if key1_1 in op_dict.get(key1_0): + raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + + key1_1 + " is repeated!") + dic_key = op_dict.get(key1_0) + dic_key[key1_1] = key2 + else: + continue + if not find_op_type: + raise RuntimeError("Not find OpType in .ini file.") + + +def check_output_exist(op_dict, is_valid): + """ + Function Description: + Check output is exist + Parameter: op_dict + Parameter: is_valid + """ + if "output0" in op_dict: + output0_dict = op_dict.get("output0") + if output0_dict.get("name", None) is None: + is_valid = False + print("output0.name is required in .ini file!") + else: + is_valid = False + print("output0 is required in .ini file!") + return is_valid + + +def check_attr_dict(attr_dict, is_valid, attr): + """ + Function Description: + Check attr_dict + Parameter: attr_dict + Parameter: is_valid + Parameter: attr + """ + attr_type = attr_dict.get("type") + value = attr_dict.get("value") + param_type = attr_dict.get("paramType") + if attr_type is None or value is None: + is_valid = False + print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) + if param_type and param_type not in ATTR_PARAMTYPE_LIST: + is_valid = False + print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) + if attr_type and attr_type not in ATTR_TYPE_LIST: + is_valid = False + print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) + return is_valid + + +def check_attr(op_dict, is_valid): + """ + Function Description: + Check attr + Parameter: op_dict + Parameter: is_valid + """ + if "attr" in op_dict: + attr_dict = op_dict.get("attr") + attr_list_str = attr_dict.get("list", None) + if attr_list_str is None: + is_valid = False + print("attr.list is required in .ini file!") + else: + attr_list = attr_list_str.split(",") + for attr_name in attr_list: + attr = "attr_" + attr_name.strip() + attr_dict = op_dict.get(attr) + if attr_dict: + is_valid = check_attr_dict(attr_dict, is_valid, attr) + else: + is_valid = False + print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) + return is_valid + + +def check_bool_flag(op_dict, is_valid): + """ + Function Description: + check_bool_flag + Parameter: op_dict + Parameter: is_valid + """ + for key in BOOL_FLAG_KEY: + if key in op_dict: + op_bool_key = op_dict.get(key) + if op_bool_key.get("flag").strip() not in BOOL_LIST: + is_valid = False + print("{0}.flag only support {1}.".format(key, BOOL_LIST)) + return is_valid + + +def check_type_format(op_info, is_valid, op_info_key): + """ + Function Description: + Check type and format + Parameter: op_info + Parameter: is_valid + Parameter: op_info_key + """ + op_info_dtype_str = op_info.get("dtype") + op_info_dtype_num = 0 + op_info_format_num = 0 + if op_info_dtype_str: + op_info_dtype = op_info_dtype_str.split(",") + op_info_dtype_num = len(op_info_dtype) + for dtype in op_info_dtype: + if dtype.strip() not in DTYPE_LIST: + is_valid = False + print("{0}.dtype not support {1}.".format(op_info_key, dtype)) + op_info_format_str = op_info.get("format") + if op_info_format_str: + op_info_format = op_info_format_str.split(",") + op_info_format_num = len(op_info_format) + for op_format in op_info_format: + if op_format.strip() not in FORMAT_LIST: + is_valid = False + print("{0}.format not support {1}.".format(op_info_key, op_format)) + if op_info_dtype_num > 0 and op_info_format_num > 0: + if op_info_dtype_num != op_info_format_num: + is_valid = False + print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) + return is_valid + + +def check_op_info(tbe_ops): + """ + Function Description: + Check info. + Parameter: tbe_ops + Return Value: is_valid + """ + print("\n\n==============check valid for ops info start==============") + required_op_input_info_keys = ["paramType", "name"] + required_op_output_info_keys = ["paramType", "name"] + param_type_valid_value = ["dynamic", "optional", "required"] + is_valid = True + for op_key in tbe_ops: + op_dict = tbe_ops[op_key] + for op_info_key in op_dict: + if op_info_key.startswith("input"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_input_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + \ + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + if op_info_key.startswith("output"): + op_input_info = op_dict[op_info_key] + missing_keys = [] + for required_op_input_info_key in required_op_output_info_keys: + if required_op_input_info_key not in op_input_info: + missing_keys.append(required_op_input_info_key) + if len(missing_keys) > 0: + print("op: " + op_key + " " + op_info_key + " missing: " + + ",".join(missing_keys)) + is_valid = False + else: + if not op_input_info["paramType"] in param_type_valid_value: + print("op: " + op_key + " " + op_info_key + + " paramType not valid, valid key:[dynamic, " + "optional, required]") + is_valid = False + is_valid = check_type_format(op_input_info, is_valid, op_info_key) + is_valid = check_attr(op_dict, is_valid) + is_valid = check_bool_flag(op_dict, is_valid) + print("==============check valid for ops info end================\n\n") + return is_valid + + +def write_json_file(tbe_ops_info, json_file_path): + """ + Save info to json file + Parameters: + ---------------- + tbe_ops_info: ops_info + json_file_path: json file path + ---------------- + """ + json_file_real_path = os.path.realpath(json_file_path) + wr_flag = os.O_WRONLY | os.O_CREAT + wr_mode = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: + # The owner have all rights，group only have read rights + os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP + + stat.S_IRUSR) + json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, + separators=(',', ':')) + print("Compile op info cfg successfully.") + + +def parse_ini_to_json(ini_file_paths, outfile_path): + """ + parse ini files to json file + Parameters: + ---------------- + ini_file_paths: list of ini file path + outfile_path: output file path + ---------------- + """ + tbe_ops_info = parse_ini_files(ini_file_paths) + if not check_op_info(tbe_ops_info): + print("Compile op info cfg failed.") + return False + write_json_file(tbe_ops_info, outfile_path) + return True + + +if __name__ == '__main__': + args = sys.argv + + OUTPUT_FILE_PATH = "tbe_ops_info.json" + ini_file_path_list = [] + + for arg in args: + if arg.endswith("ini"): + ini_file_path_list.append(arg) + OUTPUT_FILE_PATH = arg.replace(".ini", ".json") + if arg.endswith("json"): + OUTPUT_FILE_PATH = arg + + if len(ini_file_path_list) == 0: + ini_file_path_list.append("tbe_ops_info.ini") + + if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): + sys.exit(1) + sys.exit(0) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/preset_parse.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/preset_parse.py new file mode 100644 index 000000000..8f1124b1d --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/preset_parse.py @@ -0,0 +1,23 @@ +import json +import sys +import os + + +def get_config_opts(file): + src_dir = os.path.abspath(os.path.dirname(file)) + opts = '' + with open(file, 'r') as fd: + config = json.load(fd) + for conf in config: + if conf == 'configurePresets': + for node in config[conf]: + macros = node.get('cacheVariables') + if macros is not None: + for key in macros: + opts += '-D{}={} '.format(key, macros[key]['value']) + opts = opts.replace('${sourceDir}', src_dir) + print(opts) + + +if __name__ == "__main__": + get_config_opts(sys.argv[1]) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/replay_codegen.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/replay_codegen.py new file mode 100644 index 000000000..1baa364ef --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/replay_codegen.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- +""" +Created on Feb 28 20:56:45 2020 +Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. +""" + +import os +import stat +import collections +import kernel_entry as keb +from tiling_data_def_build import gen_tiling +import code_channel_infer +import const_var + +PYF_PATH = os.path.dirname(__file__) + +ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ +['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) + + +class ReplayCodeGen: + def __init__(self, replayCodeGenParams): + self.op_type = replayCodeGenParams.op_type + self.impl = replayCodeGenParams.impl + self.tiling_file = replayCodeGenParams.tiling_file + self.tiling_data_file = '' + self.kernel = replayCodeGenParams.kernel + self.entry = replayCodeGenParams.entry + self.argn = replayCodeGenParams.argn + self.batch = False + self.outdir = '' + self.data_type = 'uint8_t' + self.blknum = 32 + self.op_replay_batch = replayCodeGenParams.op_replay_batch + self.max_block_dim = replayCodeGenParams.max_block_dim + self.max_shape_size = replayCodeGenParams.max_shape_size + + def set_batch(self, is_batch): + self.batch = is_batch + + def set_outdir(self, outdir): + self.outdir = outdir + + def gen_replay(self, ops_product: str): + kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') + kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') + replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') + if self.batch: + reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') + else: + reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') + kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') + self._gen_kentry(kerentry) + self._gen_kimpl_code(kerimpl, kertmp) + self._gen_tiling_data_header() + self._gen_replay_code(replayimpl, reptmp, ops_product) + + def _gen_tiling_data_header(self): + self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') + gen_tiling(self.tiling_file, self.tiling_data_file) + + def _gen_kimpl_code(self, src, tmpfile): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__CCE_FILE__', self.impl) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_replay_code(self, src, tmpfile, ops_product: str): + with open(tmpfile, 'r') as fd: + temp = fd.read() + temp = temp.replace('__ARG_NUM__', str(self.argn)) + argdef = [] + kargs = [] + for i in range(0, self.argn): + argdef.append('{} *'.format(self.data_type)) + kargs.append('({} *)GetArg({})'.format(self.data_type, i)) + temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) + temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) + temp = temp.replace('__KERNEL_FUN__', self.entry) + core_type_infer = 'core_type' + code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ + self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) + if code_channel == code_channel_infer.CODE_VEC: + core_type_infer = '0' + elif code_channel == code_channel_infer.CODE_CUBE: + core_type_infer = '1' + temp = temp.replace('__CORE_TYPE__', core_type_infer) + # regist function + temp = temp.replace('__OPS_PRODUCT__', ops_product) + temp = temp.replace('__OPTYPE__', self.op_type) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(temp) + + def _gen_kentry(self, src): + kf = '' + pre_alloc_str = 'A' * 256 + if self.batch: + kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) + else: + kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ + self.argn, self.data_type, self.blknum) + with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(kf) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/replay_impl.temp b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/replay_impl.temp new file mode 100644 index 000000000..1d30dd865 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/replay_impl.temp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include +#include "replay_def.h" +#include "code_gen.h" +#include "replay_fun.h" +#include "register/op_check.h" +#define __ASCENDC_REPLAY_CODE__ +using namespace std; +using namespace optiling; +using namespace AscendCReplay; + +extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); +extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], + int alen[], int atlen, const char* kernelname[]); + +#define KERNEL_N 1 +#define ARG_N (__ARG_NUM__) +#define MAX_L (1024 * 1024 * 100) +#define MAX_E (1024 * 1024) + +int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) +{ + // gen type 1 : direct call codes 0: load .o file + if (param.gentype < 0 || param.gentype > 1) { + printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); + return 0; + } else if (param.gentype == 1 && param.objptr == nullptr) { + printf("Error: call replay with direct call mode, but code obj addr is null\n"); + return 0; + } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { + printf("Error: call replay with object file mode, but object file path is null\n"); + return 0; + } + // core_type 0:MIX 1:CUBE 2:VEC + if (core_type < 0 || core_type > 2) { + printf("Error: call replay core type is %d !\n", core_type); + return 0; + } + g_coreType = __CORE_TYPE__; + g_taskRation = param.task_ration; + g_tilingKey = param.tiling_key; + + unsigned char *buf, *jit; + char *kernel[KERNEL_N * 32]; + int len[KERNEL_N * 32]; + int blknum[KERNEL_N]; + int max; + block_num = param.block_dim; + g_ubBase = block_num; + uint8_t *code = (uint8_t *)malloc(MAX_L); + uint8_t *pos = code; + struct timespec tp1, tp2; + + clock_gettime(CLOCK_MONOTONIC, &tp1); + if (block_num > 32) { + printf("Error: block_num > 32\n"); + return 0; + } + //__OP_FOPEN__ + for (int i = 0; i < KERNEL_N; i++) { + for (int j = 0; j < ARG_N; j++) + AddArg(j, ARG_STEP * (j + 1)); + for (block_idx = 0; block_idx < block_num; block_idx++) { + //__OP_SET_KERNEL__ + int code_idx = i * block_num + block_idx; +#ifdef FP_CEILING + SetCtrlFloatEnable(); +#else + SetCtrlFloatDisable(); +#endif + CodeInit(pos, false); + __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); + CodeEnd(); + kernel[code_idx] = (char *)pos; + len[code_idx] = CodeLen(); + pos += len[code_idx]; + printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); + } + blknum[i] = block_num; + } + //__OP_FCLOSE__ + clock_gettime(CLOCK_MONOTONIC, &tp2); + buf = (unsigned char *)malloc(MAX_E); + int fd = open(param.entry_file, O_RDONLY); + if (fd < 0) { + printf("[error]: cannot find entry.o : %s\n", param.entry_file); + return 0; + } + uint32_t bufSize = read(fd, buf, MAX_E); + if (bufSize <= 0) { + printf("[error]: entry.o : %s is too small ! \n", param.entry_file); + } + close(fd); + jit = (unsigned char *)malloc(MAX_L); + printf("total code generated %ld\n", pos - code); + int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); + if (tp1.tv_sec != tp2.tv_sec) { + printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); + } else { + printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); + } + printf("new elf size %d\n", sz); + if (param.gentype == 0) { + fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + (void)write(fd, jit, sz); + close(fd); + free(jit); + } else if (param.gentype == 1) { + *param.objptr = (char*)jit; + } + free(buf); + free(code); + return sz; +} + +REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/tiling_data_def_build.py b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/tiling_data_def_build.py new file mode 100644 index 000000000..4c5b099b7 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/tiling_data_def_build.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# coding=utf-8 +""" +Function: +The replay funtion entry +Copyright Information: +Huawei Technologies Co., Ltd. All Rights Reserved © 2020 +""" + +import sys +import os +import stat +import re +import const_var + + +def gen_tiling(tiling_header_file: str, tiling_file_out: str): + if not os.path.exists(tiling_header_file): + print("warning: no userdef tiling header file: ", tiling_header_file) + return + print("generate tiling def header file: ", tiling_file_out) + tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() + tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) + tiling_source += '#define __{}_H__\n\n'.format(tmp_name) + tiling_source += '#include \n' + tiling_source += '#include \n\n' + tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' + end_source = "" + pattern = re.compile(r'[(](.*)[)]', re.S) + with open(tiling_header_file, 'r') as fd: + lines = fd.readlines() + for line in lines: + line = line.strip() + if (line.startswith('BEGIN_TILING_DATA_DEF')): + tiling_source += '#pragma pack(1)\n' + tiling_source += 'struct ' + struct_def = re.findall(pattern, line)[0] + tiling_source += struct_def + ' {\n' + elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('TILING_DATA_FIELD_DEF')): + field_params = re.findall(pattern, line)[0] + fds = field_params.split(',') + tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) + elif (line.startswith('END_TILING_DATA_DEF')): + tiling_source += '};\n' + tiling_source += '#pragma pack()\n\n' + tiling_source += '#ifdef __NPU_TILING__\n' + tiling_source += \ + 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ + .format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' + tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#else\n' + tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) + tiling_source += '{\n' + tiling_source += ' uint64_t *src = (uint64_t *)tiling;\n' + tiling_source += ' uint64_t *dst = (uint64_t *)const_data;\n' + tiling_source += ' for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n'\ + .format(struct_def) + tiling_source += '}\n' + tiling_source += '#endif\n\n' + end_source = ''' +#define GET_TILING_DATA(tiling_data, tiling_arg) \\ +{stru} tiling_data; \\ +Init{stru}(tiling_arg, &tiling_data)\n +'''.format(stru=struct_def) + tiling_source += end_source + tiling_source += '#endif' + with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: + ofd.write(tiling_source) + + +if __name__ == '__main__': + if len(sys.argv) <= 2: + raise RuntimeError('arguments must greater than 2') + gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/CMakeLists.txt b/operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/CMakeLists.txt new file mode 100644 index 000000000..b6be9b492 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/CMakeLists.txt @@ -0,0 +1,11 @@ +if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") + add_subdirectory(caffe_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") + add_subdirectory(tf_plugin) + endif() + if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") + add_subdirectory(onnx_plugin) + endif() +endif() diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/tf_plugin/CMakeLists.txt b/operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/tf_plugin/CMakeLists.txt new file mode 100644 index 000000000..a6aba5c20 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/tf_plugin/CMakeLists.txt @@ -0,0 +1,14 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) +add_library(cust_tf_parsers SHARED ${plugin_srcs}) +target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_tf_parsers PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) +install(TARGETS cust_tf_parsers + LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow +) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/tf_plugin/tensorflow_histogram_plugin.cc b/operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/tf_plugin/tensorflow_histogram_plugin.cc new file mode 100644 index 000000000..8e574f130 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/tf_plugin/tensorflow_histogram_plugin.cc @@ -0,0 +1,23 @@ +/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All +rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the Apache License Version 2.0. + * You may not use this file except in compliance with the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details at + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +#include "register/register.h" + +namespace domi { +// register op info to GE +REGISTER_CUSTOM_OP("Histogram") + .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW + .OriginOpType("Histogram") // name in tf module + .ParseParamsByOperatorFn(AutoMappingByOpFn); +} // namespace domi diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/CMakeLists.txt b/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/CMakeLists.txt new file mode 100644 index 000000000..40dd51cfa --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/CMakeLists.txt @@ -0,0 +1,82 @@ + +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) + +opbuild(OPS_SRC ${ops_srcs} + OUT_DIR ${ASCEND_AUTOGEN_PATH} +) + +add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) +target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) +target_compile_options(cust_op_proto PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_op_proto PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_op_proto PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME + cust_opsproto_rt2.0 +) +add_library(cust_optiling SHARED ${ops_srcs}) +target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) +target_compile_options(cust_optiling PRIVATE + -fvisibility=hidden +) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_optiling PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_optiling PRIVATE + intf_pub + exe_graph + register + tiling_api + -Wl,--whole-archive + rt2_registry + -Wl,--no-whole-archive +) +set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME + cust_opmaster_rt2.0 +) + +file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) +file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) +add_library(cust_opapi SHARED ${aclnn_src}) +if(ENABLE_CROSS_COMPILE) + target_link_directories(cust_opapi PRIVATE + ${CMAKE_COMPILE_COMPILER_LIBRARY} + ${CMAKE_COMPILE_RUNTIME_LIBRARY} + ) +endif() +target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) + +add_custom_target(optiling_compat ALL + COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ + ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so +) + +install(TARGETS cust_op_proto + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h + DESTINATION packages/vendors/${vendor_name}/op_proto/inc) +install(TARGETS cust_optiling + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) +install(TARGETS cust_opapi + LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) +install(FILES ${aclnn_inc} + DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/histogram.cpp b/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/histogram.cpp new file mode 100644 index 000000000..302867fdf --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/histogram.cpp @@ -0,0 +1,91 @@ + +#include "histogram_tiling.h" +#include "register/op_def_registry.h" +#include "tiling/platform/platform_ascendc.h" +#include + +namespace optiling { +const uint32_t BLOCK_SIZE = 32; +static ge::graphStatus TilingFunc(gert::TilingContext* context) +{ + + HistogramTilingData tiling; + uint64_t ubSize; + auto ascendcPlatform = platform_ascendc::PlatformAscendC(context->GetPlatformInfo()); + ascendcPlatform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ubSize); //获取硬件平台存储空间 UB 的内存大小 + + //获取输入shape信息 + uint32_t inputNum = context->GetInputShape(0)->GetStorageShape().GetShapeSize(); //输入数量 + uint32_t inputBytes = GetSizeByDataType(context->GetInputDesc(0)->GetDataType()); //输入类型 + uint32_t inputLength = inputBytes * inputNum; //输入长度 + //可使用的ub空间 输入3输出1，手动考虑双缓存 + uint32_t ubDataNumber = (inputBytes == 2) ? 12 : 7; + uint32_t tileBlockNum = (ubSize / BLOCK_SIZE) / ubDataNumber; //每个ub段可用的空间块数 + uint32_t tileDataNum = (tileBlockNum * BLOCK_SIZE) / inputBytes - 256; //每次处理的数据量 + uint32_t finalTileNum = inputNum / tileDataNum; //需要循环处理几次 + uint32_t TailDataNum = inputNum - (tileDataNum * finalTileNum); + finalTileNum = TailDataNum == 0 ? finalTileNum : finalTileNum + 1; //需要循环处理几次 + + int bins = *context->GetAttrs()->GetInt(0); + float min = *context->GetAttrs()->GetFloat(1); + float max = *context->GetAttrs()->GetFloat(2); + tiling.set_bins(bins); + tiling.set_min(min); + tiling.set_max(max); + + tiling.set_CoreDataNum(inputNum);//(CoreDataNum); //对齐空间后的输入数量 + tiling.set_finalTileNum(finalTileNum);//需要循环处理几次 + tiling.set_tileDataNum(tileDataNum); //每次处理的数据量 + tiling.set_TailDataNum(TailDataNum); //最后一次需要处理的数据量 + + context->SetBlockDim(1); + tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); + context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); + size_t *currentWorkspace = context->GetWorkspaceSizes(1); + currentWorkspace[0] = 0; + return ge::GRAPH_SUCCESS; +} +} + + +namespace ge { +static ge::graphStatus InferShape(gert::InferShapeContext* context) +{ + const gert::Shape* x1_shape = context->GetInputShape(0); + gert::Shape* y_shape = context->GetOutputShape(0); + *y_shape = *x1_shape; + return GRAPH_SUCCESS; +} +} + + +namespace ops { +class Histogram : public OpDef { +public: + explicit Histogram(const char* name) : OpDef(name) + { + this->Input("x") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT, ge::DT_FLOAT16, ge::DT_INT32}) + .Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); + this->Output("y") + .ParamType(REQUIRED) + .DataType({ge::DT_FLOAT, ge::DT_FLOAT16, ge::DT_INT32}) + .Format({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}) + .UnknownShapeFormat({ge::FORMAT_ND, ge::FORMAT_ND, ge::FORMAT_ND}); + this->Attr("bins").AttrType(OPTIONAL).Int(100); + this->Attr("min").AttrType(OPTIONAL).Float(0); + this->Attr("max").AttrType(OPTIONAL).Float(0); + + this->SetInferShape(ge::InferShape); + + this->AICore() + .SetTiling(optiling::TilingFunc); + this->AICore().AddConfig("ascend310b"); + + } +}; + +OP_ADD(Histogram); +} diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/histogram_tiling.h b/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/histogram_tiling.h new file mode 100644 index 000000000..4c9a1ccda --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/histogram_tiling.h @@ -0,0 +1,16 @@ + +#include "register/tilingdata_base.h" + +namespace optiling { +BEGIN_TILING_DATA_DEF(HistogramTilingData) + TILING_DATA_FIELD_DEF(uint32_t, CoreDataNum); + TILING_DATA_FIELD_DEF(uint32_t, finalTileNum); + TILING_DATA_FIELD_DEF(uint32_t, tileDataNum); + TILING_DATA_FIELD_DEF(uint32_t, TailDataNum); + TILING_DATA_FIELD_DEF(int, bins); + TILING_DATA_FIELD_DEF(float, min); + TILING_DATA_FIELD_DEF(float, max); +END_TILING_DATA_DEF; + +REGISTER_TILING_DATA_CLASS(Histogram, HistogramTilingData) +} diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_kernel/CMakeLists.txt b/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_kernel/CMakeLists.txt new file mode 100644 index 000000000..8c94a952d --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_kernel/CMakeLists.txt @@ -0,0 +1,68 @@ +# set custom compile options +if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") + add_ops_compile_options(ALL OPTIONS -g -O0) +endif() + +foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) + + # generate aic-${compute_unit}-ops-info.json + add_ops_info_target(TARGET ops_info_gen_${compute_unit} + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} + ) + + # generate ascendc impl py once + if (NOT TARGET ascendc_impl_gen) + add_ops_impl_target(TARGET ascendc_impl_gen + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl + ) + endif() + + # dynamic shape binary compile + if (${ENABLE_BINARY_PACKAGE} AND NOT ${ENABLE_CROSS_COMPILE}) + add_bin_compile_target(TARGET ascendc_bin_${compute_unit} + OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini + IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} + ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel + COMPUTE_UNIT ${compute_unit} + ) + add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) + endif() + + if (${ENABLE_CROSS_COMPILE} AND ${ENABLE_BINARY_PACKAGE}) + add_cross_compile_target( + TARGET bin_${compute_unit} + OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../kernel + INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/ + ) + endif() +endforeach() + +# generate npu_supported_ops.json +add_npu_support_target(TARGET npu_supported_ops + OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} + OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core + INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} +) + +if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) + add_subdirectory(testcases) +endif() + +# install kernel file +if (${ENABLE_SOURCE_PACKAGE}) + file(GLOB KERNEL_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/*.h + ${CMAKE_CURRENT_SOURCE_DIR}/*.py + ) + install(FILES ${KERNEL_FILES} + DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic + ) +endif() diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_kernel/histogram.cpp b/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_kernel/histogram.cpp new file mode 100644 index 000000000..2602c06fa --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_kernel/histogram.cpp @@ -0,0 +1,207 @@ +#include "kernel_operator.h" +using namespace AscendC; +constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue + +// class KernelLerp { +template class KernelLerp { + using T = TYPE_X; +public: + __aicore__ inline KernelLerp() {} + __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, + uint32_t CoreDataNum, uint32_t finalTileNum, uint32_t tileDataNum, uint32_t TailDataNum, + int bins, float min, float max) { + ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); + + this->coreDataNum = CoreDataNum; + this->tileNum = finalTileNum; + this->tileDataNum = tileDataNum; + this->tailDataNum = TailDataNum; + this->bins = bins; + this->min = min; + this->max = max; + + xGm.SetGlobalBuffer((__gm__ DTYPE_X*)x, this->coreDataNum); + yGm.SetGlobalBuffer((__gm__ DTYPE_Y*)y, this->coreDataNum); + + pipe.InitBuffer(inQueueX, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_X)); + + pipe.InitBuffer(QueueBits, (this->tileDataNum+256) * sizeof(uint8_t));// / 2); + pipe.InitBuffer(QueueZero, (this->tileDataNum+256) * sizeof(float)); + pipe.InitBuffer(QueueTmp1, (this->tileDataNum+256) * sizeof(float)); + pipe.InitBuffer(QueueTmp2, (this->tileDataNum+256) * sizeof(float)); + if(this->tileDataNum < 256) + pipe.InitBuffer(QueueBuff, 256 * sizeof(float)); + else + pipe.InitBuffer(QueueBuff, this->tileDataNum * sizeof(float)); + } + __aicore__ inline void Process() { + auto tmp1 = QueueTmp1.Get(); + auto tmp2 = QueueTmp2.Get(); + auto buff = QueueBuff.Get(); + + int32_t loopCount = this->tileNum; + + this->processDataNum = this->tileDataNum; + for (int32_t i = 0; i < loopCount; i++) + { + if (i == this->tileNum - 1) + { + this->processDataNum = this->tailDataNum; + } + + LocalTensor xLocal = inQueueX.AllocTensor(); + DataCopy(xLocal, xGm[i * this->tileDataNum], (this->processDataNum+31)/32*32); + inQueueX.EnQue(xLocal); + xLocal = inQueueX.DeQue(); + if constexpr (std::is_same_v) + { + if(this->min == 0 && this->max == 0) + { + ReduceMax(tmp1, xLocal, buff, this->processDataNum, false); + if(i == 0) + this->max = tmp1.GetValue(0); + else + { + if(this->max < tmp1.GetValue(0)) + this->max = tmp1.GetValue(0); + } + ReduceMin(tmp1, xLocal, buff, this->processDataNum, false); + if(i == 0) + this->min = tmp1.GetValue(0); + else + { + if(this->min > tmp1.GetValue(0)) + this->min = tmp1.GetValue(0); + } + } + } + else + { + Cast(tmp2, xLocal, RoundMode::CAST_NONE, this->processDataNum); + if(this->min == 0 && this->max == 0) + { + ReduceMax(tmp1, tmp2, buff, this->processDataNum, false); + if(i == 0) + this->max = tmp1.GetValue(0); + else + { + if(this->max < tmp1.GetValue(0)) + this->max = tmp1.GetValue(0); + } + ReduceMin(tmp1, tmp2, buff, this->processDataNum, false); + if(i == 0) + this->min = tmp1.GetValue(0); + else + { + if(this->min > tmp1.GetValue(0)) + this->min = tmp1.GetValue(0); + } + } + } + inQueueX.FreeTensor(xLocal); + } + + this->processDataNum = this->tileDataNum; + for (int32_t i = 0; i < loopCount; i++) { + if (i == this->tileNum - 1) { + this->processDataNum = this->tailDataNum; + } + CopyIn(i); + Compute(i); + } + } + +private: + __aicore__ inline void CopyIn(int32_t progress) + { + LocalTensor xLocal = inQueueX.AllocTensor(); + DataCopy(xLocal, xGm[progress * this->tileDataNum], (this->processDataNum+31)/32*32); + inQueueX.EnQue(xLocal); + } + __aicore__ inline void Compute(int32_t progress) + { + LocalTensor xLocal = inQueueX.DeQue(); + + auto tmp1 = QueueTmp1.Get(); + auto tmp2 = QueueTmp2.Get(); + auto buff = QueueBuff.Get(); + auto bits = QueueBits.Get(); + auto zero = QueueZero.Get(); + + float neg_min= -this->min; + float interval = (this->max - this->min); + float float_bins = this->bins; + float var1; + + Duplicate(zero, float(1), this->processDataNum); + + if constexpr (std::is_same_v) + { + Adds(tmp1, xLocal, neg_min, this->processDataNum); + } + else + { + Cast(tmp1, xLocal, RoundMode::CAST_NONE, this->processDataNum); + Adds(tmp1, tmp1, neg_min, this->processDataNum); + } + Muls(tmp1, tmp1, float_bins, this->processDataNum); + Duplicate(tmp2, (float)interval, this->processDataNum); + Div(tmp2, tmp1, tmp2, this->processDataNum); + Cast(tmp1, tmp2, RoundMode::CAST_FLOOR, this->processDataNum); + for(int32_t i=0; ibins; i++) + { + Duplicate(tmp2, (float)i, this->processDataNum); + Compare(bits, tmp2, tmp1, CMPMODE::EQ, (this->processDataNum+255)/256*256); + Select(tmp2, bits, zero, float(0), SELMODE::VSEL_TENSOR_SCALAR_MODE, this->processDataNum); + ReduceSum(tmp2, tmp2, buff, this->processDataNum); + var1 = tmp2.GetValue(0); + if(progress == 0) + yGm.SetValue(i, (DTYPE_Y)(var1)); + else + yGm.SetValue(i, (float)yGm.GetValue(i) + (var1)); + } + + Duplicate(tmp2, (float)this->max, this->processDataNum); + if constexpr (std::is_same_v) + { + Compare(bits, tmp2, xLocal, CMPMODE::EQ, (this->processDataNum+255)/256*256); + } + else + { + Cast(tmp1, xLocal, RoundMode::CAST_NONE, this->processDataNum); + Compare(bits, tmp2, tmp1, CMPMODE::EQ, (this->processDataNum+255)/256*256); + } + Select(tmp2, bits, zero, float(0), SELMODE::VSEL_TENSOR_SCALAR_MODE, this->processDataNum); + ReduceSum(tmp2, tmp2, buff, this->processDataNum); + var1 = tmp2.GetValue(0); + yGm.SetValue(this->bins-1, (float)yGm.GetValue(this->bins-1) + (var1)); + + inQueueX.FreeTensor(xLocal); + } + +private: + TPipe pipe; + TQue inQueueX; + TBuf QueueTmp1, QueueTmp2, QueueBuff, QueueBits, QueueZero; + + GlobalTensor xGm; + GlobalTensor yGm; + + uint32_t coreDataNum; + uint32_t tileNum; + uint32_t tileDataNum; + uint32_t tailDataNum; + uint32_t processDataNum; + int bins; + float min; + float max; +}; +extern "C" __global__ __aicore__ void histogram(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling) { + GET_TILING_DATA(tiling_data, tiling); + // TODO: user kernel impl + KernelLerp op; + op.Init(x, y, + tiling_data.CoreDataNum, tiling_data.finalTileNum, tiling_data.tileDataNum, tiling_data.TailDataNum, + tiling_data.bins, tiling_data.min, tiling_data.max); + op.Process(); +} \ No newline at end of file diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/scripts/install.sh b/operator_contrib/Histogram/FrameworkLaunch/Histogram/scripts/install.sh new file mode 100644 index 000000000..8468c5a25 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/scripts/install.sh @@ -0,0 +1,318 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +QUIET="y" + +while true +do + case $1 in + --quiet) + QUIET="y" + shift + ;; + --install-path=*) + INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) + INSTALL_PATH=${INSTALL_PATH%*/} + shift + ;; + --*) + shift + ;; + *) + break + ;; + esac +done + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [ -n "${INSTALL_PATH}" ]; then + if [[ ! "${INSTALL_PATH}" = /* ]]; then + log "[ERROR] use absolute path for --install-path argument" + exit 1 + fi + if [ ! -d ${INSTALL_PATH} ]; then + mkdir ${INSTALL_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${INSTALL_PATH} failed" + exit 1 + fi + fi + targetdir=${INSTALL_PATH} +elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then + if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then + mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" + fi + fi + targetdir=${ASCEND_CUSTOM_OPP_PATH} +else + if [ "x${ASCEND_OPP_PATH}" == "x" ]; then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 + fi + targetdir="${ASCEND_OPP_PATH}" +fi + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + has_same_file=-1 + for file_a in ${sourcedir}/$vendordir/$1/*; do + file_b=${file_a##*/}; + if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then + log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" + return 1 + fi + grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; + if [[ $? -eq 0 ]]; then + echo -n "${file_b} " + has_same_file=0 + fi + done + if [ 0 -eq $has_same_file ]; then + if test $QUIET = "n"; then + echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + + while true + do + read orn + if [ "$orn" = n ]; then + return 0 + elif [ "$orn" = m ]; then + break; + elif [ "$0rn" = r ]; then + [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace or merge old ops $1 files .g....." + fi + + log "copy new ops $1 files ......" + if [ -d ${targetdir}/$vendordir/$1/ ]; then + chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 + fi + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} +upgrade_proto() +{ + if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then + log "[INFO] no need to upgrade custom.proto files" + return 0 + fi + if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then + log "[INFO] create ${targetdir}/$vendordir/framework/caffe." + mkdir -p ${targetdir}/$vendordir/framework/caffe + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" + return 1 + fi + else + if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then + # 有老版本,判断是否要覆盖式安装 + if test $QUIET = "n"; then + echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ + "custom.proto file. Do you want to replace? [y/n] " + + while true + do + read yn + if [ "$yn" = n ]; then + return 0 + elif [ "$yn" = y ]; then + break; + else + echo "[ERROR] input error, please input again!" + fi + done + fi + fi + log "[INFO] replace old caffe.proto files ......" + fi + chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 + cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ + if [ $? -ne 0 ];then + log "[ERROR] copy new custom.proto failed" + return 1 + fi + log "[INFO] copy custom.proto success" + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +delete_optiling_file() +{ + if [ ! -d ${targetdir}/vendors ];then + log "[INFO] $1 not exist, no need to uninstall" + return 0 + fi + sys_info=$(uname -m) + if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then + rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so + fi + return 0 +} + +log "[INFO] copy uninstall sh success" + +if [ ! -d ${targetdir}/vendors ];then + log "[INFO] create ${targetdir}/vendors." + mkdir -p ${targetdir}/vendors + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/vendors failed" + return 1 + fi +fi +chmod u+w ${targetdir}/vendors + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +delete_optiling_file op_impl +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +upgrade_proto +if [ $? -ne 0 ];then + exit 1 +fi + +# set the set_env.bash +if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then + _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} + bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" + set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" + if [ ! -d ${bin_path} ]; then + mkdir -p ${bin_path} >> /dev/null 2>&1 + if [ $? -ne 0 ]; then + log "[ERROR] create ${bin_path} failed" + exit 1 + fi + fi + echo -e ${set_env_variable} > ${bin_path}/set_env.bash + if [ $? -ne 0 ]; then + log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" + exit 1 + else + log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ + execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" + fi +else + config_file=${targetdir}/vendors/config.ini + if [ ! -f ${config_file} ]; then + touch ${config_file} + chmod 640 ${config_file} + echo "load_priority=$vendor_name" > ${config_file} + if [ $? -ne 0 ];then + echo "echo load_priority failed" + exit 1 + fi + else + found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" + found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') + vendor=$(echo $found_vendor | tr -s ' ' ',') + if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" + fi + fi +fi + +chmod u-w ${targetdir}/vendors + +if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then + chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 +fi +if [ -f ${targetdir}/ascend_install.info ]; then + chmod -R 440 ${targetdir}/ascend_install.info +fi +if [ -f ${targetdir}/scene.info ]; then + chmod -R 440 ${targetdir}/scene.info +fi +if [ -f ${targetdir}/version.info ]; then + chmod -R 440 ${targetdir}/version.info +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/scripts/upgrade.sh b/operator_contrib/Histogram/FrameworkLaunch/Histogram/scripts/upgrade.sh new file mode 100644 index 000000000..e09173485 --- /dev/null +++ b/operator_contrib/Histogram/FrameworkLaunch/Histogram/scripts/upgrade.sh @@ -0,0 +1,151 @@ +#!/bin/bash +vendor_name=customize +targetdir=/usr/local/Ascend/opp +target_custom=0 + +sourcedir=$PWD/packages +vendordir=vendors/$vendor_name + +log() { + cur_date=`date +"%Y-%m-%d %H:%M:%S"` + echo "[runtime] [$cur_date] "$1 +} + +if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then + log "[ERROR] env ASCEND_OPP_PATH no exist" + exit 1 +fi + +targetdir=${ASCEND_OPP_PATH} + +if [ ! -d $targetdir ];then + log "[ERROR] $targetdir no exist" + exit 1 +fi + +upgrade() +{ + if [ ! -d ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 files" + return 0 + fi + + if [ ! -d ${targetdir}/$vendordir/$1 ];then + log "[INFO] create ${targetdir}/$vendordir/$1." + mkdir -p ${targetdir}/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] create ${targetdir}/$vendordir/$1 failed" + return 1 + fi + else + vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) + for i in $vendor_installed_dir;do + vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) + if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then + echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ + you want to Overlay Installation , please enter:[o]; \ + or replace directory installation , please enter: [r]; \ + or not install , please enter:[n]." + fi + while true + do + read mrn + if [ "$mrn" = m ]; then + break + elif [ "$mrn" = r ]; then + [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" + break + elif [ "$mrn" = n ]; then + return 0 + else + echo "[WARNING]: Input error, please input m or r or n to choose!" + fi + done + done + log "[INFO] replace old ops $1 files ......" + fi + + log "copy new ops $1 files ......" + cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 files failed" + return 1 + fi + + return 0 +} + +upgrade_file() +{ + if [ ! -e ${sourcedir}/$vendordir/$1 ]; then + log "[INFO] no need to upgrade ops $1 file" + return 0 + fi + + log "copy new $1 files ......" + cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 + if [ $? -ne 0 ];then + log "[ERROR] copy new $1 file failed" + return 1 + fi + + return 0 +} + +log "[INFO] copy uninstall sh success" + +echo "[ops_custom]upgrade framework" +upgrade framework +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op proto" +upgrade op_proto +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op impl" +upgrade op_impl +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade op api" +upgrade op_api +if [ $? -ne 0 ];then + exit 1 +fi + +echo "[ops_custom]upgrade version.info" +upgrade_file version.info +if [ $? -ne 0 ];then + exit 1 +fi + +config_file=${targetdir}/vendors/config.ini +found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" +found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') +vendor=$(echo $found_vendor | tr -s ' ' ',') +if [ "$vendor" != "" ]; then + sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" +fi + +changemode() +{ + if [ -d ${targetdir} ];then + chmod -R 550 ${targetdir}>/dev/null 2>&1 + fi + + return 0 +} +echo "[ops_custom]changemode..." +#changemode +if [ $? -ne 0 ];then + exit 1 +fi + +echo "SUCCESS" +exit 0 + diff --git a/operator_contrib/Histogram/README.en.md b/operator_contrib/Histogram/README.en.md new file mode 100644 index 000000000..9e885f235 --- /dev/null +++ b/operator_contrib/Histogram/README.en.md @@ -0,0 +1,164 @@ +## Histogram Custom Operator Sample Description + +This sample implements the Histogram operator using the Ascend C programming language and provides corresponding end-to-end implementations based on different operator invocation methods. + +- [FrameworkLaunch](./FrameworkLaunch): Using a framework to call the Histogram custom operator. + +The development process follows the workflow of project creation -> operator implementation -> compilation and deployment -> operator invocation. The entire process relies on the operator project: the operator kernel function and tiling are developed based on the project code framework, the operator is compiled and deployed via project compilation scripts, enabling the use of single operators or operators within third-party frameworks. + +The following invocation methods are included in this sample: + +| Invocation Method | Directory | Description | +| ----------------- | --------- | ----------- | +| [FrameworkLaunch](./FrameworkLaunch) | [AclNNInvocation](./FrameworkLaunch/AclNNInvocation) | Calling the Histogram operator through aclnn. | + +## Operator Description + +Computes the histogram of a tensor. + +The elements are sorted into equal width bins between min and max. If min and max are both zero, the minimum and maximum values of the data are used. + +Elements lower than min and higher than max and NaN elements are ignored. + +## Operator Specification + + + + + + + + + + + +
Operator TypeHistogram
Operator inputnameshapedata typeformatdefault
xyz1(b, n, 3)float32ND\
xyz2(b, m, 3)float32ND\
Operator output
dist(b, n, 3)float32ND\
indices(b, n, 3)int32ND\
kernel nameHistogram
+ +### 1. Operator Project Overview + +The operator project directory Histogram contains template files, compilation scripts, etc., as follows: + +``` +├── AclNNInvocation // Directory for invoking the Histogram operator via aclnn +│ ├── inc // Header file directory +│ │ ├── common.h // Declaration of common methods, used for reading binary files +│ │ ├── op_runner.h // Operator description declaration, including inputs/outputs, operator type, input and output descriptions +│ │ └── operator_desc.h // Operator runtime information declaration file +│ ├── input // Directory for input data generated by scripts +│ ├── output // Directory for storing operator output data and ground truth +│ ├── scripts +│ │ ├── acl.json // ACL configuration file +│ │ ├── gen_data.py // Script for generating input and ground truth data +│ │ └── verify_result.py // File for verifying results +│ ├── src +│ │ ├── CMakeLists.txt // Compilation rules +│ │ ├── common.cpp // Common functions, implementation for reading binary files +│ │ ├── main.cpp // Entry point for single operator invocation +│ │ ├── op_runner.cpp // Main process for single operator invocation +│ │ └── operator_desc.cpp // Construct operator input and output descriptions +│ ├── run.sh // Script for running the test sample +├── Histogram // Custom operator project +│ ├── cmake +│ ├── framework // Directory for implementing operator plugins, single operator model generation does not depend on plugins, not needed +│ ├── op_host // Host-side implementation +│ ├── op_kernel // Kernel-side implementation +│ ├── scripts // Directory for packaging custom operator project-related scripts +│ ├── build.sh // Compilation entry script +│ ├── CMakeLists.txt // CMakeLists.txt for the operator project +│ └── CMakePresets.json // Compilation configuration file +│ └── Histogram.json // Configuration file for generating the custom operator using msopgen +``` + +The CANN software package provides the project creation tool msopgen. The Histogram operator project can be automatically created using Histogram.json. For details, please refer to [Ascend C Operator Development](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC) > Operator Development > Operator Project Development > Creating an Operator Project. + +### 2. Compile the Operator Project + +Compile the custom operator project to generate the custom operator package. + +- Run the following commands to navigate to the ReduceCustom operator project directory. + + Use the command line to download the sample code, using the master branch as an example. + + ```bash + cd ${git_clone_path}/samples/operator_contrib/HistogramSample/Histogram + ``` + +- Modify `CMakePresets.json` to set `ASCEND_CANN_PACKAGE_PATH` to the actual path where the CANN software package is installed. + + ```json + { + …… + "configurePresets": [ + { + …… + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + // Replace with the actual installation path of the CANN software package, e.g., /home/HwHiAiUser/Ascend/ascend-toolkit/latest + }, + …… + } + ] + } + ``` + +- Run the following commands in the ReduceCustom operator project directory to compile the operator project. + + ```bash + ./build.sh + ``` + +After a successful compilation, a `build_out` directory is created in the current directory, which contains the custom operator installation package `custom_opp__.run`, for example, `custom_opp_ubuntu_x86_64.run`. + +Note: To use the dump debug feature, remove the Atlas training series product configuration from the op_host and CMakeLists.txt for Atlas 200/500 A2 inference products. + +### 3. Deploy the Operator Package + +Run the following commands in the directory containing the custom operator installation package to install the custom operator package. + +```bash +cd build_out +./custom_opp__.run +``` + +After successful execution, the relevant files in the custom operator package will be deployed to the `vendors/customize` directory in the OPP operator library of the current environment. + +### 4. Configure Environment Variables + +Please select the appropriate command to configure the environment variables according to the [installation method](https://hiascend.com/document/redirect/CannCommunityInstSoftware) of the CANN development kit package in the current environment. + +- Default path, root user installed CANN software package + + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` + +- Default path, non-root user installed CANN software package + + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` + +- Specify path `install_path`, installed CANN software package + + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ``` + +### 5. Run Operator Sample + +- [aclnn calls the Histogram operator project (test sample)](./HistogramCase) + +```bash +cd ./AclNNInvocation +./run.sh +``` + +The sample will automatically generate test data, then compile and run the aclnn sample, and finally verify the results. For details, see the `run.sh` script. + +## Update Notes + +| Date | Update Content | +| ---------- | --------------------- | +| 2024/10/24 | Uploaded Histogram code | + diff --git a/operator_contrib/Histogram/README.md b/operator_contrib/Histogram/README.md new file mode 100644 index 000000000..d56317364 --- /dev/null +++ b/operator_contrib/Histogram/README.md @@ -0,0 +1,150 @@ +## Histogram自定义算子样例说明 +本样例通过Ascend C编程语言实现了Histogram算子，并按照不同的算子调用方式分别给出了对应的端到端实现。 +- [FrameworkLaunch](./FrameworkLaunch)：使用框架调用Histogram自定义算子。 +按照工程创建->算子实现->编译部署>算子调用的流程完成算子开发。整个过程都依赖于算子工程：基于工程代码框架完成算子核函数的开发和Tiling实现，通过工程编译脚本完成算子的编译部署，继而实现单算子调用或第三方框架中的算子调用。 + +本样例中包含如下调用方式： + + + + + + +
调用方式目录描述
FrameworkLaunch AclNNInvocation通过aclnn调用的方式调用Cumsum算子。
+ +## 算子描述 +①计算张量的直方图。 +②元素被排序到最小和最大之间的等宽箱中。如果min和max都为零，则使用数据的最小值和最大值。 +③低于最小值和高于最大值的元素以及NaN元素将被忽略。 + +## 算子规格描述 + + + + + + + + + + + + +
算子类型(OpType)Cumsum
算子输入nameshapedata typeformatdefault
x\float32, float16, int32ND\
算子输出
Y\float32, float16, int32ND\
attr属性bins\int\100
min\float\0
max\float\0
核函数名Cumsum
+ + +### 1.算子工程介绍 +其中，算子工程目录Histogram包含算子实现的模板文件、编译脚本等，如下所示: +``` +├── AclNNInvocation // 通过aclnn调用的方式调用Histogram算子 +│ ├── inc // 头文件目录 +│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 +│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 +│ │ └── operator_desc.h // 算子运行相关信息声明文件，包含算子 +│ ├── input // 存放脚本生成的输入数据目录 +│ ├── output // 存放算子运行输出数据和真值数据的目录 +│ ├── scripts +│ │ ├── acl.json // acl配置文件 +│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 +│ │ └── verify_result.py // 真值对比文件 +│ ├── src +│ │ ├── CMakeLists.txt // 编译规则文件 +│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 +│ │ ├── main.cpp // 单算子调用应用的入口 +│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 +│ │ └── operator_desc.cpp // 构造算子的输入与输出描述 +│ ├── run.sh // 测试样例启动脚本 +├── Histogram //自定义算子工程 +│ ├── cmake +│ ├── framework // 算子插件实现文件目录，单算子模型文件的生成不依赖算子适配插件，无需关注 +│ ├── op_host // host侧实现文件 +│ ├── op_kernel // kernel侧实现文件 +│ ├── scripts // 自定义算子工程打包相关脚本所在目录 +│ ├── build.sh // 编译入口脚本 +│ ├── CMakeLists.txt // 算子工程的CMakeLists.txt +│ └── CMakePresets.json // 编译配置项 +│ └── Histogram.json // msopgen自定义算子生成配置文件 +``` +CANN软件包中提供了工程创建工具msopgen，Histogram算子工程可通过Histogram.json自动创建，具体请参考[Ascend C算子开发](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC)>算子开发>算子开发工程>基于自定义算子工程的算子开发>创建算子工程 章节。 + +### 2. 编译算子工程 + +编译自定义算子工程，构建生成自定义算子包。 + +- 执行如下命令，切换到算子工程ReduceCustom目录。 + + 以命令行方式下载样例代码，master分支为例。 + + ```bash + cd ${git_clone_path}/samples/operator_contrib/HistogramSample/Histogram + ``` +- 修改CMakePresets.json中ASCEND_CANN_PACKAGE_PATH为CANN软件包安装后的实际路径。 + + ```json + { + …… + "configurePresets": [ + { + …… + "ASCEND_CANN_PACKAGE_PATH": { + "type": "PATH", + "value": "/usr/local/Ascend/ascend-toolkit/latest" + //请替换为CANN软件包安装后的实际路径。eg:/home/HwHiAiUser/Ascend/ascend-toolkit/latest + }, + …… + } + ] + } + ``` +- 在算子工程ReduceCustom目录下执行如下命令，进行算子工程编译。 + + ```bash + ./build.sh + ``` + +编译成功后，会在当前目录下创建build_out目录，并在build_out目录下生成自定义算子安装包custom_opp_\_\.run，例如“custom_opp_ubuntu_x86_64.run”。 + +备注：如果要使用dump调试功能，需要移除op_host内和CMakeLists.txt内的Atlas 训练系列产品、Atlas 200/500 A2 推理产品的配置。 + +### 3. 部署算子包 + +执行如下命令，在自定义算子安装包所在路径下，安装自定义算子包。 + +```bash +cd build_out +./custom_opp__.run +``` + +命令执行成功后，自定义算子包中的相关文件将部署至当前环境的OPP算子库的vendors/customize目录中。 + +### 4. 配置环境变量 + +请根据当前环境上CANN开发套件包的[安装方式](https://hiascend.com/document/redirect/CannCommunityInstSoftware)，选择对应配置环境变量的命令。 + +- 默认路径，root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest + ``` +- 默认路径，非root用户安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest + ``` +- 指定路径install_path，安装CANN软件包 + ```bash + export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest + ``` + +### 5. 调用执行算子样例 + +- [aclnn调用Histogram算子工程(测试样例)](./HistogramCase) + +```bash +cd ./AclNNInvocation +./run.sh +``` +样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 + +## 更新说明 + | 时间 | 更新事项 | +|----|------| +| 2024/10/24 | 上传Histogram代码 | -- Gitee From c3545dadacef9d31c777ad3f96620bc5a0dbcefa Mon Sep 17 00:00:00 2001 From: appleinsky <2164757809@qq.com> Date: Thu, 24 Oct 2024 03:11:16 +0800 Subject: [PATCH 05/12] Update README.en.md --- operator_contrib/Histogram/README.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator_contrib/Histogram/README.en.md b/operator_contrib/Histogram/README.en.md index 9e885f235..c4a382195 100644 --- a/operator_contrib/Histogram/README.en.md +++ b/operator_contrib/Histogram/README.en.md @@ -69,7 +69,7 @@ The operator project directory Histogram contains template files, compilation sc │ └── Histogram.json // Configuration file for generating the custom operator using msopgen ``` -The CANN software package provides the project creation tool msopgen. The Histogram operator project can be automatically created using Histogram.json. For details, please refer to [Ascend C Operator Development](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC) > Operator Development > Operator Project Development > Creating an Operator Project. +The CANN software package provides the project creation tool msopgen. The Histogram operator project can be automatically created using Histogram.jsonh. For details, please refer to [Ascend C Operator Development](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC) > Operator Development > Operator Project Development > Creating an Operator Project. ### 2. Compile the Operator Project -- Gitee From fca821f43b394a58700038141d9eba53b3dfd327 Mon Sep 17 00:00:00 2001 From: appleinsky <2164757809@qq.com> Date: Thu, 24 Oct 2024 03:41:25 +0800 Subject: [PATCH 06/12] =?UTF-8?q?=E3=80=90S2=E7=AE=97=E5=AD=90=E6=8F=90?= =?UTF-8?q?=E4=BA=A4=E3=80=91Histogram=E7=AE=97=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../FrameworkLaunch/AclNNInvocation/inc/common.h | 0 .../FrameworkLaunch/AclNNInvocation/inc/op_runner.h | 0 .../FrameworkLaunch/AclNNInvocation/inc/operator_desc.h | 0 .../FrameworkLaunch/AclNNInvocation/run.sh | 0 .../FrameworkLaunch/AclNNInvocation/scripts/acl.json | 0 .../FrameworkLaunch/AclNNInvocation/scripts/gen_data.py | 0 .../FrameworkLaunch/AclNNInvocation/scripts/verify_result.py | 0 .../FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt | 0 .../FrameworkLaunch/AclNNInvocation/src/common.cpp | 0 .../FrameworkLaunch/AclNNInvocation/src/main.cpp | 0 .../FrameworkLaunch/AclNNInvocation/src/op_runner.cpp | 0 .../FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp | 0 .../FrameworkLaunch/Histogram/CMakeLists.txt | 0 .../FrameworkLaunch/Histogram/CMakePresets.json | 0 .../FrameworkLaunch/Histogram/Histogram.json | 0 .../FrameworkLaunch/Histogram/build.sh | 0 .../FrameworkLaunch/Histogram/cmake/config.cmake | 0 .../FrameworkLaunch/Histogram/cmake/func.cmake | 0 .../FrameworkLaunch/Histogram/cmake/intf.cmake | 0 .../FrameworkLaunch/Histogram/cmake/makeself.cmake | 0 .../FrameworkLaunch/Histogram/cmake/util/__init__.py | 0 .../Histogram/cmake/util/ascendc_bin_param_build.py | 0 .../FrameworkLaunch/Histogram/cmake/util/ascendc_impl_build.py | 0 .../FrameworkLaunch/Histogram/cmake/util/ascendc_ops_config.py | 0 .../FrameworkLaunch/Histogram/cmake/util/ascendc_replay_build.py | 0 .../FrameworkLaunch/Histogram/cmake/util/batch_replay_impl.temp | 0 .../FrameworkLaunch/Histogram/cmake/util/code_channel_infer.py | 0 .../FrameworkLaunch/Histogram/cmake/util/const_var.py | 0 .../Histogram/cmake/util/gen_impl_and_mrege_json.sh | 0 .../FrameworkLaunch/Histogram/cmake/util/gen_ops_filter.sh | 0 .../FrameworkLaunch/Histogram/cmake/util/gen_version_info.sh | 0 .../FrameworkLaunch/Histogram/cmake/util/insert_op_info.py | 0 .../Histogram/cmake/util/insert_simplified_keys.py | 0 .../FrameworkLaunch/Histogram/cmake/util/kernel_entry.py | 0 .../FrameworkLaunch/Histogram/cmake/util/kernel_impl.temp | 0 .../FrameworkLaunch/Histogram/cmake/util/makeself/COPYING | 0 .../FrameworkLaunch/Histogram/cmake/util/makeself/README.md | 0 .../FrameworkLaunch/Histogram/cmake/util/makeself/VERSION | 0 .../FrameworkLaunch/Histogram/cmake/util/makeself/make-release.sh | 0 .../Histogram/cmake/util/makeself/makeself-header.sh | 0 .../FrameworkLaunch/Histogram/cmake/util/makeself/makeself.1 | 0 .../FrameworkLaunch/Histogram/cmake/util/makeself/makeself.lsm | 0 .../FrameworkLaunch/Histogram/cmake/util/makeself/makeself.sh | 0 .../FrameworkLaunch/Histogram/cmake/util/makeself/run-tests.sh | 0 .../FrameworkLaunch/Histogram/cmake/util/merge_aicpu_info_json.sh | 0 .../FrameworkLaunch/Histogram/cmake/util/opdesc_parser.py | 0 .../FrameworkLaunch/Histogram/cmake/util/parse_ini_to_json.py | 0 .../FrameworkLaunch/Histogram/cmake/util/preset_parse.py | 0 .../FrameworkLaunch/Histogram/cmake/util/replay_codegen.py | 0 .../FrameworkLaunch/Histogram/cmake/util/replay_impl.temp | 0 .../FrameworkLaunch/Histogram/cmake/util/tiling_data_def_build.py | 0 .../FrameworkLaunch/Histogram/framework/CMakeLists.txt | 0 .../FrameworkLaunch/Histogram/framework/tf_plugin/CMakeLists.txt | 0 .../Histogram/framework/tf_plugin/tensorflow_histogram_plugin.cc | 0 .../FrameworkLaunch/Histogram/op_host/CMakeLists.txt | 0 .../FrameworkLaunch/Histogram/op_host/histogram.cpp | 0 .../FrameworkLaunch/Histogram/op_host/histogram_tiling.h | 0 .../FrameworkLaunch/Histogram/op_kernel/CMakeLists.txt | 0 .../FrameworkLaunch/Histogram/op_kernel/histogram.cpp | 0 .../FrameworkLaunch/Histogram/scripts/install.sh | 0 .../FrameworkLaunch/Histogram/scripts/upgrade.sh | 0 operator_contrib/{Histogram => HistogramSample}/README.en.md | 0 operator_contrib/{Histogram => HistogramSample}/README.md | 0 63 files changed, 0 insertions(+), 0 deletions(-) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/AclNNInvocation/inc/common.h (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/AclNNInvocation/inc/op_runner.h (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/AclNNInvocation/run.sh (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/AclNNInvocation/scripts/acl.json (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/AclNNInvocation/src/common.cpp (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/AclNNInvocation/src/main.cpp (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/CMakeLists.txt (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/CMakePresets.json (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/Histogram.json (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/build.sh (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/config.cmake (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/func.cmake (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/intf.cmake (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/makeself.cmake (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/__init__.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/ascendc_bin_param_build.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/ascendc_impl_build.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/ascendc_ops_config.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/ascendc_replay_build.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/batch_replay_impl.temp (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/code_channel_infer.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/const_var.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/gen_impl_and_mrege_json.sh (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/gen_ops_filter.sh (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/gen_version_info.sh (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/insert_op_info.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/insert_simplified_keys.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/kernel_entry.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/kernel_impl.temp (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/makeself/COPYING (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/makeself/README.md (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/makeself/VERSION (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/makeself/make-release.sh (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/makeself/makeself-header.sh (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.1 (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.lsm (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.sh (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/makeself/run-tests.sh (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/merge_aicpu_info_json.sh (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/opdesc_parser.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/parse_ini_to_json.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/preset_parse.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/replay_codegen.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/replay_impl.temp (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/cmake/util/tiling_data_def_build.py (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/framework/CMakeLists.txt (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/framework/tf_plugin/CMakeLists.txt (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/framework/tf_plugin/tensorflow_histogram_plugin.cc (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/op_host/CMakeLists.txt (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/op_host/histogram.cpp (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/op_host/histogram_tiling.h (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/op_kernel/CMakeLists.txt (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/op_kernel/histogram.cpp (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/scripts/install.sh (100%) rename operator_contrib/{Histogram => HistogramSample}/FrameworkLaunch/Histogram/scripts/upgrade.sh (100%) rename operator_contrib/{Histogram => HistogramSample}/README.en.md (100%) rename operator_contrib/{Histogram => HistogramSample}/README.md (100%) diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/common.h b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/inc/common.h similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/common.h rename to operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/inc/common.h diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/op_runner.h b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/op_runner.h rename to operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h rename to operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/run.sh b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/run.sh similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/run.sh rename to operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/run.sh diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/acl.json b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/acl.json rename to operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py rename to operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py rename to operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt rename to operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/common.cpp b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/src/common.cpp similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/common.cpp rename to operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/src/common.cpp diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/main.cpp b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/src/main.cpp similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/main.cpp rename to operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/src/main.cpp diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp rename to operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp diff --git a/operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp rename to operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/CMakeLists.txt b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/CMakeLists.txt similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/CMakeLists.txt rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/CMakeLists.txt diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/CMakePresets.json b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/CMakePresets.json similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/CMakePresets.json rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/CMakePresets.json diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/Histogram.json b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/Histogram.json similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/Histogram.json rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/Histogram.json diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/build.sh b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/build.sh similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/build.sh rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/build.sh diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/config.cmake b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/config.cmake similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/config.cmake rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/config.cmake diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/func.cmake b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/func.cmake similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/func.cmake rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/func.cmake diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/intf.cmake b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/intf.cmake similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/intf.cmake rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/intf.cmake diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/makeself.cmake b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/makeself.cmake similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/makeself.cmake rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/makeself.cmake diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/__init__.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/__init__.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/__init__.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/__init__.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_bin_param_build.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/ascendc_bin_param_build.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_bin_param_build.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/ascendc_bin_param_build.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_impl_build.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/ascendc_impl_build.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_impl_build.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/ascendc_impl_build.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_ops_config.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/ascendc_ops_config.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_ops_config.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/ascendc_ops_config.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_replay_build.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/ascendc_replay_build.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/ascendc_replay_build.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/ascendc_replay_build.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/batch_replay_impl.temp b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/batch_replay_impl.temp similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/batch_replay_impl.temp rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/batch_replay_impl.temp diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/code_channel_infer.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/code_channel_infer.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/code_channel_infer.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/code_channel_infer.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/const_var.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/const_var.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/const_var.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/const_var.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/gen_impl_and_mrege_json.sh similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_impl_and_mrege_json.sh rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/gen_impl_and_mrege_json.sh diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_ops_filter.sh b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/gen_ops_filter.sh similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_ops_filter.sh rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/gen_ops_filter.sh diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_version_info.sh b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/gen_version_info.sh similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/gen_version_info.sh rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/gen_version_info.sh diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/insert_op_info.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/insert_op_info.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/insert_op_info.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/insert_op_info.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/insert_simplified_keys.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/insert_simplified_keys.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/insert_simplified_keys.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/insert_simplified_keys.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/kernel_entry.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/kernel_entry.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/kernel_entry.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/kernel_entry.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/kernel_impl.temp b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/kernel_impl.temp similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/kernel_impl.temp rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/kernel_impl.temp diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/COPYING b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/COPYING similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/COPYING rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/COPYING diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/README.md b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/README.md similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/README.md rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/README.md diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/VERSION b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/VERSION similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/VERSION rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/VERSION diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/make-release.sh b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/make-release.sh similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/make-release.sh rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/make-release.sh diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself-header.sh b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/makeself-header.sh similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself-header.sh rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/makeself-header.sh diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.1 b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.1 similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.1 rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.1 diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.lsm b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.lsm similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.lsm rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.lsm diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.sh b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.sh similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.sh rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/makeself.sh diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/run-tests.sh b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/run-tests.sh similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/makeself/run-tests.sh rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/makeself/run-tests.sh diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/merge_aicpu_info_json.sh similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/merge_aicpu_info_json.sh rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/merge_aicpu_info_json.sh diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/opdesc_parser.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/opdesc_parser.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/opdesc_parser.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/opdesc_parser.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/parse_ini_to_json.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/parse_ini_to_json.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/parse_ini_to_json.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/parse_ini_to_json.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/preset_parse.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/preset_parse.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/preset_parse.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/preset_parse.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/replay_codegen.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/replay_codegen.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/replay_codegen.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/replay_codegen.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/replay_impl.temp b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/replay_impl.temp similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/replay_impl.temp rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/replay_impl.temp diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/tiling_data_def_build.py b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/tiling_data_def_build.py similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/cmake/util/tiling_data_def_build.py rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/cmake/util/tiling_data_def_build.py diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/CMakeLists.txt b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/framework/CMakeLists.txt similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/CMakeLists.txt rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/framework/CMakeLists.txt diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/tf_plugin/CMakeLists.txt b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/framework/tf_plugin/CMakeLists.txt similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/tf_plugin/CMakeLists.txt rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/framework/tf_plugin/CMakeLists.txt diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/tf_plugin/tensorflow_histogram_plugin.cc b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/framework/tf_plugin/tensorflow_histogram_plugin.cc similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/framework/tf_plugin/tensorflow_histogram_plugin.cc rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/framework/tf_plugin/tensorflow_histogram_plugin.cc diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/CMakeLists.txt b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/CMakeLists.txt similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/CMakeLists.txt rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/CMakeLists.txt diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/histogram.cpp b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram.cpp similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/histogram.cpp rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram.cpp diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/histogram_tiling.h b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram_tiling.h similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/op_host/histogram_tiling.h rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram_tiling.h diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_kernel/CMakeLists.txt b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_kernel/CMakeLists.txt similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/op_kernel/CMakeLists.txt rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_kernel/CMakeLists.txt diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/op_kernel/histogram.cpp b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_kernel/histogram.cpp similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/op_kernel/histogram.cpp rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_kernel/histogram.cpp diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/scripts/install.sh b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/scripts/install.sh similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/scripts/install.sh rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/scripts/install.sh diff --git a/operator_contrib/Histogram/FrameworkLaunch/Histogram/scripts/upgrade.sh b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/scripts/upgrade.sh similarity index 100% rename from operator_contrib/Histogram/FrameworkLaunch/Histogram/scripts/upgrade.sh rename to operator_contrib/HistogramSample/FrameworkLaunch/Histogram/scripts/upgrade.sh diff --git a/operator_contrib/Histogram/README.en.md b/operator_contrib/HistogramSample/README.en.md similarity index 100% rename from operator_contrib/Histogram/README.en.md rename to operator_contrib/HistogramSample/README.en.md diff --git a/operator_contrib/Histogram/README.md b/operator_contrib/HistogramSample/README.md similarity index 100% rename from operator_contrib/Histogram/README.md rename to operator_contrib/HistogramSample/README.md -- Gitee From 178a977f126aed0af494ed0603c1890b4fdbe029 Mon Sep 17 00:00:00 2001 From: appleinsky <2164757809@qq.com> Date: Fri, 1 Nov 2024 19:46:43 +0800 Subject: [PATCH 07/12] =?UTF-8?q?=E6=8F=90=E4=BA=A4Histogram?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AclNNInvocation/scripts/gen_data.py | 3 + .../AclNNInvocation/scripts/verify_result.py | 3 + .../Histogram/op_host/histogram.cpp | 13 ++++- .../Histogram/op_host/histogram_tiling.h | 10 +++- .../Histogram/op_kernel/histogram.cpp | 18 ++++-- .../HistogramSample/Histogram.json | 58 +++++++++++++++++++ 6 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 operator_contrib/HistogramSample/Histogram.json diff --git a/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py index 6994f4381..9532bcfb7 100644 --- a/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py +++ b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py @@ -1,3 +1,6 @@ +#!/usr/bin/python3 +# -*- coding:utf-8 -*- +# Copyright 2022-2023 Huawei Technologies Co., Ltd import torch import numpy as np import os diff --git a/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py index f5243eb98..e79d04013 100644 --- a/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py +++ b/operator_contrib/HistogramSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py @@ -1,3 +1,6 @@ +#!/usr/bin/python3 +# -*- coding:utf-8 -*- +# Copyright 2022-2023 Huawei Technologies Co., Ltd import os import sys import numpy as np diff --git a/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram.cpp b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram.cpp index 302867fdf..055f1aadd 100644 --- a/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram.cpp +++ b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram.cpp @@ -1,8 +1,17 @@ - +/** + * @file histogram_tiling.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ +#include #include "histogram_tiling.h" #include "register/op_def_registry.h" #include "tiling/platform/platform_ascendc.h" -#include + namespace optiling { const uint32_t BLOCK_SIZE = 32; diff --git a/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram_tiling.h b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram_tiling.h index 4c9a1ccda..cafda101a 100644 --- a/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram_tiling.h +++ b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram_tiling.h @@ -1,4 +1,12 @@ - +/** + * @file histogram_tiling.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "register/tilingdata_base.h" namespace optiling { diff --git a/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_kernel/histogram.cpp b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_kernel/histogram.cpp index 2602c06fa..049e47095 100644 --- a/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_kernel/histogram.cpp +++ b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_kernel/histogram.cpp @@ -1,12 +1,20 @@ +/** + * @file histogram.cpp + * + * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + */ #include "kernel_operator.h" using namespace AscendC; -constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue +constexpr int32_t BUFFER_NUM = 2; -// class KernelLerp { -template class KernelLerp { +template class KernelHistogram { using T = TYPE_X; public: - __aicore__ inline KernelLerp() {} + __aicore__ inline KernelHistogram() {} __aicore__ inline void Init(GM_ADDR x, GM_ADDR y, uint32_t CoreDataNum, uint32_t finalTileNum, uint32_t tileDataNum, uint32_t TailDataNum, int bins, float min, float max) { @@ -199,7 +207,7 @@ private: extern "C" __global__ __aicore__ void histogram(GM_ADDR x, GM_ADDR y, GM_ADDR workspace, GM_ADDR tiling) { GET_TILING_DATA(tiling_data, tiling); // TODO: user kernel impl - KernelLerp op; + KernelHistogram op; op.Init(x, y, tiling_data.CoreDataNum, tiling_data.finalTileNum, tiling_data.tileDataNum, tiling_data.TailDataNum, tiling_data.bins, tiling_data.min, tiling_data.max); diff --git a/operator_contrib/HistogramSample/Histogram.json b/operator_contrib/HistogramSample/Histogram.json new file mode 100644 index 000000000..76d818c73 --- /dev/null +++ b/operator_contrib/HistogramSample/Histogram.json @@ -0,0 +1,58 @@ +[ + { + "op": "Histogram", + "language": "cpp", + "input_desc": [ + { + "name": "x", + "param_type": "required", + "format": [ + "ND", + "ND", + "ND" + ], + "type": [ + "float", + "fp16", + "int32" + ] + } + ], + "output_desc": [ + { + "name": "y", + "param_type": "required", + "format": [ + "ND", + "ND", + "ND" + ], + "type": [ + "float", + "fp16", + "int32" + ] + } + ], + "attr": [ + { + "name": "bins", + "param_type": "optional", + "type": "int", + "default_value": 100 + }, + { + "name": "min", + "param_type": "optional", + "type": "float", + "default_value": 0 + }, + { + "name": "max", + "param_type": "optional", + "type": "float", + "default_value": 0 + } + ] + } +] \ No newline at end of file -- Gitee From 87c977cd404759d6655970316f30c47e1e43180f Mon Sep 17 00:00:00 2001 From: appleinsky <2164757809@qq.com> Date: Fri, 1 Nov 2024 19:56:05 +0800 Subject: [PATCH 08/12] =?UTF-8?q?=E6=96=B0=E5=A2=9EHistogram=E7=AE=97?= =?UTF-8?q?=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- operator_contrib/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/operator_contrib/README.md b/operator_contrib/README.md index 7e422aced..d1b950f7f 100644 --- a/operator_contrib/README.md +++ b/operator_contrib/README.md @@ -8,6 +8,7 @@ | [CrossSample](./CrossSample) | 基于Ascend C的Cross自定义Vector算子及调用样例 | Atlas 200/500 A2 推理产品
Atlas A2训练系列产品/Atlas 800I A2推理产品 | 8.0.RC1.alpha003 | | [FastGeluSample](./FastGeluSample) | 基于Ascend C的FastGelu自定义Vector算子及调用样例 | Atlas 200/500 A2 推理产品
Atlas A2训练系列产品/Atlas 800I A2推理产品 | 8.0.RC1.alpha003 | | [FastGeluGradSample](./FastGeluGradSample) | 基于Ascend C的FastGeluGrad自定义Vector算子及调用样例 | Atlas 200/500 A2 推理产品
Atlas A2训练系列产品/Atlas 800I A2推理产品 | 8.0.RC1.alpha003 | +| [HistogramSample](./HistogramSample) | 基于Ascend C的Histogram自定义Vector算子及调用样例 | Atlas 200/500 A2 推理产品
Atlas A2训练系列产品/Atlas 800I A2推理产品 | 8.0.RC2.alpha003 | | [GreaterEqualSample](./GreaterEqualSample) | 基于Ascend C的GreaterEqual自定义Vector算子及调用样例 | Atlas 200/500 A2
Atlas A2训练系列产品/Atlas 800I A2推理产品 | 8.0.RC1.alpha003 | | [InstanceNormSample](./InstanceNormSample) | 基于Ascend C的InstanceNorm自定义Vector算子及调用样例 | Atlas 200/500 A2 推理产品
Atlas A2训练系列产品/Atlas 800I A2推理产品 | 8.0.RC1.alpha003 | | [LayerNormCustomSample](./LayerNormCustomSample) | 基于AscendC的LayerNorm自定义算子及调用样例 | Atlas A2训练系列产品 | 8.0.RC1.alpha003 | -- Gitee From 7f5cf09d42985b25c63db4813fe6f8084527e491 Mon Sep 17 00:00:00 2001 From: appleinsky <2164757809@qq.com> Date: Fri, 1 Nov 2024 20:46:16 +0800 Subject: [PATCH 09/12] =?UTF-8?q?=20=20=20=E5=88=A0=E9=99=A4pr=5FHistogrm?= =?UTF-8?q?=E4=B8=AD=E7=9A=84threenn=E7=AE=97=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AclNNInvocation/inc/common.h | 45 - .../AclNNInvocation/inc/op_runner.h | 182 ---- .../AclNNInvocation/inc/operator_desc.h | 57 -- .../FrameworkLaunch/AclNNInvocation/run.sh | 120 --- .../AclNNInvocation/scripts/acl.json | 1 - .../AclNNInvocation/scripts/gen_data.py | 72 -- .../AclNNInvocation/scripts/verify_result.py | 26 - .../scripts/verify_result_int.py | 26 - .../AclNNInvocation/src/CMakeLists.txt | 68 -- .../AclNNInvocation/src/common.cpp | 79 -- .../AclNNInvocation/src/main.cpp | 171 ---- .../AclNNInvocation/src/op_runner.cpp | 486 ----------- .../AclNNInvocation/src/operator_desc.cpp | 56 -- .../FrameworkLaunch/ThreeNN/CMakeLists.txt | 69 -- .../FrameworkLaunch/ThreeNN/CMakePresets.json | 63 -- .../FrameworkLaunch/ThreeNN/ThreeNN.json | 50 -- .../FrameworkLaunch/ThreeNN/build.sh | 80 -- .../ThreeNN/cmake/config.cmake | 25 - .../FrameworkLaunch/ThreeNN/cmake/func.cmake | 192 ---- .../FrameworkLaunch/ThreeNN/cmake/intf.cmake | 26 - .../ThreeNN/cmake/makeself.cmake | 17 - .../ThreeNN/cmake/three_nn.cpp | 127 --- .../ThreeNN/cmake/util/__init__.py | 8 - .../cmake/util/ascendc_bin_param_build.py | 127 --- .../ThreeNN/cmake/util/ascendc_impl_build.py | 501 ----------- .../ThreeNN/cmake/util/ascendc_ops_config.py | 120 --- .../cmake/util/ascendc_replay_build.py | 65 -- .../ThreeNN/cmake/util/batch_replay_impl.temp | 117 --- .../ThreeNN/cmake/util/code_channel_infer.py | 58 -- .../ThreeNN/cmake/util/const_var.py | 33 - .../cmake/util/gen_impl_and_mrege_json.sh | 20 - .../ThreeNN/cmake/util/gen_ops_filter.sh | 62 -- .../ThreeNN/cmake/util/gen_version_info.sh | 6 - .../ThreeNN/cmake/util/insert_op_info.py | 36 - .../cmake/util/insert_simplified_keys.py | 242 ------ .../ThreeNN/cmake/util/kernel_entry.py | 115 --- .../ThreeNN/cmake/util/kernel_impl.temp | 10 - .../ThreeNN/cmake/util/makeself/COPYING | 339 -------- .../ThreeNN/cmake/util/makeself/README.md | 246 ------ .../ThreeNN/cmake/util/makeself/VERSION | 1 - .../cmake/util/makeself/make-release.sh | 9 - .../cmake/util/makeself/makeself-header.sh | 660 -------------- .../ThreeNN/cmake/util/makeself/makeself.1 | 110 --- .../ThreeNN/cmake/util/makeself/makeself.lsm | 16 - .../ThreeNN/cmake/util/makeself/makeself.sh | 822 ------------------ .../ThreeNN/cmake/util/makeself/run-tests.sh | 8 - .../cmake/util/merge_aicpu_info_json.sh | 31 - .../ThreeNN/cmake/util/opdesc_parser.py | 298 ------- .../ThreeNN/cmake/util/parse_ini_to_json.py | 338 ------- .../ThreeNN/cmake/util/preset_parse.py | 23 - .../ThreeNN/cmake/util/replay_codegen.py | 105 --- .../ThreeNN/cmake/util/replay_impl.temp | 120 --- .../cmake/util/tiling_data_def_build.py | 87 -- .../ThreeNN/framework/CMakeLists.txt | 11 - .../framework/tf_plugin/CMakeLists.txt | 14 - .../tf_plugin/tensorflow_three_nn_plugin.cc | 23 - .../ThreeNN/op_host/CMakeLists.txt | 82 -- .../ThreeNN/op_host/three_nn.cpp | 109 --- .../ThreeNN/op_host/three_nn_tiling.h | 16 - .../ThreeNN/op_kernel/CMakeLists.txt | 68 -- .../ThreeNN/op_kernel/three_nn.cpp | 166 ---- .../ThreeNN/scripts/install.sh | 318 ------- .../ThreeNN/scripts/upgrade.sh | 151 ---- operator_contrib/ThreeNNSample/README.en.md | 160 ---- operator_contrib/ThreeNNSample/README.md | 145 --- 65 files changed, 8034 deletions(-) delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh delete mode 100644 operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh delete mode 100644 operator_contrib/ThreeNNSample/README.en.md delete mode 100644 operator_contrib/ThreeNNSample/README.md diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h deleted file mode 100644 index 7036813a9..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/common.h +++ /dev/null @@ -1,45 +0,0 @@ -/** -* @file common.h -* -* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#ifndef COMMON_H -#define COMMON_H - -#include -#include -#include -#include -#include - -#include "acl/acl.h" - -#define SUCCESS 0 -#define FAILED 1 - -#define INFO_LOG(fmt, args...) fprintf(stdout, "[INFO] " fmt "\n", ##args) -#define WARN_LOG(fmt, args...) fprintf(stdout, "[WARN] " fmt "\n", ##args) -#define ERROR_LOG(fmt, args...) fprintf(stderr, "[ERROR] " fmt "\n", ##args) - -/** - * @brief Read data from file - * @param [in] filePath: file path - * @param [out] fileSize: file size - * @return read result - */ -bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize); - -/** - * @brief Write data to file - * @param [in] filePath: file path - * @param [in] buffer: data to write to file - * @param [in] size: size to write - * @return write result - */ -bool WriteFile(const std::string &filePath, const void *buffer, size_t size); - -#endif // COMMON_H diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h deleted file mode 100644 index bf2a9ef43..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/op_runner.h +++ /dev/null @@ -1,182 +0,0 @@ -/** -* @file op_runner.h -* -* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#ifndef OP_RUNNER_H -#define OP_RUNNER_H - -#include "aclnn/acl_meta.h" -#include "acl/acl.h" -#include "common.h" -#include "operator_desc.h" - -/** - * Op Runner - */ -class OpRunner { -public: - /** - * @brief Constructor - * @param [in] opDesc: op description - */ - explicit OpRunner(OperatorDesc *opDesc); - - /** - * @brief Destructor - */ - virtual ~OpRunner(); - - /** - * @brief Init op runner - */ - bool Init(); - - /** - * @brief Get number of inputs - * @return number of inputs - */ - const size_t NumInputs(); - - /** - * @brief Get number of outputs - * @return number of outputs - */ - const size_t NumOutputs(); - - /** - * @brief Get input size by index - * @param [in] index: input index - * @return size of the input - */ - const size_t GetInputSize(size_t index) const; - const size_t GetInputNumDims(size_t index) const; - aclDataType GetInputDataType(size_t index) const; - aclFormat GetInputFormat(size_t index) const; - - /** - * @brief Get output size by index - * @param [in] index: output index - * @return size of the output - */ - size_t GetOutputSize(size_t index) const; - const size_t GetOutputNumDims(size_t index) const; - aclDataType GetOutputDataType(size_t index) const; - aclFormat GetOutputFormat(size_t index) const; - - /** - * @brief Get input element count by index - * @param i[in] ndex: input index - * @return element count of the input - */ - size_t GetInputElementCount(size_t index) const; - - /** - * @brief Get output element count by index - * @param [in] index: output index - * @return element count of the output - */ - size_t GetOutputElementCount(size_t index) const; - - /** - * @brief Get input shape by index - * @param [in] index: input index - * @return shape of the output - */ - std::vector GetInputShape(size_t index) const; - - /** - * @brief Get output shape by index - * @param [in] index: output index - * @return shape of the output - */ - std::vector GetOutputShape(size_t index) const; - - /** - * @brief Get input buffer(host memory) by index - * @tparam T: data type - * @param [in] index: input index - * @return host address of the input - */ - template - T *GetInputBuffer(size_t index) - { - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return nullptr; - } - return reinterpret_cast(hostInputs_[index]); - } - - /** - * @brief Get output buffer(host memory) by index - * @tparam T: data type - * @param [in] index: output index - * @return host address of the output - */ - template - const T *GetOutputBuffer(size_t index) - { - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return nullptr; - } - - return reinterpret_cast(hostOutputs_[index]); - } - - /** - * @brief Print readable input by index - * @param [in] index: input index - * @param [in] elementsPerRow: number of elements per row - */ - void PrintInput(size_t index, size_t elementsPerRow = 16); - - /** - * @brief Print readable output by index - * @param [in] index: output index - * @param [in] elementsPerRow: number of elements per row - */ - void PrintOutput(size_t index, size_t elementsPerRow = 16); - - /** - * @brief Compile static op - * @return compile result - */ - bool CompileStaticOp(); - - /** - * @brief Compile dynamic op - * @return compile result - */ - bool CompileDynamicOp(); - - /** - * @brief Run op - * @return run result - */ - bool RunOp(); - -private: - size_t numInputs_; - size_t numOutputs_; - - std::vector inputBuffers_; - std::vector outputBuffers_; - - std::vector devInputs_; - std::vector devOutputs_; - - std::vector hostInputs_; - std::vector hostOutputs_; - - std::vector inputTensor_; - std::vector outputTensor_; - OperatorDesc *opDesc_; -}; - -#endif // OP_RUNNER_H diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h deleted file mode 100644 index 4cbdf07e0..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/inc/operator_desc.h +++ /dev/null @@ -1,57 +0,0 @@ -/** -* @file operator_desc.h -* -* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#ifndef OPERATOR_DESC_H -#define OPERATOR_DESC_H - -#include -#include - -#include "acl/acl.h" - -/** - * Op description - */ -struct OperatorDesc { - /** - * Constructor - */ - explicit OperatorDesc(); - - /** - * Destructor - */ - virtual ~OperatorDesc(); - - /** - * Add an input tensor description - * @param [in] dataType: data type - * @param [in] numDims: number of dims - * @param [in] dims: dims - * @param [in] format: format - * @return OperatorDesc - */ - OperatorDesc &AddInputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); - - /** - * Add an output tensor description - * @param [in] dataType: data type - * @param [in] numDims: number of dims - * @param [in] dims: dims - * @param [in] format: format - * @return OperatorDesc - */ - OperatorDesc &AddOutputTensorDesc(aclDataType dataType, int numDims, const int64_t *dims, aclFormat format); - - std::string opType; - std::vector inputDesc; - std::vector outputDesc; -}; - -#endif // OPERATOR_DESC_H diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh deleted file mode 100644 index 96ca9c3b1..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/run.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/bin/bash -export ASCEND_SLOG_PRINT_TO_STDOUT=0 -export ASCEND_GLOBAL_LOG_LEVEL=0 - -CURRENT_DIR=$( - cd $(dirname ${BASH_SOURCE:-$0}) - pwd -) -cd $CURRENT_DIR - -# 导出环境变量 -SHORT=v:, -LONG=dtype:, -OPTS=$(getopt -a --options $SHORT --longoptions $LONG -- "$@") -eval set -- "$OPTS" -while : -do - case "$1" in - # float16, float, int32 - (-v | --dtype) - DTYPE="$2" - shift 2;; - (--) - shift; - break;; - (*) - echo "[ERROR] Unexpected option: $1"; - break;; - esac -done - -if [ ! $ASCEND_HOME_DIR ]; then - if [ -d "$HOME/Ascend/ascend-toolkit/latest" ]; then - export ASCEND_HOME_DIR=$HOME/Ascend/ascend-toolkit/latest - else - export ASCEND_HOME_DIR=/usr/local/Ascend/ascend-toolkit/latest - fi -fi -source $ASCEND_HOME_DIR/bin/setenv.bash - -export DDK_PATH=$ASCEND_HOME_DIR -arch=$(uname -m) -export NPU_HOST_LIB=$ASCEND_HOME_DIR/${arch}-linux/lib64 - -function main { - # 1. 清除算子输出和日志文件 - - rm ./output/execute_op - rm ./input/*.bin - rm ./output/*.bin - # rm -rf ./output/output*.bin > /dev/null - - # 2. 生成或复用输入数据和真值数据 - if [ -d "./input" ]; then - if [ "$(ls -A "./input")" ]; then - echo "已存在测试数据" - else - echo "生成测试数据" - cd $CURRENT_DIR - python3 scripts/gen_data.py - fi - else - echo "生成测试数据" - cd $CURRENT_DIR - python3 scripts/gen_data.py - fi - - if [ $? -ne 0 ]; then - echo "ERROR: generate input data failed!" - return 1 - fi - echo "INFO: generate input data success!" - - # 3. 编译或复用acl可执行文件 - if [ -e "./output/execute_op" ]; then - echo "可执行存在" - else - echo "可执行不存在" - cd $CURRENT_DIR; rm -rf build; mkdir -p build; cd build - cmake ../src - if [ $? -ne 0 ]; then - echo "ERROR: cmake failed!" - return 1 - fi - echo "INFO: cmake success!" - make - if [ $? -ne 0 ]; then - echo "ERROR: make failed!" - return 1 - fi - echo "INFO: make success!" - fi - - # 4. 运行可执行文件 - cd $CURRENT_DIR/output - echo "INFO: execute op!" - timeout 30 ./execute_op - - if [ $? -ne 0 ]; then - echo "ERROR: acl executable run failed! please check your project!" - return 1 - fi - echo "INFO: acl executable run success!" - - # 5. 比较真值文件 - cd $CURRENT_DIR - ret_dist=`python3 scripts/verify_result.py output/output_dist.bin output/golden_dist.bin` - ret_idx=`python3 scripts/verify_result_int.py output/output_idx.bin output/golden_idx.bin` - echo "verify dist $ret_dist" - echo "verify idx $ret_idx" - if [ "x$ret_dist" == "xtest pass" ] && [ "x$ret_idx" == "xtest pass" ]; then - echo "" - echo "#####################################" - echo "INFO: you have passed the Precision!" - echo "#####################################" - echo "" - fi -} - -main diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json deleted file mode 100644 index 9e26dfeeb..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/acl.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py deleted file mode 100644 index c723df735..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/gen_data.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/python3 -# -*- coding:utf-8 -*- -# Copyright 2022-2023 Huawei Technologies Co., Ltd -import numpy as np -import os -import torch - -def run_three_nn(b, n, m, xyz1, xyz2): - dist = np.zeros_like(xyz1, dtype=np.float32) - idx = np.zeros_like(xyz1, dtype=np.int32) - - for ii in range(b): - for jj in range(n): - x1 = xyz1[ii, jj, 0] - y1 = xyz1[ii, jj, 1] - z1 = xyz1[ii, jj, 2] - best1 = 1e40 - best2 = 1e40 - best3 = 1e40 - besti1 = 0 - besti2 = 0 - besti3 = 0 - for kk in range(m): - x2 = xyz2[ii, kk, 0] - y2 = xyz2[ii, kk, 1] - z2 = xyz2[ii, kk, 2] - d = (x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1) + (z2 - z1) * (z2 - z1) - if (d < best1): - best3 = best2 - besti3 = besti2 - best2 = best1 - besti2 = besti1 - best1 = d - besti1 = kk - elif (d < best2): - best3 = best2 - besti3 = besti2 - best2 = d - besti2 = kk - elif (d < best3): - best3 = d - besti3 = kk - dist[ii, jj, 0] = best1 - idx[ii, jj, 0] = besti1 - dist[ii, jj, 1] = best2 - idx[ii, jj, 1] = besti2 - dist[ii, jj, 2] = best3 - idx[ii, jj, 2] = besti3 - - return dist, idx - -def gen_golden_data_simple(): - b = 5 - n = 160 - m = 256 - xyz1_shape = [b,n,3] - xyz2_shape = [b,m,3] - input_xyz1 = np.random.uniform(-10, 10, xyz1_shape).astype(np.float32) - input_xyz2 = np.random.uniform(-10, 10, xyz2_shape).astype(np.float32) - dist, idx = run_three_nn(b, n, m, input_xyz1, input_xyz2) - os.system("mkdir -p input") - os.system("mkdir -p output") - print("dist = ",dist) - print("idx = ",idx) - input_xyz1.tofile("./input/input_xyz1.bin") - input_xyz2.tofile("./input/input_xyz2.bin") - dist.tofile("./output/golden_dist.bin") - idx.tofile("./output/golden_idx.bin") - - -if __name__ == "__main__": - gen_golden_data_simple() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py deleted file mode 100644 index d917ab413..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import sys -import numpy as np -np.set_printoptions(suppress=True) - -loss = 1e-4 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 -minimum = 10e-10 - -def verify_result(real_result, golden): - real_result = np.fromfile(real_result, dtype=np.float32) # 从bin文件读取实际运算结果 - golden = np.fromfile(golden, dtype=np.float32) # 从bin文件读取预期运算结果 - result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 - deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 - result_atol = np.less_equal(result, loss) # 计算绝对误差 - result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 - if not result_rtol.all() and not result_atol.all(): - if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 - print("[ERROR] result error") - print(real_result) - print(golden) - return False - print("test pass") - return True - -if __name__ == '__main__': - verify_result(sys.argv[1],sys.argv[2]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py deleted file mode 100644 index 9cbc3482b..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/scripts/verify_result_int.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import sys -import numpy as np -np.set_printoptions(suppress=True) - -loss = 1e-4 # 容忍偏差，一般fp16要求绝对误差和相对误差均不超过千分之一 -minimum = 10e-10 - -def verify_result(real_result, golden): - real_result = np.fromfile(real_result, dtype=np.int32) # 从bin文件读取实际运算结果 - golden = np.fromfile(golden, dtype=np.int32) # 从bin文件读取预期运算结果 - result = np.abs(real_result - golden) # 计算运算结果和预期结果偏差 - deno = np.maximum(np.abs(real_result), np.abs(golden)) # 获取最大值并组成新数组 - result_atol = np.less_equal(result, loss) # 计算绝对误差 - result_rtol = np.less_equal(result / np.add(deno, minimum), loss) # 计算相对误差 - if not result_rtol.all() and not result_atol.all(): - if np.sum(result_rtol == False) > real_result.size * loss and np.sum(result_atol == False) > real_result.size * loss: # 误差超出预期时返回打印错误，返回对比失败 - print("[ERROR] result error") - print(real_result) - print(golden) - return False - print("test pass") - return True - -if __name__ == '__main__': - verify_result(sys.argv[1],sys.argv[2]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt deleted file mode 100644 index ee73e930f..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/CMakeLists.txt +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) Huawei Technologies Co., Ltd. 2020. All rights reserved. - -# CMake lowest version requirement -cmake_minimum_required(VERSION 3.5.1) - -# project information -project(acl_execute_add) - -# Compile options -add_compile_options(-std=c++11) - -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "../output") -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "../output") - -set(INC_PATH $ENV{DDK_PATH}) - -if (NOT DEFINED ENV{DDK_PATH}) - set(INC_PATH "/usr/local/Ascend/ascend-toolkit/latest") - message(STATUS "set default INC_PATH: ${INC_PATH}") -else () - message(STATUS "env INC_PATH: ${INC_PATH}") -endif() - -set(CUST_PKG_PATH "${INC_PATH}/opp/vendors/customize/op_api") - -set(LIB_PATH $ENV{NPU_HOST_LIB}) - -# Dynamic libraries in the stub directory can only be used for compilation -if (NOT DEFINED ENV{NPU_HOST_LIB}) - set(LIB_PATH "/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64/stub/") - set(LIB_PATH1 "/usr/local/Ascend/ascend-toolkit/latest/atc/lib64/stub/") - message(STATUS "set default LIB_PATH: ${LIB_PATH}") -else () - message(STATUS "env LIB_PATH: ${LIB_PATH}") -endif() - -# Header path -include_directories( - ${INC_PATH}/runtime/include - ${INC_PATH}/atc/include - ../inc - ${CUST_PKG_PATH}/include - ${BUILT_PKG_PATH}/include/aclnnop -) - -# add host lib path -link_directories( - ${LIB_PATH} - ${LIB_PATH1} - ${CUST_PKG_PATH}/lib -) - -add_executable(execute_op - operator_desc.cpp - op_runner.cpp - main.cpp - common.cpp -) - -target_link_libraries(execute_op - ascendcl - cust_opapi - acl_op_compiler - nnopbase - stdc++ -) - -install(TARGETS execute_op DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp deleted file mode 100644 index 47a39cd5f..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/common.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/** -* @file common.cpp -* -* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#include "common.h" - -#include -#include -#include -#include - -extern bool g_isDevice; - -bool ReadFile(const std::string &filePath, size_t fileSize, void *buffer, size_t bufferSize) -{ - struct stat sBuf; - int fileStatus = stat(filePath.data(), &sBuf); - if (fileStatus == -1) { - ERROR_LOG("failed to get file %s", filePath.c_str()); - return false; - } - if (S_ISREG(sBuf.st_mode) == 0) { - ERROR_LOG("%s is not a file, please enter a file", filePath.c_str()); - return false; - } - - std::ifstream file; - file.open(filePath, std::ios::binary); - if (!file.is_open()) { - ERROR_LOG("Open file failed. path = %s", filePath.c_str()); - return false; - } - - std::filebuf *buf = file.rdbuf(); - size_t size = buf->pubseekoff(0, std::ios::end, std::ios::in); - if (size == 0) { - ERROR_LOG("file size is 0"); - file.close(); - return false; - } - if (size > bufferSize) { - ERROR_LOG("file size is larger than buffer size"); - file.close(); - return false; - } - buf->pubseekpos(0, std::ios::in); - buf->sgetn(static_cast(buffer), size); - fileSize = size; - file.close(); - return true; -} - -bool WriteFile(const std::string &filePath, const void *buffer, size_t size) -{ - if (buffer == nullptr) { - ERROR_LOG("Write file failed. buffer is nullptr"); - return false; - } - - int fd = open(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWRITE); - if (fd < 0) { - ERROR_LOG("Open file failed. path = %s", filePath.c_str()); - return false; - } - - auto writeSize = write(fd, buffer, size); - (void) close(fd); - if (writeSize != size) { - ERROR_LOG("Write file Failed."); - return false; - } - - return true; -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp deleted file mode 100644 index 7536fdb6e..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/main.cpp +++ /dev/null @@ -1,171 +0,0 @@ -/** -* @file main.cpp -* -* Copyright (C) 2023. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#include -#include -#include -#include -#include - -#include "acl/acl.h" -#include "op_runner.h" - -#include "common.h" - -bool g_isDevice = false; -int deviceId = 0; - -OperatorDesc CreateOpDesc() -{ - // define operator - int64_t b = 5; - int64_t n = 160; - int64_t m = 256; - std::vector xyz1_shape {b,n,3}; - std::vector xyz2_shape {b,m,3}; - std::vector out_shape {b,n,3}; - aclDataType dataType = ACL_FLOAT; - aclDataType indicesType = ACL_INT32; - aclFormat format = ACL_FORMAT_ND; - OperatorDesc opDesc; - opDesc.AddInputTensorDesc(dataType, xyz1_shape.size(), xyz1_shape.data(), format); - opDesc.AddInputTensorDesc(dataType, xyz2_shape.size(), xyz2_shape.data(), format); - opDesc.AddOutputTensorDesc(dataType, out_shape.size(), out_shape.data(), format); - opDesc.AddOutputTensorDesc(indicesType, out_shape.size(), out_shape.data(), format); - return opDesc; -} - -bool SetInputData(OpRunner &runner) -{ - size_t fileSize = 0; - ReadFile("../input/input_xyz1.bin", fileSize, runner.GetInputBuffer(0), runner.GetInputSize(0)); - ReadFile("../input/input_xyz2.bin", fileSize, runner.GetInputBuffer(1), runner.GetInputSize(1)); - INFO_LOG("Set input success"); - return true; -} - -bool ProcessOutputData(OpRunner &runner) -{ - WriteFile("../output/output_dist.bin", runner.GetOutputBuffer(0), runner.GetOutputSize(0)); - WriteFile("../output/output_idx.bin", runner.GetOutputBuffer(1), runner.GetOutputSize(1)); - INFO_LOG("Write output success"); - return true; -} - -void DestoryResource() -{ - bool flag = false; - if (aclrtResetDevice(deviceId) != ACL_SUCCESS) { - ERROR_LOG("Reset device %d failed", deviceId); - flag = true; - } - INFO_LOG("Reset Device success"); - if (aclFinalize() != ACL_SUCCESS) { - ERROR_LOG("Finalize acl failed"); - flag = true; - } - if (flag) { - ERROR_LOG("Destory resource failed"); - } else { - INFO_LOG("Destory resource success"); - } -} - -bool InitResource() -{ - std::string output = "../output"; - if (access(output.c_str(), 0) == -1) { - int ret = mkdir(output.c_str(), 0700); - if (ret == 0) { - INFO_LOG("Make output directory successfully"); - } - else { - ERROR_LOG("Make output directory fail"); - return false; - } - } - - // acl.json is dump or profiling config file - if (aclInit("../scripts/acl.json") != ACL_SUCCESS) { - ERROR_LOG("acl init failed"); - return false; - } - - if (aclrtSetDevice(deviceId) != ACL_SUCCESS) { - ERROR_LOG("Set device failed. deviceId is %d", deviceId); - (void)aclFinalize(); - return false; - } - INFO_LOG("Set device[%d] success", deviceId); - - // runMode is ACL_HOST which represents app is running in host - // runMode is ACL_DEVICE which represents app is running in device - aclrtRunMode runMode; - if (aclrtGetRunMode(&runMode) != ACL_SUCCESS) { - ERROR_LOG("Get run mode failed"); - DestoryResource(); - return false; - } - g_isDevice = (runMode == ACL_DEVICE); - INFO_LOG("Get RunMode[%d] success", runMode); - - return true; -} - -bool RunOp() -{ - // create op desc - OperatorDesc opDesc = CreateOpDesc(); - - // create Runner - OpRunner opRunner(&opDesc); - if (!opRunner.Init()) { - ERROR_LOG("Init OpRunner failed"); - return false; - } - - // Load inputs - if (!SetInputData(opRunner)) { - ERROR_LOG("Set input data failed"); - return false; - } - - // Run op - if (!opRunner.RunOp()) { - ERROR_LOG("Run op failed"); - return false; - } - - // process output data - if (!ProcessOutputData(opRunner)) { - ERROR_LOG("Process output data failed"); - return false; - } - - INFO_LOG("Run op success"); - return true; -} - -int main(int argc, char **argv) -{ - if (!InitResource()) { - ERROR_LOG("Init resource failed"); - return FAILED; - } - INFO_LOG("Init resource success"); - - if (!RunOp()) { - DestoryResource(); - return FAILED; - } - - DestoryResource(); - - return SUCCESS; -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp deleted file mode 100644 index 16bbc21d0..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/op_runner.cpp +++ /dev/null @@ -1,486 +0,0 @@ -/** -* @file op_runner.cpp -* -* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#include "op_runner.h" -#include "aclnn_three_nn.h" -#include -#include -#include "acl/acl_op_compiler.h" -#include "common.h" - -using namespace std; - -extern bool g_isDevice; - -OpRunner::OpRunner(OperatorDesc *opDesc) : opDesc_(opDesc) -{ - numInputs_ = opDesc->inputDesc.size(); - numOutputs_ = opDesc->outputDesc.size(); -} - -OpRunner::~OpRunner() -{ - for (size_t i = 0; i < numInputs_; ++i) { - auto ret = aclDestroyTensor(inputTensor_[i]); - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free InputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - ret = aclDestroyDataBuffer(inputBuffers_[i]); - - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free inputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - ret = aclrtFree(devInputs_[i]); - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free devInputs[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - if (g_isDevice) { - ret = aclrtFree(hostInputs_[i]); - } else { - ret = aclrtFreeHost(hostInputs_[i]); - } - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free hostInputs[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - } - - for (size_t i = 0; i < numOutputs_; ++i) { - auto ret = aclDestroyTensor(outputTensor_[i]); - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free outputTensor[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - ret = aclDestroyDataBuffer(outputBuffers_[i]); - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free outputBuffers[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - ret = aclrtFree(devOutputs_[i]); - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free devOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - if (g_isDevice) { - ret = aclrtFree(hostOutputs_[i]); - } else { - ret = aclrtFreeHost(hostOutputs_[i]); - } - if (ret != ACL_SUCCESS) { - ERROR_LOG("Free hostOutputs[%d]error code is %d", static_cast(i), static_cast(ret)); - exit(EXIT_FAILURE); - } - } -} - -bool OpRunner::Init() -{ - for (size_t i = 0; i < numInputs_; ++i) { - auto size = GetInputSize(i); - void *devMem = nullptr; - if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory for input[%zu] failed", i); - return false; - } - devInputs_.emplace_back(devMem); - inputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); - - void *hostInput = nullptr; - if (g_isDevice) { - if (aclrtMalloc(&hostInput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory for input[%zu] failed", i); - return false; - } - } else { - if (aclrtMallocHost(&hostInput, size) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory for input[%zu] failed", i); - return false; - } - } - if (hostInput == nullptr) { - ERROR_LOG("Malloc memory for input[%zu] failed", i); - return false; - } - hostInputs_.emplace_back(hostInput); - - aclTensor *inputTensor = aclCreateTensor(GetInputShape(i).data(), GetInputNumDims(i), GetInputDataType(i), - nullptr, 0, GetInputFormat(i), GetInputShape(i).data(), GetInputNumDims(i), devInputs_[i]); - if (inputTensor == nullptr) { - ERROR_LOG("Create Tensor for input[%zu] failed", i); - return false; - } - inputTensor_.emplace_back(inputTensor); - } - - for (size_t i = 0; i < numOutputs_; ++i) { - auto size = GetOutputSize(i); - void *devMem = nullptr; - if (aclrtMalloc(&devMem, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory for output[%zu] failed", i); - return false; - } - devOutputs_.emplace_back(devMem); - outputBuffers_.emplace_back(aclCreateDataBuffer(devMem, size)); - - void *hostOutput = nullptr; - if (g_isDevice) { - if (aclrtMalloc(&hostOutput, size, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory for output[%zu] failed", i); - return false; - } - } else { - if (aclrtMallocHost(&hostOutput, size) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory for output[%zu] failed", i); - return false; - } - } - if (hostOutput == nullptr) { - ERROR_LOG("Malloc host memory for output[%zu] failed", i); - return false; - } - hostOutputs_.emplace_back(hostOutput); - - aclTensor *outputTensor = aclCreateTensor(GetOutputShape(i).data(), GetOutputNumDims(i), GetOutputDataType(i), - nullptr, 0, GetOutputFormat(i), GetOutputShape(i).data(), GetOutputNumDims(i), devOutputs_[i]); - if (outputTensor == nullptr) { - ERROR_LOG("Create Tensor for output[%zu] failed", i); - return false; - } - outputTensor_.emplace_back(outputTensor); - } - - return true; -} - -const size_t OpRunner::NumInputs() -{ - return numInputs_; -} - -const size_t OpRunner::NumOutputs() -{ - return numOutputs_; -} - -const size_t OpRunner::GetInputSize(size_t index) const -{ - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return 0; - } - - return aclGetTensorDescSize(opDesc_->inputDesc[index]); -} - -const size_t OpRunner::GetInputNumDims(size_t index) const -{ - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return 0; - } - - return aclGetTensorDescNumDims(opDesc_->inputDesc[index]); -} - -aclDataType OpRunner::GetInputDataType(size_t index) const -{ - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return ACL_DT_UNDEFINED; - } - - return aclGetTensorDescType(opDesc_->inputDesc[index]); -} - -aclFormat OpRunner::GetInputFormat(size_t index) const -{ - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return ACL_FORMAT_UNDEFINED; - } - - return aclGetTensorDescFormat(opDesc_->inputDesc[index]); -} - -std::vector OpRunner::GetInputShape(size_t index) const -{ - std::vector ret; - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return ret; - } - - auto desc = opDesc_->inputDesc[index]; - for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { - int64_t dimSize; - if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { - ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); - ret.clear(); - return ret; - } - ret.emplace_back(dimSize); - } - - return ret; -} - -size_t OpRunner::GetOutputSize(size_t index) const -{ - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return 0; - } - - return aclGetTensorDescSize(opDesc_->outputDesc[index]); -} - -const size_t OpRunner::GetOutputNumDims(size_t index) const -{ - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return 0; - } - - return aclGetTensorDescNumDims(opDesc_->outputDesc[index]); -} - -aclDataType OpRunner::GetOutputDataType(size_t index) const -{ - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return ACL_DT_UNDEFINED; - } - - return aclGetTensorDescType(opDesc_->outputDesc[index]); -} - - -aclFormat OpRunner::GetOutputFormat(size_t index) const -{ - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return ACL_FORMAT_UNDEFINED; - } - - return aclGetTensorDescFormat(opDesc_->outputDesc[index]); -} - -std::vector OpRunner::GetOutputShape(size_t index) const -{ - std::vector ret; - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return ret; - } - - auto desc = opDesc_->outputDesc[index]; - for (size_t i = 0; i < aclGetTensorDescNumDims(desc); ++i) { - int64_t dimSize; - if (aclGetTensorDescDimV2(desc, i, &dimSize) != ACL_SUCCESS) { - ERROR_LOG("get dims from tensor desc failed. dims index = %zu", i); - ret.clear(); - return ret; - } - ret.emplace_back(dimSize); - } - return ret; -} - -size_t OpRunner::GetInputElementCount(size_t index) const -{ - if (index >= opDesc_->inputDesc.size()) { - ERROR_LOG("index out of range. index = %zu, numInputs = %zu", index, numInputs_); - return 0; - } - - return aclGetTensorDescElementCount(opDesc_->inputDesc[index]); -} - -size_t OpRunner::GetOutputElementCount(size_t index) const -{ - if (index >= opDesc_->outputDesc.size()) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return 0; - } - - return aclGetTensorDescElementCount(opDesc_->outputDesc[index]); -} - -bool OpRunner::RunOp() -{ - for (size_t i = 0; i < numInputs_; ++i) { - auto size = GetInputSize(i); - aclrtMemcpyKind kind = ACL_MEMCPY_HOST_TO_DEVICE; - if (g_isDevice) { - kind = ACL_MEMCPY_DEVICE_TO_DEVICE; - } - if (aclrtMemcpy(devInputs_[i], size, hostInputs_[i], size, kind) != ACL_SUCCESS) { - ERROR_LOG("Copy input[%zu] failed", i); - return false; - } - INFO_LOG("Copy input[%zu] success", i); - } - - aclrtStream stream = nullptr; - if (aclrtCreateStream(&stream) != ACL_SUCCESS) { - ERROR_LOG("Create stream failed"); - return false; - } - INFO_LOG("Create stream success"); - - size_t workspaceSize = 0; - aclOpExecutor *handle = nullptr; - - auto ret = aclnnThreeNNGetWorkspaceSize(inputTensor_[0], inputTensor_[1], outputTensor_[0], outputTensor_[1], - &workspaceSize, &handle); - if (ret != ACL_SUCCESS) { - (void)aclrtDestroyStream(stream); - ERROR_LOG("Get Operator Workspace failed. error code is %d", static_cast(ret)); - return false; - } - INFO_LOG("Execute GetWorkspaceSize success, workspace size %lu", workspaceSize); - - void *workspace = nullptr; - if (workspaceSize != 0) { - if (aclrtMalloc(&workspace, workspaceSize, ACL_MEM_MALLOC_HUGE_FIRST) != ACL_SUCCESS) { - ERROR_LOG("Malloc device memory failed"); - } - } - ret = aclnnThreeNN(workspace, workspaceSize, handle, stream); - - if (ret != ACL_SUCCESS) { - (void)aclrtDestroyStream(stream); - ERROR_LOG("Execute Operator failed. error code is %d", static_cast(ret)); - return false; - } - INFO_LOG("Execute Operator success"); - - ret = aclrtSynchronizeStreamWithTimeout(stream, 5000); - if (ret != SUCCESS) { - ERROR_LOG("Synchronize stream failed. error code is %d", static_cast(ret)); - (void)aclrtDestroyStream(stream); - return false; - } - INFO_LOG("Synchronize stream success"); - - for (size_t i = 0; i < numOutputs_; ++i) { - auto size = GetOutputSize(i); - aclrtMemcpyKind kind = ACL_MEMCPY_DEVICE_TO_HOST; - if (g_isDevice) { - kind = ACL_MEMCPY_DEVICE_TO_DEVICE; - } - if (aclrtMemcpy(hostOutputs_[i], size, devOutputs_[i], size, kind) != ACL_SUCCESS) { - INFO_LOG("Copy output[%zu] success", i); - (void)aclrtDestroyStream(stream); - return false; - } - INFO_LOG("Copy output[%zu] success", i); - } - - (void)aclrtDestroyStream(stream); - return true; -} - - -template -void DoPrintData(const T *data, size_t count, size_t elementsPerRow) -{ - assert(elementsPerRow != 0); - for (size_t i = 0; i < count; ++i) { - std::cout << std::setw(10) << data[i]; - if (i % elementsPerRow == elementsPerRow - 1) { - std::cout << std::endl; - } - } -} - -void DoPrintFp16Data(const aclFloat16 *data, size_t count, size_t elementsPerRow) -{ - assert(elementsPerRow != 0); - for (size_t i = 0; i < count; ++i) { - std::cout << std::setw(10) << std::setprecision(4) << aclFloat16ToFloat(data[i]); - if (i % elementsPerRow == elementsPerRow - 1) { - std::cout << std::endl; - } - } -} - -void PrintData(const void *data, size_t count, aclDataType dataType, size_t elementsPerRow) -{ - if (data == nullptr) { - ERROR_LOG("Print data failed. data is nullptr"); - return; - } - - switch (dataType) { - case ACL_BOOL: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_INT8: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_UINT8: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_INT16: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_UINT16: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_INT32: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_UINT32: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_INT64: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_UINT64: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_FLOAT16: - DoPrintFp16Data(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_FLOAT: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - case ACL_DOUBLE: - DoPrintData(reinterpret_cast(data), count, elementsPerRow); - break; - default: - ERROR_LOG("Unsupported type: %d", dataType); - } -} - -void OpRunner::PrintInput(size_t index, size_t numElementsPerRow) -{ - if (index >= numInputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numInputs_); - return; - } - - auto desc = opDesc_->inputDesc[index]; - PrintData(hostInputs_[index], GetInputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); -} - -void OpRunner::PrintOutput(size_t index, size_t numElementsPerRow) -{ - if (index >= numOutputs_) { - ERROR_LOG("index out of range. index = %zu, numOutputs = %zu", index, numOutputs_); - return; - } - - auto desc = opDesc_->outputDesc[index]; - PrintData(hostOutputs_[index], GetOutputElementCount(index), aclGetTensorDescType(desc), numElementsPerRow); -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp deleted file mode 100644 index a7a1ed3bd..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/AclNNInvocation/src/operator_desc.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/** -* @file operator_desc.cpp -* -* Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -*/ -#include "common.h" -#include "operator_desc.h" - -using namespace std; - -OperatorDesc::OperatorDesc() {} - -OperatorDesc::~OperatorDesc() -{ - for (auto *desc : inputDesc) { - aclDestroyTensorDesc(desc); - } - - for (auto *desc : outputDesc) { - aclDestroyTensorDesc(desc); - } - -} - -OperatorDesc &OperatorDesc::AddInputTensorDesc(aclDataType dataType, - int numDims, - const int64_t *dims, - aclFormat format) -{ - aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); - if (desc == nullptr) { - ERROR_LOG("create tensor failed"); - return *this; - } - inputDesc.emplace_back(desc); - return *this; -} - -OperatorDesc &OperatorDesc::AddOutputTensorDesc(aclDataType dataType, - int numDims, - const int64_t *dims, - aclFormat format) -{ - aclTensorDesc *desc = aclCreateTensorDesc(dataType, numDims, dims, format); - if (desc == nullptr) { - ERROR_LOG("create tensor failed"); - return *this; - } - - outputDesc.emplace_back(desc); - return *this; -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt deleted file mode 100644 index 584132d80..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakeLists.txt +++ /dev/null @@ -1,69 +0,0 @@ -cmake_minimum_required(VERSION 3.16.0) -project(opp) -if(ENABLE_CROSS_COMPILE) - if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL x86_64) - set(CROSS_COMPILE_PLATFORM aarch64) - else() - set(CROSS_COMPILE_PLATFORM x86_64) - endif() - set(PLATFORM ${CMAKE_SYSTEM_PROCESSOR}) - set(CMAKE_COMPILE_COMPILER_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/linux/${CROSS_COMPILE_PLATFORM}/) - set(CMAKE_COMPILE_RUNTIME_LIBRARY ${ASCEND_CANN_PACKAGE_PATH}/${PLATFORM}-linux/devlib/${CROSS_COMPILE_PLATFORM}/) - set(CMAKE_SYSTEM_PROCESSOR ${CROSS_COMPILE_PLATFORM}) - set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) - set(CMAKE_CXX_COMPILER ${CMAKE_CROSS_PLATFORM_COMPILER}) -else() - set(CMAKE_COMPILE ${CMAKE_CXX_COMPILER}) -endif() - -include(cmake/config.cmake) -include(cmake/func.cmake) -include(cmake/intf.cmake) - -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/framework) - add_subdirectory(framework) -endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_host) - add_subdirectory(op_host) -endif() -if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/op_kernel) - add_subdirectory(op_kernel) -endif() -if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) - add_subdirectory(testcases) -endif() - -# modify vendor_name in install.sh and upgrade.sh -add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh - COMMAND mkdir -p ${CMAKE_BINARY_DIR}/scripts - COMMAND cp -r ${CMAKE_SOURCE_DIR}/scripts/* ${CMAKE_BINARY_DIR}/scripts/ - COMMAND sed -i "s/vendor_name=customize/vendor_name=${vendor_name}/g" ${CMAKE_BINARY_DIR}/scripts/* -) -add_custom_target(modify_vendor ALL DEPENDS ${CMAKE_BINARY_DIR}/scripts/install.sh ${CMAKE_BINARY_DIR}/scripts/upgrade.sh) -install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/ DESTINATION . FILE_PERMISSIONS OWNER_EXECUTE OWNER_READ GROUP_READ) - -install(FILES ${CMAKE_SOURCE_DIR}/custom.proto DESTINATION packages OPTIONAL) - -get_system_info(SYSTEM_INFO) - -# gen version.info -add_custom_target(gen_version_info ALL - COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/cmake/util/gen_version_info.sh ${ASCEND_CANN_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR} -) - -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/version.info - DESTINATION packages/vendors/${vendor_name}/) - -# CPack config -set(CPACK_PACKAGE_NAME ${CMAKE_PROJECT_NAME}) -set(CPACK_PACKAGE_VERSION ${CMAKE_PROJECT_VERSION}) -set(CPACK_PACKAGE_DESCRIPTION "CPack opp project") -set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "CPack opp project") -set(CPACK_PACKAGE_DIRECTORY ${CMAKE_INSTALL_PREFIX}) -set(CPACK_PACKAGE_FILE_NAME "custom_opp_${SYSTEM_INFO}.run") -set(CPACK_GENERATOR External) -set(CPACK_CMAKE_GENERATOR "Unix Makefiles") -set(CPACK_EXTERNAL_ENABLE_STAGING TRUE) -set(CPACK_EXTERNAL_PACKAGE_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/makeself.cmake) -set(CPACK_EXTERNAL_BUILT_PACKAGES ${CPACK_PACKAGE_DIRECTORY}/_CPack_Packages/Linux/External/${CPACK_PACKAGE_FILE_NAME}/${CPACK_PACKAGE_FILE_NAME}) -include(CPack) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json deleted file mode 100644 index 8b7181000..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/CMakePresets.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "version": 1, - "cmakeMinimumRequired": { - "major": 3, - "minor": 19, - "patch": 0 - }, - "configurePresets": [ - { - "name": "default", - "displayName": "Default Config", - "description": "Default build using Unix Makefiles generator", - "generator": "Unix Makefiles", - "binaryDir": "${sourceDir}/build_out", - "cacheVariables": { - "CMAKE_BUILD_TYPE": { - "type": "STRING", - "value": "Release" - }, - "ENABLE_SOURCE_PACKAGE": { - "type": "BOOL", - "value": "True" - }, - "ENABLE_BINARY_PACKAGE": { - "type": "BOOL", - "value": "True" - }, - "ASCEND_COMPUTE_UNIT": { - "type": "STRING", - "value": "ascend310b" - }, - "ENABLE_TEST": { - "type": "BOOL", - "value": "True" - }, - "vendor_name": { - "type": "STRING", - "value": "customize" - }, - "ASCEND_CANN_PACKAGE_PATH": { - "type": "PATH", - "value": "/usr/local/Ascend/ascend-toolkit/latest" - }, - "ASCEND_PYTHON_EXECUTABLE": { - "type": "STRING", - "value": "python3" - }, - "CMAKE_INSTALL_PREFIX": { - "type": "PATH", - "value": "${sourceDir}/build_out" - }, - "ENABLE_CROSS_COMPILE": { - "type": "BOOL", - "value": "False" - }, - "CMAKE_CROSS_PLATFORM_COMPILER": { - "type": "PATH", - "value": "/usr/bin/aarch64-linux-gnu-g++" - } - } - } - ] -} \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json deleted file mode 100644 index f35459443..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/ThreeNN.json +++ /dev/null @@ -1,50 +0,0 @@ -[ - { - "op": "ThreeNN", - "language": "cpp", - "input_desc": [ - { - "name": "xyz1", - "param_type": "required", - "format": [ - "ND" - ], - "type": [ - "float" - ] - }, - { - "name": "xyz2", - "param_type": "required", - "format": [ - "ND" - ], - "type": [ - "float" - ] - } - ], - "output_desc": [ - { - "name": "dist", - "param_type": "required", - "format": [ - "ND" - ], - "type": [ - "float" - ] - }, - { - "name": "indices", - "param_type": "required", - "format": [ - "ND" - ], - "type": [ - "int32" - ] - } - ] - } -] \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh deleted file mode 100644 index 27d1968d1..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/build.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -script_path=$(realpath $(dirname $0)) - - -mkdir -p build_out -rm -rf build_out/* -cd build_out - -opts=$(python3 $script_path/cmake/util/preset_parse.py $script_path/CMakePresets.json) -ENABLE_CROSS="-DENABLE_CROSS_COMPILE=True" -ENABLE_BINARY="-DENABLE_BINARY_PACKAGE=True" -cmake_version=$(cmake --version | grep "cmake version" | awk '{print $3}') - -cmake_run_package() -{ - target=$1 - cmake --build . --target $target -j16 - if [ $? -ne 0 ]; then exit 1; fi - - if [ $target = "package" ]; then - if test -d ./op_kernel/binary ; then - ./cust*.run - if [ $? -ne 0 ]; then exit 1; fi - cmake --build . --target binary -j16 - if [ $? -ne 0 ]; then - echo "[ERROR] Kernel compile failed, the run package will not be generated." - rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; - fi - cmake --build . --target $target -j16 - fi - fi -} - -if [[ $opts =~ $ENABLE_CROSS ]] && [[ $opts =~ $ENABLE_BINARY ]] -then - target=package - if [ "$1"x != ""x ]; then target=$1; fi - if [ "$cmake_version" \< "3.19.0" ] ; then - cmake .. $opts -DENABLE_CROSS_COMPILE=0 - else - cmake .. --preset=default -DENABLE_CROSS_COMPILE=0 - fi - cmake_run_package $target - cp -r kernel ../ - rm -rf * - if [ "$cmake_version" \< "3.19.0" ] ; then - cmake .. $opts - else - cmake .. --preset=default - fi - - cmake --build . --target $target -j16 - if [ $? -ne 0 ]; then - echo "[ERROR] Kernel compile failed, the run package will not be generated." - rm -rf ./cust*.run && rm -rf ./cust*.run.json && exit 1; - fi - if [ $target = "package" ]; then - if test -d ./op_kernel/binary ; then - ./cust*.run - fi - fi - rm -rf ../kernel - -else - target=package - if [ "$1"x != ""x ]; then target=$1; fi - if [ "$cmake_version" \< "3.19.0" ] ; then - cmake .. $opts - else - cmake .. --preset=default - fi - cmake_run_package $target -fi - - -# for debug -# cd build_out -# make -# cpack -# verbose append -v \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake deleted file mode 100644 index 886119daa..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/config.cmake +++ /dev/null @@ -1,25 +0,0 @@ - -set(CMAKE_CXX_FLAGS_DEBUG "") -set(CMAKE_CXX_FLAGS_RELEASE "") - -if (NOT DEFINED vendor_name) - set(vendor_name customize CACHE STRING "") -endif() -if (NOT DEFINED ASCEND_CANN_PACKAGE_PATH) - set(ASCEND_CANN_PACKAGE_PATH /usr/local/Ascend/latest CACHE PATH "") -endif() -if (NOT DEFINED ASCEND_PYTHON_EXECUTABLE) - set(ASCEND_PYTHON_EXECUTABLE python3 CACHE STRING "") -endif() -if (NOT DEFINED ASCEND_COMPUTE_UNIT) - message(FATAL_ERROR "ASCEND_COMPUTE_UNIT not set in CMakePreset.json ! -") -endif() -set(ASCEND_TENSOR_COMPILER_PATH ${ASCEND_CANN_PACKAGE_PATH}/compiler) -set(ASCEND_CCEC_COMPILER_PATH ${ASCEND_TENSOR_COMPILER_PATH}/ccec_compiler/bin) -set(ASCEND_AUTOGEN_PATH ${CMAKE_BINARY_DIR}/autogen) -set(ASCEND_FRAMEWORK_TYPE tensorflow) -file(MAKE_DIRECTORY ${ASCEND_AUTOGEN_PATH}) -set(CUSTOM_COMPILE_OPTIONS "custom_compile_options.ini") -execute_process(COMMAND rm -rf ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} - COMMAND touch ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS}) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake deleted file mode 100644 index 4179dfd25..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/func.cmake +++ /dev/null @@ -1,192 +0,0 @@ - -function(get_system_info SYSTEM_INFO) - if (UNIX) - execute_process(COMMAND grep -i ^id= /etc/os-release OUTPUT_VARIABLE TEMP) - string(REGEX REPLACE "\n|id=|ID=|\"" "" SYSTEM_NAME ${TEMP}) - set(${SYSTEM_INFO} ${SYSTEM_NAME}_${CMAKE_SYSTEM_PROCESSOR} PARENT_SCOPE) - elseif (WIN32) - message(STATUS "System is Windows. Only for pre-build.") - else () - message(FATAL_ERROR "${CMAKE_SYSTEM_NAME} not support.") - endif () -endfunction() - -function(opbuild) - message(STATUS "Opbuild generating sources") - cmake_parse_arguments(OPBUILD "" "OUT_DIR;PROJECT_NAME;ACCESS_PREFIX" "OPS_SRC" ${ARGN}) - execute_process(COMMAND ${CMAKE_COMPILE} -g -fPIC -shared -std=c++11 ${OPBUILD_OPS_SRC} -D_GLIBCXX_USE_CXX11_ABI=0 - -I ${ASCEND_CANN_PACKAGE_PATH}/include -L ${ASCEND_CANN_PACKAGE_PATH}/lib64 -lexe_graph -lregister -ltiling_api - -o ${OPBUILD_OUT_DIR}/libascend_all_ops.so - RESULT_VARIABLE EXEC_RESULT - OUTPUT_VARIABLE EXEC_INFO - ERROR_VARIABLE EXEC_ERROR - ) - if (${EXEC_RESULT}) - message("build ops lib info: ${EXEC_INFO}") - message("build ops lib error: ${EXEC_ERROR}") - message(FATAL_ERROR "opbuild run failed!") - endif() - set(proj_env "") - set(prefix_env "") - if (NOT "${OPBUILD_PROJECT_NAME}x" STREQUAL "x") - set(proj_env "OPS_PROJECT_NAME=${OPBUILD_PROJECT_NAME}") - endif() - if (NOT "${OPBUILD_ACCESS_PREFIX}x" STREQUAL "x") - set(prefix_env "OPS_DIRECT_ACCESS_PREFIX=${OPBUILD_ACCESS_PREFIX}") - endif() - execute_process(COMMAND ${proj_env} ${prefix_env} ${ASCEND_CANN_PACKAGE_PATH}/toolkit/tools/opbuild/op_build - ${OPBUILD_OUT_DIR}/libascend_all_ops.so ${OPBUILD_OUT_DIR} - RESULT_VARIABLE EXEC_RESULT - OUTPUT_VARIABLE EXEC_INFO - ERROR_VARIABLE EXEC_ERROR - ) - if (${EXEC_RESULT}) - message("opbuild ops info: ${EXEC_INFO}") - message("opbuild ops error: ${EXEC_ERROR}") - endif() - message(STATUS "Opbuild generating sources - done") -endfunction() - -function(add_ops_info_target) - cmake_parse_arguments(OPINFO "" "TARGET;OPS_INFO;OUTPUT;INSTALL_DIR" "" ${ARGN}) - get_filename_component(opinfo_file_path "${OPINFO_OUTPUT}" DIRECTORY) - add_custom_command(OUTPUT ${OPINFO_OUTPUT} - COMMAND mkdir -p ${opinfo_file_path} - COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/parse_ini_to_json.py - ${OPINFO_OPS_INFO} ${OPINFO_OUTPUT} - ) - add_custom_target(${OPINFO_TARGET} ALL - DEPENDS ${OPINFO_OUTPUT} - ) - install(FILES ${OPINFO_OUTPUT} - DESTINATION ${OPINFO_INSTALL_DIR} - ) -endfunction() - -function(add_ops_compile_options OP_TYPE) - cmake_parse_arguments(OP_COMPILE "" "OP_TYPE" "COMPUTE_UNIT;OPTIONS" ${ARGN}) - file(APPEND ${ASCEND_AUTOGEN_PATH}/${CUSTOM_COMPILE_OPTIONS} - "${OP_TYPE},${OP_COMPILE_COMPUTE_UNIT},${OP_COMPILE_OPTIONS}\n") -endfunction() - -function(add_ops_impl_target) - cmake_parse_arguments(OPIMPL "" "TARGET;OPS_INFO;IMPL_DIR;OUT_DIR;INSTALL_DIR" "OPS_BATCH;OPS_ITERATE" ${ARGN}) - add_custom_command(OUTPUT ${OPIMPL_OUT_DIR}/.impl_timestamp - COMMAND mkdir -m 700 -p ${OPIMPL_OUT_DIR}/dynamic - COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py - ${OPIMPL_OPS_INFO} - \"${OPIMPL_OPS_BATCH}\" \"${OPIMPL_OPS_ITERATE}\" - ${OPIMPL_IMPL_DIR} - ${OPIMPL_OUT_DIR}/dynamic - ${ASCEND_AUTOGEN_PATH} - - COMMAND rm -rf ${OPIMPL_OUT_DIR}/.impl_timestamp - COMMAND touch ${OPIMPL_OUT_DIR}/.impl_timestamp - DEPENDS ${OPIMPL_OPS_INFO} - ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_impl_build.py - ) - add_custom_target(${OPIMPL_TARGET} ALL - DEPENDS ${OPIMPL_OUT_DIR}/.impl_timestamp) - if (${ENABLE_SOURCE_PACKAGE}) - install(DIRECTORY ${OPIMPL_OUT_DIR}/dynamic - DESTINATION ${OPIMPL_INSTALL_DIR} - ) - endif() -endfunction() - -function(add_npu_support_target) - cmake_parse_arguments(NPUSUP "" "TARGET;OPS_INFO_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) - get_filename_component(npu_sup_file_path "${NPUSUP_OUT_DIR}" DIRECTORY) - add_custom_command(OUTPUT ${NPUSUP_OUT_DIR}/npu_supported_ops.json - COMMAND mkdir -p ${NPUSUP_OUT_DIR} - COMMAND ${CMAKE_SOURCE_DIR}/cmake/util/gen_ops_filter.sh - ${NPUSUP_OPS_INFO_DIR} - ${NPUSUP_OUT_DIR} - ) - add_custom_target(npu_supported_ops ALL - DEPENDS ${NPUSUP_OUT_DIR}/npu_supported_ops.json - ) - install(FILES ${NPUSUP_OUT_DIR}/npu_supported_ops.json - DESTINATION ${NPUSUP_INSTALL_DIR} - ) -endfunction() - -function(add_bin_compile_target) - cmake_parse_arguments(BINCMP "" "TARGET;OPS_INFO;COMPUTE_UNIT;IMPL_DIR;ADP_DIR;OUT_DIR;INSTALL_DIR" "" ${ARGN}) - file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/src) - file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin) - file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/gen) - execute_process(COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_bin_param_build.py - ${BINCMP_OPS_INFO} ${BINCMP_OUT_DIR}/gen ${BINCMP_COMPUTE_UNIT} - RESULT_VARIABLE EXEC_RESULT - OUTPUT_VARIABLE EXEC_INFO - ERROR_VARIABLE EXEC_ERROR - ) - if (${EXEC_RESULT}) - message("ops binary compile scripts gen info: ${EXEC_INFO}") - message("ops binary compile scripts gen error: ${EXEC_ERROR}") - message(FATAL_ERROR "ops binary compile scripts gen failed!") - endif() - if (NOT TARGET binary) - add_custom_target(binary) - endif() - add_custom_target(${BINCMP_TARGET} - COMMAND cp -r ${BINCMP_IMPL_DIR}/*.* ${BINCMP_OUT_DIR}/src - ) - add_custom_target(${BINCMP_TARGET}_gen_ops_config - COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/insert_simplified_keys.py -p ${BINCMP_OUT_DIR}/bin - COMMAND ${ASCEND_PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/cmake/util/ascendc_ops_config.py -p ${BINCMP_OUT_DIR}/bin - -s ${BINCMP_COMPUTE_UNIT} - ) - add_dependencies(binary ${BINCMP_TARGET}_gen_ops_config) - file(GLOB bin_scripts ${BINCMP_OUT_DIR}/gen/*.sh) - foreach(bin_script ${bin_scripts}) - get_filename_component(bin_file ${bin_script} NAME_WE) - string(REPLACE "-" ";" bin_sep ${bin_file}) - list(GET bin_sep 0 op_type) - list(GET bin_sep 1 op_file) - list(GET bin_sep 2 op_index) - if (NOT TARGET ${BINCMP_TARGET}_${op_file}_copy) - file(MAKE_DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file}) - add_custom_target(${BINCMP_TARGET}_${op_file}_copy - COMMAND cp ${BINCMP_ADP_DIR}/${op_file}.py ${BINCMP_OUT_DIR}/src/${op_type}.py - ) - install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} - DESTINATION ${BINCMP_INSTALL_DIR}/${BINCMP_COMPUTE_UNIT} OPTIONAL - ) - install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json - DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT}/ OPTIONAL - ) - endif() - add_custom_target(${BINCMP_TARGET}_${op_file}_${op_index} - COMMAND export HI_PYTHON=${ASCEND_PYTHON_EXECUTABLE} && bash ${bin_script} ${BINCMP_OUT_DIR}/src/${op_type}.py ${BINCMP_OUT_DIR}/bin/${op_file} && echo $(MAKE) - WORKING_DIRECTORY ${BINCMP_OUT_DIR} - ) - add_dependencies(${BINCMP_TARGET}_${op_file}_${op_index} ${BINCMP_TARGET} ${BINCMP_TARGET}_${op_file}_copy) - add_dependencies(${BINCMP_TARGET}_gen_ops_config ${BINCMP_TARGET}_${op_file}_${op_index}) - endforeach() - install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json - DESTINATION ${BINCMP_INSTALL_DIR}/config/${BINCMP_COMPUTE_UNIT} OPTIONAL - ) - - install(DIRECTORY ${BINCMP_OUT_DIR}/bin/${op_file} - DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/${BINCMP_COMPUTE_UNIT} OPTIONAL - ) - install(FILES ${BINCMP_OUT_DIR}/bin/binary_info_config.json - DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL - ) - install(FILES ${BINCMP_OUT_DIR}/bin/${op_file}.json - DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/../build_out/kernel/config/${BINCMP_COMPUTE_UNIT} OPTIONAL - ) - -endfunction() - -function(add_cross_compile_target) - cmake_parse_arguments(CROSSMP "" "TARGET;OUT_DIR;INSTALL_DIR" "" ${ARGN}) - add_custom_target(${CROSSMP_TARGET} ALL - DEPENDS ${CROSSMP_OUT_DIR} - ) - install(DIRECTORY ${CROSSMP_OUT_DIR} - DESTINATION ${CROSSMP_INSTALL_DIR} - ) -endfunction() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake deleted file mode 100644 index 2f362c396..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/intf.cmake +++ /dev/null @@ -1,26 +0,0 @@ - -add_library(intf_pub INTERFACE) -target_compile_options(intf_pub INTERFACE - -fPIC - -fvisibility=hidden - -fvisibility-inlines-hidden - $<$:-O2> - $<$:-O0 -g> - $<$:-std=c++11> - $<$,$>:-ftrapv -fstack-check> - $<$:-pthread -Wfloat-equal -Wshadow -Wformat=2 -Wno-deprecated -Wextra> - $,-fstack-protector-strong,-fstack-protector-all> -) -target_compile_definitions(intf_pub INTERFACE - _GLIBCXX_USE_CXX11_ABI=0 - $<$:_FORTIFY_SOURCE=2> -) -target_include_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/include) -target_link_options(intf_pub INTERFACE - $<$,EXECUTABLE>:-pie> - $<$:-s> - -Wl,-z,relro - -Wl,-z,now - -Wl,-z,noexecstack -) -target_link_directories(intf_pub INTERFACE ${ASCEND_CANN_PACKAGE_PATH}/lib64) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake deleted file mode 100644 index 48c565bfb..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/makeself.cmake +++ /dev/null @@ -1,17 +0,0 @@ -execute_process(COMMAND chmod +x ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh) -execute_process(COMMAND ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself.sh - --header ${CMAKE_CURRENT_LIST_DIR}/util/makeself/makeself-header.sh - --help-header ./help.info - --gzip --complevel 4 --nomd5 --sha256 - ./ ${CPACK_PACKAGE_FILE_NAME} "version:1.0" ./install.sh - WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} - RESULT_VARIABLE EXEC_RESULT - ERROR_VARIABLE EXEC_ERROR -) -if (NOT "${EXEC_RESULT}x" STREQUAL "0x") - message(FATAL_ERROR "CPack Command error: ${EXEC_RESULT}\n${EXEC_ERROR}") -endif() -execute_process(COMMAND cp ${CPACK_EXTERNAL_BUILT_PACKAGES} ${CPACK_PACKAGE_DIRECTORY}/ - COMMAND echo "Copy ${CPACK_EXTERNAL_BUILT_PACKAGES} to ${CPACK_PACKAGE_DIRECTORY}/" - WORKING_DIRECTORY ${CPACK_TEMPORARY_DIRECTORY} -) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp deleted file mode 100644 index 113e0d5cd..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/three_nn.cpp +++ /dev/null @@ -1,127 +0,0 @@ -#include "kernel_operator.h" -using namespace AscendC; -constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue - -class KernelFastGeluGrad { -public: - __aicore__ inline KernelFastGeluGrad() {} - __aicore__ inline void Init(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, - uint32_t CoreDataNum, uint32_t finalTileNum, uint32_t tileDataNum, uint32_t TailDataNum, - uint32_t xyz2DataNum) { - ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); - - this->coreDataNum = CoreDataNum; - this->tileNum = finalTileNum; - this->tileDataNum = tileDataNum; - this->tailDataNum = TailDataNum; - this->xyz2DataNum = xyz2DataNum; - - xyz1Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ1*)xyz1, this->coreDataNum); - xyz2Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ2*)xyz2, this->xyz2DataNum); - distGm.SetGlobalBuffer((__gm__ DTYPE_DIST*)dist, this->coreDataNum); - indicesGm.SetGlobalBuffer((__gm__ DTYPE_INDICES*)indices, this->coreDataNum); - - // pipe.InitBuffer(inQueueXYZ1, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_XYZ1)); - // pipe.InitBuffer(inQueueXYZ2, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_XYZ2)); - // pipe.InitBuffer(outQueueDIST, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_DIST)); - // pipe.InitBuffer(outQueueINDICES, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_INDICES)); - - } - __aicore__ inline void Process() { - float ux,uy,uz; - float x,y,z; - float best1, best2, best3; - int besti1, besti2, besti3; - for(uint32_t i=0; icoreDataNum; i+=3) - { - ux = xyz1Gm.GetValue(i); - uy = xyz1Gm.GetValue(i+1); - uz = xyz1Gm.GetValue(i+2); - for(j=0; jxyz2DataNum; j+=3) - { - x = xyz2Gm.GetValue(j); - y = xyz2Gm.GetValue(j+1); - z = xyz2Gm.GetValue(j+2); - } - } - // SetValue(count*M*N + i*N+j, xGm.GetValue(count*M*N + i*N+j)); - } -// __aicore__ inline void Process() { - -// int32_t loopCount = this->tileNum; -// this->processDataNum = this->tileDataNum; -// for (int32_t i = 0; i < loopCount; i++) { -// if (i == this->tileNum - 1) { -// this->processDataNum = this->tailDataNum; -// } -// CopyIn(i); -// Compute(i); -// CopyOut(i); -// } -// } - -// private: -// __aicore__ inline void CopyIn(int32_t progress) -// { -// LocalTensor xyz1Local = inQueueXYZ1.AllocTensor(); -// LocalTensor xyz2Local = inQueueXYZ2.AllocTensor(); - -// DataCopy(xyz1Local, xyz1Gm[progress * this->tileDataNum], this->processDataNum); -// DataCopy(xyz2Local, xyz2Gm[progress * this->tileDataNum], this->processDataNum); - -// inQueueXYZ1.EnQue(xyz1Local); -// inQueueXYZ2.EnQue(xyz2Local); -// } -// __aicore__ inline void Compute(int32_t progress) -// { -// LocalTensor xyz1Local = inQueueXYZ1.DeQue(); -// LocalTensor xyz2Local = inQueueXYZ2.DeQue(); -// LocalTensor distLocal = outQueueDIST.AllocTensor(); -// LocalTensor indicesLocal = outQueueINDICES.AllocTensor(); - - -// outQueueDIST.EnQue(distLocal); -// outQueueINDICES.EnQue(indicesLocal); -// inQueueXYZ1.FreeTensor(xyz2Local); -// inQueueXYZ2.FreeTensor(xyz2Local); -// } -// __aicore__ inline void CopyOut(int32_t progress) -// { -// LocalTensor distLocal = outQueueDIST.DeQue(); -// LocalTensor indicesLocal = outQueueINDICES.DeQue(); -// DataCopy(distGm[progress * this->tileDataNum], distLocal, this->processDataNum); -// DataCopy(indicesGm[progress * this->tileDataNum], indicesLocal, this->processDataNum); -// outQueueDIST.FreeTensor(distLocal); -// outQueueINDICES.FreeTensor(indicesLocal); -// } - -private: - TPipe pipe; - TBuf tmpBuffer, signbitBuffer; - TQue inQueueXYZ1, inQueueXYZ2; - TQue outQueueDIST, outQueueINDICES; - GlobalTensor xyz1Gm; - GlobalTensor xyz2Gm; - GlobalTensor distGm; - GlobalTensor indicesGm; - uint32_t coreDataNum; - uint32_t tileNum; - uint32_t tileDataNum; - uint32_t tailDataNum; - uint32_t processDataNum; - uint32_t xyz2DataNum; - // LocalTensor signbit; -}; - -extern "C" __global__ __aicore__ void three_nn(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling) { - GET_TILING_DATA(tiling_data, tiling); - // TODO: user kernel impl - KernelFastGeluGrad op; - op.Init(xyz1, xyz2, dist, indices, - tiling_data.CoreDataNum, tiling_data.finalTileNum, tiling_data.tileDataNum, tiling_data.TailDataNum, tiling_data.xyz2DataNum); - op.Process(); -} -// extern "C" __global__ __aicore__ void three_nn(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling) { -// GET_TILING_DATA(tiling_data, tiling); -// // TODO: user kernel impl -// } \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py deleted file mode 100644 index c4ddc893a..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import sys -import os - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(PYF_PATH) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py deleted file mode 100644 index 63b15c900..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_bin_param_build.py +++ /dev/null @@ -1,127 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import sys -import os -import json -import hashlib -import const_var -import opdesc_parser - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - - -class BinParamBuilder(opdesc_parser.OpDesc): - def __init__(self: any, op_type: str): - super().__init__(op_type) - self.soc = '' - self.out_path = '' - - def set_soc_version(self: any, soc: str): - self.soc = soc - - def set_out_path(self: any, out_path: str): - self.out_path = out_path - - def gen_input_json(self: any): - key_map = {} - count = len(self.input_dtype[0].split(',')) - for i in range(0, count): - inputs = [] - outputs = [] - attrs = [] - op_node = {} - for idx in range(0, len(self.input_name)): - idtypes = self.input_dtype[idx].split(',') - ifmts = self.input_fmt[idx].split(',') - itype = self.input_type[idx] - para = {} - para['name'] = self.input_name[idx][:-5] - para['index'] = idx - para['dtype'] = idtypes[i] - para['format'] = ifmts[i] - para['paramType'] = itype - para['shape'] = [-2] - if itype == 'dynamic': - inputs.append([para]) - else: - inputs.append(para) - for idx in range(0, len(self.output_name)): - odtypes = self.output_dtype[idx].split(',') - ofmts = self.output_fmt[idx].split(',') - otype = self.output_type[idx] - para = {} - para['name'] = self.output_name[idx][:-5] - para['index'] = idx - para['dtype'] = odtypes[i] - para['format'] = ofmts[i] - para['paramType'] = otype - para['shape'] = [-2] - if otype == 'dynamic': - outputs.append([para]) - else: - outputs.append(para) - for attr in self.attr_list: - att = {} - att['name'] = attr - atype = self.attr_val.get(attr).get('type').lower() - att['dtype'] = atype - att['value'] = const_var.ATTR_DEF_VAL.get(atype) - attrs.append(att) - op_node['bin_filename'] = '' - op_node['inputs'] = inputs - op_node['outputs'] = outputs - if len(attrs) > 0: - op_node['attrs'] = attrs - param = {} - param['op_type'] = self.op_type - param['op_list'] = [op_node] - objstr = json.dumps(param, indent=' ') - md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() - while key_map.get(md5sum) is not None: - objstr += '1' - md5sum = hashlib.md5(objstr.encode('utf-8')).hexdigest() - key_map[md5sum] = md5sum - bin_file = self.op_type + '_' + md5sum - op_node['bin_filename'] = bin_file - param_file = os.path.join(self.out_path, bin_file + '_param.json') - param_file = os.path.realpath(param_file) - with os.fdopen(os.open(param_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: - json.dump(param, fd, indent=' ') - self._write_buld_cmd(param_file, bin_file, i) - - def _write_buld_cmd(self: any, param_file: str, bin_file: str, index: int): - hard_soc = const_var.SOC_MAP_EXT.get(self.soc) - if not hard_soc: - hard_soc = self.soc.capitalize() - name_com = [self.op_type, self.op_file, str(index)] - compile_file = os.path.join(self.out_path, '-'.join(name_com) + '.sh') - compile_file = os.path.realpath(compile_file) - with os.fdopen(os.open(compile_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: - fd.write('#!/bin/bash\n') - fd.write('echo "[{}] Generating {} ..."\n'.format(hard_soc, bin_file)) - cmd = const_var.BIN_CMD.format(fun=self.op_intf, soc=hard_soc, param=param_file, impl='""') - fd.write(cmd) - chk = const_var.CHK_CMD.format(res_file=bin_file + '.json') - fd.write(chk) - chk = const_var.CHK_CMD.format(res_file=bin_file + '.o') - fd.write(chk) - fd.write('echo "[{}] Generating {} Done"\n'.format(hard_soc, bin_file)) - - -def gen_bin_param_file(cfgfile: str, out_dir: str, soc: str): - op_descs = opdesc_parser.get_op_desc(cfgfile, [], [], BinParamBuilder, None) - for op_desc in op_descs: - op_desc.set_soc_version(soc) - op_desc.set_out_path(out_dir) - op_desc.gen_input_json() - - -if __name__ == '__main__': - if len(sys.argv) <= 3: - raise RuntimeError('arguments must greater than 3') - gen_bin_param_file(sys.argv[1], sys.argv[2], sys.argv[3]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py deleted file mode 100644 index 647801c07..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_impl_build.py +++ /dev/null @@ -1,501 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import sys -import os -import re -import opdesc_parser -import const_var - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - -IMPL_HEAD = ''' -import os, sys -import ctypes -import json -import shutil -from tbe.common.platform import get_soc_spec -from tbe.common.utils import para_check -from tbe.tikcpp import compile_op, replay_op, check_op_cap, generalize_op_params, get_code_channel, OpInfo -from tbe.common.buildcfg import get_default_build_config -from impl.util.platform_adapter import tbe_register -from tbe.common.buildcfg import get_current_build_config -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - -DTYPE_MAP = {{"float32": ["DT_FLOAT", "float"], - "float16": ["DT_FLOAT16", "half"], - "int8": ["DT_INT8", "int8_t"], - "int16": ["DT_INT16", "int16_t"], - "int32": ["DT_INT32", "int32_t"], - "int64": ["DT_INT64", "int64_t"], - "uint1": ["DT_UINT1", "uint8_t"], - "uint8": ["DT_UINT8", "uint8_t"], - "uint16": ["DT_UINT16", "uint16_t"], - "uint32": ["DT_UINT32", "uint32_t"], - "uint64": ["DT_UINT64", "uint64_t"], - "bool": ["DT_BOOL", "bool"], - "double": ["DT_DOUBLE", "double"], - "dual": ["DT_DUAL", "unknown"], - "dual_sub_int8": ["DT_DUAL_SUB_INT8", "unknown"], - "dual_sub_uint8": ["DT_DUAL_SUB_UINT8", "unknown"], - "string": ["DT_STRING", "unknown"], - "complex64": ["DT_COMPLEX64", "unknown"], - "complex128": ["DT_COMPLEX128", "unknown"], - "qint8": ["DT_QINT8", "unknown"], - "qint16": ["DT_QINT16", "unknown"], - "qint32": ["DT_QINT32", "unknown"], - "quint8": ["DT_QUINT8", "unknown"], - "quint16": ["DT_QUINT16", "unknown"], - "resource": ["DT_RESOURCE", "unknown"], - "string_ref": ["DT_STRING_REF", "unknown"], - "int4": ["DT_INT4", "int4b_t"], - "bfloat16": ["DT_BF16", "bfloat16_t"]}} - -def add_dtype_fmt_option_single(x, x_n, is_ref: bool = False): - options = [] - x_fmt = x.get("format") - x_dtype = x.get("dtype") - x_n_in_kernel = x_n + '_REF' if is_ref else x_n - options.append("-DDTYPE_{{n}}={{t}}".format(n=x_n_in_kernel, t=DTYPE_MAP.get(x_dtype)[1])) - options.append("-DORIG_DTYPE_{{n}}={{ot}}".format(n=x_n_in_kernel, ot=DTYPE_MAP.get(x_dtype)[0])) - options.append("-DFORMAT_{{n}}=FORMAT_{{f}}".format(n=x_n_in_kernel, f=x_fmt)) - return options - -def get_dtype_fmt_options(__inputs__, __outputs__): - options = [] - input_names = {} - output_names = {} - unique_param_name_set = set() - for idx, x in enumerate(__inputs__): - if x is None: - continue - x_n = input_names[idx].upper() - unique_param_name_set.add(x_n) - options += add_dtype_fmt_option_single(x, x_n) - - for idx, x in enumerate(__outputs__): - if x is None: - continue - x_n = output_names[idx].upper() - if x_n in unique_param_name_set: - options += add_dtype_fmt_option_single(x, x_n, True) - else: - options += add_dtype_fmt_option_single(x, x_n) - return options - -def load_dso(so_path): - try: - ctypes.CDLL(so_path) - except OSError as error : - print(error) - raise RuntimeError("cannot open %s" %(so_path)) - else: - print("load so succ ", so_path) - -def get_shortsoc_compile_option(compile_option_list: list, shortsoc:str): - compile_options = [] - if shortsoc in compile_option_list: - compile_options = compile_option_list[shortsoc] - elif '__ALLSOC__' in compile_option_list: - compile_options = compile_option_list['__ALLSOC__'] - return compile_options - -def get_kernel_source(src_file, dir_snake, dir_ex): - src_ex = os.path.join(PYF_PATH, "..", "ascendc", dir_ex, src_file) - if os.path.exists(src_ex): - return src_ex - src = os.path.join(PYF_PATH, "..", "ascendc", dir_snake, src_file) - if os.path.exists(src): - return src - src = os.path.join(PYF_PATH, src_file) - if os.path.exists(src): - return src - return src_ex - -''' - -IMPL_API = ''' -@tbe_register.register_operator("{}", trans_bool_to_s8=False) -@para_check.check_op_params({}) -def {}({}, kernel_name="{}", impl_mode=""): - if get_current_build_config("enable_op_prebuild"): - return - __inputs__, __outputs__, __attrs__ = _build_args({}) - options = get_dtype_fmt_options(__inputs__, __outputs__) - options += ["-x", "cce"] - bisheng = shutil.which("bisheng") - if bisheng != None: - bisheng_path = os.path.dirname(bisheng) - tikcpp_path = os.path.realpath(os.path.join(bisheng_path, "..", "..", "tikcpp")) - else: - tikcpp_path = os.path.realpath("/usr/local/Ascend/latest/compiler/tikcpp") - options.append("-I" + tikcpp_path) - options.append("-I" + os.path.join(tikcpp_path, "tikcfw")) - options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "impl")) - options.append("-I" + os.path.join(tikcpp_path, "tikcfw", "interface")) - options.append("-I" + os.path.join(PYF_PATH, "..", "ascendc", "common")) - if impl_mode == "high_performance": - options.append("-DHIGH_PERFORMANCE=1") - elif impl_mode == "high_precision": - options.append("-DHIGH_PRECISION=1") - if get_default_build_config("enable_deterministic_mode") == 1: - options.append("-DDETEMINISTIC_MODE=1") - - custom_compile_options = {}, - custom_all_compile_options = {}, - soc_version = get_soc_spec("SOC_VERSION") - soc_short = get_soc_spec("SHORT_SOC_VERSION").lower() - custom_compile_options_soc = get_shortsoc_compile_option(custom_compile_options[0], soc_short) - custom_all_compile_options_soc = get_shortsoc_compile_option(custom_all_compile_options[0], soc_short) - options += custom_all_compile_options_soc - options += custom_compile_options_soc - - origin_func_name = "{}" - ascendc_src_dir_ex = "{}" - ascendc_src_dir = "{}" - ascendc_src_file = "{}" - src = get_kernel_source(ascendc_src_file, ascendc_src_dir, ascendc_src_dir_ex) -''' - -REPLAY_OP_API = ''' - print("start replay Acend C Operator {}, kernel name is {}") - tikreplay_codegen_path = tikcpp_path + "/tikreplaylib/lib" - tikreplay_stub_path = tikcpp_path + "/tikreplaylib/lib/" + soc_version - print("start load libtikreplaylib_codegen.so and libtikreplaylib_stub.so") - codegen_so_path = tikreplay_codegen_path + "/libtikreplaylib_codegen.so" - replaystub_so_path = tikreplay_stub_path + "/libtikreplaylib_stub.so" - if PYF_PATH.endswith("dynamic"): - op_replay_path = os.path.join(PYF_PATH, "..", "..", "op_replay") - else: - op_replay_path = os.path.join(PYF_PATH, "..", "op_replay") - replayapi_so_path = os.path.join(op_replay_path, "libreplay_{}_" + soc_short + ".so") - load_dso(codegen_so_path) - load_dso(replaystub_so_path) - load_dso(replayapi_so_path) - op_type = "{}" - entry_obj = os.path.join(op_replay_path, "{}_entry_" + soc_short + ".o") - code_channel = get_code_channel(src, kernel_name, op_type, options) - op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ - attrs = __attrs__, impl_mode = impl_mode) - res, msg = replay_op(op_info, entry_obj, code_channel, src, options) - if not res: - print("call replay op failed for %s and get into call compile op" %(msg)) - compile_op(src, origin_func_name, op_info, options, code_channel, '{}') -''' - -COMPILE_OP_API = ''' - print("start compile Ascend C operator {}. kernel name is " + kernel_name) - op_type = "{}" - code_channel = get_code_channel(src, kernel_name, op_type, options) - op_info = OpInfo(kernel_name = kernel_name, op_type = op_type, inputs = __inputs__, outputs = __outputs__,\\ - attrs = __attrs__, impl_mode = impl_mode, origin_inputs=[{}], origin_outputs = [{}], param_type_dynamic = {},\\ - mc2_ctx = None, param_type_list = {}) - compile_op(src, origin_func_name, op_info, options, code_channel, '{}') -''' - -SUP_API = ''' -def {}({}, impl_mode=""): - __inputs__, __outputs__, __attrs__ = _build_args({}) - ret_str = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) - ret_dict = json.loads(ret_str) - err_code = ret_dict.get("ret_code") - sup = "Unknown" - reason = "Unknown reason" - if err_code is not None: - if err_code is 0: - sup = "True" - reason = "" - elif err_code is 1: - sup = "False" - reason = ret_dict.get("reason") - else: - sup = "Unknown" - reason = ret_dict.get("reason") - return sup, reason -''' -CAP_API = ''' -def {}({}, impl_mode=""): - __inputs__, __outputs__, __attrs__ = _build_args({}) - result = check_op_cap("{}", "{}", __inputs__, __outputs__, __attrs__) - return result.decode("utf-8") -''' -GLZ_API = ''' -@tbe_register.register_param_generalization("{}") -def {}_generalization({}, generalize_config=None): - __inputs__, __outputs__, __attrs__ = _build_args({}) - ret_str = generalize_op_params("{}", __inputs__, __outputs__, __attrs__, generalize_config) - return [json.loads(ret_str)] -''' - -ATTR_DEFAULT = {'bool': 'False', 'int': '0', 'float': '0.0', 'list_int': '[]', - 'list_float': '[]', 'list_bool': '[]', 'list_list_int': '[[]]', 'str': ''} - - -def _get_snake_str(s, i): - if s[i - 1] != '_': - if not s[i - 1].isupper(): - return "_" - elif s[i - 1].isupper() and (i + 1) < len(s) and s[i + 1].islower(): - return "_" - return "" - return "" - - -def optype_snake(origin_str): - temp_str = origin_str[0].lower() + origin_str[1:] - new_str = re.sub(r'([A-Z])', r'_\1', temp_str).lower() - return new_str - - -def optype_snake_ex(s): - snake_case = "" - for i, c in enumerate(s): - if i == 0: - snake_case += c.lower() - elif c.isupper(): - snake_case += _get_snake_str(s, i) - snake_case += c.lower() - else: - snake_case += c - return snake_case - - -class AdpBuilder(opdesc_parser.OpDesc): - def __init__(self: any, op_type: str): - self.argsdefv = [] - self.op_compile_option:str = '{}' - super().__init__(op_type) - - - def write_adapt(self: any, impl_path, path: str, op_compile_option_all:list = None): - self._build_paradefault() - if impl_path != "": - src_file = os.path.join(impl_path, self.op_file + '.cpp') - if not os.path.exists(src_file): - return - out_path = os.path.abspath(path) - if self.dynamic_shape and not out_path.endswith('dynamic'): - out_path = os.path.join(path, 'dynamic') - os.makedirs(out_path, mode=0o700, exist_ok=True) - adpfile = os.path.join(out_path, self.op_file + '.py') - self._gen_op_compile_option(op_compile_option_all) - with os.fdopen(os.open(adpfile, const_var.WFLAGS, const_var.WMODES), 'w') as fd: - self._write_head(fd) - self._write_argparse(fd) - self._write_impl(fd) - if self.op_chk_support: - self._write_cap('check_supported', fd) - self._write_cap('get_op_support_info', fd) - if self.op_fmt_sel: - self._write_cap('op_select_format', fd) - self._write_cap('get_op_specific_info', fd) - if self.op_range_limit == 'limited' or self.op_range_limit == 'dynamic': - self._write_glz(fd) - - - def _gen_op_compile_option(self:any, op_compile_option_all:list =None): - if op_compile_option_all is not None: - if self.op_type in op_compile_option_all: - self.op_compile_option = op_compile_option_all[self.op_type] - elif "__all__" in op_compile_option_all: - self.op_compile_option = op_compile_option_all["__all__"] - - - def _ip_argpack(self: any, default: bool = True) -> list: - args = [] - for i in range(len(self.input_name)): - arg = self.input_name[i] - if default and self.argsdefv[i] is not None: - arg += '=' + self.argsdefv[i] - args.append(arg) - return args - - def _op_argpack(self: any, default: bool = True) -> list: - args = [] - argidx = len(self.input_name) - for i in range(len(self.output_name)): - arg = self.output_name[i] - if default and self.argsdefv[i + argidx] is not None: - arg += '=' + self.argsdefv[i + argidx] - args.append(arg) - return args - - def _attr_argpack(self: any, default: bool = True) -> list: - args = [] - argidx = len(self.input_name) + len(self.output_name) - for i in range(len(self.attr_list)): - att = self.attr_list[i] - arg = att - if default and self.argsdefv[i + argidx] is not None: - if self.attr_val.get(att).get('type') == 'str': - arg += '="' + self.argsdefv[i + argidx] + '"' - elif self.attr_val.get(att).get('type') == 'bool': - arg += '=' + self.argsdefv[i + argidx].capitalize() - else: - arg += '=' + self.argsdefv[i + argidx] - args.append(arg) - return args - - def _build_paralist(self: any, default: bool = True) -> str: - args = [] - args.extend(self._ip_argpack(default)) - args.extend(self._op_argpack(default)) - args.extend(self._attr_argpack(default)) - return ', '.join(args) - - def _io_parachk(self: any, types: list, type_name: str) -> list: - chk = [] - for iot in types: - if iot == 'optional': - ptype = 'OPTION' - else: - ptype = iot.upper() - chk.append('para_check.{}_{}'.format(ptype, type_name)) - return chk - - def _attr_parachk(self: any) -> list: - chk = [] - for att in self.attr_list: - if self.attr_val.get(att).get('paramType') == 'optional': - pt = 'OPTION' - else: - pt = self.attr_val.get(att).get('paramType').upper() - att_type = self.attr_val.get(att).get('type').upper() - chk.append('para_check.{}_ATTR_{}'.format(pt, att_type)) - return chk - - def _build_parachk(self: any) -> str: - chk = [] - chk.extend(self._io_parachk(self.input_type, 'INPUT')) - chk.extend(self._io_parachk(self.output_type, 'OUTPUT')) - chk.extend(self._attr_parachk()) - chk.append('para_check.KERNEL_NAME') - return ', '.join(chk) - - def _build_paradefault(self: any): - optional = False - argtypes = [] - argtypes.extend(self.input_type) - argtypes.extend(self.output_type) - for atype in argtypes: - if atype == 'optional': - optional = True - if optional: - self.argsdefv.append('None') - else: - self.argsdefv.append(None) - for attr in self.attr_list: - atype = self.attr_val.get(attr).get('paramType') - if atype == 'optional': - optional = True - attrval = self.attr_val.get(attr).get('defaultValue') - if attrval is not None: - optional = True - if type == "bool": - attrval = attrval.capitalize() - elif type == "str": - attrval = "\"" + attrval + "\"" - self.argsdefv.append(attrval) - continue - if optional: - self.argsdefv.append(ATTR_DEFAULT.get(self.attr_val.get(attr).get('type'))) - else: - self.argsdefv.append(None) - - def _write_head(self: any, fd: object): - fd.write(IMPL_HEAD.format(self.input_ori_name, self.output_ori_name)) - - def _write_argparse(self: any, fd: object): - args = self._build_paralist(False) - fd.write('def _build_args({}):\n'.format(args)) - fd.write(' __inputs__ = []\n') - fd.write(' for arg in [{}]:\n'.format(', '.join(self.input_name))) - fd.write(' if arg != None:\n') - fd.write(' if isinstance(arg, (list, tuple)):\n') - fd.write(' if len(arg) == 0:\n') - fd.write(' continue\n') - fd.write(' __inputs__.append(arg[0])\n') - fd.write(' else:\n') - fd.write(' __inputs__.append(arg)\n') - fd.write(' else:\n') - fd.write(' __inputs__.append(arg)\n') - fd.write(' __outputs__ = []\n') - fd.write(' for arg in [{}]:\n'.format(', '.join(self.output_name))) - fd.write(' if arg != None:\n') - fd.write(' if isinstance(arg, (list, tuple)):\n') - fd.write(' if len(arg) == 0:\n') - fd.write(' continue\n') - fd.write(' __outputs__.append(arg[0])\n') - fd.write(' else:\n') - fd.write(' __outputs__.append(arg)\n') - fd.write(' else:\n') - fd.write(' __outputs__.append(arg)\n') - fd.write(' __attrs__ = []\n') - for attr in self.attr_list: - fd.write(' if {} != None:\n'.format(attr)) - fd.write(' attr = {}\n') - fd.write(' attr["name"] = "{}"\n'.format(attr)) - fd.write(' attr["dtype"] = "{}"\n'.format(self.attr_val.get(attr).get('type'))) - fd.write(' attr["value"] = {}\n'.format(attr)) - fd.write(' __attrs__.append(attr)\n') - fd.write(' return __inputs__, __outputs__, __attrs__\n') - - def _write_impl(self: any, fd: object): - argsdef = self._build_paralist() - argsval = self._build_paralist(False) - pchk = self._build_parachk() - if len(self.kern_name) > 0: - kern_name = self.kern_name - else: - kern_name = self.op_intf - src = self.op_file + '.cpp' - fd.write(IMPL_API.format(self.op_type, pchk, self.op_intf, argsdef, kern_name, argsval,\ - self.custom_compile_options, self.custom_all_compile_options, self.op_intf,\ - optype_snake_ex(self.op_type), optype_snake(self.op_type), src)) - if self.op_replay_flag: - fd.write(REPLAY_OP_API.format(self.op_type, kern_name, self.op_file, self.op_type, self.op_file,\ - self.op_compile_option)) - else: - fd.write(COMPILE_OP_API.format(self.op_type, self.op_type, ', '.join(self.input_name),\ - ', '.join(self.output_name), False, self.input_type + self.output_type, self.op_compile_option)) - - def _write_cap(self: any, cap_name: str, fd: object): - argsdef = self._build_paralist() - argsval = self._build_paralist(False) - if cap_name == 'check_supported': - fd.write(SUP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) - else: - fd.write(CAP_API.format(cap_name, argsdef, argsval, cap_name, self.op_type)) - - def _write_glz(self: any, fd: object): - argsdef = self._build_paralist() - argsval = self._build_paralist(False) - fd.write(GLZ_API.format(self.op_type, self.op_intf, argsdef, argsval, self.op_type)) - - -def write_scripts(cfgfile: str, cfgs: dict, dirs: dict, ops: list = None, op_compile_option:list = None): - batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') - iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') - file_map = {} - op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, AdpBuilder,\ - ops, dirs.get(const_var.AUTO_GEN_DIR)) - for op_desc in op_descs: - op_desc.write_adapt(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), op_compile_option) - file_map[op_desc.op_type] = op_desc.op_file - return file_map - -if __name__ == '__main__': - if len(sys.argv) <= 6: - raise RuntimeError('arguments must greater equal than 6') - rep_cfg = {} - rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] - rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] - cfg_dir = {} - cfg_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] - cfg_dir[const_var.CFG_OUT_DIR] = sys.argv[5] - cfg_dir[const_var.AUTO_GEN_DIR] = sys.argv[6] - write_scripts(cfgfile=sys.argv[1], cfgs=rep_cfg, dirs=cfg_dir) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py deleted file mode 100644 index b138047af..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_ops_config.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import sys -import os -import glob -import json -import argparse -import const_var - - -def load_json(json_file: str): - with open(json_file, encoding='utf-8') as file: - json_content = json.load(file) - return json_content - - -def get_specified_suffix_file(root_dir, suffix): - specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) - all_suffix_files = glob.glob(specified_suffix, recursive=True) - return all_suffix_files - - -def add_simplified_config(op_type, key, core_type, task_ration, objfile, config): - simple_cfg = config.get('binary_info_config.json') - op_cfg = simple_cfg.get(op_type) - if not op_cfg: - op_cfg = {} - op_cfg['dynamicRankSupport'] = True - op_cfg['simplifiedKeyMode'] = 0 - op_cfg['binaryList'] = [] - simple_cfg[op_type] = op_cfg - bin_list = op_cfg.get('binaryList') - if core_type == 0 and task_ration == "tilingKey": - bin_list.append({'coreType': core_type, 'simplifiedKey': key, \ - 'multiKernelType':1, 'binPath': objfile}) - else: - bin_list.append({'coreType': core_type, 'simplifiedKey': key, 'binPath': objfile}) - - -def add_op_config(op_file, bin_info, config): - op_cfg = config.get(op_file) - if not op_cfg: - op_cfg = {} - op_cfg['binList'] = [] - config[op_file] = op_cfg - op_cfg.get('binList').append(bin_info) - - -def gen_ops_config(json_file, soc, config): - core_type_map = {'MIX': 0, 'AiCore': 1, 'VectorCore': 2, 'MIX_AICORE':3, 'MIX_VECTOR_CORE':4} - contents = load_json(json_file) - if ('binFileName' not in contents) or ('supportInfo' not in contents): - return - json_base_name = os.path.basename(json_file) - op_dir = os.path.basename(os.path.dirname(json_file)) - support_info = contents.get('supportInfo') - bin_name = contents.get('binFileName') - bin_suffix = contents.get('binFileSuffix') - task_ration = contents.get('taskRation') - core_type = core_type_map.get(contents.get("coreType")) - bin_file_name = bin_name + bin_suffix - op_type = bin_name.split('_')[0] - op_file = op_dir + '.json' - bin_info = {} - keys = support_info.get('simplifiedKey') - if keys: - bin_info['simplifiedKey'] = keys - for key in keys: - add_simplified_config(op_type, key, core_type, task_ration, os.path.join(soc, op_dir, bin_file_name), \ - config) - bin_info['staticKey'] = support_info.get('staticKey') - bin_info['int64Mode'] = support_info.get('int64Mode') - bin_info['inputs'] = support_info.get('inputs') - bin_info['outputs'] = support_info.get('outputs') - if support_info.get('attrs'): - bin_info['attrs'] = support_info.get('attrs') - bin_info['binInfo'] = {'jsonFilePath': os.path.join(soc, op_dir, json_base_name)} - add_op_config(op_file, bin_info, config) - - -def gen_all_config(root_dir, soc): - suffix = 'json' - config = {} - config['binary_info_config.json'] = {} - all_json_files = get_specified_suffix_file(root_dir, suffix) - for _json in all_json_files: - gen_ops_config(_json, soc, config) - for cfg_key in config.keys(): - cfg_file = os.path.join(root_dir, cfg_key) - with os.fdopen(os.open(cfg_file, const_var.WFLAGS, const_var.WMODES), 'w') as fd: - json.dump(config.get(cfg_key), fd, indent=' ') - - -def args_prase(): - parser = argparse.ArgumentParser() - parser.add_argument('-p', - '--path', - nargs='?', - required=True, - help='Parse the path of the json file.') - parser.add_argument('-s', - '--soc', - nargs='?', - required=True, - help='Parse the soc_version of ops.') - return parser.parse_args() - - -def main(): - args = args_prase() - gen_all_config(args.path, args.soc) - - -if __name__ == '__main__': - main() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py deleted file mode 100644 index 1cac7d911..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/ascendc_replay_build.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import sys -import os -import opdesc_parser -import replay_codegen -import const_var -from replay_codegen import ReplayCodeGenParams - -PYF_PATH = os.path.dirname(os.path.realpath(__file__)) - - -class ReplayBuilder(opdesc_parser.OpDesc): - def __init__(self: any, op_type: str): - super().__init__(op_type) - - def gen_replay_source(self: any, impl_path: str, out_path: str, ops_product: str): - if not self.op_replay_flag: - print('{} replay not enabled'.format(self.op_type)) - return - argn = len(self.input_name) + len(self.output_name) + 1 - if self.op_replay_batch: - print('{} replay in batch mode'.format(self.op_type)) - else: - print('{} replay in normal mode'.format(self.op_type)) - if impl_path.endswith('op_kernel'): - implf = os.path.join(impl_path, self.op_file + '.cpp') - tiling_file = os.path.join(impl_path, "../op_host", self.op_file + '_tiling.h') - else: - if self.dynamic_shape: - dyn_path = 'dynamic' - else: - dyn_path = '' - implf = os.path.join(impl_path, dyn_path, self.op_file + '.cpp') - tiling_file = os.path.join(impl_path, "../../op_tiling", self.op_file + '_tiling.h') - rep_conf = replay_codegen.ReplayCodeGen(ReplayCodeGenParams(self.op_type, implf, tiling_file, self.op_file, \ - self.op_intf, argn, self.op_replay_batch, self.max_block_dim, self.max_shape_size)) - rep_conf.set_batch(self.op_replay_batch) - rep_conf.set_outdir(out_path) - rep_conf.gen_replay(ops_product) - - -def gen_replay(cfgfile: str, cfgs: dict, dirs: dict, ops_product: str, ops: list = None): - batch_lists = cfgs.get(const_var.REPLAY_BATCH).split(';') - iterator_lists = cfgs.get(const_var.REPLAY_ITERATE).split(';') - op_descs = opdesc_parser.get_op_desc(cfgfile, batch_lists, iterator_lists, ReplayBuilder, ops) - for op_desc in op_descs: - op_desc.gen_replay_source(dirs.get(const_var.CFG_IMPL_DIR), dirs.get(const_var.CFG_OUT_DIR), ops_product) - - -if __name__ == '__main__': - if len(sys.argv) <= 6: - raise RuntimeError('arguments must greater than 6') - rep_cfg = {} - rep_cfg[const_var.REPLAY_BATCH] = sys.argv[2] - rep_cfg[const_var.REPLAY_ITERATE] = sys.argv[3] - rep_dir = {} - rep_dir[const_var.CFG_IMPL_DIR] = sys.argv[4] - rep_dir[const_var.CFG_OUT_DIR] = sys.argv[5] - gen_replay(sys.argv[1], rep_cfg, rep_dir, sys.argv[6]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp deleted file mode 100644 index 0e8834664..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/batch_replay_impl.temp +++ /dev/null @@ -1,117 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include "replay_def.h" -#include "code_gen.h" -#include "replay_fun.h" -#include "register/op_check.h" -#define __ASCENDC_REPLAY_CODE__ -#include - -using namespace std; -using namespace optiling; -using namespace AscendCReplay; - -extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); -extern "C" int elf_batch_append(char *elf, uint32_t elfSize, char *jit, int kernum, char *atext[], int alen[], - int atlen, const char* kernelname[]); - -#define KERNEL_N 1 -#define ARG_N (__ARG_NUM__) -#define MAX_L (1024 * 1024 * 100) -#define MAX_E (1024 * 1024) - -int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) -{ - // gen type 1 : direct call codes 0: load .o file - if (param.gentype < 0 || param.gentype > 1) { - printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); - return 0; - } else if (param.gentype == 1 && param.objptr == nullptr) { - printf("Error: call replay with direct call mode, but code obj addr is null\n"); - return 0; - } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { - printf("Error: call replay with object file mode, but object file path is null\n"); - return 0; - } - // core_type 0:MIX 1:CUBE 2:VEC - if (core_type < 0 || core_type > 2) { - printf("Error: call replay core type is %d !\n", core_type); - return 0; - } - g_coreType = __CORE_TYPE__; - g_taskRation = param.task_ration; - g_tilingKey = param.tiling_key; - - unsigned char *buf, *jit; - char *kernel[KERNEL_N]; - int len[KERNEL_N]; - block_idx = 0; - block_num = param.block_dim; - g_ubBase = block_num; - uint8_t *code = (uint8_t *)malloc(MAX_L); - uint8_t *pos = code; - struct timespec tp1, tp2; - - clock_gettime(CLOCK_MONOTONIC, &tp1); - if (block_num > 32) { - printf("Error: block_num > 32\n"); - return 0; - } - //__OP_FOPEN__ - for (int i = 0; i < KERNEL_N; i++) { - //__OP_SET_KERNEL__ - for (int j = 0; j < ARG_N; j++) - AddArg(j, ARG_STEP * (j + 1)); -#ifdef FP_CEILING - SetCtrlFloatEnable(); -#else - SetCtrlFloatDisable(); -#endif - CodeInit(pos, true); - __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); - CodeEnd(); - kernel[i] = (char *)pos; - len[i] = CodeLen(); - pos += len[i]; - } - //__OP_FCLOSE__ - clock_gettime(CLOCK_MONOTONIC, &tp2); - buf = (unsigned char *)malloc(MAX_E); - int fd = open(param.entry_file, O_RDONLY); - if (fd < 0) { - printf("[error]: cannot find entry.o : %s\n", param.entry_file); - return 0; - } - uint32_t bufSize = read(fd, buf, MAX_E); - if (bufSize <= 0) { - printf("[error]: entry.o : %s is too small ! \n", param.entry_file); - } - close(fd); - jit = (unsigned char *)malloc(MAX_L); - printf("total code generated %ld\n", pos - code); - int sz = elf_batch_append((char *)buf, bufSize, (char *)jit, KERNEL_N, kernel, len, pos - code, ¶m.kernel_name); - if (tp1.tv_sec != tp2.tv_sec) { - printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); - } else { - printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); - } - printf("new elf size %d\n", sz); - if (param.gentype == 0) { - fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); - (void)write(fd, jit, sz); - close(fd); - free(jit); - } else if (param.gentype == 1) { - *param.objptr = (char*)jit; - } - free(buf); - free(code); - return sz; -} - -REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py deleted file mode 100644 index d4e6a540c..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/code_channel_infer.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" -import os -import stat -import ctypes -import collections -import shutil -import subprocess -import copy - -"""CODE_* is used to cube/vector api is called in operator code -CODE_MIX means both cube and vector api is called -CODE_CUBE means only cube api is called -CODE_VEC means only vector api is called -""" -CODE_MIX = 0 -CODE_CUBE = 1 -CODE_VEC = 2 - - -def _is_v220(op_product: str): - """return if current soc version is V220 - - Returns: - res: True means V220 - """ - if op_product in ["ascend910b", "ascend910c"]: - return True - return False - - -InfoCodeChanelParams = collections.namedtuple('InfoCodeChanelParams',\ -['src_file', 'tiling_header', 'kernel_name', 'outdir', 'op_product', 'compile_options']) - - -def infer_code_channel(params: InfoCodeChanelParams): - """get code channel for v220, return CODE_MIX if soc version is not V220 - - Args: - src_file (str): AscendC operator code file - src_file (str): AscendC operator tiling header file - kernel_name (str): kernel function name - optype (str): operator type - compile_options (list): compile options for bisheng cmd - - Raises: - Exception: if not exist L1/L0/UB if code, it's not a aicore code - - Returns: - res (int): CODE_MIX/CODE_CUBE/CODE_VEC - """ - if not _is_v220(params.op_product): - return CODE_MIX - return CODE_VEC diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py deleted file mode 100644 index 3e71f5ab8..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/const_var.py +++ /dev/null @@ -1,33 +0,0 @@ - -#!/usr/bin/env python -# coding=utf-8 -""" -Function: -The replay funtion entry -Copyright Information: -Huawei Technologies Co., Ltd. All Rights Reserved © 2020 -""" - -import os -import stat - - -REPLAY_BATCH = 'batch' -REPLAY_ITERATE = 'iterate' -CFG_IMPL_DIR = 'impl_dir' -CFG_OUT_DIR = 'out_dir' -AUTO_GEN_DIR = 'auto_gen_dir' -WFLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC -WMODES = stat.S_IWUSR | stat.S_IRUSR -SOC_MAP_EXT = {'ascend310p': 'Ascend310P3', 'ascend310b': 'Ascend310B1', - 'ascend910': 'Ascend910A', 'ascend910b': 'Ascend910B1', 'ascend910c': 'Ascend910C1'} -BIN_CMD = 'opc $1 --main_func={fun} --input_param={param} --soc_version={soc} \ ---output=$2 --impl_mode={impl} --simplified_key_mode=0 --op_mode=dynamic\n' -CHK_CMD = ''' -if ! test -f $2/{res_file} ; then - echo "$2/{res_file} not generated!" - exit 1 -fi -''' -ATTR_DEF_VAL = {'str' : '', 'int': 0, 'float': 0.0, 'bool': False, 'list_bool': [], - 'list_int': [], 'list_float': [], 'list_list_int': [[]]} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh deleted file mode 100644 index 492034e49..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_impl_and_mrege_json.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/bash - -project_path=$1 -build_path=$2 -vendor_name=customize -if [[ ! -d "$project_path" ]]; then - echo "[ERROR] No projcet path is provided" - exit 1 -fi - -if [[ ! -d "$build_path" ]]; then - echo "[ERROR] No build path is provided" - exit 1 -fi - -# copy aicpu kernel so operators -if [[ -d "${project_path}/cpukernel/aicpu_kernel_lib" ]]; then - cp -f ${project_path}/cpukernel/aicpu_kernel_lib/* ${build_path}/makepkg/packages/vendors/$vendor_name/op_impl/cpu/aicpu_kernel/impl - rm -rf ${project_path}/cpukernel/aicpu_kernel_lib -fi diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh deleted file mode 100644 index d4c27d17f..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_ops_filter.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -# Description: Generate npu_supported_ops.json -# ============================================================================== - -if [[ -z "$1" ]]; then - echo -e "[ERROR] No source dir provided" - exit 1 -fi - -if [[ -z "$2" ]]; then - echo -e "[ERROR] No destination dir provided" - exit 1 -fi - -src=$1 -dest_file=$2/npu_supported_ops.json - -if [ -f "$dest_file" ];then - chmod u+w $dest_file -fi - -echo $* - -add_ops() { - name=$1 - isHeavy=$2 - file=$3 - grep -w "\"$name\"" ${file} >/dev/null - if [ $? == 0 ];then - return - fi - echo " \"${name}\": {" >> ${file} - echo " \"isGray\": false," >> ${file} - echo " \"isHeavy\": ${isHeavy}" >> ${file} - echo " }," >> ${file} -} - -echo "{" > ${dest_file} -ini_files=$(find ${src} -name "*.ini") -for file in ${ini_files} ; do - name=$(grep '^\[' ${file} | sed 's/\[//g' | sed 's/]//g' | sed 's/\r//g') - grep 'heavyOp.flag' ${file} >/dev/null - if [ $? == 0 ];then - isHeavy=$(grep 'heavyOp.flag' ${file} | awk -F= '{print $2}') - else - isHeavy="false" - fi - for op in ${name} ; do - add_ops ${op} "false" ${dest_file} - done -done -echo "}" >> ${dest_file} -file_count=$(cat ${dest_file} | wc -l) -line=$(($file_count-1)) -sed -i "${line}{s/,//g}" ${dest_file} - -chmod 640 "${dest_file}" -echo -e "[INFO] Succed generated ${dest_file}" - -exit 0 - diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh deleted file mode 100644 index a06cfc78d..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/gen_version_info.sh +++ /dev/null @@ -1,6 +0,0 @@ -ascend_install_dir=$1 -gen_file_dir=$2 - -# create version.info -compiler_version=$(grep "Version" -w ${ascend_install_dir}/compiler/version.info | awk -F = '{print $2}') -echo "custom_opp_compiler_version=${compiler_version}" > ${gen_file_dir}/version.info \ No newline at end of file diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py deleted file mode 100644 index 28ba08757..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_op_info.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" -import json -import os -import sys -import stat -import const_var - - -if __name__ == '__main__': - if len(sys.argv) != 3: - print(sys.argv) - print('argv error, inert_op_info.py your_op_file lib_op_file') - sys.exit(2) - - with open(sys.argv[1], 'r') as load_f: - insert_operator = json.load(load_f) - - all_operators = {} - if os.path.exists(sys.argv[2]): - if os.path.getsize(sys.argv[2]) != 0: - with open(sys.argv[2], 'r') as load_f: - all_operators = json.load(load_f) - - for k in insert_operator.keys(): - if k in all_operators.keys(): - print('replace op:[', k, '] success') - else: - print('insert op:[', k, '] success') - all_operators[k] = insert_operator[k] - - with os.fdopen(os.open(sys.argv[2], const_var.WFLAGS, const_var.WMODES), 'w') as json_file: - json_file.write(json.dumps(all_operators, indent=4)) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py deleted file mode 100644 index 7e8dc2c55..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/insert_simplified_keys.py +++ /dev/null @@ -1,242 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import os -import re -import glob -import json -import argparse - - -DATA_TPYE_DICT = { - 'float32': 0, - 'float16': 1, - 'int8': 2, - 'int16': 6, - 'uint16': 7, - 'uint8': 4, - 'int32': 3, - 'int64': 9, - 'uint32': 8, - 'uint64': 10, - 'bool': 12, - 'double': 11, - 'complex64': 16, - 'complex128': 17, - 'qint8': 18, - 'qint16': 19, - 'qint32': 20, - 'quint8': 21, - 'quint16': 22, - 'resource': 23, - 'string': 24, - 'dual': 25, - 'variant': 26, - 'bf16': 27, - 'bfloat16': 27, - 'undefined': 28, - 'int4': 29, - 'uint1': 30, - 'int2': 31 -} - -FORMAT_DICT = { - 'NCHW': 0, - 'NHWC': 1, - 'ND': 2, - 'NC1HWC0': 3, - 'FRACTAL_Z': 4, - 'NC1C0HWPAD': 5, - 'NHWC1C0': 6, - 'FSR_NCHW': 7, - 'FRACTAL_DECONV': 8, - 'C1HWNC0': 9, - 'FRACTAL_DECONV_TRANSPOSE': 10, - 'FRACTAL_DECONV_SP_STRIDE_TRANS': 11, - 'NC1HWC0_C04': 12, - 'FRACTAL_Z_C04': 13, - 'CHWN': 14, - 'FRACTAL_DECONV_SP_STRIDE8_TRANS': 15, - 'HWCN': 16, - 'NC1KHKWHWC0': 17, - 'BN_WEIGHT': 18, - 'FILTER_HWCK': 19, - 'HASHTABLE_LOOKUP_LOOKUPS': 20, - 'HASHTABLE_LOOKUP_KEYS': 21, - 'HASHTABLE_LOOKUP_VALUE': 22, - 'HASHTABLE_LOOKUP_OUTPUT': 23, - 'HASHTABLE_LOOKUP_HITS': 24, - 'C1HWNCoC0': 25, - 'MD': 26, - 'NDHWC': 27, - 'FRACTAL_ZZ': 28, - 'FRACTAL_NZ': 29, - 'NCDHW': 30, - 'DHWCN': 31, - 'NDC1HWC0': 32, - 'FRACTAL_Z_3D': 33, - 'CN': 34, - 'NC': 35, - 'DHWNC': 36, - 'FRACTAL_Z_3D_TRANSPOSE': 37, - 'FRACTAL_ZN_LSTM': 38, - 'FRACTAL_Z_G': 39, - 'RESERVED': 40, - 'ALL': 41, - 'NULL': 42, - 'ND_RNN_BIAS': 43, - 'FRACTAL_ZN_RNN': 44, - 'NYUV': 45, - 'NYUV_A': 46 -} - - -def load_json(json_file: str): - with open(json_file, encoding='utf-8') as file: - json_content = json.load(file) - return json_content - - -def get_specified_suffix_file(root_dir, suffix): - specified_suffix = os.path.join(root_dir, '**/*.{}'.format(suffix)) - all_suffix_files = glob.glob(specified_suffix, recursive=True) - return all_suffix_files - - -def get_deterministic_value(support_info): - deterministic_key = 'deterministic' - if deterministic_key not in support_info: - return 0 - deterministic_value = support_info.get(deterministic_key) - if deterministic_value == 'true': - return 1 - else: - return 0 - - -def get_precision_value(support_info): - precision_key = 'implMode' - precision_value = support_info.get(precision_key) - if precision_value == 'high_performance': - _value = 1 - elif precision_value == 'high_precision': - _value = 2 - else: - _value = 0 - return _value - - -def get_overflow_value(support_info): - return 0 - - -def get_parameters(info): - if info: - if 'dtype' in info: - data_type = info['dtype'] - data_type_value = DATA_TPYE_DICT.get(data_type) - else: - data_type_value = 0 - if 'format' in info: - _format = info['format'] - _format_value = FORMAT_DICT.get(_format) - else: - _format_value = 0 - else: - data_type_value = 0 - _format_value = 0 - return str(data_type_value), str(_format_value) - - -def get_dynamic_parameters(info): - # 动态输入时只需获取第一个参数 - return get_parameters(info[0]) - - -def get_all_parameters(support_info, _type): - result_list = list() - info_lists = support_info.get(_type) - if info_lists: - for _info in info_lists: - # 输入为列表时是动态输入 - if isinstance(_info, (list, tuple)): - data_type_value, _format_value = get_dynamic_parameters(_info) - else: - data_type_value, _format_value = get_parameters(_info) - result_list.append("{},{}".format(data_type_value, _format_value)) - return result_list - - -def get_all_input_parameters(support_info): - result = get_all_parameters(support_info, 'inputs') - return '/'.join(result) - - -def insert_content_into_file(input_file, content): - with open(input_file, 'r+') as file: - lines = file.readlines() - for index, line in enumerate(lines): - match_result = re.search(r'"staticKey":', line) - if match_result: - count = len(line) - len(line.lstrip()) - new_content = "{}{}".format(' ' * count, content) - # 插入到前一行，防止插入最后时还需要考虑是否添加逗号 - lines.insert(index, new_content) - break - file.seek(0) - file.write(''.join(lines)) - - -def insert_simplified_keys(json_file): - contents = load_json(json_file) - # 不存在'binFileName'或者'supportInfo'字段时，非需要替换的解析json文件 - if ('binFileName' not in contents) or ('supportInfo' not in contents): - return - support_info = contents.get('supportInfo') - bin_file_name = contents.get('binFileName') - # 'simplifiedKey'字段已经存在时，直接返回，不重复生成 - if 'simplifiedKey' in support_info: - return - op_type = bin_file_name.split('_')[0] - deterministic = str(get_deterministic_value(support_info)) - precision = str(get_precision_value(support_info)) - overflow = str(get_overflow_value(support_info)) - input_parameters = get_all_input_parameters(support_info) - key = '{}/d={},p={},o={}/{}/'.format( - op_type, - deterministic, - precision, - overflow, - input_parameters) - result = '"simplifiedKey": "' + key + '",\n' - insert_content_into_file(json_file, result) - - -def insert_all_simplified_keys(root_dir): - suffix = 'json' - all_json_files = get_specified_suffix_file(root_dir, suffix) - for _json in all_json_files: - insert_simplified_keys(_json) - - -def args_prase(): - parser = argparse.ArgumentParser() - parser.add_argument('-p', - '--path', - nargs='?', - required=True, - help='Parse the path of the json file.') - return parser.parse_args() - - -def main(): - args = args_prase() - insert_all_simplified_keys(args.path) - - -if __name__ == '__main__': - main() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py deleted file mode 100644 index 2b77c970d..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_entry.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - - -def gen_fun_def(title, kernel, argn, arg_type, arg_name): - entry = [] - entry.append(title) - entry.append(kernel) - entry.append('(') - args = [] - for i in range(0, argn): - args.append(arg_type + ' ' + arg_name + str(i)) - entry.append(', '.join(args)) - entry.append(')') - return ' '.join(entry) - - -def gen_batch_kernel_body(fname, argn, arg_name): - body = [] - body.append('{') - fun = [] - fun.append(fname) - fun.append('(') - args = [] - for i in range(0, argn): - args.append(arg_name + str(i)) - fun.append(', '.join(args)) - fun.append(');') - body.append(' '.join(fun)) - body.append('}') - return '\n'.join(body) - - -def gen_mc_kernel_body(kn, argn, arg_name, blknum): - body = [] - body.append('{') - body.append(' switch(block_idx) {') - for blk in range(0, blknum): - fun = [] - fun.append('{}_blk{:02d}'.format(kn, blk)) - fun.append('(') - args = [] - for i in range(0, argn): - args.append(arg_name + str(i)) - fun.append(', '.join(args)) - fun.append(')') - body.append(' case {}: {}; break;'.format(blk, ' '.join(fun))) - body.append(' default: break;') - body.append(' }') - body.append('}') - return '\n'.join(body) - - -def gen_proc_body(argn, arg_name): - body = [] - body.append('{') - args = [] - for i in range(0, argn): - args.append(arg_name + str(i)) - body.append('uint64_t __x = (uint64_t)' + ' + (uint64_t)'.join(args) + ';') - body.append('__asm__ ("NOP");') - body.append('__asm__ ("NOP");') - body.append('__asm__ ("NOP");') - body.append('}') - return '\n'.join(body) - - -def batch_code_gen(kn, argn, argt): - codes = [] - kernel_name = kn - proc_name = kernel_name + '_percore' - arg_num = int(argn) - data_type = argt - arg_type = '__gm__ ' + data_type + '* __restrict__' - arg_name = 'arg' - kernel_title = 'extern \"C\" __global__ __aicore__ void' - proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' - codes.append('#ifndef __aicore__') - codes.append('#define __aicore__ [aicore]') - codes.append('#endif') - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') - codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) - codes.append(gen_batch_kernel_body(proc_name, arg_num, arg_name)) - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) - codes.append(gen_proc_body(arg_num, arg_name)) - return '\n'.join(codes) + '\n' - - -def mc_code_gen(kn, argn, argt, blknum): - codes = [] - kernel_name = kn - core_num = int(blknum) - arg_num = int(argn) - data_type = argt - arg_type = '__gm__ ' + data_type + '* __restrict__' - arg_name = 'arg' - kernel_title = 'extern \"C\" __global__ __aicore__ void' - proc_title = 'extern \"C\" __attribute__((noinline)) __aicore__ void' - codes.append('#ifndef __aicore__') - codes.append('#define __aicore__ [aicore]') - codes.append('#endif') - for i in range(0, core_num): - proc_name = '{}_blk{:02d}'.format(kernel_name, i) - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name) + ';') - codes.append(gen_fun_def(kernel_title, kernel_name, arg_num, arg_type, arg_name)) - codes.append(gen_mc_kernel_body(kernel_name, arg_num, arg_name, core_num)) - for i in range(0, core_num): - proc_name = '{}_blk{:02d}'.format(kernel_name, i) - codes.append(gen_fun_def(proc_title, proc_name, arg_num, arg_type, arg_name)) - codes.append(gen_proc_body(arg_num, arg_name)) - return '\n'.join(codes) + '\n' diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp deleted file mode 100644 index 5079a1043..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/kernel_impl.temp +++ /dev/null @@ -1,10 +0,0 @@ -#include -#include -#include -#include -#include -#include "replay_def.h" -#include "code_gen.h" -#include "replay_fun.h" -#define __ASCENDC_REPLAY_CODE__ -#include "__CCE_FILE__" diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING deleted file mode 100644 index d159169d1..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/COPYING +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md deleted file mode 100644 index b41f01682..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/README.md +++ /dev/null @@ -1,246 +0,0 @@ -[![License: GPL v2](https://img.shields.io/badge/License-GPL%20v2-blue.svg)](https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) -![Build Status](https://github.com/megastep/makeself/workflows/CI/badge.svg) - -# makeself - Make self-extractable archives on Unix - -[makeself.sh][1] is a small shell script that generates a self-extractable -compressed tar archive from a directory. The resulting file appears as a shell script -(many of those have a **.run** suffix), and can be launched as is. The archive -will then uncompress itself to a temporary directory and an optional arbitrary -command will be executed (for example an installation script). This is pretty -similar to archives generated with WinZip Self-Extractor in the Windows world. -Makeself archives also include checksums for integrity self-validation (CRC -and/or MD5/SHA256 checksums). - -The makeself.sh script itself is used only to create the archives from a -directory of files. The resultant archive is actually a compressed (using -gzip, bzip2, or compress) TAR archive, with a small shell script stub at the -beginning. This small stub performs all the steps of extracting the files, -running the embedded command, and removing the temporary files when done. -All the user has to do to install the software contained in such an -archive is to "run" the archive, i.e **sh nice-software.run**. I recommend -using the ".run" (which was introduced by some Makeself archives released by -Loki Software) or ".sh" suffix for such archives not to confuse the users, -so that they will know they are actually shell scripts (with quite a lot of binary data -attached to them though!). - -I am trying to keep the code of this script as portable as possible, i.e it is -not relying on any bash-specific features and only calls commands that are -installed on any functioning UNIX-compatible system. This script as well as -the archives it generates should run on any Unix flavor, with any compatible -Bourne shell, provided of course that the compression programs are available. - -As of version 2.1, Makeself has been rewritten and tested on the following -platforms : - - * Linux (all distributions) - * Sun Solaris (8 and above) - * HP-UX (tested on 11.0 and 11i on HPPA RISC) - * SCO OpenUnix and OpenServer - * IBM AIX 5.1L - * macOS (Darwin) - * SGI IRIX 6.5 - * FreeBSD - * UnicOS / Cray - * Cygwin (Windows) - -If you successfully run Makeself and/or archives created with it on another -system, then please [let me know][2]! - -Examples of publicly available archives made using makeself are : - - * Game patches and installers for [Id Software][3] games like Quake 3 for Linux or Return To Castle Wolfenstein ; - * All game patches released by [Loki Software][4] for the Linux version of popular games ; - * The [nVidia drivers][5] for Linux - * The installer for the Linux version of [Google Earth][6] - * The [VirtualBox][7] installers for Linux - * The [Makeself][1] distribution itself ;-) - * and countless others... - -**Important note for Apache users:** By default, most Web servers will think that Makeself archives are regular text files and thus they may show up as text in a Web browser. The correct way to prevent this is to add a MIME type for this file format, like so (in httpd.conf) : - -`AddType application/x-makeself .run` - -**Important note for certain GNU/Linux distributions:** Archives created with Makeself prior to v2.1.2 were using an old syntax for the _head_ and _tail_ Unix commands that is being progressively obsoleted in their GNU forms. Therefore you may have problems uncompressing some of these archives. A workaround for this is to set the environment variable $_POSIX2_VERSION to enable the old syntax, i.e. : - -`export _POSIX2_VERSION=199209` - -## Usage - -The syntax of makeself is the following: - -``` -makeself.sh [args] archive_dir file_name label startup_script [script_args] -``` - - * _args_ are optional options for Makeself. The available ones are : - - * **`--version`** : Prints the version number on stdout, then exits immediately - * **`--gzip`** : Use gzip for compression (the default on platforms on which gzip is commonly available, like Linux) - * **`--bzip2`** : Use bzip2 instead of gzip for better compression. The bzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. - * **`--pbzip2`** : Use pbzip2 instead of gzip for better and faster compression on machines having multiple CPUs. The pbzip2 command must be available in the command path. It is recommended that the archive prefix be set to something like '.bz2.run', so that potential users know that they'll need bzip2 to extract it. - * **`--xz`** : Use xz instead of gzip for better compression. The xz command must be available in the command path. It is recommended that the archive prefix be set to something like '.xz.run' for the archive, so that potential users know that they'll need xz to extract it. - * **`--lzo`** : Use lzop instead of gzip for better compression. The lzop command must be available in the command path. It is recommended that the archive prefix be set to something like `.lzo.run` for the archive, so that potential users know that they'll need lzop to extract it. - * **`--lz4`** : Use lz4 instead of gzip for better compression. The lz4 command must be available in the command path. It is recommended that the archive prefix be set to something like '.lz4.run' for the archive, so that potential users know that they'll need lz4 to extract it. - * **`--zstd`** : Use zstd instead of gzip for better compression. The zstd command must be available in the command path. It is recommended that the archive prefix be set to something like '.zstd.run' for the archive, so that potential users know that they'll need zstd to extract it. - * **`--pigz`** : Use pigz for compression. - * **`--base64`** : Encode the archive to ASCII in Base64 format instead of compressing (base64 command required). - * **`--gpg-encrypt`** : Encrypt the archive using `gpg -ac -z $COMPRESS_LEVEL`. This will prompt for a password to encrypt with. Assumes that potential users have `gpg` installed. - * **`--ssl-encrypt`** : Encrypt the archive using `openssl aes-256-cbc -a -salt`. This will prompt for a password to encrypt with. Assumes that the potential users have the OpenSSL tools installed. - * **`--compress`** : Use the UNIX `compress` command to compress the data. This should be the default on all platforms that don't have gzip available. - * **`--nocomp`** : Do not use any compression for the archive, which will then be an uncompressed TAR. - * **`--complevel`** : Specify the compression level for gzip, bzip2, pbzip2, zstd, xz, lzo or lz4. (defaults to 9) - * **`--threads`** : Specify the number of threads to be used by compressors that support parallelization. Omit to use compressor's default. Most useful (and required) for opting into xz's threading, usually with `--threads=0` for all available cores. pbzip2 and pigz are parallel by default, and setting this value allows limiting the number of threads they use. - * **`--notemp`** : The generated archive will not extract the files to a temporary directory, but in a new directory created in the current directory. This is better to distribute software packages that may extract and compile by themselves (i.e. launch the compilation through the embedded script). - * **`--current`** : Files will be extracted to the current directory, instead of in a subdirectory. This option implies `--notemp` above. - * **`--follow`** : Follow the symbolic links inside of the archive directory, i.e. store the files that are being pointed to instead of the links themselves. - * **`--append`** _(new in 2.1.x)_: Append data to an existing archive, instead of creating a new one. In this mode, the settings from the original archive are reused (compression type, label, embedded script), and thus don't need to be specified again on the command line. - * **`--header`** : Makeself uses a separate file to store the header stub, called `makeself-header.sh`. By default, it is assumed that it is stored in the same location as makeself.sh. This option can be used to specify its actual location if it is stored someplace else. - * **`--cleanup`** : Specify a script that is run when execution is interrupted or finishes successfully. The script is executed with the same environment and initial `script_args` as `startup_script`. - * **`--copy`** : Upon extraction, the archive will first extract itself to a temporary directory. The main application of this is to allow self-contained installers stored in a Makeself archive on a CD, when the installer program will later need to unmount the CD and allow a new one to be inserted. This prevents "Filesystem busy" errors for installers that span multiple CDs. - * **`--nox11`** : Disable the automatic spawning of a new terminal in X11. - * **`--nowait`** : When executed from a new X11 terminal, disable the user prompt at the end of the script execution. - * **`--nomd5`** and **`--nocrc`** : Disable the creation of a MD5 / CRC checksum for the archive. This speeds up the extraction process if integrity checking is not necessary. - * **`--sha256`** : Adds a SHA256 checksum for the archive. This is in addition to the MD5 / CRC checksums unless `--nomd5` is also used. - * **`--lsm` _file_** : Provide and LSM file to makeself, that will be embedded in the generated archive. LSM files are describing a software package in a way that is easily parseable. The LSM entry can then be later retrieved using the `--lsm` argument to the archive. An example of a LSM file is provided with Makeself. - * **`--tar-format opt`** : Specify the tar archive format (default is ustar); you may use any value accepted by your tar command (such as posix, v7, etc). - * **`--tar-extra opt`** : Append more options to the tar command line. - - For instance, in order to exclude the `.git` directory from the packaged archive directory using the GNU `tar`, one can use `makeself.sh --tar-extra "--exclude=.git" ...` - - * **`--keep-umask`** : Keep the umask set to shell default, rather than overriding when executing self-extracting archive. - * **`--packaging-date date`** : Use provided string as the packaging date instead of the current date. - * **`--license`** : Append a license file. - * **`--nooverwrite`** : Do not extract the archive if the specified target directory already exists. - * **`--help-header file`** : Add a header to the archive's `--help` output. - * `archive_dir` is the name of the directory that contains the files to be archived - * `file_name` is the name of the archive to be created - * `label` is an arbitrary text string describing the package. It will be displayed while extracting the files. - * `startup_script` is the command to be executed _from within_ the directory of extracted files. Thus, if you wish to execute a program contained in this directory, you must prefix your command with `./`. For example, `./program` will be fine. The `script_args` are additional arguments for this command. - -Here is an example, assuming the user has a package image stored in a **/home/joe/mysoft**, and he wants to generate a self-extracting package named -**mysoft.sh**, which will launch the "setup" script initially stored in /home/joe/mysoft : - -`makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup -` - -Here is also how I created the [makeself.run][9] archive which contains the Makeself distribution : - -`makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" ` - -Archives generated with Makeself can be passed the following arguments: - - * **`--keep`** : Prevent the files to be extracted in a temporary directory that will be removed after the embedded script's execution. The files will then be extracted in the current working directory and will stay here until you remove them. - * **`--verbose`** : Will prompt the user before executing the embedded command - * **`--target dir`** : Allows to extract the archive in an arbitrary place. - * **`--nox11`** : Do not spawn a X11 terminal. - * **`--confirm`** : Prompt the user for confirmation before running the embedded command. - * **`--info`** : Print out general information about the archive (does not extract). - * **`--lsm`** : Print out the LSM entry, if it is present. - * **`--list`** : List the files in the archive. - * **`--check`** : Check the archive for integrity using the embedded checksums. Does not extract the archive. - * **`--nochown`** : By default, a `chown -R` command is run on the target directory after extraction, so that all files belong to the current user. This is mostly needed if you are running as root, as tar will then try to recreate the initial user ownerships. You may disable this behavior with this flag. - * **`--tar`** : Run the tar command on the contents of the archive, using the following arguments as parameter for the command. - * **`--noexec`** : Do not run the embedded script after extraction. - * **`--noexec-cleanup`** : Do not run the embedded cleanup script. - * **`--nodiskspace`** : Do not check for available disk space before attempting to extract. - * **`--cleanup-args`** : Specify arguments to be passed to the cleanup script. Wrap value in quotes to specify multiple arguments. - -Any subsequent arguments to the archive will be passed as additional arguments to the embedded command. You must explicitly use the `--` special command-line construct before any such options to make sure that Makeself will not try to interpret them. - -## Startup Script - -The startup script must be a regular Shell script. - -Within the startup script, you can use the `$USER_PWD` variable to get the path of the folder from which the self-extracting script is executed. This is especially useful to access files that are located in the same folder as the script, as shown in the example below. - -`my-self-extracting-script.sh --fooBarFileParameter foo.bar` - -## Building and Testing - -Clone the git repo and execute `git submodule update --init --recursive` to obtain all submodules. - -* To make a release: `make` -* To run all tests: `make test` - -## Maven Usage - -Makeself is now supported by the following maven plugin [makeself-maven-plugin](https://github.com/hazendaz/makeself-maven-plugin). Please refer to project for usage and report any bugs in regards to maven plugin on that project. - -## License - -Makeself itself is covered by the [GNU General Public License][8] (GPL) version 2 and above. Archives generated by Makeself don't have to be placed under this license (although I encourage it ;-)), since the archive itself is merely data for Makeself. - -## Contributing - -I will gladly consider merging your pull requests on the [GitHub][10] repository. However, please keep the following in mind: - - * One of the main purposes of Makeself is portability. Do not submit patches that will break supported platforms. The more platform-agnostic, the better. - * Please explain clearly what the purpose of the patch is, and how you achieved it. - -## Download - -Get the latest official distribution [here][9] (version 2.4.2). - -The latest development version can be grabbed from [GitHub][10]. Feel free to submit any patches there through the fork and pull request process. - -## Version history - - * **v1.0:** Initial public release - * **v1.1:** The archive can be passed parameters that will be passed on to the embedded script, thanks to John C. Quillan - * **v1.2:** Cosmetic updates, support for bzip2 compression and non-temporary archives. Many ideas thanks to Francois Petitjean. - * **v1.3:** More patches from Bjarni R. Einarsson and Francois Petitjean: Support for no compression (`--nocomp`), script is no longer mandatory, automatic launch in an xterm, optional verbose output, and -target archive option to indicate where to extract the files. - * **v1.4:** Many patches from Francois Petitjean: improved UNIX compatibility, automatic integrity checking, support of LSM files to get info on the package at run time.. - * **v1.5.x:** A lot of bugfixes, and many other patches, including automatic verification through the usage of checksums. Version 1.5.5 was the stable release for a long time, even though the Web page didn't get updated ;-). Makeself was also officially made a part of the [Loki Setup installer][11], and its source is being maintained as part of this package. - * **v2.0:** Complete internal rewrite of Makeself. The command-line parsing was vastly improved, the overall maintenance of the package was greatly improved by separating the stub from makeself.sh. Also Makeself was ported and tested to a variety of Unix platforms. - * **v2.0.1:** First public release of the new 2.0 branch. Prior versions are officially obsoleted. This release introduced the `--copy` argument that was introduced in response to a need for the [UT2K3][12] Linux installer. - * **v2.1.0:** Big change : Makeself can now support multiple embedded tarballs, each stored separately with their own checksums. An existing archive can be updated with the `--append` flag. Checksums are also better managed, and the `--nochown` option for archives appeared. - * **v2.1.1:** Fixes related to the Unix compression (compress command). Some Linux distributions made the insane choice to make it unavailable, even though gzip is capable of uncompressing these files, plus some more bugfixes in the extraction and checksum code. - * **v2.1.2:** Some bug fixes. Use head -n to avoid problems with POSIX conformance. - * **v2.1.3:** Bug fixes with the command line when spawning terminals. Added `--tar`, `--noexec` for archives. Added `--nomd5` and `--nocrc` to avoid creating checksums in archives. The embedded script is now run through "eval". The `--info` output now includes the command used to create the archive. A man page was contributed by Bartosz Fenski. - * **v2.1.4:** Fixed `--info` output. Generate random directory name when extracting files to . to avoid problems. Better handling of errors with wrong permissions for the directory containing the files. Avoid some race conditions, Unset the $CDPATH variable to avoid problems if it is set. Better handling of dot files in the archive directory. - * **v2.1.5:** Made the md5sum detection consistent with the header code. Check for the presence of the archive directory. Added `--encrypt` for symmetric encryption through gpg (Eric Windisch). Added support for the digest command on Solaris 10 for MD5 checksums. Check for available disk space before extracting to the target directory (Andreas Schweitzer). Allow extraction to run asynchronously (patch by Peter Hatch). Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo). - * **v2.1.6:** Replaced one dot per file progress with a realtime progress percentage and a spinning cursor. Added `--noprogress` to prevent showing the progress during the decompression. Added `--target` dir to allow extracting directly to a target directory. (Guy Baconniere) - * **v2.2.0:** First major new release in years! Includes many bugfixes and user contributions. Please look at the [project page on Github][10] for all the details. - * **v2.3.0:** Support for archive encryption via GPG or OpenSSL. Added LZO and LZ4 compression support. Options to set the packaging date and stop the umask from being overriden. Optionally ignore check for available disk space when extracting. New option to check for root permissions before extracting. - * **v2.3.1:** Various compatibility updates. Added unit tests for Travis CI in the GitHub repo. New `--tar-extra`, `--untar-extra`, `--gpg-extra`, `--gpg-asymmetric-encrypt-sign` options. - * **v2.4.0:** Added optional support for SHA256 archive integrity checksums. - * **v2.4.2:** New --cleanup and --cleanup-args arguments for cleanup scripts. Added threading support for supported compressors. Now supports zstd compression. - * **v2.4.3:** Make explicit POSIX tar archives for increased compatibility. - * **v2.4.4:** Fixed various compatibility issues (no longer use POSIX tar archives), Github Actions to check on Solaris and FreeBSD. - * **v2.4.5:** Added `--tar-format` option to set the tar archive format (default is ustar) - -## Links - - * Check out the ["Loki Setup"][11] installer, used to install many Linux games and other applications, and of which I am the co-author. Since the demise of Loki, I am now the official maintainer of the project, and it is now being hosted here on GitHub. - * Bjarni R. Einarsson also wrote the **setup.sh** installer script, inspired by Makeself. [Check it out !][14] - -## Contact - -This script was written by [Stéphane Peter][15] (megastep at megastep.org). Any enhancements and suggestions are welcome. - -Contributions were included from John C. Quillan, Bjarni R. Einarsson, -Francois Petitjean, Ryan C. Gordon, and many contributors on GitHub. If you think I forgot -your name, don't hesitate to contact me. - -This project is now hosted on GitHub. Feel free to submit patches and bug reports on the [project page][10]. - -* * * - -[Stephane Peter][2] - - [1]: http://makeself.io/ - [2]: mailto:megastep@megastep.org - [3]: http://www.idsoftware.com/ - [4]: http://www.lokigames.com/products/myth2/updates.php3 - [5]: http://www.nvidia.com/ - [6]: http://earth.google.com/ - [7]: http://www.virtualbox.org/ - [8]: http://www.gnu.org/copyleft/gpl.html - [9]: https://github.com/megastep/makeself/releases/download/release-2.4.5/makeself-2.4.5.run - [10]: https://github.com/megastep/makeself - [11]: https://github.com/megastep/loki_setup/ - [12]: http://www.unrealtournament2003.com/ - [13]: http://www.icculus.org/ - [14]: http://bre.klaki.net/programs/setup.sh/ - [15]: https://stephanepeter.com/ diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION deleted file mode 100644 index 59aa62c1f..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/VERSION +++ /dev/null @@ -1 +0,0 @@ -2.4.5 diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh deleted file mode 100644 index b5692d490..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/make-release.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -# -# Create a distributable archive of the current version of Makeself - -VER=`cat VERSION` -mkdir -p /tmp/makeself-$VER release -cp -pPR makeself* test README.md COPYING VERSION .gitmodules /tmp/makeself-$VER/ -./makeself.sh --notemp /tmp/makeself-$VER release/makeself-$VER.run "Makeself v$VER" echo "Makeself has extracted itself" - diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh deleted file mode 100644 index 940903148..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself-header.sh +++ /dev/null @@ -1,660 +0,0 @@ -cat << EOF > "$archname" -#!/bin/bash -# This script was generated using Makeself $MS_VERSION -# The license covering this archive and its contents, if any, is wholly independent of the Makeself license (GPL) -# 2022.3.19-Modified the MS_Help function and some options -# Huawei Technologies Co., Ltd. - -ORIG_UMASK=\`umask\` - -CRCsum="$CRCsum" -MD5="$MD5sum" -SHA="$SHAsum" -SIGNATURE="$Signature" -TMPROOT=\${TMPDIR:="\$HOME"} -if ! test -d "\$TMPROOT"; then - TMPROOT="\$PWD" -fi -export TMPDIR="\$TMPROOT" -USER_PWD="\$PWD" -if ! test -d "\$USER_PWD"; then - exit 1 -fi -export USER_PWD -ARCHIVE_DIR=\`dirname "\$0"\` -export ARCHIVE_DIR - -name_of_file="\$0 " -pwd_of_file="\$PWD" -label="$LABEL" -script="$SCRIPT" -scriptargs="$SCRIPTARGS" -cleanup_script="${CLEANUP_SCRIPT}" -licensetxt="$LICENSE" -helpheader='$HELPHEADER' -targetdir="$archdirname" -filesizes="$filesizes" -totalsize="$totalsize" -keep="$KEEP" -nooverwrite="$NOOVERWRITE" -quiet="n" -accept="n" -nodiskspace="n" -export_conf="$EXPORT_CONF" -decrypt_cmd="$DECRYPT_CMD" -skip="$SKIP" - -print_cmd_arg="" -if type printf > /dev/null; then - print_cmd="printf" -elif test -x /usr/ucb/echo; then - print_cmd="/usr/ucb/echo" -else - print_cmd="echo" -fi - -if test -d /usr/xpg4/bin; then - PATH=/usr/xpg4/bin:\$PATH - export PATH -fi - -if test -d /usr/sfw/bin; then - PATH=\$PATH:/usr/sfw/bin - export PATH -fi - -unset CDPATH - -MS_Printf() -{ - \$print_cmd \$print_cmd_arg "\$1" -} - -MS_PrintLicense() -{ - PAGER=\${PAGER:=more} - if test x"\$licensetxt" != x; then - PAGER_PATH=\`exec <&- 2>&-; which \$PAGER || command -v \$PAGER || type \$PAGER\` - if test -x "\$PAGER_PATH"; then - echo "\$licensetxt" | \$PAGER - else - echo "\$licensetxt" - fi - if test x"\$accept" != xy; then - while true - do - MS_Printf "Please type y to accept, n otherwise: " - read yn - if test x"\$yn" = xn; then - keep=n - eval \$finish; exit 1 - break; - elif test x"\$yn" = xy; then - break; - fi - done - fi - fi -} - -MS_diskspace() -{ - ( - df -kP "\$1" | tail -1 | awk '{ if (\$4 ~ /%/) {print \$3} else {print \$4} }' - ) -} - -MS_dd() -{ - blocks=\`expr \$3 / 1024\` - bytes=\`expr \$3 % 1024\` - # Test for ibs, obs and conv feature - if dd if=/dev/zero of=/dev/null count=1 ibs=512 obs=512 conv=sync 2> /dev/null; then - dd if="\$1" ibs=\$2 skip=1 obs=1024 conv=sync 2> /dev/null | \\ - { test \$blocks -gt 0 && dd ibs=1024 obs=1024 count=\$blocks ; \\ - test \$bytes -gt 0 && dd ibs=1 obs=1024 count=\$bytes ; } 2> /dev/null - else - dd if="\$1" bs=\$2 skip=1 2> /dev/null - fi -} - -MS_dd_Progress() -{ - if test x"\$noprogress" = xy; then - MS_dd "\$@" - return \$? - fi - file="\$1" - offset=\$2 - length=\$3 - pos=0 - bsize=4194304 - while test \$bsize -gt \$length; do - bsize=\`expr \$bsize / 4\` - done - blocks=\`expr \$length / \$bsize\` - bytes=\`expr \$length % \$bsize\` - ( - dd ibs=\$offset skip=1 2>/dev/null - pos=\`expr \$pos \+ \$bsize\` - MS_Printf " 0%% " 1>&2 - if test \$blocks -gt 0; then - while test \$pos -le \$length; do - dd bs=\$bsize count=1 2>/dev/null - pcent=\`expr \$length / 100\` - pcent=\`expr \$pos / \$pcent\` - if test \$pcent -lt 100; then - MS_Printf "\b\b\b\b\b\b\b" 1>&2 - if test \$pcent -lt 10; then - MS_Printf " \$pcent%% " 1>&2 - else - MS_Printf " \$pcent%% " 1>&2 - fi - fi - pos=\`expr \$pos \+ \$bsize\` - done - fi - if test \$bytes -gt 0; then - dd bs=\$bytes count=1 2>/dev/null - fi - MS_Printf "\b\b\b\b\b\b\b" 1>&2 - MS_Printf " 100%% " 1>&2 - ) < "\$file" -} - -MS_Help() -{ - cat << EOH >&2 -Usage: \$0 [options] -Options: - --help | -h Print this message - --info Print embedded info : title, default target directory, embedded script ... - --list Print the list of files in the archive - --check Checks integrity and version dependency of the archive - --quiet Quiet install mode, skip human-computer interactions - --nox11 Do not spawn an xterm - --noexec Do not run embedded script - --extract= Extract directly to a target directory (absolute or relative) - Usually used with --noexec to just extract files without running - --tar arg1 [arg2 ...] Access the contents of the archive through the tar command -\${helpheader} -EOH -} - -MS_Verify_Sig() -{ - GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` - MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` - test -x "\$GPG_PATH" || GPG_PATH=\`exec <&- 2>&-; which gpg || command -v gpg || type gpg\` - test -x "\$MKTEMP_PATH" || MKTEMP_PATH=\`exec <&- 2>&-; which mktemp || command -v mktemp || type mktemp\` - offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` - temp_sig=\`mktemp -t XXXXX\` - echo \$SIGNATURE | base64 --decode > "\$temp_sig" - gpg_output=\`MS_dd "\$1" \$offset \$totalsize | LC_ALL=C "\$GPG_PATH" --verify "\$temp_sig" - 2>&1\` - gpg_res=\$? - rm -f "\$temp_sig" - if test \$gpg_res -eq 0 && test \`echo \$gpg_output | grep -c Good\` -eq 1; then - if test \`echo \$gpg_output | grep -c \$sig_key\` -eq 1; then - test x"\$quiet" = xn && echo "GPG signature is good" >&2 - else - echo "GPG Signature key does not match" >&2 - exit 2 - fi - else - test x"\$quiet" = xn && echo "GPG signature failed to verify" >&2 - exit 2 - fi -} - -MS_Check() -{ - OLD_PATH="\$PATH" - PATH=\${GUESS_MD5_PATH:-"\$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} - MD5_ARG="" - MD5_PATH=\`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum\` - test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which md5 || command -v md5 || type md5\` - test -x "\$MD5_PATH" || MD5_PATH=\`exec <&- 2>&-; which digest || command -v digest || type digest\` - PATH="\$OLD_PATH" - - SHA_PATH=\`exec <&- 2>&-; which shasum || command -v shasum || type shasum\` - test -x "\$SHA_PATH" || SHA_PATH=\`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum\` - - if test x"\$quiet" = xn; then - MS_Printf "Verifying archive integrity..." - fi - offset=\`head -n "\$skip" "\$1" | wc -c | tr -d " "\` - fsize=\`cat "\$1" | wc -c | tr -d " "\` - if test \$totalsize -ne \`expr \$fsize - \$offset\`; then - echo " Unexpected archive size." >&2 - exit 2 - fi - verb=\$2 - i=1 - for s in \$filesizes - do - crc=\`echo \$CRCsum | cut -d" " -f\$i\` - if test -x "\$SHA_PATH"; then - if test x"\`basename \$SHA_PATH\`" = xshasum; then - SHA_ARG="-a 256" - fi - sha=\`echo \$SHA | cut -d" " -f\$i\` - if test x"\$sha" = x0000000000000000000000000000000000000000000000000000000000000000; then - test x"\$verb" = xy && echo " \$1 does not contain an embedded SHA256 checksum." >&2 - else - shasum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$SHA_PATH \$SHA_ARG" | cut -b-64\`; - if test x"\$shasum" != x"\$sha"; then - echo "Error in SHA256 checksums: \$shasum is different from \$sha" >&2 - exit 2 - elif test x"\$quiet" = xn; then - MS_Printf " SHA256 checksums are OK." >&2 - fi - crc="0000000000"; - fi - fi - if test -x "\$MD5_PATH"; then - if test x"\`basename \$MD5_PATH\`" = xdigest; then - MD5_ARG="-a md5" - fi - md5=\`echo \$MD5 | cut -d" " -f\$i\` - if test x"\$md5" = x00000000000000000000000000000000; then - test x"\$verb" = xy && echo " \$1 does not contain an embedded MD5 checksum." >&2 - else - md5sum=\`MS_dd_Progress "\$1" \$offset \$s | eval "\$MD5_PATH \$MD5_ARG" | cut -b-32\`; - if test x"\$md5sum" != x"\$md5"; then - echo "Error in MD5 checksums: \$md5sum is different from \$md5" >&2 - exit 2 - elif test x"\$quiet" = xn; then - MS_Printf " MD5 checksums are OK." >&2 - fi - crc="0000000000"; verb=n - fi - fi - if test x"\$crc" = x0000000000; then - test x"\$verb" = xy && echo " \$1 does not contain a CRC checksum." >&2 - else - sum1=\`MS_dd_Progress "\$1" \$offset \$s | CMD_ENV=xpg4 cksum | awk '{print \$1}'\` - if test x"\$sum1" != x"\$crc"; then - echo "Error in checksums: \$sum1 is different from \$crc" >&2 - exit 2 - elif test x"\$quiet" = xn; then - MS_Printf " CRC checksums are OK." >&2 - fi - fi - i=\`expr \$i + 1\` - offset=\`expr \$offset + \$s\` - done - if test x"\$quiet" = xn; then - echo " All good." - fi -} - -MS_Decompress() -{ - if test x"\$decrypt_cmd" != x""; then - { eval "\$decrypt_cmd" || echo " ... Decryption failed." >&2; } | eval "$GUNZIP_CMD" - else - eval "$GUNZIP_CMD" - fi - - if test \$? -ne 0; then - echo " ... Decompression failed." >&2 - fi -} - -UnTAR() -{ - if test x"\$quiet" = xn; then - tar \$1vf - $UNTAR_EXTRA 2>&1 || { echo " ... Extraction failed." >&2; kill -15 \$$; } - else - tar \$1f - $UNTAR_EXTRA 2>&1 || { echo Extraction failed. >&2; kill -15 \$$; } - fi -} - -MS_exec_cleanup() { - if test x"\$cleanup" = xy && test x"\$cleanup_script" != x""; then - cleanup=n - cd "\$tmpdir" - eval "\"\$cleanup_script\" \$scriptargs \$cleanupargs" - fi -} - -MS_cleanup() -{ - echo 'Signal caught, cleaning up' >&2 - MS_exec_cleanup - cd "\$TMPROOT" - rm -rf "\$tmpdir" - eval \$finish; exit 15 -} - -Script_Args_Check() -{ - script_supported_args=\$(echo \${helpheader} | grep -o -E "\-\-[^ ]+" | awk -F"=" {'print \$1'}) - arg_to_test=\$(echo \$1|awk -F"=" {'print \$1'}) - - for arg in \${script_supported_args}; - do - if test x"\$arg_to_test" = x"\$arg" ;then - return - fi - done - - MS_Help - exit 1 -} - -finish=true -xterm_loop= -noprogress=$NOPROGRESS -nox11=$NOX11 -copy=$COPY -ownership=$OWNERSHIP -verbose=n -cleanup=y -cleanupargs= -sig_key= - -initargs="\$@" - -while [ -n "\$*" ] -do - case "\$1" in - -h | --help) - MS_Help - exit 0 - ;; - -q | --quiet) - quiet=y - noprogress=y - shift - ;; - --info) - echo Identification: "\$label" - echo Target directory: "\$targetdir" - echo Uncompressed size: $USIZE KB - echo Compression: $COMPRESS - if test x"$ENCRYPT" != x""; then - echo Encryption: $ENCRYPT - fi - echo Date of packaging: $DATE - echo Built with Makeself version $MS_VERSION - echo Build command was: "$MS_COMMAND" - if test x"\$script" != x; then - echo Script run after extraction: - echo " " \$script \$scriptargs - fi - if test x"$copy" = xcopy; then - echo "Archive will copy itself to a temporary location" - fi - if test x"$NEED_ROOT" = xy; then - echo "Root permissions required for extraction" - fi - if test x"$KEEP" = xy; then - echo "directory \$targetdir is permanent" - else - echo "\$targetdir will be removed after extraction" - fi - exit 0 - ;; - --list) - echo Target directory: \$targetdir - offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` - for s in \$filesizes - do - MS_dd "\$0" \$offset \$s | MS_Decompress | UnTAR t - offset=\`expr \$offset + \$s\` - done - exit 0 - ;; - --tar) - offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` - arg1="\$2" - shift 2 || { MS_Help; exit 1; } - for s in \$filesizes - do - MS_dd "\$0" \$offset \$s | MS_Decompress | tar "\$arg1" - "\$@" - offset=\`expr \$offset + \$s\` - done - exit 0 - ;; - --check) - MS_Check "\$0" y - scriptargs="\$scriptargs \$1" - shift - ;; - --noexec) - script="" - cleanup_script="" - shift - ;; - --extract=*) - keep=y - targetdir=\`echo \$1 | cut -d"=" -f2 \` - if ! shift; then MS_Help; exit 1; fi - ;; - --nox11) - nox11=y - shift - ;; - --xwin) - if test "$NOWAIT" = n; then - finish="echo Press Return to close this window...; read junk" - fi - xterm_loop=1 - shift - ;; - --phase2) - copy=phase2 - shift - ;; - --repack | --repack-path=*) - Script_Args_Check \$1 - scriptargs="\$scriptargs '\$1'" - shift - if [[ ! "\$1" =~ ^-.* ]]; then - scriptargs="\$scriptargs '\$1'" - shift - fi - ;; - *) - Script_Args_Check \$1 - scriptargs="\$scriptargs '\$1'" - shift - ;; - esac -done - -quiet_para="" -if test x"\$quiet" = xy; then - quiet_para="--quiet " -fi -scriptargs="--\$name_of_file""--\"\$pwd_of_file\""" \$quiet_para""\$scriptargs" - -if test x"\$quiet" = xy -a x"\$verbose" = xy; then - echo Cannot be verbose and quiet at the same time. >&2 - exit 1 -fi - -if test x"$NEED_ROOT" = xy -a \`id -u\` -ne 0; then - echo "Administrative privileges required for this archive (use su or sudo)" >&2 - exit 1 -fi - -if test x"\$copy" \!= xphase2; then - MS_PrintLicense -fi - -case "\$copy" in -copy) - tmpdir="\$TMPROOT"/makeself.\$RANDOM.\`date +"%y%m%d%H%M%S"\`.\$\$ - mkdir "\$tmpdir" || { - echo "Could not create temporary directory \$tmpdir" >&2 - exit 1 - } - SCRIPT_COPY="\$tmpdir/makeself" - echo "Copying to a temporary location..." >&2 - cp "\$0" "\$SCRIPT_COPY" - chmod +x "\$SCRIPT_COPY" - cd "\$TMPROOT" - exec "\$SCRIPT_COPY" --phase2 -- \$initargs - ;; -phase2) - finish="\$finish ; rm -rf \`dirname \$0\`" - ;; -esac - -if test x"\$nox11" = xn; then - if tty -s; then # Do we have a terminal? - : - else - if test x"\$DISPLAY" != x -a x"\$xterm_loop" = x; then # No, but do we have X? - if xset q > /dev/null 2>&1; then # Check for valid DISPLAY variable - GUESS_XTERMS="xterm gnome-terminal rxvt dtterm eterm Eterm xfce4-terminal lxterminal kvt konsole aterm terminology" - for a in \$GUESS_XTERMS; do - if type \$a >/dev/null 2>&1; then - XTERM=\$a - break - fi - done - chmod a+x \$0 || echo Please add execution rights on \$0 - if test \`echo "\$0" | cut -c1\` = "/"; then # Spawn a terminal! - exec \$XTERM -e "\$0 --xwin \$initargs" - else - exec \$XTERM -e "./\$0 --xwin \$initargs" - fi - fi - fi - fi -fi - -if test x"\$targetdir" = x.; then - tmpdir="." -else - if test x"\$keep" = xy; then - if test x"\$nooverwrite" = xy && test -d "\$targetdir"; then - echo "Target directory \$targetdir already exists, aborting." >&2 - exit 1 - fi - if test x"\$quiet" = xn; then - echo "Creating directory \$targetdir" >&2 - fi - tmpdir="\$targetdir" - dashp="-p" - else - tmpdir="\$TMPROOT/selfgz\$\$\$RANDOM" - dashp="" - fi - mkdir \$dashp "\$tmpdir" || { - echo 'Cannot create target directory' \$tmpdir >&2 - echo 'You should try option --extract=' >&2 - eval \$finish - exit 1 - } -fi - -location="\`pwd\`" -if test x"\$SETUP_NOCHECK" != x1; then - MS_Check "\$0" -fi -offset=\`head -n "\$skip" "\$0" | wc -c | tr -d " "\` - -if test x"\$verbose" = xy; then - MS_Printf "About to extract $USIZE KB in \$tmpdir ... Proceed ? [Y/n] " - read yn - if test x"\$yn" = xn; then - eval \$finish; exit 1 - fi -fi - -if test x"\$quiet" = xn; then - # Decrypting with openssl will ask for password, - # the prompt needs to start on new line - if test x"$ENCRYPT" = x"openssl"; then - echo "Decrypting and uncompressing \$label..." - else - MS_Printf "Uncompressing \$label" - fi -fi -res=3 -if test x"\$keep" = xn; then - trap MS_cleanup 1 2 3 15 -fi - -if test x"\$nodiskspace" = xn; then - leftspace=\`MS_diskspace "\$tmpdir"\` - if test -n "\$leftspace"; then - if test "\$leftspace" -lt $USIZE; then - echo - echo "Not enough space left in "\`dirname \$tmpdir\`" (\$leftspace KB) to decompress \$0 ($USIZE KB)" >&2 - if test x"\$keep" = xn; then - echo "Consider setting TMPDIR to a directory with more free space." - fi - eval \$finish; exit 1 - fi - fi -fi - -for s in \$filesizes -do - if MS_dd_Progress "\$0" \$offset \$s | MS_Decompress | ( cd "\$tmpdir"; umask \$ORIG_UMASK ; UnTAR xp ) 1>/dev/null; then - if test x"\$ownership" = xy; then - (cd "\$tmpdir"; chown -R \`id -u\` .; chgrp -R \`id -g\` .) - fi - else - echo >&2 - echo "Unable to decompress \$0" >&2 - eval \$finish; exit 1 - fi - offset=\`expr \$offset + \$s\` -done -if test x"\$quiet" = xn; then - echo -fi - -cd "\$tmpdir" -res=0 -if test x"\$script" != x; then - if test x"\$export_conf" = x"y"; then - MS_BUNDLE="\$0" - MS_LABEL="\$label" - MS_SCRIPT="\$script" - MS_SCRIPTARGS="\$scriptargs" - MS_ARCHDIRNAME="\$archdirname" - MS_KEEP="\$KEEP" - MS_NOOVERWRITE="\$NOOVERWRITE" - MS_COMPRESS="\$COMPRESS" - MS_CLEANUP="\$cleanup" - export MS_BUNDLE MS_LABEL MS_SCRIPT MS_SCRIPTARGS - export MS_ARCHDIRNAME MS_KEEP MS_NOOVERWRITE MS_COMPRESS - fi - - if test x"\$verbose" = x"y"; then - yn="x" - while test x"\$yn" != x -a x"\$yn" != xy -a x"\$yn" != xY -a x"\$yn" != xn -a x"\$yn" != xN - do - MS_Printf "OK to execute: \$script \$scriptargs \$* ? [Y/n] " - read yn - if test x"\$yn" = x -o x"\$yn" = xy -o x"\$yn" = xY; then - eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$?; - elif test x"\$yn" = xn -o x"\$yn" = xN; then - echo "Unable to decompress \$script ,because of aborting! ";res=\$? - else - echo "Input value is unacceptable,please try again." - fi - done - else - eval "\"\$script\" \$scriptargs \"\\\$@\""; res=\$? - fi - if test "\$res" -ne 0; then - test x"\$verbose" = xy && echo "The program '\$script' returned an error code (\$res)" >&2 - fi -fi - -MS_exec_cleanup - -if test x"\$keep" = xn; then - cd "\$TMPROOT" - rm -rf "\$tmpdir" -fi -eval \$finish; exit \$res -EOF diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 deleted file mode 100644 index 81bf6e4ff..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.1 +++ /dev/null @@ -1,110 +0,0 @@ -.TH "MAKESELF" "1" "2.4.5" -.SH "NAME" -makeself \- An utility to generate self-extractable archives. -.SH "SYNTAX" -.B makeself [\fIoptions\fP] archive_dir file_name label -.B [\fIstartup_script\fP] [\fIargs\fP] -.SH "DESCRIPTION" -This program is a free (GPL) utility designed to create self-extractable -archives from a directory. -.SH "OPTIONS" -The following options are supported. -.TP 15 -.B -v, --version -Prints out the makeself version number and exits. -.TP -.B -h, --help -Print out help information. -.TP -.B --tar-quietly -Suppress verbose output from the tar command -.TP -.B --quiet -Do not print any messages other than errors -.TP -.B --gzip -Compress using gzip (default if detected). -.TP -.B --bzip2 -Compress using bzip2. -.TP -.B --pbzip2 -Compress using pbzip2. -.TP -.B --xz -Compress using xz. -.TP -.B --lzo -Compress using lzop. -.TP -.B --lz4 -Compress using lz4. -.TP -.B --compress -Compress using the UNIX 'compress' command. -.TP -.B --nocomp -Do not compress the data. -.TP -.B --complevel lvl -Specify the compression level for gzip,bzip2,pbzui2,xz,lzo or lz4 -.TP -.B --notemp -The archive will create archive_dir in the current directory and -uncompress in ./archive_dir. -.TP -.B --copy -Upon extraction, the archive will first copy itself to a temporary directory. -.TP -.B --append -Append more files to an existing makeself archive. The label and startup scripts will then be ignored. -.TP -.B --current -Files will be extracted to the current directory. Both --current and --target dir imply --notemp. -.TP -.B --target dir -Extract directly to a target directory. Directory path can be either absolute or relative. -.TP -.B --header file -Specify location of the header script. -.TP -.B --cleanup file -Specify a cleanup script that executes on interrupt and when finished successfully. -.TP -.B --follow -Follow the symlinks in the archive. -.TP -.B --noprogress -Do not show the progress during the decompression. -.TP -.B --nox11 -Disable automatic spawn of an xterm if running in X11. -.TP -.B --nowait -Do not wait for user input after executing embedded program from an xterm. -.TP -.B --nomd5 -Do not create a MD5 checksum for the archive. -.TP -.B --nocrc -Do not create a CRC32 checksum for the archive. -.TP -.B --lsm file -LSM file describing the package. -.B --packaging-date date -Use provided string as the packaging date instead of the current date. -.SH "EXAMPLES" -Here is an example, assuming the user has a package image stored in a /home/joe/mysoft, -and he wants to generate a self-extracting package named mysoft.sh, which will launch -the "setup" script initially stored in /home/joe/mysoft: -.TP -makeself.sh /home/joe/mysoft mysoft.sh "Joe's Nice Software Package" ./setup -.TP -Here is also how I created the makeself.run archive which contains the Makeself distribution: -.TP -makeself.sh --notemp makeself makeself.run "Makeself by Stephane Peter" echo "Makeself has extracted itself" -.SH "AUTHORS" -Makeself has been written by Stéphane Peter . -.BR -This man page was originally written by Bartosz Fenski for the -Debian GNU/Linux distribution (but it may be used by others). diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm deleted file mode 100644 index 3c4cea8c1..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.lsm +++ /dev/null @@ -1,16 +0,0 @@ -Begin3 -Title: makeself.sh -Version: 2.4.5 -Description: makeself.sh is a shell script that generates a self-extractable - tar.gz archive from a directory. The resulting file appears as a shell - script, and can be launched as is. The archive will then uncompress - itself to a temporary directory and an arbitrary command will be - executed (for example an installation script). This is pretty similar - to archives generated with WinZip Self-Extractor in the Windows world. -Keywords: Installation archive tar winzip -Author: Stephane Peter (megastep@megastep.org) -Maintained-by: Stephane Peter (megastep@megastep.org) -Original-site: https://makeself.io/ -Platform: Unix -Copying-policy: GPL -End diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh deleted file mode 100644 index c8ea56597..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/makeself.sh +++ /dev/null @@ -1,822 +0,0 @@ -#!/bin/sh -# -# Makeself version 2.4.x -# by Stephane Peter -# -# Utility to create self-extracting tar.gz archives. -# The resulting archive is a file holding the tar.gz archive with -# a small Shell script stub that uncompresses the archive to a temporary -# directory and then executes a given script from withing that directory. -# -# Makeself home page: https://makeself.io/ -# -# Version 2.0 is a rewrite of version 1.0 to make the code easier to read and maintain. -# -# Version history : -# - 1.0 : Initial public release -# - 1.1 : The archive can be passed parameters that will be passed on to -# the embedded script, thanks to John C. Quillan -# - 1.2 : Package distribution, bzip2 compression, more command line options, -# support for non-temporary archives. Ideas thanks to Francois Petitjean -# - 1.3 : More patches from Bjarni R. Einarsson and Francois Petitjean: -# Support for no compression (--nocomp), script is no longer mandatory, -# automatic launch in an xterm, optional verbose output, and -target -# archive option to indicate where to extract the files. -# - 1.4 : Improved UNIX compatibility (Francois Petitjean) -# Automatic integrity checking, support of LSM files (Francois Petitjean) -# - 1.5 : Many bugfixes. Optionally disable xterm spawning. -# - 1.5.1 : More bugfixes, added archive options -list and -check. -# - 1.5.2 : Cosmetic changes to inform the user of what's going on with big -# archives (Quake III demo) -# - 1.5.3 : Check for validity of the DISPLAY variable before launching an xterm. -# More verbosity in xterms and check for embedded command's return value. -# Bugfix for Debian 2.0 systems that have a different "print" command. -# - 1.5.4 : Many bugfixes. Print out a message if the extraction failed. -# - 1.5.5 : More bugfixes. Added support for SETUP_NOCHECK environment variable to -# bypass checksum verification of archives. -# - 1.6.0 : Compute MD5 checksums with the md5sum command (patch from Ryan Gordon) -# - 2.0 : Brand new rewrite, cleaner architecture, separated header and UNIX ports. -# - 2.0.1 : Added --copy -# - 2.1.0 : Allow multiple tarballs to be stored in one archive, and incremental updates. -# Added --nochown for archives -# Stopped doing redundant checksums when not necesary -# - 2.1.1 : Work around insane behavior from certain Linux distros with no 'uncompress' command -# Cleaned up the code to handle error codes from compress. Simplified the extraction code. -# - 2.1.2 : Some bug fixes. Use head -n to avoid problems. -# - 2.1.3 : Bug fixes with command line when spawning terminals. -# Added --tar for archives, allowing to give arbitrary arguments to tar on the contents of the archive. -# Added --noexec to prevent execution of embedded scripts. -# Added --nomd5 and --nocrc to avoid creating checksums in archives. -# Added command used to create the archive in --info output. -# Run the embedded script through eval. -# - 2.1.4 : Fixed --info output. -# Generate random directory name when extracting files to . to avoid problems. (Jason Trent) -# Better handling of errors with wrong permissions for the directory containing the files. (Jason Trent) -# Avoid some race conditions (Ludwig Nussel) -# Unset the $CDPATH variable to avoid problems if it is set. (Debian) -# Better handling of dot files in the archive directory. -# - 2.1.5 : Made the md5sum detection consistent with the header code. -# Check for the presence of the archive directory -# Added --encrypt for symmetric encryption through gpg (Eric Windisch) -# Added support for the digest command on Solaris 10 for MD5 checksums -# Check for available disk space before extracting to the target directory (Andreas Schweitzer) -# Allow extraction to run asynchronously (patch by Peter Hatch) -# Use file descriptors internally to avoid error messages (patch by Kay Tiong Khoo) -# - 2.1.6 : Replaced one dot per file progress with a realtime progress percentage and a spining cursor (Guy Baconniere) -# Added --noprogress to prevent showing the progress during the decompression (Guy Baconniere) -# Added --target dir to allow extracting directly to a target directory (Guy Baconniere) -# - 2.2.0 : Many bugfixes, updates and contributions from users. Check out the project page on Github for the details. -# - 2.3.0 : Option to specify packaging date to enable byte-for-byte reproducibility. (Marc Pawlowsky) -# - 2.4.0 : Optional support for SHA256 checksums in archives. -# - 2.4.2 : Add support for threads for several compressors. (M. Limber) -# Added zstd support. -# - 2.4.3 : Make explicit POSIX tar archives for increased compatibility. -# - 2.4.5 : Added --tar-format to override ustar tar archive format -# -# (C) 1998-2021 by Stephane Peter -# -# This software is released under the terms of the GNU GPL version 2 and above -# Please read the license at http://www.gnu.org/copyleft/gpl.html -# Self-extracting archives created with this script are explictly NOT released under the term of the GPL -# - -MS_VERSION=2.4.5 -MS_COMMAND="$0" -unset CDPATH - -for f in ${1+"$@"}; do - MS_COMMAND="$MS_COMMAND \\\\ - \\\"$f\\\"" -done - -# For Solaris systems -if test -d /usr/xpg4/bin; then - PATH=/usr/xpg4/bin:$PATH - export PATH -fi - -# Procedures - -MS_Usage() -{ - echo "Usage: $0 [args] archive_dir file_name label startup_script [script_args]" - echo "args can be one or more of the following :" - echo " --version | -v : Print out Makeself version number and exit" - echo " --help | -h : Print out this help message" - echo " --tar-quietly : Suppress verbose output from the tar command" - echo " --quiet | -q : Do not print any messages other than errors." - echo " --gzip : Compress using gzip (default if detected)" - echo " --pigz : Compress with pigz" - echo " --zstd : Compress with zstd" - echo " --bzip2 : Compress using bzip2 instead of gzip" - echo " --pbzip2 : Compress using pbzip2 instead of gzip" - echo " --xz : Compress using xz instead of gzip" - echo " --lzo : Compress using lzop instead of gzip" - echo " --lz4 : Compress using lz4 instead of gzip" - echo " --compress : Compress using the UNIX 'compress' command" - echo " --complevel lvl : Compression level for gzip pigz zstd xz lzo lz4 bzip2 and pbzip2 (default 9)" - echo " --threads thds : Number of threads to be used by compressors that support parallelization." - echo " Omit to use compressor's default. Most useful (and required) for opting" - echo " into xz's threading, usually with '--threads=0' for all available cores." - echo " pbzip2 and pigz are parallel by default, and setting this value allows" - echo " limiting the number of threads they use." - echo " --base64 : Instead of compressing, encode the data using base64" - echo " --gpg-encrypt : Instead of compressing, encrypt the data using GPG" - echo " --gpg-asymmetric-encrypt-sign" - echo " : Instead of compressing, asymmetrically encrypt and sign the data using GPG" - echo " --gpg-extra opt : Append more options to the gpg command line" - echo " --ssl-encrypt : Instead of compressing, encrypt the data using OpenSSL" - echo " --ssl-passwd pass : Use the given password to encrypt the data using OpenSSL" - echo " --ssl-pass-src src : Use the given src as the source of password to encrypt the data" - echo " using OpenSSL. See \"PASS PHRASE ARGUMENTS\" in man openssl." - echo " If this option is not supplied, the user will be asked to enter" - echo " encryption password on the current terminal." - echo " --ssl-no-md : Do not use \"-md\" option not supported by older OpenSSL." - echo " --nochown : Do not give the target folder to the current user (default)" - echo " --chown : Give the target folder to the current user recursively" - echo " --nocomp : Do not compress the data" - echo " --notemp : The archive will create archive_dir in the" - echo " current directory and uncompress in ./archive_dir" - echo " --needroot : Check that the root user is extracting the archive before proceeding" - echo " --copy : Upon extraction, the archive will first copy itself to" - echo " a temporary directory" - echo " --append : Append more files to an existing Makeself archive" - echo " The label and startup scripts will then be ignored" - echo " --target dir : Extract directly to a target directory" - echo " directory path can be either absolute or relative" - echo " --nooverwrite : Do not extract the archive if the specified target directory exists" - echo " --current : Files will be extracted to the current directory" - echo " Both --current and --target imply --notemp" - echo " --tar-format opt : Specify a tar archive format (default is ustar)" - echo " --tar-extra opt : Append more options to the tar command line" - echo " --untar-extra opt : Append more options to the during the extraction of the tar archive" - echo " --nomd5 : Don't calculate an MD5 for archive" - echo " --nocrc : Don't calculate a CRC for archive" - echo " --sha256 : Compute a SHA256 checksum for the archive" - echo " --header file : Specify location of the header script" - echo " --cleanup file : Specify a cleanup script that executes on interrupt and when finished successfully." - echo " --follow : Follow the symlinks in the archive" - echo " --noprogress : Do not show the progress during the decompression" - echo " --nox11 : Disable automatic spawn of a xterm" - echo " --nowait : Do not wait for user input after executing embedded" - echo " program from an xterm" - echo " --sign passphrase : Signature private key to sign the package with" - echo " --lsm file : LSM file describing the package" - echo " --license file : Append a license file" - echo " --help-header file : Add a header to the archive's --help output" - echo " --packaging-date date" - echo " : Use provided string as the packaging date" - echo " instead of the current date." - echo - echo " --keep-umask : Keep the umask set to shell default, rather than overriding when executing self-extracting archive." - echo " --export-conf : Export configuration variables to startup_script" - echo - echo "Do not forget to give a fully qualified startup script name" - echo "(i.e. with a ./ prefix if inside the archive)." - exit 1 -} - -# Default settings -if type gzip >/dev/null 2>&1; then - COMPRESS=gzip -elif type compress >/dev/null 2>&1; then - COMPRESS=compress -else - echo "ERROR: missing commands: gzip, compress" >&2 - MS_Usage -fi -ENCRYPT=n -PASSWD="" -PASSWD_SRC="" -OPENSSL_NO_MD=n -COMPRESS_LEVEL=9 -DEFAULT_THREADS=123456 # Sentinel value -THREADS=$DEFAULT_THREADS -KEEP=n -CURRENT=n -NOX11=n -NOWAIT=n -APPEND=n -TAR_QUIETLY=n -KEEP_UMASK=n -QUIET=n -NOPROGRESS=n -COPY=none -NEED_ROOT=n -TAR_ARGS=rvf -TAR_FORMAT=ustar -TAR_EXTRA="" -GPG_EXTRA="" -DU_ARGS=-ks -HEADER=`dirname "$0"`/makeself-header.sh -SIGNATURE="" -TARGETDIR="" -NOOVERWRITE=n -DATE=`LC_ALL=C date` -EXPORT_CONF=n -SHA256=n -OWNERSHIP=n -SIGN=n -GPG_PASSPHRASE="" - -# LSM file stuff -LSM_CMD="echo No LSM. >> \"\$archname\"" - -while true -do - case "$1" in - --version | -v) - echo Makeself version $MS_VERSION - exit 0 - ;; - --pbzip2) - COMPRESS=pbzip2 - shift - ;; - --bzip2) - COMPRESS=bzip2 - shift - ;; - --gzip) - COMPRESS=gzip - shift - ;; - --pigz) - COMPRESS=pigz - shift - ;; - --zstd) - COMPRESS=zstd - shift - ;; - --xz) - COMPRESS=xz - shift - ;; - --lzo) - COMPRESS=lzo - shift - ;; - --lz4) - COMPRESS=lz4 - shift - ;; - --compress) - COMPRESS=compress - shift - ;; - --base64) - COMPRESS=base64 - shift - ;; - --gpg-encrypt) - COMPRESS=gpg - shift - ;; - --gpg-asymmetric-encrypt-sign) - COMPRESS=gpg-asymmetric - shift - ;; - --gpg-extra) - GPG_EXTRA="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --ssl-encrypt) - ENCRYPT=openssl - shift - ;; - --ssl-passwd) - PASSWD=$2 - shift 2 || { MS_Usage; exit 1; } - ;; - --ssl-pass-src) - PASSWD_SRC=$2 - shift 2 || { MS_Usage; exit 1; } - ;; - --ssl-no-md) - OPENSSL_NO_MD=y - shift - ;; - --nocomp) - COMPRESS=none - shift - ;; - --complevel) - COMPRESS_LEVEL="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --threads) - THREADS="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --nochown) - OWNERSHIP=n - shift - ;; - --chown) - OWNERSHIP=y - shift - ;; - --notemp) - KEEP=y - shift - ;; - --copy) - COPY=copy - shift - ;; - --current) - CURRENT=y - KEEP=y - shift - ;; - --tar-format) - TAR_FORMAT="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --tar-extra) - TAR_EXTRA="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --untar-extra) - UNTAR_EXTRA="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --target) - TARGETDIR="$2" - KEEP=y - shift 2 || { MS_Usage; exit 1; } - ;; - --sign) - SIGN=y - GPG_PASSPHRASE="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --nooverwrite) - NOOVERWRITE=y - shift - ;; - --needroot) - NEED_ROOT=y - shift - ;; - --header) - HEADER="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --cleanup) - CLEANUP_SCRIPT="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --license) - # We need to escape all characters having a special meaning in double quotes - LICENSE=$(sed 's/\\/\\\\/g; s/"/\\\"/g; s/`/\\\`/g; s/\$/\\\$/g' "$2") - shift 2 || { MS_Usage; exit 1; } - ;; - --follow) - TAR_ARGS=rvhf - DU_ARGS=-ksL - shift - ;; - --noprogress) - NOPROGRESS=y - shift - ;; - --nox11) - NOX11=y - shift - ;; - --nowait) - NOWAIT=y - shift - ;; - --nomd5) - NOMD5=y - shift - ;; - --sha256) - SHA256=y - shift - ;; - --nocrc) - NOCRC=y - shift - ;; - --append) - APPEND=y - shift - ;; - --lsm) - LSM_CMD="cat \"$2\" >> \"\$archname\"" - shift 2 || { MS_Usage; exit 1; } - ;; - --packaging-date) - DATE="$2" - shift 2 || { MS_Usage; exit 1; } - ;; - --help-header) - HELPHEADER=`sed -e "s/'/'\\\\\''/g" $2` - shift 2 || { MS_Usage; exit 1; } - [ -n "$HELPHEADER" ] && HELPHEADER="$HELPHEADER -" - ;; - --tar-quietly) - TAR_QUIETLY=y - shift - ;; - --keep-umask) - KEEP_UMASK=y - shift - ;; - --export-conf) - EXPORT_CONF=y - shift - ;; - -q | --quiet) - QUIET=y - shift - ;; - -h | --help) - MS_Usage - ;; - -*) - echo Unrecognized flag : "$1" - MS_Usage - ;; - *) - break - ;; - esac -done - -if test $# -lt 1; then - MS_Usage -else - if test -d "$1"; then - archdir="$1" - else - echo "Directory $1 does not exist." >&2 - exit 1 - fi -fi -archname="$2" - -if test "$QUIET" = "y" || test "$TAR_QUIETLY" = "y"; then - if test "$TAR_ARGS" = "rvf"; then - TAR_ARGS="rf" - elif test "$TAR_ARGS" = "rvhf"; then - TAR_ARGS="rhf" - fi -fi - -if test "$APPEND" = y; then - if test $# -lt 2; then - MS_Usage - fi - - # Gather the info from the original archive - OLDENV=`sh "$archname" --dumpconf` - if test $? -ne 0; then - echo "Unable to update archive: $archname" >&2 - exit 1 - else - eval "$OLDENV" - OLDSKIP=`expr $SKIP + 1` - fi -else - if test "$KEEP" = n -a $# = 3; then - echo "ERROR: Making a temporary archive with no embedded command does not make sense!" >&2 - echo >&2 - MS_Usage - fi - # We don't want to create an absolute directory unless a target directory is defined - if test "$CURRENT" = y; then - archdirname="." - elif test x"$TARGETDIR" != x; then - archdirname="$TARGETDIR" - else - archdirname=`basename "$1"` - fi - - if test $# -lt 3; then - MS_Usage - fi - - LABEL="$3" - SCRIPT="$4" - test "x$SCRIPT" = x || shift 1 - shift 3 - SCRIPTARGS="$*" -fi - -if test "$KEEP" = n -a "$CURRENT" = y; then - echo "ERROR: It is A VERY DANGEROUS IDEA to try to combine --notemp and --current." >&2 - exit 1 -fi - -case $COMPRESS in -gzip) - GZIP_CMD="gzip -c$COMPRESS_LEVEL" - GUNZIP_CMD="gzip -cd" - ;; -pigz) - GZIP_CMD="pigz -$COMPRESS_LEVEL" - if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated - GZIP_CMD="$GZIP_CMD --processes $THREADS" - fi - GUNZIP_CMD="gzip -cd" - ;; -zstd) - GZIP_CMD="zstd -$COMPRESS_LEVEL" - if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated - GZIP_CMD="$GZIP_CMD --threads=$THREADS" - fi - GUNZIP_CMD="zstd -cd" - ;; -pbzip2) - GZIP_CMD="pbzip2 -c$COMPRESS_LEVEL" - if test $THREADS -ne $DEFAULT_THREADS; then # Leave as the default if threads not indicated - GZIP_CMD="$GZIP_CMD -p$THREADS" - fi - GUNZIP_CMD="bzip2 -d" - ;; -bzip2) - GZIP_CMD="bzip2 -$COMPRESS_LEVEL" - GUNZIP_CMD="bzip2 -d" - ;; -xz) - GZIP_CMD="xz -c$COMPRESS_LEVEL" - # Must opt-in by specifying a value since not all versions of xz support threads - if test $THREADS -ne $DEFAULT_THREADS; then - GZIP_CMD="$GZIP_CMD --threads=$THREADS" - fi - GUNZIP_CMD="xz -d" - ;; -lzo) - GZIP_CMD="lzop -c$COMPRESS_LEVEL" - GUNZIP_CMD="lzop -d" - ;; -lz4) - GZIP_CMD="lz4 -c$COMPRESS_LEVEL" - GUNZIP_CMD="lz4 -d" - ;; -base64) - GZIP_CMD="base64" - GUNZIP_CMD="base64 --decode -i -" - ;; -gpg) - GZIP_CMD="gpg $GPG_EXTRA -ac -z$COMPRESS_LEVEL" - GUNZIP_CMD="gpg -d" - ENCRYPT="gpg" - ;; -gpg-asymmetric) - GZIP_CMD="gpg $GPG_EXTRA -z$COMPRESS_LEVEL -es" - GUNZIP_CMD="gpg --yes -d" - ENCRYPT="gpg" - ;; -compress) - GZIP_CMD="compress -fc" - GUNZIP_CMD="(type compress >/dev/null 2>&1 && compress -fcd || gzip -cd)" - ;; -none) - GZIP_CMD="cat" - GUNZIP_CMD="cat" - ;; -esac - -if test x"$ENCRYPT" = x"openssl"; then - if test x"$APPEND" = x"y"; then - echo "Appending to existing archive is not compatible with OpenSSL encryption." >&2 - fi - - ENCRYPT_CMD="openssl enc -aes-256-cbc -salt" - DECRYPT_CMD="openssl enc -aes-256-cbc -d" - - if test x"$OPENSSL_NO_MD" != x"y"; then - ENCRYPT_CMD="$ENCRYPT_CMD -md sha256" - DECRYPT_CMD="$DECRYPT_CMD -md sha256" - fi - - if test -n "$PASSWD_SRC"; then - ENCRYPT_CMD="$ENCRYPT_CMD -pass $PASSWD_SRC" - elif test -n "$PASSWD"; then - ENCRYPT_CMD="$ENCRYPT_CMD -pass pass:$PASSWD" - fi -fi - -tmpfile="${TMPDIR:-/tmp}/mkself$$" - -if test -f "$HEADER"; then - oldarchname="$archname" - archname="$tmpfile" - # Generate a fake header to count its lines - SKIP=0 - . "$HEADER" - SKIP=`cat "$tmpfile" |wc -l` - # Get rid of any spaces - SKIP=`expr $SKIP` - rm -f "$tmpfile" - if test "$QUIET" = "n"; then - echo "Header is $SKIP lines long" >&2 - fi - archname="$oldarchname" -else - echo "Unable to open header file: $HEADER" >&2 - exit 1 -fi - -if test "$QUIET" = "n"; then - echo -fi - -if test "$APPEND" = n; then - if test -f "$archname"; then - echo "WARNING: Overwriting existing file: $archname" >&2 - fi -fi - -USIZE=`du $DU_ARGS "$archdir" | awk '{print $1}'` - -if test "." = "$archdirname"; then - if test "$KEEP" = n; then - archdirname="makeself-$$-`date +%Y%m%d%H%M%S`" - fi -fi - -test -d "$archdir" || { echo "Error: $archdir does not exist."; rm -f "$tmpfile"; exit 1; } -if test "$QUIET" = "n"; then - echo "About to compress $USIZE KB of data..." - echo "Adding files to archive named \"$archname\"..." -fi - -# See if we have GNU tar -TAR=`exec <&- 2>&-; which gtar || command -v gtar || type gtar` -test -x "$TAR" || TAR=tar - -tmparch="${TMPDIR:-/tmp}/mkself$$.tar" -( - if test "$APPEND" = "y"; then - tail -n "+$OLDSKIP" "$archname" | eval "$GUNZIP_CMD" > "$tmparch" - fi - cd "$archdir" - # "Determining if a directory is empty" - # https://www.etalabs.net/sh_tricks.html - find . \ - \( \ - ! -type d \ - -o \ - \( -links 2 -exec sh -c ' - is_empty () ( - cd "$1" - set -- .[!.]* ; test -f "$1" && return 1 - set -- ..?* ; test -f "$1" && return 1 - set -- * ; test -f "$1" && return 1 - return 0 - ) - is_empty "$0"' {} \; \ - \) \ - \) -print \ - | LC_ALL=C sort \ - | sed 's/./\\&/g' \ - | xargs $TAR $TAR_EXTRA --format $TAR_FORMAT -$TAR_ARGS "$tmparch" -) || { - echo "ERROR: failed to create temporary archive: $tmparch" - rm -f "$tmparch" "$tmpfile" - exit 1 -} - -USIZE=`du $DU_ARGS "$tmparch" | awk '{print $1}'` - -eval "$GZIP_CMD" <"$tmparch" >"$tmpfile" || { - echo "ERROR: failed to create temporary file: $tmpfile" - rm -f "$tmparch" "$tmpfile" - exit 1 -} -rm -f "$tmparch" - -if test x"$ENCRYPT" = x"openssl"; then - echo "About to encrypt archive \"$archname\"..." - { eval "$ENCRYPT_CMD -in $tmpfile -out ${tmpfile}.enc" && mv -f ${tmpfile}.enc $tmpfile; } || \ - { echo Aborting: could not encrypt temporary file: "$tmpfile".; rm -f "$tmpfile"; exit 1; } -fi - -fsize=`cat "$tmpfile" | wc -c | tr -d " "` - -# Compute the checksums - -shasum=0000000000000000000000000000000000000000000000000000000000000000 -md5sum=00000000000000000000000000000000 -crcsum=0000000000 - -if test "$NOCRC" = y; then - if test "$QUIET" = "n"; then - echo "skipping crc at user request" - fi -else - crcsum=`CMD_ENV=xpg4 cksum < "$tmpfile" | sed -e 's/ /Z/' -e 's/ /Z/' | cut -dZ -f1` - if test "$QUIET" = "n"; then - echo "CRC: $crcsum" - fi -fi - -if test "$SHA256" = y; then - SHA_PATH=`exec <&- 2>&-; which shasum || command -v shasum || type shasum` - if test -x "$SHA_PATH"; then - shasum=`eval "$SHA_PATH -a 256" < "$tmpfile" | cut -b-64` - else - SHA_PATH=`exec <&- 2>&-; which sha256sum || command -v sha256sum || type sha256sum` - shasum=`eval "$SHA_PATH" < "$tmpfile" | cut -b-64` - fi - if test "$QUIET" = "n"; then - if test -x "$SHA_PATH"; then - echo "SHA256: $shasum" - else - echo "SHA256: none, SHA command not found" - fi - fi -fi -if test "$NOMD5" = y; then - if test "$QUIET" = "n"; then - echo "Skipping md5sum at user request" - fi -else - # Try to locate a MD5 binary - OLD_PATH=$PATH - PATH=${GUESS_MD5_PATH:-"$OLD_PATH:/bin:/usr/bin:/sbin:/usr/local/ssl/bin:/usr/local/bin:/opt/openssl/bin"} - MD5_ARG="" - MD5_PATH=`exec <&- 2>&-; which md5sum || command -v md5sum || type md5sum` - test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which md5 || command -v md5 || type md5` - test -x "$MD5_PATH" || MD5_PATH=`exec <&- 2>&-; which digest || command -v digest || type digest` - PATH=$OLD_PATH - if test -x "$MD5_PATH"; then - if test `basename ${MD5_PATH}`x = digestx; then - MD5_ARG="-a md5" - fi - md5sum=`eval "$MD5_PATH $MD5_ARG" < "$tmpfile" | cut -b-32` - if test "$QUIET" = "n"; then - echo "MD5: $md5sum" - fi - else - if test "$QUIET" = "n"; then - echo "MD5: none, MD5 command not found" - fi - fi -fi -if test "$SIGN" = y; then - GPG_PATH=`exec <&- 2>&-; which gpg || command -v gpg || type gpg` - if test -x "$GPG_PATH"; then - SIGNATURE=`$GPG_PATH --pinentry-mode=loopback --batch --yes --passphrase "$GPG_PASSPHRASE" --output - --detach-sig $tmpfile | base64 | tr -d \\\\n` - if test "$QUIET" = "n"; then - echo "Signature: $SIGNATURE" - fi - else - echo "Missing gpg command" >&2 - fi -fi - -totalsize=0 -for size in $fsize; -do - totalsize=`expr $totalsize + $size` -done - -if test "$APPEND" = y; then - mv "$archname" "$archname".bak || exit - - # Prepare entry for new archive - filesizes="$fsize" - CRCsum="$crcsum" - MD5sum="$md5sum" - SHAsum="$shasum" - Signature="$SIGNATURE" - # Generate the header - . "$HEADER" - # Append the new data - cat "$tmpfile" >> "$archname" - - chmod +x "$archname" - rm -f "$archname".bak - if test "$QUIET" = "n"; then - echo "Self-extractable archive \"$archname\" successfully updated." - fi -else - filesizes="$fsize" - CRCsum="$crcsum" - MD5sum="$md5sum" - SHAsum="$shasum" - Signature="$SIGNATURE" - - # Generate the header - . "$HEADER" - - # Append the compressed tar data after the stub - if test "$QUIET" = "n"; then - echo - fi - cat "$tmpfile" >> "$archname" - chmod +x "$archname" - if test "$QUIET" = "n"; then - echo Self-extractable archive \"$archname\" successfully created. - fi -fi -rm -f "$tmpfile" diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh deleted file mode 100644 index 31ee16511..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/makeself/run-tests.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -# Run every available test - Bash needed -cd test -for test in *test; -do - echo "Running test $test ..." - bash $test || { echo "*** ERROR: Test '$test' failed!"; exit 1; } -done diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh deleted file mode 100644 index a977bd51d..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/merge_aicpu_info_json.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -project_path=$1 -build_path=$2 -vendor_name=customize -echo $@ -if [[ ! -d "$project_path" ]]; then - echo "[ERROR] No projcet path is provided" - exit 1 -fi - -if [[ ! -d "$build_path" ]]; then - echo "[ERROR] No build path is provided" - exit 1 -fi - -if [[ ! -d "$ASCEND_OPP_PATH" ]]; then - echo "[ERROR] No opp install path is provided" - exit 1 -fi -custom_exist_info_json=$ASCEND_OPP_PATH/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json -custom_new_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/cust_aicpu_kernel.json -temp_info_json=$build_path/makepkg/packages/vendors/$vendor_name/op_impl/cpu/config/temp_cust_aicpu_kernel.json - -if [[ -f "$custom_exist_info_json" ]] && [[ -f "$custom_new_info_json" ]]; then - cp -f $custom_exist_info_json $temp_info_json - chmod +w $temp_info_json - python3 ${project_path}/cmake/util/insert_op_info.py ${custom_new_info_json} ${temp_info_json} - cp -f $temp_info_json $custom_new_info_json - rm -f $temp_info_json -fi diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py deleted file mode 100644 index 0d3221602..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/opdesc_parser.py +++ /dev/null @@ -1,298 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import sys -import os - - -OP_ALL = '__ALLOP__' -SOC_ALL = '__ALLSOC__' -SOC_TO_SHORT_SOC_MAP = { - "ascend910a": "ascend910", - "ascend910proa": "ascend910", - "ascend910b": "ascend910", - "ascend910prob": "ascend910", - "ascend910premiuma": "ascend910", - "ascend910b1": "ascend910b", - "ascend910b2": "ascend910b", - "ascend910b2c": "ascend910b", - "ascend910b3": "ascend910b", - "ascend910b4": "ascend910b", - "ascend910c1": "ascend910c", - "ascend910c2": "ascend910c", - "ascend910c3": "ascend910c", - "ascend910c4": "ascend910c", - "ascend310p1": "ascend310p", - "ascend310p3": "ascend310p", - "ascend310p3vir01": "ascend310p", - "ascend310p3vir02": "ascend310p", - "ascend310p3vir04": "ascend310p", - "ascend310p3vir08": "ascend310p", - "ascend310b1": "ascend310b", - "bs9sx1aa": "bs9sx1a" -} -CONFLICT_KEYWORDS = { - "and", "as", "assert", "break", "class", "continue", "def", "del", "elif", "else", - "except", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", - "not", "or", "pass", "raise", "return", "try", "while", "with", "yield", "False", - "None", "True", "nonlocal", "arg", "__inputs__", "__outputs__", "options", "bisheng", - "bisheng_path", "tikcpp_path", "impl_mode", "custom_compile_options", - "custom_all_compile_options", "soc_version", "soc_short", "custom_compile_options_soc", - "custom_all_compile_options_soc", "origin_func_name", "ascendc_src_dir_ex", - "ascendc_src_dir", "ascendc_src_file", "src", "op_type", "code_channel", "op_info", - "compile_op", "get_code_channel", "result", "__attrs__", "isinstance", "attr", - "get_current_build_config", "_build_args", "get_dtype_fmt_options", "shutil", "os", - "get_kernel_source" -} - - -class OpDesc: - def __init__(self: any, op_type: str): - self.op_type = op_type - self.attr_list = [] - self.attr_val = {} - self.input_name = [] - self.input_ori_name = [] - self.input_type = [] - self.input_dtype = [] - self.input_fmt = [] - self.output_name = [] - self.output_ori_name = [] - self.output_type = [] - self.output_dtype = [] - self.output_fmt = [] - self.op_fmt_sel = False - self.op_chk_support = False - self.op_intf = '' - self.kern_name = '' - self.op_file = '' - self.op_replay_flag = False - self.op_replay_batch = False - self.input_idx = -1 - self.output_idx = -1 - self.max_block_dim = 32 - self.max_shape_size = 268435456 - self.dynamic_shape = False - self.op_range_limit = '' - self.custom_compile_options = {} - self.custom_all_compile_options = {} - - @staticmethod - def _parse_digit(conf: str) -> int: - return int(conf.split('=')[1]) - - @staticmethod - def _parse_flag(conf: str) -> bool: - if 'true' == conf.split('=')[1]: - return True - return False - - @staticmethod - def _parse_str(conf: str) -> str: - return conf.split('=')[1] - - @staticmethod - def _parse_list(conf: str) -> list: - return conf.split('=')[1].split(',') - - def parse_input(self: any, conf: str): - if conf.startswith('input{}.name'.format(int(self.input_idx) + 1)): - self.input_idx += 1 - self.input_ori_name.append(self._parse_str(conf)) - self.input_name.append(self.input_ori_name[-1] + '_in__') - elif conf.startswith('input{}.paramType'.format(int(self.input_idx))): - self.input_type.append(self._parse_str(conf)) - elif conf.startswith('input{}.dtype'.format(int(self.input_idx))): - self.input_dtype.append(self._parse_str(conf)) - elif conf.startswith('input{}.format'.format(int(self.input_idx))): - self.input_fmt.append(self._parse_str(conf)) - else: - return - - def parse_output(self: any, conf: str): - if conf.startswith('output{}.name'.format(int(self.output_idx) + 1)): - self.output_idx += 1 - self.output_ori_name.append(self._parse_str(conf)) - self.output_name.append(self.output_ori_name[-1] + '_out_') - elif conf.startswith('output{}.paramType'.format(int(self.output_idx))): - self.output_type.append(self._parse_str(conf)) - elif conf.startswith('output{}.dtype'.format(int(self.output_idx))): - self.output_dtype.append(self._parse_str(conf)) - elif conf.startswith('output{}.format'.format(int(self.output_idx))): - self.output_fmt.append(self._parse_str(conf)) - else: - return - - def parse_op_format(self: any, conf: str): - self.op_fmt_sel = self._parse_flag(conf) - - def parse_check_support(self: any, conf: str): - self.op_chk_support = self._parse_flag(conf) - - def parse_range_limit(self: any, conf: str): - self.op_range_limit = self._parse_str(conf) - - def parse_kern_name(self: any, conf: str): - self.kern_name = self._parse_str(conf) - - def parse_op_intf(self: any, conf: str): - self.op_intf = self._parse_str(conf) - - def parse_op_file(self: any, conf: str): - self.op_file = self._parse_str(conf) - - def parse_dynamic_shape(self: any, conf: str): - self.dynamic_shape = self._parse_flag(conf) - - def parse_attr_list(self: any, conf: str): - self.attr_list = self._parse_list(conf) - intersection_element = set(self.attr_list) & CONFLICT_KEYWORDS - if intersection_element: - raise Exception(f'The attribute name: {intersection_element} in op: {{\'{self.op_type}\'}} conflicts with\ - the built-in variable name. Use a complex name or prefix the operator name.') - - @staticmethod - def _camel_to_snake(camel_case_str: str): - snake_case_str = '' - for i, c in enumerate(camel_case_str): - if i == 0: - snake_case_str += c.lower() - elif c.isupper(): - snake_case_str += '_' + c.lower() - else: - snake_case_str += c - return snake_case_str - - def parse_attr_val(self: any, conf: str): - for attr in self.attr_list: - if self.attr_val.get(attr) is None: - self.attr_val[attr] = {} - if conf.startswith('attr_{}.type'.format(attr)): - self.attr_val.get(attr)['type'] = self._camel_to_snake(self._parse_str(conf)) - elif conf.startswith('attr_{}.paramType'.format(attr)): - self.attr_val.get(attr)['paramType'] = self._parse_str(conf) - elif conf.startswith('attr_{}.defaultValue'.format(attr)): - self.attr_val.get(attr)['defaultValue'] = self._parse_str(conf) - - def parse_replay_val(self: any, batch_list: list, iterator_list: list): - if self.op_type in batch_list: - self.op_replay_flag = True - self.op_replay_batch = True - elif self.op_type in iterator_list: - self.op_replay_flag = True - self.op_replay_batch = False - - -def _is_op_type_in_opdesc(op_descs: list, op_type: str): - for op in op_descs: - if op_type == op.op_type: - return True - return False - - -def _set_all_options_to_opdescs(op_descs, soc_ver_compile_options): - for op in op_descs: - op.custom_all_compile_options = soc_ver_compile_options - - -def _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options): - for op in op_descs: - if op.op_type != op_type: - continue - op.custom_compile_options = soc_ver_compile_options - - -def _trans_soc_ver_to_short(soc_ver: str): - low_soc_ver = soc_ver.lower() - if low_soc_ver not in SOC_TO_SHORT_SOC_MAP: - print(f'WARNING: caution: {soc_ver} will trans into ascend910, if not your intention,' - f'use ascend910b1~4 instead') - return SOC_TO_SHORT_SOC_MAP[low_soc_ver] - - -def _get_op_custom_options(op_descs: list, auto_gen_dir: str): - if auto_gen_dir is None: - return - file = os.path.join(auto_gen_dir, "custom_compile_options.ini") - if not os.path.exists(file): - print(f'WARNING: cannot find {auto_gen_dir}/custom_compile_options.ini') - return - with open(file, 'r') as fd: - lines = fd.readlines() - for line in lines: - param_list = str.split(line.rstrip('\n'), ',') - if len(param_list) != 3: - raise Exception(f'ERROR: custom compile option {param_list} len is not 3') - op_type = param_list[0] - if op_type.upper() == 'ALL': - op_type = OP_ALL - if op_type != OP_ALL and _is_op_type_in_opdesc(op_descs, op_type) == False: - print(f'WARNING: op: {op_type} are not exists in this project') - continue - soc_ver_compile_options = {} - soc_ver = param_list[1] - options_str = param_list[2] - options = str.split(options_str, ';') - if soc_ver == '': - soc_ver_compile_options[SOC_ALL] = options - else: - soc_ver_list = str.split(soc_ver, ';') - for ver in soc_ver_list: - short_ver = _trans_soc_ver_to_short(ver) - soc_ver_compile_options[short_ver] = options - if op_type == OP_ALL: - _set_all_options_to_opdescs(op_descs, soc_ver_compile_options) - else: - _set_options_to_opdesc(op_descs, op_type, soc_ver_compile_options) - - -def get_op_desc(file: str, batch_list: list, iterator_list: list, builder: any, - op_type: list, auto_gen_dir: str = None) -> list: - op_descs = [] - op_match = False - with open (file, 'r') as fd: - lines = fd.readlines() - for line in lines: - line = line.strip() - if line.startswith('['): - name = line[1:-1] - if op_type is None or name in op_type: - op_match = True - op_desc = builder(name) - op_desc.parse_replay_val(batch_list, iterator_list) - op_descs.append(op_desc) - else: - op_match = False - if op_type is not None and len(op_descs) == len(op_type): - return op_descs - continue - if not op_match: - continue - if line.startswith('input'): - op_desc.parse_input(line) - elif line.startswith('output'): - op_desc.parse_output(line) - elif line.startswith('dynamicFormat.flag'): - op_desc.parse_op_format(line) - elif line.startswith('needCheckSupport.flag'): - op_desc.parse_check_support(line) - elif line.startswith('rangeLimit.value'): - op_desc.parse_range_limit(line) - elif line.startswith('opInterface.value'): - op_desc.parse_op_intf(line) - elif line.startswith('kernel.name'): - op_desc.parse_kern_name(line) - elif line.startswith('opFile.value'): - op_desc.parse_op_file(line) - elif line.startswith('dynamicShapeSupport.flag'): - op_desc.parse_dynamic_shape(line) - elif line.startswith('attr.list'): - op_desc.parse_attr_list(line) - elif line.startswith('attr_'): - op_desc.parse_attr_val(line) - _get_op_custom_options(op_descs, auto_gen_dir) - return op_descs diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py deleted file mode 100644 index ac931e5d3..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/parse_ini_to_json.py +++ /dev/null @@ -1,338 +0,0 @@ -# Copyright 2020-2021 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -parser ini to json -""" - -import json -import os -import stat -import sys - - -ATTR_TYPE_LIST = ["int", "float", "bool", "str", "listInt", "listFloat", "listBool", "listStr", "listListInt", - "type", "listType", "tensor", "listTensor"] -ATTR_PARAMTYPE_LIST = ["optional", "required"] -BOOL_FLAG_KEY = ["dynamicFormat", "dynamicShapeSupport", "dynamicRankSupport", "precision_reduce", "heavyOp", - "needCheckSupport", "enableVectorCore"] -BOOL_LIST = ["true", "false"] -DTYPE_LIST = ["float16", "float", "float32", "int8", "int16", "int32", "uint8", "uint16", "uint32", "bool", - "int64", "uint64", "qint8", "qint16", "qint32", "quint8", "quint16", "double", "complex64", - "complex128", "string", "resource", "dual", "dual_sub_int8", "dual_sub_uint8", "string_ref", - "int4", "bfloat16", "uint1"] -FORMAT_LIST = ["NCHW", "NHWC", "ND", "NC1HWC0", "FRACTAL_Z", "NC1C0HWPAD", "NHWC1C0", "FSR_NCHW", "FRACTAL_DECONV", - "C1HWNC0", "FRACTAL_DECONV_TRANSPOSE", "FRACTAL_DECONV_SP_STRIDE_TRANS", "NC1HWC0_C04", - "FRACTAL_Z_C04", "CHWN", "FRACTAL_DECONV_SP_STRIDE8_TRANS", "HWCN", "NC1KHKWHWC0", "BN_WEIGHT", - "FILTER_HWCK", "HASHTABLE_LOOKUP_LOOKUPS", "HASHTABLE_LOOKUP_KEYS", "HASHTABLE_LOOKUP_VALUE", - "HASHTABLE_LOOKUP_OUTPUT", "HASHTABLE_LOOKUP_HITS", "C1HWNCoC0", "MD", "NDHWC", "FRACTAL_ZZ", - "FRACTAL_NZ", "NCDHW", "DHWCN", "NDC1HWC0", "FRACTAL_Z_3D", "CN", "NC", "DHWNC", - "FRACTAL_Z_3D_TRANSPOSE", "FRACTAL_ZN_LSTM", "FRACTAL_ZN_RNN", "FRACTAL_Z_G", "NULL"] - - -def parse_ini_files(ini_files): - """ - parse ini files to json - Parameters: - ---------------- - ini_files:input file list - return:ops_info - ---------------- - """ - tbe_ops_info = {} - for ini_file in ini_files: - check_file_size(ini_file) - parse_ini_to_obj(ini_file, tbe_ops_info) - return tbe_ops_info - - -def check_file_size(input_file): - try: - file_size = os.path.getsize(input_file) - except OSError as os_error: - print('[ERROR] Failed to open "%s". %s' % (input_file, str(os_error))) - raise OSError from os_error - if file_size > 10*1024*1024: - print('[WARN] The size of %s exceeds 10MB, it may take more time to run, please wait.' % input_file) - - -def parse_ini_to_obj(ini_file, tbe_ops_info): - """ - parse ini file to json obj - Parameters: - ---------------- - ini_file:ini file path - tbe_ops_info:ops_info - ---------------- - """ - with open(ini_file) as ini_file: - lines = ini_file.readlines() - op_dict = {} - op_name = "" - find_op_type = False - for line in lines: - line = line.rstrip() - if line == "": - continue - if line.startswith("["): - if line.endswith("]"): - op_name = line[1:-1] - op_dict = {} - tbe_ops_info[op_name] = op_dict - find_op_type = True - elif "=" in line: - key1 = line[:line.index("=")] - key2 = line[line.index("=")+1:] - key1_0, key1_1 = key1.split(".") - if key1_0 not in op_dict: - op_dict[key1_0] = {} - if key1_1 in op_dict.get(key1_0): - raise RuntimeError("Op:" + op_name + " " + key1_0 + " " + - key1_1 + " is repeated!") - dic_key = op_dict.get(key1_0) - dic_key[key1_1] = key2 - else: - continue - if not find_op_type: - raise RuntimeError("Not find OpType in .ini file.") - - -def check_output_exist(op_dict, is_valid): - """ - Function Description: - Check output is exist - Parameter: op_dict - Parameter: is_valid - """ - if "output0" in op_dict: - output0_dict = op_dict.get("output0") - if output0_dict.get("name", None) is None: - is_valid = False - print("output0.name is required in .ini file!") - else: - is_valid = False - print("output0 is required in .ini file!") - return is_valid - - -def check_attr_dict(attr_dict, is_valid, attr): - """ - Function Description: - Check attr_dict - Parameter: attr_dict - Parameter: is_valid - Parameter: attr - """ - attr_type = attr_dict.get("type") - value = attr_dict.get("value") - param_type = attr_dict.get("paramType") - if attr_type is None or value is None: - is_valid = False - print("If attr.list is exist, {0}.type and {0}.value is required".format(attr)) - if param_type and param_type not in ATTR_PARAMTYPE_LIST: - is_valid = False - print("{0}.paramType only support {1}.".format(attr, ATTR_PARAMTYPE_LIST)) - if attr_type and attr_type not in ATTR_TYPE_LIST: - is_valid = False - print("{0}.type only support {1}.".format(attr, ATTR_TYPE_LIST)) - return is_valid - - -def check_attr(op_dict, is_valid): - """ - Function Description: - Check attr - Parameter: op_dict - Parameter: is_valid - """ - if "attr" in op_dict: - attr_dict = op_dict.get("attr") - attr_list_str = attr_dict.get("list", None) - if attr_list_str is None: - is_valid = False - print("attr.list is required in .ini file!") - else: - attr_list = attr_list_str.split(",") - for attr_name in attr_list: - attr = "attr_" + attr_name.strip() - attr_dict = op_dict.get(attr) - if attr_dict: - is_valid = check_attr_dict(attr_dict, is_valid, attr) - else: - is_valid = False - print("%s is required in .ini file, when attr.list is %s!" % (attr, attr_list_str)) - return is_valid - - -def check_bool_flag(op_dict, is_valid): - """ - Function Description: - check_bool_flag - Parameter: op_dict - Parameter: is_valid - """ - for key in BOOL_FLAG_KEY: - if key in op_dict: - op_bool_key = op_dict.get(key) - if op_bool_key.get("flag").strip() not in BOOL_LIST: - is_valid = False - print("{0}.flag only support {1}.".format(key, BOOL_LIST)) - return is_valid - - -def check_type_format(op_info, is_valid, op_info_key): - """ - Function Description: - Check type and format - Parameter: op_info - Parameter: is_valid - Parameter: op_info_key - """ - op_info_dtype_str = op_info.get("dtype") - op_info_dtype_num = 0 - op_info_format_num = 0 - if op_info_dtype_str: - op_info_dtype = op_info_dtype_str.split(",") - op_info_dtype_num = len(op_info_dtype) - for dtype in op_info_dtype: - if dtype.strip() not in DTYPE_LIST: - is_valid = False - print("{0}.dtype not support {1}.".format(op_info_key, dtype)) - op_info_format_str = op_info.get("format") - if op_info_format_str: - op_info_format = op_info_format_str.split(",") - op_info_format_num = len(op_info_format) - for op_format in op_info_format: - if op_format.strip() not in FORMAT_LIST: - is_valid = False - print("{0}.format not support {1}.".format(op_info_key, op_format)) - if op_info_dtype_num > 0 and op_info_format_num > 0: - if op_info_dtype_num != op_info_format_num: - is_valid = False - print("The number of {0}.dtype not match the number of {0}.format.".format(op_info_key)) - return is_valid - - -def check_op_info(tbe_ops): - """ - Function Description: - Check info. - Parameter: tbe_ops - Return Value: is_valid - """ - print("\n\n==============check valid for ops info start==============") - required_op_input_info_keys = ["paramType", "name"] - required_op_output_info_keys = ["paramType", "name"] - param_type_valid_value = ["dynamic", "optional", "required"] - is_valid = True - for op_key in tbe_ops: - op_dict = tbe_ops[op_key] - for op_info_key in op_dict: - if op_info_key.startswith("input"): - op_input_info = op_dict[op_info_key] - missing_keys = [] - for required_op_input_info_key in required_op_input_info_keys: - if required_op_input_info_key not in op_input_info: - missing_keys.append(required_op_input_info_key) - if len(missing_keys) > 0: - print("op: " + op_key + " " + op_info_key + " missing: " + - ",".join(missing_keys)) - is_valid = False - else: - if not op_input_info["paramType"] in param_type_valid_value: - print("op: " + op_key + " " + op_info_key + \ - " paramType not valid, valid key:[dynamic, " - "optional, required]") - is_valid = False - is_valid = check_type_format(op_input_info, is_valid, op_info_key) - if op_info_key.startswith("output"): - op_input_info = op_dict[op_info_key] - missing_keys = [] - for required_op_input_info_key in required_op_output_info_keys: - if required_op_input_info_key not in op_input_info: - missing_keys.append(required_op_input_info_key) - if len(missing_keys) > 0: - print("op: " + op_key + " " + op_info_key + " missing: " + - ",".join(missing_keys)) - is_valid = False - else: - if not op_input_info["paramType"] in param_type_valid_value: - print("op: " + op_key + " " + op_info_key + - " paramType not valid, valid key:[dynamic, " - "optional, required]") - is_valid = False - is_valid = check_type_format(op_input_info, is_valid, op_info_key) - is_valid = check_attr(op_dict, is_valid) - is_valid = check_bool_flag(op_dict, is_valid) - print("==============check valid for ops info end================\n\n") - return is_valid - - -def write_json_file(tbe_ops_info, json_file_path): - """ - Save info to json file - Parameters: - ---------------- - tbe_ops_info: ops_info - json_file_path: json file path - ---------------- - """ - json_file_real_path = os.path.realpath(json_file_path) - wr_flag = os.O_WRONLY | os.O_CREAT - wr_mode = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(json_file_real_path, wr_flag, wr_mode), 'w') as file_path: - # The owner have all rights，group only have read rights - os.chmod(json_file_real_path, stat.S_IWUSR + stat.S_IRGRP - + stat.S_IRUSR) - json.dump(tbe_ops_info, file_path, sort_keys=True, indent=4, - separators=(',', ':')) - print("Compile op info cfg successfully.") - - -def parse_ini_to_json(ini_file_paths, outfile_path): - """ - parse ini files to json file - Parameters: - ---------------- - ini_file_paths: list of ini file path - outfile_path: output file path - ---------------- - """ - tbe_ops_info = parse_ini_files(ini_file_paths) - if not check_op_info(tbe_ops_info): - print("Compile op info cfg failed.") - return False - write_json_file(tbe_ops_info, outfile_path) - return True - - -if __name__ == '__main__': - args = sys.argv - - OUTPUT_FILE_PATH = "tbe_ops_info.json" - ini_file_path_list = [] - - for arg in args: - if arg.endswith("ini"): - ini_file_path_list.append(arg) - OUTPUT_FILE_PATH = arg.replace(".ini", ".json") - if arg.endswith("json"): - OUTPUT_FILE_PATH = arg - - if len(ini_file_path_list) == 0: - ini_file_path_list.append("tbe_ops_info.ini") - - if not parse_ini_to_json(ini_file_path_list, OUTPUT_FILE_PATH): - sys.exit(1) - sys.exit(0) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py deleted file mode 100644 index 8f1124b1d..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/preset_parse.py +++ /dev/null @@ -1,23 +0,0 @@ -import json -import sys -import os - - -def get_config_opts(file): - src_dir = os.path.abspath(os.path.dirname(file)) - opts = '' - with open(file, 'r') as fd: - config = json.load(fd) - for conf in config: - if conf == 'configurePresets': - for node in config[conf]: - macros = node.get('cacheVariables') - if macros is not None: - for key in macros: - opts += '-D{}={} '.format(key, macros[key]['value']) - opts = opts.replace('${sourceDir}', src_dir) - print(opts) - - -if __name__ == "__main__": - get_config_opts(sys.argv[1]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py deleted file mode 100644 index 1baa364ef..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_codegen.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python -# -*- coding: UTF-8 -*- -""" -Created on Feb 28 20:56:45 2020 -Copyright (c) Huawei Technologies Co., Ltd. 2020-2021. All rights reserved. -""" - -import os -import stat -import collections -import kernel_entry as keb -from tiling_data_def_build import gen_tiling -import code_channel_infer -import const_var - -PYF_PATH = os.path.dirname(__file__) - -ReplayCodeGenParams = collections.namedtuple('ReplayCodeGenParams',\ -['op_type', 'impl', 'tiling_file', 'kernel', 'entry', 'argn', 'op_replay_batch', 'max_block_dim', 'max_shape_size']) - - -class ReplayCodeGen: - def __init__(self, replayCodeGenParams): - self.op_type = replayCodeGenParams.op_type - self.impl = replayCodeGenParams.impl - self.tiling_file = replayCodeGenParams.tiling_file - self.tiling_data_file = '' - self.kernel = replayCodeGenParams.kernel - self.entry = replayCodeGenParams.entry - self.argn = replayCodeGenParams.argn - self.batch = False - self.outdir = '' - self.data_type = 'uint8_t' - self.blknum = 32 - self.op_replay_batch = replayCodeGenParams.op_replay_batch - self.max_block_dim = replayCodeGenParams.max_block_dim - self.max_shape_size = replayCodeGenParams.max_shape_size - - def set_batch(self, is_batch): - self.batch = is_batch - - def set_outdir(self, outdir): - self.outdir = outdir - - def gen_replay(self, ops_product: str): - kerentry = os.path.join(self.outdir, self.kernel + '_entry.cce') - kerimpl = os.path.join(self.outdir, self.kernel + '_impl.cpp') - replayimpl = os.path.join(self.outdir, self.kernel + '_replay.cpp') - if self.batch: - reptmp = os.path.join(PYF_PATH, 'batch_replay_impl.temp') - else: - reptmp = os.path.join(PYF_PATH, 'replay_impl.temp') - kertmp = os.path.join(PYF_PATH, 'kernel_impl.temp') - self._gen_kentry(kerentry) - self._gen_kimpl_code(kerimpl, kertmp) - self._gen_tiling_data_header() - self._gen_replay_code(replayimpl, reptmp, ops_product) - - def _gen_tiling_data_header(self): - self.tiling_data_file = os.path.join(self.outdir, self.kernel + '_tiling_data.h') - gen_tiling(self.tiling_file, self.tiling_data_file) - - def _gen_kimpl_code(self, src, tmpfile): - with open(tmpfile, 'r') as fd: - temp = fd.read() - temp = temp.replace('__CCE_FILE__', self.impl) - with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: - ofd.write(temp) - - def _gen_replay_code(self, src, tmpfile, ops_product: str): - with open(tmpfile, 'r') as fd: - temp = fd.read() - temp = temp.replace('__ARG_NUM__', str(self.argn)) - argdef = [] - kargs = [] - for i in range(0, self.argn): - argdef.append('{} *'.format(self.data_type)) - kargs.append('({} *)GetArg({})'.format(self.data_type, i)) - temp = temp.replace('__ARGS_DEF__', ', '.join(argdef)) - temp = temp.replace('__KERNEL_ARGS__', ', '.join(kargs)) - temp = temp.replace('__KERNEL_FUN__', self.entry) - core_type_infer = 'core_type' - code_channel = code_channel_infer.infer_code_channel(code_channel_infer.InfoCodeChanelParams(self.impl,\ - self.tiling_data_file, self.kernel, self.outdir, ops_product, None)) - if code_channel == code_channel_infer.CODE_VEC: - core_type_infer = '0' - elif code_channel == code_channel_infer.CODE_CUBE: - core_type_infer = '1' - temp = temp.replace('__CORE_TYPE__', core_type_infer) - # regist function - temp = temp.replace('__OPS_PRODUCT__', ops_product) - temp = temp.replace('__OPTYPE__', self.op_type) - with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: - ofd.write(temp) - - def _gen_kentry(self, src): - kf = '' - pre_alloc_str = 'A' * 256 - if self.batch: - kf += keb.batch_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str), self.argn, self.data_type) - else: - kf += keb.mc_code_gen("K{:02d}_{}{}".format(0, self.entry, pre_alloc_str),\ - self.argn, self.data_type, self.blknum) - with os.fdopen(os.open(src, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: - ofd.write(kf) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp deleted file mode 100644 index 1d30dd865..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/replay_impl.temp +++ /dev/null @@ -1,120 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include "replay_def.h" -#include "code_gen.h" -#include "replay_fun.h" -#include "register/op_check.h" -#define __ASCENDC_REPLAY_CODE__ -using namespace std; -using namespace optiling; -using namespace AscendCReplay; - -extern "C" void __KERNEL_FUN__ (__ARGS_DEF__, const char *); -extern "C" int elf_append(char *elf, uint32_t elfSize, char *jit, int kernum, int blknum[], char *atext[], - int alen[], int atlen, const char* kernelname[]); - -#define KERNEL_N 1 -#define ARG_N (__ARG_NUM__) -#define MAX_L (1024 * 1024 * 100) -#define MAX_E (1024 * 1024) - -int __KERNEL_FUN___replay___OPS_PRODUCT__(ReplayFuncParam& param, const int core_type) -{ - // gen type 1 : direct call codes 0: load .o file - if (param.gentype < 0 || param.gentype > 1) { - printf("Error: call replay gen type is %d, should only be 1 or 0\n", param.gentype); - return 0; - } else if (param.gentype == 1 && param.objptr == nullptr) { - printf("Error: call replay with direct call mode, but code obj addr is null\n"); - return 0; - } else if (param.gentype == 0 && param.output_kernel_file == nullptr) { - printf("Error: call replay with object file mode, but object file path is null\n"); - return 0; - } - // core_type 0:MIX 1:CUBE 2:VEC - if (core_type < 0 || core_type > 2) { - printf("Error: call replay core type is %d !\n", core_type); - return 0; - } - g_coreType = __CORE_TYPE__; - g_taskRation = param.task_ration; - g_tilingKey = param.tiling_key; - - unsigned char *buf, *jit; - char *kernel[KERNEL_N * 32]; - int len[KERNEL_N * 32]; - int blknum[KERNEL_N]; - int max; - block_num = param.block_dim; - g_ubBase = block_num; - uint8_t *code = (uint8_t *)malloc(MAX_L); - uint8_t *pos = code; - struct timespec tp1, tp2; - - clock_gettime(CLOCK_MONOTONIC, &tp1); - if (block_num > 32) { - printf("Error: block_num > 32\n"); - return 0; - } - //__OP_FOPEN__ - for (int i = 0; i < KERNEL_N; i++) { - for (int j = 0; j < ARG_N; j++) - AddArg(j, ARG_STEP * (j + 1)); - for (block_idx = 0; block_idx < block_num; block_idx++) { - //__OP_SET_KERNEL__ - int code_idx = i * block_num + block_idx; -#ifdef FP_CEILING - SetCtrlFloatEnable(); -#else - SetCtrlFloatDisable(); -#endif - CodeInit(pos, false); - __KERNEL_FUN__(__KERNEL_ARGS__, param.tiling_data); - CodeEnd(); - kernel[code_idx] = (char *)pos; - len[code_idx] = CodeLen(); - pos += len[code_idx]; - printf("kernel %d core %ld code generated len %d\n", i, block_idx, len[code_idx]); - } - blknum[i] = block_num; - } - //__OP_FCLOSE__ - clock_gettime(CLOCK_MONOTONIC, &tp2); - buf = (unsigned char *)malloc(MAX_E); - int fd = open(param.entry_file, O_RDONLY); - if (fd < 0) { - printf("[error]: cannot find entry.o : %s\n", param.entry_file); - return 0; - } - uint32_t bufSize = read(fd, buf, MAX_E); - if (bufSize <= 0) { - printf("[error]: entry.o : %s is too small ! \n", param.entry_file); - } - close(fd); - jit = (unsigned char *)malloc(MAX_L); - printf("total code generated %ld\n", pos - code); - int sz = elf_append((char *)buf, bufSize, (char *)jit, KERNEL_N, blknum, kernel, len, pos - code, ¶m.kernel_name); - if (tp1.tv_sec != tp2.tv_sec) { - printf("%ld NS\n", tp2.tv_nsec + 1000000000 - tp1.tv_nsec); - } else { - printf("%ld NS\n", tp2.tv_nsec - tp1.tv_nsec); - } - printf("new elf size %d\n", sz); - if (param.gentype == 0) { - fd = open(param.output_kernel_file, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); - (void)write(fd, jit, sz); - close(fd); - free(jit); - } else if (param.gentype == 1) { - *param.objptr = (char*)jit; - } - free(buf); - free(code); - return sz; -} - -REG_REPLAY_FUNC(__OPTYPE__, __OPS_PRODUCT__, __KERNEL_FUN___replay___OPS_PRODUCT__); diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py deleted file mode 100644 index 4c5b099b7..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/cmake/util/tiling_data_def_build.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python -# coding=utf-8 -""" -Function: -The replay funtion entry -Copyright Information: -Huawei Technologies Co., Ltd. All Rights Reserved © 2020 -""" - -import sys -import os -import stat -import re -import const_var - - -def gen_tiling(tiling_header_file: str, tiling_file_out: str): - if not os.path.exists(tiling_header_file): - print("warning: no userdef tiling header file: ", tiling_header_file) - return - print("generate tiling def header file: ", tiling_file_out) - tmp_name = os.path.splitext(os.path.basename(tiling_header_file))[0].upper() - tiling_source = '#ifndef __{}_H__\n'.format(tmp_name) - tiling_source += '#define __{}_H__\n\n'.format(tmp_name) - tiling_source += '#include \n' - tiling_source += '#include \n\n' - tiling_source += '#include "kernel_tiling/kernel_tiling.h"\n\n' - end_source = "" - pattern = re.compile(r'[(](.*)[)]', re.S) - with open(tiling_header_file, 'r') as fd: - lines = fd.readlines() - for line in lines: - line = line.strip() - if (line.startswith('BEGIN_TILING_DATA_DEF')): - tiling_source += '#pragma pack(1)\n' - tiling_source += 'struct ' - struct_def = re.findall(pattern, line)[0] - tiling_source += struct_def + ' {\n' - elif (line.startswith('TILING_DATA_FIELD_DEF_ARR')): - field_params = re.findall(pattern, line)[0] - fds = field_params.split(',') - tiling_source += ' {} {}[{}] = {{}};\n'.format(fds[0].strip(), fds[2].strip(), fds[1].strip()) - elif (line.startswith('TILING_DATA_FIELD_DEF_STRUCT')): - field_params = re.findall(pattern, line)[0] - fds = field_params.split(',') - tiling_source += ' {} {};\n'.format(fds[0].strip(), fds[1].strip()) - elif (line.startswith('TILING_DATA_FIELD_DEF')): - field_params = re.findall(pattern, line)[0] - fds = field_params.split(',') - tiling_source += ' {} {} = 0;\n'.format(fds[0].strip(), fds[1].strip()) - elif (line.startswith('END_TILING_DATA_DEF')): - tiling_source += '};\n' - tiling_source += '#pragma pack()\n\n' - tiling_source += '#ifdef __NPU_TILING__\n' - tiling_source += \ - 'inline [aicore] void Init{stru}(const __gm__ uint8_t* tiling, {stru}* const_data)\n'\ - .format(stru=struct_def) - tiling_source += '{\n' - tiling_source += ' const __gm__ uint32_t *src = (const __gm__ uint32_t *)tiling;\n' - tiling_source += ' uint32_t *dst = (uint32_t *)const_data;\n' - tiling_source += ' for (auto i = 0; i < sizeof({}) / 4; i++) *(dst + i) = *(src + i);\n'\ - .format(struct_def) - tiling_source += '}\n' - tiling_source += '#else\n' - tiling_source += 'inline void Init{stru}(uint8_t* tiling, {stru}* const_data)\n'.format(stru=struct_def) - tiling_source += '{\n' - tiling_source += ' uint64_t *src = (uint64_t *)tiling;\n' - tiling_source += ' uint64_t *dst = (uint64_t *)const_data;\n' - tiling_source += ' for (auto i = 0; i < sizeof({}) / 8; i++) *(dst + i) = *(src + i);\n'\ - .format(struct_def) - tiling_source += '}\n' - tiling_source += '#endif\n\n' - end_source = ''' -#define GET_TILING_DATA(tiling_data, tiling_arg) \\ -{stru} tiling_data; \\ -Init{stru}(tiling_arg, &tiling_data)\n -'''.format(stru=struct_def) - tiling_source += end_source - tiling_source += '#endif' - with os.fdopen(os.open(tiling_file_out, const_var.WFLAGS, const_var.WMODES), 'w') as ofd: - ofd.write(tiling_source) - - -if __name__ == '__main__': - if len(sys.argv) <= 2: - raise RuntimeError('arguments must greater than 2') - gen_tiling(sys.argv[1], sys.argv[2]) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt deleted file mode 100644 index b6be9b492..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/mindspore") - if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/caffe_plugin") - add_subdirectory(caffe_plugin) - endif() - if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tf_plugin") - add_subdirectory(tf_plugin) - endif() - if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/onnx_plugin") - add_subdirectory(onnx_plugin) - endif() -endif() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt deleted file mode 100644 index a6aba5c20..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ - -aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} plugin_srcs) -add_library(cust_tf_parsers SHARED ${plugin_srcs}) -target_compile_definitions(cust_tf_parsers PRIVATE google=ascend_private) -if(ENABLE_CROSS_COMPILE) - target_link_directories(cust_tf_parsers PRIVATE - ${CMAKE_COMPILE_COMPILER_LIBRARY} - ${CMAKE_COMPILE_RUNTIME_LIBRARY} - ) -endif() -target_link_libraries(cust_tf_parsers PRIVATE intf_pub graph) -install(TARGETS cust_tf_parsers - LIBRARY DESTINATION packages/vendors/${vendor_name}/framework/tensorflow -) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc deleted file mode 100644 index 1809e029a..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/framework/tf_plugin/tensorflow_three_nn_plugin.cc +++ /dev/null @@ -1,23 +0,0 @@ -/* Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All -rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the Apache License Version 2.0. - * You may not use this file except in compliance with the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Apache License for more details at - * http://www.apache.org/licenses/LICENSE-2.0 - */ - -#include "register/register.h" - -namespace domi { -// register op info to GE -REGISTER_CUSTOM_OP("ThreeNN") - .FrameworkType(TENSORFLOW) // type: CAFFE, TENSORFLOW - .OriginOpType("ThreeNN") // name in tf module - .ParseParamsByOperatorFn(AutoMappingByOpFn); -} // namespace domi diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt deleted file mode 100644 index 40dd51cfa..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/CMakeLists.txt +++ /dev/null @@ -1,82 +0,0 @@ - -aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} ops_srcs) - -opbuild(OPS_SRC ${ops_srcs} - OUT_DIR ${ASCEND_AUTOGEN_PATH} -) - -add_library(cust_op_proto SHARED ${ops_srcs} ${ASCEND_AUTOGEN_PATH}/op_proto.cc) -target_compile_definitions(cust_op_proto PRIVATE OP_PROTO_LIB) -target_compile_options(cust_op_proto PRIVATE - -fvisibility=hidden -) -if(ENABLE_CROSS_COMPILE) - target_link_directories(cust_op_proto PRIVATE - ${CMAKE_COMPILE_COMPILER_LIBRARY} - ${CMAKE_COMPILE_RUNTIME_LIBRARY} - ) -endif() -target_link_libraries(cust_op_proto PRIVATE - intf_pub - exe_graph - register - tiling_api - -Wl,--whole-archive - rt2_registry - -Wl,--no-whole-archive -) -set_target_properties(cust_op_proto PROPERTIES OUTPUT_NAME - cust_opsproto_rt2.0 -) -add_library(cust_optiling SHARED ${ops_srcs}) -target_compile_definitions(cust_optiling PRIVATE OP_TILING_LIB) -target_compile_options(cust_optiling PRIVATE - -fvisibility=hidden -) -if(ENABLE_CROSS_COMPILE) - target_link_directories(cust_optiling PRIVATE - ${CMAKE_COMPILE_COMPILER_LIBRARY} - ${CMAKE_COMPILE_RUNTIME_LIBRARY} - ) -endif() -target_link_libraries(cust_optiling PRIVATE - intf_pub - exe_graph - register - tiling_api - -Wl,--whole-archive - rt2_registry - -Wl,--no-whole-archive -) -set_target_properties(cust_optiling PROPERTIES OUTPUT_NAME - cust_opmaster_rt2.0 -) - -file(GLOB aclnn_src ${ASCEND_AUTOGEN_PATH}/aclnn_*.cpp) -file(GLOB aclnn_inc ${ASCEND_AUTOGEN_PATH}/aclnn_*.h) -add_library(cust_opapi SHARED ${aclnn_src}) -if(ENABLE_CROSS_COMPILE) - target_link_directories(cust_opapi PRIVATE - ${CMAKE_COMPILE_COMPILER_LIBRARY} - ${CMAKE_COMPILE_RUNTIME_LIBRARY} - ) -endif() -target_link_libraries(cust_opapi PRIVATE intf_pub ascendcl nnopbase) - -add_custom_target(optiling_compat ALL - COMMAND ln -sf lib/linux/${CMAKE_SYSTEM_PROCESSOR}/$ - ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so -) - -install(TARGETS cust_op_proto - LIBRARY DESTINATION packages/vendors/${vendor_name}/op_proto/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) -install(FILES ${ASCEND_AUTOGEN_PATH}/op_proto.h - DESTINATION packages/vendors/${vendor_name}/op_proto/inc) -install(TARGETS cust_optiling - LIBRARY DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling/lib/linux/${CMAKE_SYSTEM_PROCESSOR}) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liboptiling.so - DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/op_tiling) -install(TARGETS cust_opapi - LIBRARY DESTINATION packages/vendors/${vendor_name}/op_api/lib) -install(FILES ${aclnn_inc} - DESTINATION packages/vendors/${vendor_name}/op_api/include) diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp deleted file mode 100644 index 027ef0850..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn.cpp +++ /dev/null @@ -1,109 +0,0 @@ - -#include "three_nn_tiling.h" -#include "register/op_def_registry.h" -#include "tiling/platform/platform_ascendc.h" -#include - -namespace optiling { -const uint32_t BLOCK_SIZE = 32*3*4; -static ge::graphStatus TilingFunc(gert::TilingContext* context) -{ - ThreeNNTilingData tiling; - - uint64_t ubSize; - auto ascendcPlatform = platform_ascendc::PlatformAscendC(context->GetPlatformInfo()); - // auto socVersion = ascendcPlatform.GetSocVersion(); - ascendcPlatform.GetCoreMemSize(platform_ascendc::CoreMemType::UB, ubSize); //获取硬件平台存储空间 UB 的内存大小 - uint32_t shapeB = context->GetInputTensor(0)->GetOriginShape().GetDim(0); - uint32_t shapeN = context->GetInputTensor(0)->GetOriginShape().GetDim(1); - uint32_t shapeM = context->GetInputTensor(1)->GetOriginShape().GetDim(1); - - //获取输入shape信息 - uint32_t inputNum = context->GetInputShape(1)->GetStorageShape().GetShapeSize()/shapeB; //输入数量 xyz2 - uint32_t inputBytes = GetSizeByDataType(context->GetInputDesc(1)->GetDataType()); //输入类型 - uint32_t inputLength = inputBytes * inputNum; //输入长度 - - //可使用的ub空间 输入3输出1，手动考虑双缓存 - uint32_t ubDataNumber = 4;//(inputBytes == 2) ? 10 : 6; - - // The number of 32B data blocks that can be used for each data. DOUBLE BUFFER is already counted here - uint32_t tileBlockNum = (ubSize / BLOCK_SIZE) / ubDataNumber; //每个ub段可用的空间块数 - uint32_t tileDataNum = 1440; - - // Input data for 32B alignment - uint32_t inputLengthAlgin32 = (((inputLength + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE); //输入长度 对齐处理 - uint32_t everyCoreInputBlockNum = inputLengthAlgin32 / BLOCK_SIZE;// 输入数据需要多少空间块 - - uint32_t TileNum = inputNum / tileDataNum; - uint32_t finalTileNum = (inputNum % tileDataNum) == 0 ? TileNum : TileNum + 1; //需要循环处理几次 - // Tail block calculation for chunks of data - uint32_t TailDataNum = inputNum - (tileDataNum * TileNum);//CoreDataNum - (tileDataNum * TileNum); - TailDataNum = TailDataNum == 0 ? tileDataNum : TailDataNum; //最后一次需要处理的数据量 - - tiling.set_CoreDataNum((inputNum*shapeB+31)/32*32); //对齐空间后的输入数量 - tiling.set_finalTileNum(finalTileNum);//需要循环处理几次 - tiling.set_tileDataNum(tileDataNum); //每个b处理的数据量 - tiling.set_TailDataNum(TailDataNum); //最后一次需要处理的数据量 - - tiling.set_shapeB(shapeB); - tiling.set_shapeN(shapeN); - tiling.set_shapeM(shapeM); - - context->SetBlockDim(1); - tiling.SaveToBuffer(context->GetRawTilingData()->GetData(), context->GetRawTilingData()->GetCapacity()); - context->GetRawTilingData()->SetDataSize(tiling.GetDataSize()); - size_t *currentWorkspace = context->GetWorkspaceSizes(1); - currentWorkspace[0] = 0; - return ge::GRAPH_SUCCESS; -} -} - - -namespace ge { -static ge::graphStatus InferShape(gert::InferShapeContext* context) -{ - const gert::Shape* x1_shape = context->GetInputShape(0); - gert::Shape* y_shape = context->GetOutputShape(0); - *y_shape = *x1_shape; - return GRAPH_SUCCESS; -} -} - - -namespace ops { -class ThreeNN : public OpDef { -public: - explicit ThreeNN(const char* name) : OpDef(name) - { - this->Input("xyz1") - .ParamType(REQUIRED) - .DataType({ge::DT_FLOAT}) - .Format({ge::FORMAT_ND}) - .UnknownShapeFormat({ge::FORMAT_ND}); - this->Input("xyz2") - .ParamType(REQUIRED) - .DataType({ge::DT_FLOAT}) - .Format({ge::FORMAT_ND}) - .UnknownShapeFormat({ge::FORMAT_ND}); - this->Output("dist") - .ParamType(REQUIRED) - .DataType({ge::DT_FLOAT}) - .Format({ge::FORMAT_ND}) - .UnknownShapeFormat({ge::FORMAT_ND}); - this->Output("indices") - .ParamType(REQUIRED) - .DataType({ge::DT_INT32}) - .Format({ge::FORMAT_ND}) - .UnknownShapeFormat({ge::FORMAT_ND}); - - this->SetInferShape(ge::InferShape); - - this->AICore() - .SetTiling(optiling::TilingFunc); - this->AICore().AddConfig("ascend310b"); - - } -}; - -OP_ADD(ThreeNN); -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h deleted file mode 100644 index 5da548fed..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_host/three_nn_tiling.h +++ /dev/null @@ -1,16 +0,0 @@ - -#include "register/tilingdata_base.h" - -namespace optiling { -BEGIN_TILING_DATA_DEF(ThreeNNTilingData) - TILING_DATA_FIELD_DEF(uint32_t, CoreDataNum); - TILING_DATA_FIELD_DEF(uint32_t, finalTileNum); - TILING_DATA_FIELD_DEF(uint32_t, tileDataNum); - TILING_DATA_FIELD_DEF(uint32_t, TailDataNum); - - TILING_DATA_FIELD_DEF(uint32_t, shapeB); - TILING_DATA_FIELD_DEF(uint32_t, shapeN); - TILING_DATA_FIELD_DEF(uint32_t, shapeM); -END_TILING_DATA_DEF; -REGISTER_TILING_DATA_CLASS(ThreeNN, ThreeNNTilingData) -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt deleted file mode 100644 index 8c94a952d..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/CMakeLists.txt +++ /dev/null @@ -1,68 +0,0 @@ -# set custom compile options -if ("${CMAKE_BUILD_TYPE}x" STREQUAL "Debugx") - add_ops_compile_options(ALL OPTIONS -g -O0) -endif() - -foreach(compute_unit ${ASCEND_COMPUTE_UNIT}) - - # generate aic-${compute_unit}-ops-info.json - add_ops_info_target(TARGET ops_info_gen_${compute_unit} - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core/${compute_unit}/aic-${compute_unit}-ops-info.json - OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini - INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/config/${compute_unit} - ) - - # generate ascendc impl py once - if (NOT TARGET ascendc_impl_gen) - add_ops_impl_target(TARGET ascendc_impl_gen - OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini - IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} - OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe - INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl - ) - endif() - - # dynamic shape binary compile - if (${ENABLE_BINARY_PACKAGE} AND NOT ${ENABLE_CROSS_COMPILE}) - add_bin_compile_target(TARGET ascendc_bin_${compute_unit} - OPS_INFO ${ASCEND_AUTOGEN_PATH}/aic-${compute_unit}-ops-info.ini - IMPL_DIR ${CMAKE_CURRENT_SOURCE_DIR} - ADP_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/dynamic - OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/binary/${compute_unit} - INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/kernel - COMPUTE_UNIT ${compute_unit} - ) - add_dependencies(ascendc_bin_${compute_unit} ascendc_impl_gen) - endif() - - if (${ENABLE_CROSS_COMPILE} AND ${ENABLE_BINARY_PACKAGE}) - add_cross_compile_target( - TARGET bin_${compute_unit} - OUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../kernel - INSTALL_DIR packages/vendors/${vendor_name}/op_impl/ai_core/tbe/ - ) - endif() -endforeach() - -# generate npu_supported_ops.json -add_npu_support_target(TARGET npu_supported_ops - OPS_INFO_DIR ${ASCEND_AUTOGEN_PATH} - OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/tbe/op_info_cfg/ai_core - INSTALL_DIR packages/vendors/${vendor_name}/framework/${ASCEND_FRAMEWORK_TYPE} -) - -if(ENABLE_TEST AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/testcases) - add_subdirectory(testcases) -endif() - -# install kernel file -if (${ENABLE_SOURCE_PACKAGE}) - file(GLOB KERNEL_FILES - ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/*.h - ${CMAKE_CURRENT_SOURCE_DIR}/*.py - ) - install(FILES ${KERNEL_FILES} - DESTINATION packages/vendors/${vendor_name}/op_impl/ai_core/tbe/${vendor_name}_impl/dynamic - ) -endif() diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp deleted file mode 100644 index 9d60d095b..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/op_kernel/three_nn.cpp +++ /dev/null @@ -1,166 +0,0 @@ -#define K_MAX_SHAPE_DIM 0 -#include "kernel_operator.h" -#include -using namespace AscendC; -constexpr int32_t BUFFER_NUM = 2; // tensor num for each queue - -class KernelFastGeluGrad { -public: - __aicore__ inline KernelFastGeluGrad() {} - __aicore__ inline void Init(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, - uint32_t CoreDataNum, uint32_t finalTileNum, uint32_t tileDataNum, uint32_t TailDataNum, - uint32_t shapeB, uint32_t shapeN, uint32_t shapeM) { - ASSERT(GetBlockNum() != 0 && "block dim can not be zero!"); - - this->coreDataNum = CoreDataNum; - this->tileNum = finalTileNum; - this->tileDataNum = tileDataNum; - this->tailDataNum = TailDataNum; - - this->shapeB = shapeB; - this->shapeN = shapeN; - this->shapeM = shapeM; - - xyz1Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ1*)xyz1, this->coreDataNum); - xyz2Gm.SetGlobalBuffer((__gm__ DTYPE_XYZ2*)xyz2, this->coreDataNum); - distGm.SetGlobalBuffer((__gm__ DTYPE_DIST*)dist, this->coreDataNum); - indicesGm.SetGlobalBuffer((__gm__ DTYPE_INDICES*)indices, this->coreDataNum); - - pipe.InitBuffer(inQueueXYZ2, BUFFER_NUM, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - pipe.InitBuffer(QueueTmp1, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - pipe.InitBuffer(QueueTmp2, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - pipe.InitBuffer(QueueTmp3, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - - pipe.InitBuffer(QueueTmp10, 32 * sizeof(DTYPE_XYZ2)); - - pipe.InitBuffer(QueueTmpX, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - pipe.InitBuffer(QueueTmpY, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - pipe.InitBuffer(QueueTmpZ, this->tileDataNum * sizeof(DTYPE_XYZ2)+128); - - pipe.InitBuffer(QueueMask, sizeof(uint32_t)*(128+36)); - - } - __aicore__ inline void Process() { - int32_t loopCount; - int32_t b=0; - - LocalTensor mask_scl = QueueMask.Get(); - auto tmpx = QueueTmpX.Get(); - auto tmpy = QueueTmpY.Get(); - auto tmpz = QueueTmpZ.Get(); - - uint64_t mask1[2] = {10540996613548315209, 5270498306774157604}; - uint64_t mask2[2] = {2635249153387078802, 10540996613548315209}; - uint64_t mask3[2] = {5270498306774157604, 2635249153387078802}; - uint32_t scalar = 1227133513; - Duplicate(mask_scl, scalar, mask1, 2, 1, 8 ); - - scalar = 2454267026; - Duplicate(mask_scl, scalar, mask2, 2, 1, 8 ); - - scalar = 613566756; - Duplicate(mask_scl, scalar, mask3, 2, 1, 8 ); - - this->processDataNum = this->tailDataNum; - for(int32_t b=0; bshapeB; b++) - { - CopyIn(b, 0); - Compute(b, 0, mask_scl, tmpx, tmpy, tmpz); - } - } - -private: - __aicore__ inline void CopyIn(int32_t b, int32_t progress) - { - LocalTensor xyz2Local = inQueueXYZ2.AllocTensor(); - DataCopy(xyz2Local, xyz2Gm[3*b*this->shapeM], (3*this->shapeM+31)/32*32); - inQueueXYZ2.EnQue(xyz2Local); - } - __aicore__ inline void Compute(int32_t b, int32_t progress, LocalTensor mask_scl, LocalTensor tmpx, LocalTensor tmpy, LocalTensor tmpz) - { - LocalTensor xyz2Local = inQueueXYZ2.DeQue(); - auto tmp1 = QueueTmp1.Get(); - auto tmp2 = QueueTmp2.Get(); - auto tmp3 = QueueTmp3.Get(); - auto tmp10 = QueueTmp10.Get(); - - float ux,uy,uz; - int besti1, besti2, besti3; - uint32_t calculate_num = this->shapeM; - uint32_t mask = this->shapeM*3; - uint64_t rsvdCnt = 0; - GatherMask (tmpx, xyz2Local, mask_scl, true, mask, { 1, 1, 8, 8 }, rsvdCnt); - GatherMask (tmpy, xyz2Local, mask_scl[16], true, mask, { 1, 1, 8, 8 }, rsvdCnt); - GatherMask (tmpz, xyz2Local, mask_scl[8], true, mask, { 1, 1, 8, 8 }, rsvdCnt); - - uint32_t b_xyz1_index = 3*(b*this->shapeN); - for(uint32_t i=0; i < this->shapeN; i++) - { - ux = -xyz1Gm.GetValue(b_xyz1_index); - uy = -xyz1Gm.GetValue(b_xyz1_index+1); - uz = -xyz1Gm.GetValue(b_xyz1_index+2); - - Adds(tmp3, tmpx, ux, calculate_num); - Mul(tmp2, tmp3, tmp3, calculate_num); - - Adds(tmp3, tmpy, uy, calculate_num); - Mul(tmp1, tmp3, tmp3, calculate_num); - - Add(tmp3, tmp2, tmp1, calculate_num); - - Adds(tmp2, tmpz, uz, calculate_num); - Mul(tmp1, tmp2, tmp2, calculate_num); - - Add(tmp3, tmp3, tmp1, calculate_num); - - ReduceMin(tmp2, tmp3, tmp1, calculate_num, true); - besti1 = (tmp2.ReinterpretCast()).GetValue(1); - tmp3.SetValue(besti1, 3.402823466e+38F); - - ReduceMin(tmp2[8], tmp3, tmp1, calculate_num, true); - besti2 = (tmp2.ReinterpretCast()).GetValue(9); - tmp3.SetValue(besti2, 3.402823466e+38F); - - ReduceMin(tmp2[16], tmp3, tmp1, calculate_num, true); - besti3 = (tmp2.ReinterpretCast()).GetValue(17); - - indicesGm.SetValue(b_xyz1_index, besti1); - indicesGm.SetValue(b_xyz1_index+1, besti2); - indicesGm.SetValue(b_xyz1_index+2, besti3); - - distGm.SetValue(b_xyz1_index, tmp2.GetValue(0)); - distGm.SetValue(b_xyz1_index+1, tmp2.GetValue(8)); - distGm.SetValue(b_xyz1_index+2, tmp2.GetValue(16)); - b_xyz1_index+=3; - } - inQueueXYZ2.FreeTensor(xyz2Local); - } - -private: - TPipe pipe; - TQue inQueueXYZ2; - TBuf QueueTmp1, QueueTmp2, QueueTmp3, QueueTmpX, QueueTmpY, QueueTmpZ, QueueMask, QueueTmp10; - GlobalTensor xyz1Gm; - GlobalTensor xyz2Gm; - GlobalTensor distGm; - GlobalTensor indicesGm; - uint32_t coreDataNum; - uint32_t tileNum; - uint32_t tileDataNum; - uint32_t tailDataNum; - uint32_t processDataNum; - uint32_t xyz2DataNum; - uint32_t shapeB; - uint32_t shapeN; - uint32_t shapeM; -}; - -extern "C" __global__ __aicore__ void three_nn(GM_ADDR xyz1, GM_ADDR xyz2, GM_ADDR dist, GM_ADDR indices, GM_ADDR workspace, GM_ADDR tiling) { - GET_TILING_DATA(tiling_data, tiling); - // TODO: user kernel impl - KernelFastGeluGrad op; - op.Init(xyz1, xyz2, dist, indices, - tiling_data.CoreDataNum, tiling_data.finalTileNum, tiling_data.tileDataNum, tiling_data.TailDataNum, - tiling_data.shapeB, tiling_data.shapeN, tiling_data.shapeM); - op.Process(); -} diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh deleted file mode 100644 index 8468c5a25..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/install.sh +++ /dev/null @@ -1,318 +0,0 @@ -#!/bin/bash -vendor_name=customize -targetdir=/usr/local/Ascend/opp -target_custom=0 - -sourcedir=$PWD/packages -vendordir=vendors/$vendor_name - -QUIET="y" - -while true -do - case $1 in - --quiet) - QUIET="y" - shift - ;; - --install-path=*) - INSTALL_PATH=$(echo $1 | cut -d"=" -f2-) - INSTALL_PATH=${INSTALL_PATH%*/} - shift - ;; - --*) - shift - ;; - *) - break - ;; - esac -done - -log() { - cur_date=`date +"%Y-%m-%d %H:%M:%S"` - echo "[runtime] [$cur_date] "$1 -} - -if [ -n "${INSTALL_PATH}" ]; then - if [[ ! "${INSTALL_PATH}" = /* ]]; then - log "[ERROR] use absolute path for --install-path argument" - exit 1 - fi - if [ ! -d ${INSTALL_PATH} ]; then - mkdir ${INSTALL_PATH} >> /dev/null 2>&1 - if [ $? -ne 0 ]; then - log "[ERROR] create ${INSTALL_PATH} failed" - exit 1 - fi - fi - targetdir=${INSTALL_PATH} -elif [ -n "${ASCEND_CUSTOM_OPP_PATH}" ]; then - if [ ! -d ${ASCEND_CUSTOM_OPP_PATH} ]; then - mkdir -p ${ASCEND_CUSTOM_OPP_PATH} >> /dev/null 2>&1 - if [ $? -ne 0 ]; then - log "[ERROR] create ${ASCEND_CUSTOM_OPP_PATH} failed" - fi - fi - targetdir=${ASCEND_CUSTOM_OPP_PATH} -else - if [ "x${ASCEND_OPP_PATH}" == "x" ]; then - log "[ERROR] env ASCEND_OPP_PATH no exist" - exit 1 - fi - targetdir="${ASCEND_OPP_PATH}" -fi - -if [ ! -d $targetdir ];then - log "[ERROR] $targetdir no exist" - exit 1 -fi - -upgrade() -{ - if [ ! -d ${sourcedir}/$vendordir/$1 ]; then - log "[INFO] no need to upgrade ops $1 files" - return 0 - fi - - if [ ! -d ${targetdir}/$vendordir/$1 ];then - log "[INFO] create ${targetdir}/$vendordir/$1." - mkdir -p ${targetdir}/$vendordir/$1 - if [ $? -ne 0 ];then - log "[ERROR] create ${targetdir}/$vendordir/$1 failed" - return 1 - fi - else - has_same_file=-1 - for file_a in ${sourcedir}/$vendordir/$1/*; do - file_b=${file_a##*/}; - if [ "ls ${targetdir}/$vendordir/$1" = "" ]; then - log "[INFO] ${targetdir}/$vendordir/$1 is empty !!" - return 1 - fi - grep -q $file_b <<<`ls ${targetdir}/$vendordir/$1`; - if [[ $? -eq 0 ]]; then - echo -n "${file_b} " - has_same_file=0 - fi - done - if [ 0 -eq $has_same_file ]; then - if test $QUIET = "n"; then - echo "[INFO]: has old version in ${targetdir}/$vendordir/$1, \ - you want to Overlay Installation , please enter:[o]; \ - or replace directory installation , please enter: [r]; \ - or not install , please enter:[n]." - - while true - do - read orn - if [ "$orn" = n ]; then - return 0 - elif [ "$orn" = m ]; then - break; - elif [ "$0rn" = r ]; then - [ -n "${targetdir}/$vendordir/$1/" ] && rm -rf "${targetdir}/$vendordir/$1"/* - break; - else - echo "[ERROR] input error, please input again!" - fi - done - fi - fi - log "[INFO] replace or merge old ops $1 files .g....." - fi - - log "copy new ops $1 files ......" - if [ -d ${targetdir}/$vendordir/$1/ ]; then - chmod -R +w "$targetdir/$vendordir/$1/" >/dev/null 2>&1 - fi - cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ - if [ $? -ne 0 ];then - log "[ERROR] copy new $1 files failed" - return 1 - fi - - return 0 -} -upgrade_proto() -{ - if [ ! -f ${sourcedir}/$vendordir/custom.proto ]; then - log "[INFO] no need to upgrade custom.proto files" - return 0 - fi - if [ ! -d ${targetdir}/$vendordir/framework/caffe ];then - log "[INFO] create ${targetdir}/$vendordir/framework/caffe." - mkdir -p ${targetdir}/$vendordir/framework/caffe - if [ $? -ne 0 ];then - log "[ERROR] create ${targetdir}/$vendordir/framework/caffe failed" - return 1 - fi - else - if [ -f ${targetdir}/$vendordir/framework/caffe/custom.proto ]; then - # 有老版本,判断是否要覆盖式安装 - if test $QUIET = "n"; then - echo "[INFO] ${targetdir}/$vendordir/framework/caffe has old version"\ - "custom.proto file. Do you want to replace? [y/n] " - - while true - do - read yn - if [ "$yn" = n ]; then - return 0 - elif [ "$yn" = y ]; then - break; - else - echo "[ERROR] input error, please input again!" - fi - done - fi - fi - log "[INFO] replace old caffe.proto files ......" - fi - chmod -R +w "$targetdir/$vendordir/framework/caffe/" >/dev/null 2>&1 - cp -rf ${sourcedir}/$vendordir/custom.proto ${targetdir}/$vendordir/framework/caffe/ - if [ $? -ne 0 ];then - log "[ERROR] copy new custom.proto failed" - return 1 - fi - log "[INFO] copy custom.proto success" - - return 0 -} - -upgrade_file() -{ - if [ ! -e ${sourcedir}/$vendordir/$1 ]; then - log "[INFO] no need to upgrade ops $1 file" - return 0 - fi - - log "copy new $1 files ......" - cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 - if [ $? -ne 0 ];then - log "[ERROR] copy new $1 file failed" - return 1 - fi - - return 0 -} - -delete_optiling_file() -{ - if [ ! -d ${targetdir}/vendors ];then - log "[INFO] $1 not exist, no need to uninstall" - return 0 - fi - sys_info=$(uname -m) - if [ ! -d ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/lib/linux/${sys_info} ];then - rm -rf ${sourcedir}/$vendordir/$1/ai_core/tbe/op_tiling/liboptiling.so - fi - return 0 -} - -log "[INFO] copy uninstall sh success" - -if [ ! -d ${targetdir}/vendors ];then - log "[INFO] create ${targetdir}/vendors." - mkdir -p ${targetdir}/vendors - if [ $? -ne 0 ];then - log "[ERROR] create ${targetdir}/vendors failed" - return 1 - fi -fi -chmod u+w ${targetdir}/vendors - -echo "[ops_custom]upgrade framework" -upgrade framework -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade op proto" -upgrade op_proto -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade version.info" -upgrade_file version.info -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade op impl" -delete_optiling_file op_impl -upgrade op_impl -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade op api" -upgrade op_api -if [ $? -ne 0 ];then - exit 1 -fi - -upgrade_proto -if [ $? -ne 0 ];then - exit 1 -fi - -# set the set_env.bash -if [ -n "${INSTALL_PATH}" ] && [ -d ${INSTALL_PATH} ]; then - _ASCEND_CUSTOM_OPP_PATH=${targetdir}/${vendordir} - bin_path="${_ASCEND_CUSTOM_OPP_PATH}/bin" - set_env_variable="#!/bin/bash\nexport ASCEND_CUSTOM_OPP_PATH=${_ASCEND_CUSTOM_OPP_PATH}:\${ASCEND_CUSTOM_OPP_PATH}" - if [ ! -d ${bin_path} ]; then - mkdir -p ${bin_path} >> /dev/null 2>&1 - if [ $? -ne 0 ]; then - log "[ERROR] create ${bin_path} failed" - exit 1 - fi - fi - echo -e ${set_env_variable} > ${bin_path}/set_env.bash - if [ $? -ne 0 ]; then - log "[ERROR] write ASCEND_CUSTOM_OPP_PATH to set_env.bash failed" - exit 1 - else - log "[INFO] using requirements: when custom module install finished or before you run the custom module, \ - execute the command [ source ${bin_path}/set_env.bash ] to set the environment path" - fi -else - config_file=${targetdir}/vendors/config.ini - if [ ! -f ${config_file} ]; then - touch ${config_file} - chmod 640 ${config_file} - echo "load_priority=$vendor_name" > ${config_file} - if [ $? -ne 0 ];then - echo "echo load_priority failed" - exit 1 - fi - else - found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" - found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') - vendor=$(echo $found_vendor | tr -s ' ' ',') - if [ "$vendor" != "" ]; then - sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" - fi - fi -fi - -chmod u-w ${targetdir}/vendors - -if [ -d ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/ ]; then - chmod -R 440 ${targetdir}/$vendordir/op_impl/cpu/aicpu_kernel/impl/* >/dev/null 2>&1 -fi -if [ -f ${targetdir}/ascend_install.info ]; then - chmod -R 440 ${targetdir}/ascend_install.info -fi -if [ -f ${targetdir}/scene.info ]; then - chmod -R 440 ${targetdir}/scene.info -fi -if [ -f ${targetdir}/version.info ]; then - chmod -R 440 ${targetdir}/version.info -fi - -echo "SUCCESS" -exit 0 - diff --git a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh b/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh deleted file mode 100644 index e09173485..000000000 --- a/operator_contrib/ThreeNNSample/FrameworkLaunch/ThreeNN/scripts/upgrade.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash -vendor_name=customize -targetdir=/usr/local/Ascend/opp -target_custom=0 - -sourcedir=$PWD/packages -vendordir=vendors/$vendor_name - -log() { - cur_date=`date +"%Y-%m-%d %H:%M:%S"` - echo "[runtime] [$cur_date] "$1 -} - -if [[ "x${ASCEND_OPP_PATH}" == "x" ]];then - log "[ERROR] env ASCEND_OPP_PATH no exist" - exit 1 -fi - -targetdir=${ASCEND_OPP_PATH} - -if [ ! -d $targetdir ];then - log "[ERROR] $targetdir no exist" - exit 1 -fi - -upgrade() -{ - if [ ! -d ${sourcedir}/$vendordir/$1 ]; then - log "[INFO] no need to upgrade ops $1 files" - return 0 - fi - - if [ ! -d ${targetdir}/$vendordir/$1 ];then - log "[INFO] create ${targetdir}/$vendordir/$1." - mkdir -p ${targetdir}/$vendordir/$1 - if [ $? -ne 0 ];then - log "[ERROR] create ${targetdir}/$vendordir/$1 failed" - return 1 - fi - else - vendor_installed_dir=$(ls "$targetdir/vendors" 2> /dev/null) - for i in $vendor_installed_dir;do - vendor_installed_file=$(ls "$vendor_installed_dir/$vendor_name/$i" 2> /dev/null) - if [ "$i" = "$vendor_name" ] && [ "$vendor_installed_file" != "" ]; then - echo "[INFO]: $vendor_name custom opp package has been installed on the path $vendor_installed_dir, \ - you want to Overlay Installation , please enter:[o]; \ - or replace directory installation , please enter: [r]; \ - or not install , please enter:[n]." - fi - while true - do - read mrn - if [ "$mrn" = m ]; then - break - elif [ "$mrn" = r ]; then - [ -n "$vendor_installed_file"] && rm -rf "$vendor_installed_file" - break - elif [ "$mrn" = n ]; then - return 0 - else - echo "[WARNING]: Input error, please input m or r or n to choose!" - fi - done - done - log "[INFO] replace old ops $1 files ......" - fi - - log "copy new ops $1 files ......" - cp -rf ${sourcedir}/$vendordir/$1/* $targetdir/$vendordir/$1/ - if [ $? -ne 0 ];then - log "[ERROR] copy new $1 files failed" - return 1 - fi - - return 0 -} - -upgrade_file() -{ - if [ ! -e ${sourcedir}/$vendordir/$1 ]; then - log "[INFO] no need to upgrade ops $1 file" - return 0 - fi - - log "copy new $1 files ......" - cp -f ${sourcedir}/$vendordir/$1 $targetdir/$vendordir/$1 - if [ $? -ne 0 ];then - log "[ERROR] copy new $1 file failed" - return 1 - fi - - return 0 -} - -log "[INFO] copy uninstall sh success" - -echo "[ops_custom]upgrade framework" -upgrade framework -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade op proto" -upgrade op_proto -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade op impl" -upgrade op_impl -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade op api" -upgrade op_api -if [ $? -ne 0 ];then - exit 1 -fi - -echo "[ops_custom]upgrade version.info" -upgrade_file version.info -if [ $? -ne 0 ];then - exit 1 -fi - -config_file=${targetdir}/vendors/config.ini -found_vendors="$(grep -w "load_priority" "$config_file" | cut --only-delimited -d"=" -f2-)" -found_vendor=$(echo $found_vendors | sed "s/$vendor_name//g" | tr ',' ' ') -vendor=$(echo $found_vendor | tr -s ' ' ',') -if [ "$vendor" != "" ]; then - sed -i "/load_priority=$found_vendors/s@load_priority=$found_vendors@load_priority=$vendor_name,$vendor@g" "$config_file" -fi - -changemode() -{ - if [ -d ${targetdir} ];then - chmod -R 550 ${targetdir}>/dev/null 2>&1 - fi - - return 0 -} -echo "[ops_custom]changemode..." -#changemode -if [ $? -ne 0 ];then - exit 1 -fi - -echo "SUCCESS" -exit 0 - diff --git a/operator_contrib/ThreeNNSample/README.en.md b/operator_contrib/ThreeNNSample/README.en.md deleted file mode 100644 index 19000a924..000000000 --- a/operator_contrib/ThreeNNSample/README.en.md +++ /dev/null @@ -1,160 +0,0 @@ -## ThreeNN Custom Operator Sample Description - -This sample implements the ThreeNN operator using the Ascend C programming language and provides corresponding end-to-end implementations based on different operator invocation methods. - -- [FrameworkLaunch](./FrameworkLaunch): Using a framework to call the ThreeNN custom operator. - -The development process follows the workflow of project creation -> operator implementation -> compilation and deployment -> operator invocation. The entire process relies on the operator project: the operator kernel function and tiling are developed based on the project code framework, the operator is compiled and deployed via project compilation scripts, enabling the use of single operators or operators within third-party frameworks. - -The following invocation methods are included in this sample: - -| Invocation Method | Directory | Description | -| ----------------- | --------- | ----------- | -| [FrameworkLaunch](./FrameworkLaunch) | [AclNNInvocation](./FrameworkLaunch/AclNNInvocation) | Calling the ThreeNN operator through aclnn. | - -## Operator Description - -For each point in the target point set, the ThreeNN operator finds the distances and coordinates of the three closest points from a reference set. The returned points are sorted in ascending order of squared distance. - -## Operator Specification - - - - - - - - - - - -
Operator TypeThreeNN
Operator inputnameshapedata typeformatdefault
xyz1(b, n, 3)float32ND\
xyz2(b, m, 3)float32ND\
Operator output
dist(b, n, 3)float32ND\
indices(b, n, 3)int32ND\
kernel nameThreeNN
- -### 1. Operator Project Overview - -The operator project directory ThreeNN contains template files, compilation scripts, etc., as follows: - -``` -├── AclNNInvocation // Directory for invoking the ThreeNN operator via aclnn -│ ├── inc // Header file directory -│ │ ├── common.h // Declaration of common methods, used for reading binary files -│ │ ├── op_runner.h // Operator description declaration, including inputs/outputs, operator type, input and output descriptions -│ │ └── operator_desc.h // Operator runtime information declaration file -│ ├── input // Directory for input data generated by scripts -│ ├── output // Directory for storing operator output data and ground truth -│ ├── scripts -│ │ ├── acl.json // ACL configuration file -│ │ ├── gen_data.py // Script for generating input and ground truth data -│ │ └── verify_result.py // File for verifying results -│ ├── src -│ │ ├── CMakeLists.txt // Compilation rules -│ │ ├── common.cpp // Common functions, implementation for reading binary files -│ │ ├── main.cpp // Entry point for single operator invocation -│ │ ├── op_runner.cpp // Main process for single operator invocation -│ │ └── operator_desc.cpp // Construct operator input and output descriptions -│ ├── run.sh // Script for running the test sample -├── ThreeNN // Custom operator project -│ ├── cmake -│ ├── framework // Directory for implementing operator plugins, single operator model generation does not depend on plugins, not needed -│ ├── op_host // Host-side implementation -│ ├── op_kernel // Kernel-side implementation -│ ├── scripts // Directory for packaging custom operator project-related scripts -│ ├── build.sh // Compilation entry script -│ ├── CMakeLists.txt // CMakeLists.txt for the operator project -│ └── CMakePresets.json // Compilation configuration file -│ └── ThreeNN.json // Configuration file for generating the custom operator using msopgen -``` - -The CANN software package provides the project creation tool msopgen. The ThreeNN operator project can be automatically created using ThreeNN.json. For details, please refer to [Ascend C Operator Development](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC) > Operator Development > Operator Project Development > Creating an Operator Project. - -### 2. Compile the Operator Project - -Compile the custom operator project to generate the custom operator package. - -- Run the following commands to navigate to the ReduceCustom operator project directory. - - Use the command line to download the sample code, using the master branch as an example. - - ```bash - cd ${git_clone_path}/samples/operator_contrib/ThreeNNSample/ThreeNN - ``` - -- Modify `CMakePresets.json` to set `ASCEND_CANN_PACKAGE_PATH` to the actual path where the CANN software package is installed. - - ```json - { - …… - "configurePresets": [ - { - …… - "ASCEND_CANN_PACKAGE_PATH": { - "type": "PATH", - "value": "/usr/local/Ascend/ascend-toolkit/latest" - // Replace with the actual installation path of the CANN software package, e.g., /home/HwHiAiUser/Ascend/ascend-toolkit/latest - }, - …… - } - ] - } - ``` - -- Run the following commands in the ReduceCustom operator project directory to compile the operator project. - - ```bash - ./build.sh - ``` - -After a successful compilation, a `build_out` directory is created in the current directory, which contains the custom operator installation package `custom_opp__.run`, for example, `custom_opp_ubuntu_x86_64.run`. - -Note: To use the dump debug feature, remove the Atlas training series product configuration from the op_host and CMakeLists.txt for Atlas 200/500 A2 inference products. - -### 3. Deploy the Operator Package - -Run the following commands in the directory containing the custom operator installation package to install the custom operator package. - -```bash -cd build_out -./custom_opp__.run -``` - -After successful execution, the relevant files in the custom operator package will be deployed to the `vendors/customize` directory in the OPP operator library of the current environment. - -### 4. Configure Environment Variables - -Please select the appropriate command to configure the environment variables according to the [installation method](https://hiascend.com/document/redirect/CannCommunityInstSoftware) of the CANN development kit package in the current environment. - -- Default path, root user installed CANN software package - - ```bash - export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest - ``` - -- Default path, non-root user installed CANN software package - - ```bash - export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest - ``` - -- Specify path `install_path`, installed CANN software package - - ```bash - export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest - ``` - -### 5. Run Operator Sample - -- [aclnn calls the ThreeNN operator project (test sample)](./ThreeNNCase) - -```bash -cd ./AclNNInvocation -./run.sh -``` - -The sample will automatically generate test data, then compile and run the aclnn sample, and finally verify the results. For details, see the `run.sh` script. - -## Update Notes - -| Date | Update Content | -| ---------- | --------------------- | -| 2024/10/24 | Uploaded ThreeNN code | - diff --git a/operator_contrib/ThreeNNSample/README.md b/operator_contrib/ThreeNNSample/README.md deleted file mode 100644 index 39f6afce4..000000000 --- a/operator_contrib/ThreeNNSample/README.md +++ /dev/null @@ -1,145 +0,0 @@ -## ThreeNN自定义算子样例说明 -本样例通过Ascend C编程语言实现了ThreeNN算子，并按照不同的算子调用方式分别给出了对应的端到端实现。 -- [FrameworkLaunch](./FrameworkLaunch)：使用框架调用ThreeNN自定义算子。 -按照工程创建->算子实现->编译部署>算子调用的流程完成算子开发。整个过程都依赖于算子工程：基于工程代码框架完成算子核函数的开发和Tiling实现，通过工程编译脚本完成算子的编译部署，继而实现单算子调用或第三方框架中的算子调用。 - -本样例中包含如下调用方式： - - - - - - -
调用方式目录描述
FrameworkLaunch AclNNInvocation通过aclnn调用的方式调用ThreeNN算子。
- -## 算子描述 -对于目标点集中每个点，找到比较点集中距离其最近的3个点的距离、坐标，返回的三个点会按照距离平方升序。 - -## 算子规格描述 - - - - - - - - - - -
算子类型(OpType)ThreeNN
算子输入nameshapedata typeformatdefault
xyz1(b, n, 3)float32ND\
xyz2(b, m, 3)float32ND\
算子输出
dist(b, n, 3)float32ND\
indices(b, n, 3)int32ND\
核函数名ThreeNN
- -### 1.算子工程介绍 -其中，算子工程目录ThreeNN包含算子实现的模板文件、编译脚本等，如下所示: -``` -├── AclNNInvocation // 通过aclnn调用的方式调用ThreeNN算子 -│ ├── inc // 头文件目录 -│ │ ├── common.h // 声明公共方法类，用于读取二进制文件 -│ │ ├── op_runner.h // 算子描述声明文件，包含算子输入/输出，算子类型以及输入描述与输出描述 -│ │ └── operator_desc.h // 算子运行相关信息声明文件，包含算子 -│ ├── input // 存放脚本生成的输入数据目录 -│ ├── output // 存放算子运行输出数据和真值数据的目录 -│ ├── scripts -│ │ ├── acl.json // acl配置文件 -│ │ ├── gen_data.py // 输入数据和真值数据生成脚本 -│ │ └── verify_result.py // 真值对比文件 -│ ├── src -│ │ ├── CMakeLists.txt // 编译规则文件 -│ │ ├── common.cpp // 公共函数，读取二进制文件函数的实现文件 -│ │ ├── main.cpp // 单算子调用应用的入口 -│ │ ├── op_runner.cpp // 单算子调用主体流程实现文件 -│ │ └── operator_desc.cpp // 构造算子的输入与输出描述 -│ ├── run.sh // 测试样例启动脚本 -├── ThreeNN //自定义算子工程 -│ ├── cmake -│ ├── framework // 算子插件实现文件目录，单算子模型文件的生成不依赖算子适配插件，无需关注 -│ ├── op_host // host侧实现文件 -│ ├── op_kernel // kernel侧实现文件 -│ ├── scripts // 自定义算子工程打包相关脚本所在目录 -│ ├── build.sh // 编译入口脚本 -│ ├── CMakeLists.txt // 算子工程的CMakeLists.txt -│ └── CMakePresets.json // 编译配置项 -│ └── ThreeNN.json // msopgen自定义算子生成配置文件 -``` -CANN软件包中提供了工程创建工具msopgen，ThreeNN算子工程可通过ThreeNN.json自动创建，具体请参考[Ascend C算子开发](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC)>算子开发>算子开发工程>基于自定义算子工程的算子开发>创建算子工程 章节。 - -### 2. 编译算子工程 - -编译自定义算子工程，构建生成自定义算子包。 - -- 执行如下命令，切换到算子工程ReduceCustom目录。 - - 以命令行方式下载样例代码，master分支为例。 - - ```bash - cd ${git_clone_path}/samples/operator_contrib/ThreeNNSample/ThreeNN - ``` -- 修改CMakePresets.json中ASCEND_CANN_PACKAGE_PATH为CANN软件包安装后的实际路径。 - - ```json - { - …… - "configurePresets": [ - { - …… - "ASCEND_CANN_PACKAGE_PATH": { - "type": "PATH", - "value": "/usr/local/Ascend/ascend-toolkit/latest" - //请替换为CANN软件包安装后的实际路径。eg:/home/HwHiAiUser/Ascend/ascend-toolkit/latest - }, - …… - } - ] - } - ``` -- 在算子工程ReduceCustom目录下执行如下命令，进行算子工程编译。 - - ```bash - ./build.sh - ``` - -编译成功后，会在当前目录下创建build_out目录，并在build_out目录下生成自定义算子安装包custom_opp_\_\.run，例如“custom_opp_ubuntu_x86_64.run”。 - -备注：如果要使用dump调试功能，需要移除op_host内和CMakeLists.txt内的Atlas 训练系列产品、Atlas 200/500 A2 推理产品的配置。 - -### 3. 部署算子包 - -执行如下命令，在自定义算子安装包所在路径下，安装自定义算子包。 - -```bash -cd build_out -./custom_opp__.run -``` - -命令执行成功后，自定义算子包中的相关文件将部署至当前环境的OPP算子库的vendors/customize目录中。 - -### 4. 配置环境变量 - -请根据当前环境上CANN开发套件包的[安装方式](https://hiascend.com/document/redirect/CannCommunityInstSoftware)，选择对应配置环境变量的命令。 - -- 默认路径，root用户安装CANN软件包 - ```bash - export ASCEND_INSTALL_PATH=/usr/local/Ascend/ascend-toolkit/latest - ``` -- 默认路径，非root用户安装CANN软件包 - ```bash - export ASCEND_INSTALL_PATH=$HOME/Ascend/ascend-toolkit/latest - ``` -- 指定路径install_path，安装CANN软件包 - ```bash - export ASCEND_INSTALL_PATH=${install_path}/ascend-toolkit/latest - ``` - -### 5. 调用执行算子样例 - -- [aclnn调用ThreeNN算子工程(测试样例)](./ThreeNNCase) - -```bash -cd ./AclNNInvocation -./run.sh -``` -样例执行过程中会自动生成测试数据，然后编译与运行aclnn样例，最后检验运行结果。具体过程可参见run.sh脚本。 - -## 更新说明 - | 时间 | 更新事项 | -|----|------| -| 2024/10/24 | 上传ThreeNN代码 | -- Gitee From beff6997092726a252df70a971bd7907a761b8d4 Mon Sep 17 00:00:00 2001 From: appleinsky <2164757809@qq.com> Date: Fri, 1 Nov 2024 20:55:02 +0800 Subject: [PATCH 10/12] =?UTF-8?q?=E4=BF=AE=E6=94=B9readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- operator_contrib/HistogramSample/README.md | 29 +++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/operator_contrib/HistogramSample/README.md b/operator_contrib/HistogramSample/README.md index d56317364..f5f433f38 100644 --- a/operator_contrib/HistogramSample/README.md +++ b/operator_contrib/HistogramSample/README.md @@ -32,8 +32,14 @@ 核函数名Cumsum - -### 1.算子工程介绍 +## 支持的产品型号 +本样例支持如下产品型号： +- Atlas 训练系列产品 +- Atlas 推理系列产品（Ascend 310P处理器） +- Atlas A2训练系列产品/Atlas 800I A2推理产品 +- Atlas 200/500 A2推理产品 + +## 目录结构介绍 其中，算子工程目录Histogram包含算子实现的模板文件、编译脚本等，如下所示: ``` ├── AclNNInvocation // 通过aclnn调用的方式调用Histogram算子 @@ -67,6 +73,23 @@ ``` CANN软件包中提供了工程创建工具msopgen，Histogram算子工程可通过Histogram.json自动创建，具体请参考[Ascend C算子开发](https://hiascend.com/document/redirect/CannCommunityOpdevAscendC)>算子开发>算子开发工程>基于自定义算子工程的算子开发>创建算子工程 章节。 +## 环境要求 +编译运行此样例前，请参考[《CANN软件安装指南》](https://hiascend.com/document/redirect/CannCommunityInstSoftware)完成开发运行环境的部署。 + +## 编译运行样例算子 + +### 1. 准备：获取样例代码 + + 可以使用以下两种方式下载，请选择其中一种进行源码准备。 + + - 命令行方式下载（下载时间较长，但步骤简单）。 + + ```bash + # 开发环境，非root用户命令行中执行以下命令下载源码仓。git_clone_path为用户自己创建的某个目录。 + cd ${git_clone_path} + git clone https://gitee.com/ascend/samples.git + ``` + ### 2. 编译算子工程 编译自定义算子工程，构建生成自定义算子包。 @@ -136,7 +159,7 @@ cd build_out ### 5. 调用执行算子样例 -- [aclnn调用Histogram算子工程(测试样例)](./HistogramCase) +- [aclnn调用Histogram算子工程(测试样例)](./FrameworkLaunch/AclNNInvocation/) ```bash cd ./AclNNInvocation -- Gitee From 078953fbadac43a92c715f9df86ad0a96707c3e3 Mon Sep 17 00:00:00 2001 From: appleinsky <2164757809@qq.com> Date: Fri, 1 Nov 2024 21:03:39 +0800 Subject: [PATCH 11/12] =?UTF-8?q?=E4=BF=AE=E6=94=B9readme?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- operator_contrib/HistogramSample/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operator_contrib/HistogramSample/README.md b/operator_contrib/HistogramSample/README.md index f5f433f38..83fcdf729 100644 --- a/operator_contrib/HistogramSample/README.md +++ b/operator_contrib/HistogramSample/README.md @@ -94,7 +94,7 @@ CANN软件包中提供了工程创建工具msopgen，Histogram算子工程可通 编译自定义算子工程，构建生成自定义算子包。 -- 执行如下命令，切换到算子工程ReduceCustom目录。 +- 执行如下命令，切换到算子工程Histogram目录。 以命令行方式下载样例代码，master分支为例。 @@ -119,7 +119,7 @@ CANN软件包中提供了工程创建工具msopgen，Histogram算子工程可通 ] } ``` -- 在算子工程ReduceCustom目录下执行如下命令，进行算子工程编译。 +- 在算子工程Histogram目录下执行如下命令，进行算子工程编译。 ```bash ./build.sh -- Gitee From f2130068a3c22bc46aede571a4f9173d36e8bbe8 Mon Sep 17 00:00:00 2001 From: appleinsky <2164757809@qq.com> Date: Mon, 4 Nov 2024 16:03:20 +0800 Subject: [PATCH 12/12] =?UTF-8?q?=E6=8F=90=E4=BA=A4OAT.3=20=E8=AE=B8?= =?UTF-8?q?=E5=8F=AF=E8=AF=81=E5=A4=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Histogram/op_host/histogram.cpp | 14 +++++++++++++- .../Histogram/op_host/histogram_tiling.h | 12 ++++++++++++ .../Histogram/op_kernel/histogram.cpp | 12 ++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram.cpp b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram.cpp index 055f1aadd..306ed7ddf 100644 --- a/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram.cpp +++ b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram.cpp @@ -1,8 +1,20 @@ /** - * @file histogram_tiling.cpp + * @file histogram.cpp * * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. diff --git a/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram_tiling.h b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram_tiling.h index cafda101a..41ce880e3 100644 --- a/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram_tiling.h +++ b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_host/histogram_tiling.h @@ -3,6 +3,18 @@ * * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. diff --git a/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_kernel/histogram.cpp b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_kernel/histogram.cpp index 049e47095..fe0b4295c 100644 --- a/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_kernel/histogram.cpp +++ b/operator_contrib/HistogramSample/FrameworkLaunch/Histogram/op_kernel/histogram.cpp @@ -3,6 +3,18 @@ * * Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -- Gitee