main.ipynb · 华科OpenHarmony技术俱乐部/MindAcc

加入 Gitee

与超过 1200万开发者一起发现、参与优秀开源项目，私有仓库也完全免费：）

文件

克隆/下载

main.ipynb 28.68 KB

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import & Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import gradio as gr\n",
    "import json\n",
    "import logging\n",
    "import numpy as np\n",
    "import os\n",
    "import onnx\n",
    "import onnxruntime\n",
    "import onnxdumper\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import shutil\n",
    "import re\n",
    "\n",
    "logger = logging.getLogger(\"[MindAcc]\")\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "\n",
    "# mslite 环境变量配置 doc: https://www.mindspore.cn/lite/docs/zh-CN/master/train/converter_train.html\n",
    "PACKAGE_ROOT_PATH = Path(\"mindspore-lite-2.3.1-linux-x64\")\n",
    "converter = PACKAGE_ROOT_PATH / \"tools/converter/converter/converter_lite\"\n",
    "benchmark = PACKAGE_ROOT_PATH / \"tools/benchmark/benchmark\"\n",
    "os.environ[\"LD_LIBRARY_PATH\"] = (\n",
    "    str(PACKAGE_ROOT_PATH / \"runtime/lib\")\n",
    "    + \":\"\n",
    "    + str(PACKAGE_ROOT_PATH / \"tools/converter/lib\")\n",
    ")\n",
    "# mslite benchmark配置 doc: https://www.mindspore.cn/lite/docs/zh-CN/master/tools/benchmark_tool.html#dump功能\n",
    "mslite_benchmark_config = {\n",
    "    \"common_dump_settings\": {\n",
    "        \"dump_mode\": 0,\n",
    "        \"path\": f\"{os.getcwd()}/output\",\n",
    "        \"net_name\": \"resnet50\",\n",
    "        \"input_output\": 0,\n",
    "        \"kernels\": [],\n",
    "    },\n",
    "}\n",
    "# 写入文件\n",
    "with open('dump_config.json', 'w') as f:\n",
    "    json.dump(mslite_benchmark_config, f)\n",
    "os.environ[\"MINDSPORE_DUMP_CONFIG\"] = f\"{os.getcwd()}/dump_config.json\"\n",
    "ms_dump_path = Path(\"output\")/mslite_benchmark_config[\"common_dump_settings\"][\"net_name\"]\n",
    "# 创建目录\n",
    "for folder in [Path(\"input\"), Path(\"model\"), Path(\"output\"), ms_dump_path]:\n",
    "    if not folder.exists():\n",
    "        folder.mkdir(parents=True)\n",
    "# 记录当前dump目录结构\n",
    "last_dump_dirs = [f for f in ms_dump_path.iterdir() if f.is_dir()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MindAcc Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MindaccModel:\n",
    "    def __init__(self):\n",
    "        self.path: Path = None\n",
    "        self.name: str = None\n",
    "        self.onnx_session: onnxdumper.InferenceSession = None\n",
    "        self.input_nodes: list[onnxruntime.NodeArg] = None\n",
    "        self.output_nodes: list[onnxruntime.NodeArg] = None\n",
    "        self.precision: np.dtype = None\n",
    "        self.onnx_input: dict = None\n",
    "        self.ms_output_path: Path = None\n",
    "        self.onnx_output_path: Path = \"onnx_dumpinrun.npz\"\n",
    "        self.compare_map = {}\n",
    "        self.compare_result: pd.DataFrame = None\n",
    "\n",
    "    def load(self, model_file: gr.File):\n",
    "        self.path = Path(model_file)\n",
    "        self.name = self.path.stem\n",
    "        # 使用 onnxdumper 的带导出的推理会话\n",
    "        self.onnx_session = onnxdumper.InferenceSession(self.path)\n",
    "        self.input_nodes = self.onnx_session.get_inputs()\n",
    "        self.output_nodes = self.onnx_session.get_outputs()\n",
    "        # 获取输入精度\n",
    "        self.precision = onnx.load(self.path).graph.input[0].type.tensor_type\n",
    "        self.precision = onnx.helper.tensor_dtype_to_np_dtype(self.precision.elem_type)\n",
    "\n",
    "    def run_ms_converter(self, optimize = \"\") -> Path:\n",
    "        cmd = f\"{converter} --fmk=ONNX --modelFile={self.path} --outputFile={self.path} {\"--optimize=\" + optimize if optimize else \"\"}\"\n",
    "        # cmd = f\"{converter} --fmk=ONNX --optimize=none --modelFile={self.path} --outputFile={model}\"\n",
    "        logger.info(\"Run convert cmd:{}\".format(cmd))\n",
    "        os.system(cmd)\n",
    "        logger.info(\"Convert done, output model:{}\".format(self.path))\n",
    "        return Path(str(self.path) + \".ms\")\n",
    "\n",
    "    def input_generate(self, seed = 0) -> dict:\n",
    "        # 设置随机数种子\n",
    "        np.random.seed(seed)\n",
    "        random_input = {}\n",
    "        # 先考虑单输入的情况\n",
    "        # inputs_nodes = ort_session.get_inputs()\n",
    "        # for input_node in inputs_nodes:\n",
    "        #     input_data = np.random.random(input_node.shape).astype(precision)\n",
    "        #     input_data.tofile(f\"mslite_input.bin\")\n",
    "        #     random_input[input_node.name] = input_data\n",
    "        #     logger.info(\"input node generated: {} {}\".format(input_node.name, input_node.shape))\n",
    "        input_node = mindacc_model.input_nodes[0]\n",
    "        input_data = np.random.random(input_node.shape).astype(mindacc_model.precision)\n",
    "        # 4维输入时 nchw to nhwc\n",
    "        if len(input_data.shape) == 4:\n",
    "            ms_input_data = np.transpose(input_data, (0, 2, 3, 1))\n",
    "        else :\n",
    "            ms_input_data = input_data\n",
    "        ms_input_data.tofile(\"input/mslite_input.bin\")\n",
    "        random_input[input_node.name] = input_data\n",
    "        logger.info(\n",
    "            \"input node generated: {} {}\".format(input_node.name, input_node.shape)\n",
    "        )\n",
    "        self.onnx_input = random_input\n",
    "        return random_input\n",
    "    \n",
    "    def input_load(self, input_data: gr.File) -> None:      # 按mslite的输入格式加载数据\n",
    "        ms_input_data = np.fromfile(input_data, dtype=self.precision)\n",
    "        ms_input_data.tofile(\"input/mslite_input.bin\")\n",
    "        # 4维输入时 nhwc to nchw\n",
    "        if len(ms_input_data.shape) == 4:\n",
    "            onnx_input_data = np.transpose(input_data, (0, 3, 1, 2))\n",
    "        else:\n",
    "            onnx_input_data = ms_input_data\n",
    "        self.onnx_input = {self.input_nodes[0].name: onnx_input_data}\n",
    "\n",
    "    def run_ms_dump(self) -> Path:\n",
    "        global last_dump_dirs\n",
    "        cmd = f\"{benchmark.resolve()} --modelFile={str(self.path)+\".ms\"} --inDataFile=input/mslite_input.bin\"\n",
    "        logger.info(\"Run benchmark cmd:{}\".format(cmd))\n",
    "        os.system(cmd)\n",
    "        logger.info(\"Benchmark done\")\n",
    "        now_dump_dirs = [f for f in ms_dump_path.iterdir() if f.is_dir()]\n",
    "        # 获取最新的dump文件夹\n",
    "        new_dump_dir = list(set(now_dump_dirs) - set(last_dump_dirs))\n",
    "        logger.info(\"Dump dir:{}\".format(new_dump_dir))\n",
    "        last_dump_dirs = now_dump_dirs\n",
    "        self.ms_output_path = new_dump_dir[0] if new_dump_dir else None\n",
    "        return self.ms_output_path\n",
    "        \n",
    "    def run_onnx_dump(self, dump_path = \"onnx_dumpinrun.npz\") -> dict:\n",
    "        self.onnx_output_path = dump_path\n",
    "        onnx_outputs = self.onnx_session.run([x.name for x in self.output_nodes], self.onnx_input, dump_path=dump_path)\n",
    "        logger.info(\"onnx inference done\")\n",
    "        return onnx_outputs\n",
    "\n",
    "# 单例模式\n",
    "mindacc_model = MindaccModel()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MindAcc Mapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MindAccMapper:\n",
    "    \n",
    "    @classmethod\n",
    "    def get_ms_bin_info(cls, ms_bin_file: Path):\n",
    "         # 从文件名中正则解析出shape和dtype   {op_name}_{input_output_index}_{shape}_{data_type}_{format}.bin\n",
    "        MS_DATATYPE_TO_NP = {\n",
    "            \"Float64\": np.float64,\n",
    "            \"Float32\": np.float32,\n",
    "            \"Float16\": np.float16,\n",
    "            \"Int64\": np.int64,\n",
    "            \"Int32\": np.int32,\n",
    "            \"Int8\": np.int8,\n",
    "            \"UInt64\": np.uint64,\n",
    "            \"UInt8\": np.uint8,\n",
    "            \"Bool\": np.bool_,\n",
    "        }\n",
    "        ms_out_pattern = re.compile(\n",
    "           r\"(?P<op_name>\\w+)_(?P<io_flag>(input|output))_(?P<io_index>\\d+)_(shape)(_(?P<shape>(\\d+(_\\d+)*)))?_(?P<data_type>[^_]+)(_(?P<layout>\\w+))?\"\n",
    "        )\n",
    "        shape_str_to_np = lambda shape_str: (tuple(map(int, shape_str.split(\"_\"))) if shape_str else ())\n",
    "        try:\n",
    "            node_match = ms_out_pattern.match(ms_bin_file.stem).groupdict()\n",
    "            node_match[\"shape\"] = shape_str_to_np(node_match[\"shape\"])\n",
    "            node_match[\"data_type\"] = MS_DATATYPE_TO_NP[node_match[\"data_type\"]]\n",
    "            return node_match\n",
    "        except Exception as e:\n",
    "            logger.warning(f\"Can't parse file {ms_bin_file.name}, error: {e}\")\n",
    "\n",
    "    @classmethod\n",
    "    def read_ms_output(cls, ms_output_path: Path):\n",
    "        \"\"\"\n",
    "        解析结果格式如：\n",
    "        {1: {'shape': (6,), 'data_type': <class 'numpy.int32'>, 'layout': 'NCHW', 'file': PosixPath('/home/liuhaoyi/oh24/mindacc/output/ssd12/9/    TopK_717_output_1_shape_6_Int32_NCHW.bin')}, 0: {'shape':   (6,), 'data_type': <class 'numpy.float32'>, 'layout': 'NCHW', 'file': PosixPath('/home/liuhaoyi/oh24/   mindacc/output/ssd12/9/TopK_717_output_0_shape_6_Float32_NCHW.bin')}}\n",
    "        \"\"\"\n",
    "\n",
    "        ms_output = defaultdict(dict)\n",
    "        for file in ms_output_path.iterdir():\n",
    "            if file.is_file() and file.suffix == \".bin\":\n",
    "                x = MindAccMapper.get_ms_bin_info(file)\n",
    "                if x[\"io_flag\"] == \"output\":\n",
    "                    index = int(x[\"io_index\"])\n",
    "                    ms_output[x[\"op_name\"]][index] = {\n",
    "                        \"shape\": x[\"shape\"],\n",
    "                        \"data_type\": x[\"data_type\"],\n",
    "                        \"layout\": x[\"layout\"],\n",
    "                        \"file\": file,\n",
    "                    }\n",
    "                # 比较时再读取数据\n",
    "                # ms_output[node_name][\"data\"] = np.fromfile(\n",
    "                #     file, dtype=MS_DATATYPE_TO_NP[x[\"data_type\"]]\n",
    "                # )\n",
    "        return ms_output\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        onnx_model: str,\n",
    "        onnx_dump_file: Path,\n",
    "        ms_dump_dir: Path,\n",
    "        extra_rules: dict = {},\n",
    "    ) -> None:\n",
    "        self.onnx_model = onnx.load(onnx_model)\n",
    "        self.onnx_dump = np.load(onnx_dump_file)\n",
    "        self.ms_dump = MindAccMapper.read_ms_output(Path(ms_dump_dir))\n",
    "        self.extra_rules = extra_rules\n",
    "        self.map = {}\n",
    "    \n",
    "    def simple_map(self):\n",
    "        hwc2chw = lambda shape: (shape[0], shape[3], shape[1], shape[2]) if len(shape) == 4 else shape\n",
    "        chw2hwc = lambda shape: (shape[0], shape[2], shape[3], shape[1]) if len(shape) == 4 else shape\n",
    "        dtype_matcher = lambda dtype1, dtype2: (dtype1 == dtype2 or \n",
    "                                        (dtype1 in [np.int32, np.int64] and dtype2 in [np.int32, np.int64]))\n",
    "        shape_matcher = lambda shape1, shape2: (shape1 == shape2 or\n",
    "                                        (len(shape1) == 4 and len(shape2) == 4 and hwc2chw(shape1) == shape2) or\n",
    "                                        (len(shape1) == 4 and len(shape2) == 4 and chw2hwc(shape1) == shape2))\n",
    "        \n",
    "        \n",
    "        for ms_node in self.ms_dump:\n",
    "            # 在 onnx model 中找到对应的 node\n",
    "            onnx_node = next((x for x in self.onnx_model.graph.node if x.name in ms_node), None)\n",
    "            if onnx_node is None:\n",
    "                logger.warning(f\"Can't find node {ms_node} in onnx model\")\n",
    "                continue\n",
    "            # 为每一个 output 找到对应的 onnx output\n",
    "            onnx_outputs = onnx_node.output\n",
    "            matched_onnx = []\n",
    "            for i, ms_output_info in self.ms_dump[ms_node].items():\n",
    "                for j, onnx_output in enumerate(onnx_outputs):\n",
    "                    # 条件1: 形状匹配\n",
    "                    shape_match = shape_matcher(ms_output_info[\"shape\"], self.onnx_dump[onnx_output].shape)\n",
    "                    # 条件2: 数据类型匹配\n",
    "                    dtype_match = dtype_matcher(ms_output_info[\"data_type\"], self.onnx_dump[onnx_output].dtype)\n",
    "                    # 条件3: ONNX输出未匹配\n",
    "                    not_matched = onnx_output not in matched_onnx\n",
    "                    # 满足所有条件\n",
    "                    match_condition = shape_match and dtype_match and not_matched\n",
    "                    # if ms_node == \"Shape_384_fusion\":\n",
    "                    #     print(ms_output_info[\"shape\"], self.onnx_dump[onnx_output].shape, ms_output_info[\"data_type\"], \"onnx:\",self.onnx_dump[onnx_output].dtype, match_condition)\n",
    "                    if match_condition:\n",
    "                        matched_onnx.append(onnx_output)\n",
    "                        self.ms_dump[ms_node][i][\"onnx_output\"] = onnx_output\n",
    "\n",
    "        self.map = {}\n",
    "        for ms_node in self.ms_dump:\n",
    "            for i, ms_output_info in self.ms_dump[ms_node].items():\n",
    "                if \"onnx_output\" in ms_output_info:\n",
    "                    file_name=ms_output_info['file'].name\n",
    "                    self.map[file_name] =ms_output_info['onnx_output']\n",
    "                else:\n",
    "                    # logger.warning(f\"Can't find node {ms_node} output {i} in onnx model\")\n",
    "                    pass\n",
    "        return self.map\n",
    "    \n",
    "    def get_map_result(self):\n",
    "        # 获取映射成功率\n",
    "        maped_count = len(self.map)\n",
    "        all_count = sum([len(self.ms_dump[node]) for node in self.ms_dump])\n",
    "        map_rate = maped_count / all_count\n",
    "        # 获取未映射列表\n",
    "        unmap_list = []\n",
    "        for ms_node in self.ms_dump:\n",
    "            for i, ms_output_info in self.ms_dump[ms_node].items():\n",
    "                if \"onnx_output\" not in ms_output_info:\n",
    "                    file_name=ms_output_info['file'].name\n",
    "                    unmap_list.append(file_name)\n",
    "        # 获取映射列表详情\n",
    "        map_list = {}\n",
    "        for i in self.map:\n",
    "            tuple_to_str = lambda t: '_'.join(map(str, t))\n",
    "            v = f\"{self.map[i]}_{tuple_to_str(self.onnx_dump[self.map[i]].shape)}_{self.onnx_dump[self.map[i]].dtype}\"\n",
    "            map_list[i] = v\n",
    "        return maped_count, all_count, map_rate, unmap_list, map_list\n",
    "            \n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MindAcc Analyzer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MindAccAnalyzer:\n",
    "    analyzers = {\n",
    "        \"cosine_similarity\": lambda a, b: np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)),\n",
    "        \"relative_euclidean_distance\": lambda a, b: np.linalg.norm(a - b) / np.linalg.norm(a),\n",
    "        \"max_absolute_error\": lambda a, b: np.max(np.abs(a - b)),\n",
    "        \"mean_absolute_error\": lambda a, b: np.mean(np.abs(a - b)),\n",
    "        \"root_mean_square_error\": lambda a, b: np.sqrt(np.mean(np.square(a - b))),\n",
    "        \"max_relative_error\": lambda a, b: np.max(np.abs(a - b) / np.abs(a)),\n",
    "        \"mean_relative_error\": lambda a, b: np.mean(np.abs(a - b) / np.abs(a)),\n",
    "        \"accumulated_relative_error\": lambda a, b: np.mean(np.abs(a - b) / np.abs(a)),\n",
    "        \"standard_deviation\": lambda a, b: np.std(np.abs(a - b) / np.abs(a)),\n",
    "        \"kullback_leibler_divergence\": lambda a, b: np.sum(a * np.log(a / b))\n",
    "    }\n",
    "    # MeanAbsoluteError趋于0，RootMeanSquareError趋于0，说明测量值与真实值越近似。\n",
    "    # MeanAbsoluteError趋于0，RootMeanSquareError越大，说明存在局部过大的异常值。\n",
    "    # MeanAbsoluteError越大，RootMeanSquareError等于或近似MeanAbsoluteError，说明整体偏差越集中。\n",
    "    # MeanAbsoluteError越大，RootMeanSquareError越大于MeanAbsoluteError，说明存在整体偏差，且整体偏差分布分散。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradio Interface"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_model(model_file: gr.File, optimize: str):\n",
    "    # 保存模型文件到./model目录\n",
    "    if not model_file:\n",
    "        return None, None\n",
    "    model_file = shutil.copy(model_file, \"./model\")\n",
    "    mindacc_model.load(model_file)\n",
    "    model_info = f\"模型路径: {mindacc_model.path}\\n模型名称: {mindacc_model.name}\\n输入节点: {[str(x) for x in mindacc_model.input_nodes]}\\n输出节点: {[str(x) for x in mindacc_model.output_nodes]}\\n输入精度: {mindacc_model.precision}\"\n",
    "\n",
    "    converted_model = mindacc_model.run_ms_converter(optimize = optimize)\n",
    "    converted_model = str(converted_model)\n",
    "\n",
    "    return  model_info, gr.File(label=\"转换后的模型\", value=converted_model, visible=True)\n",
    "\n",
    "def process_input(input_file: gr.File):\n",
    "    mindacc_model.input_load(input_file)\n",
    "\n",
    "def random_input(seed):\n",
    "    # 生成随机输入\n",
    "    mindacc_model.input_generate(seed=seed)\n",
    "    onnx_shape = mindacc_model.input_nodes[0].shape\n",
    "    # # 4维输入时 nchw to nhwc\n",
    "    ms_shape = onnx_shape if len(onnx_shape) != 4 else (onnx_shape[0], onnx_shape[2], onnx_shape[3], onnx_shape[1])\n",
    "    return \"input/mslite_input.bin\", ms_shape\n",
    "\n",
    "def run_infer():\n",
    "    # 运行推理\n",
    "    ms_bench_dir=mindacc_model.run_ms_dump()\n",
    "    mindacc_model.run_onnx_dump(dump_path = ms_bench_dir/\"onnx_dump.npz\")\n",
    "    return f\"mslite模型输出文件夹: {str(ms_bench_dir)}, onnx模型输出文件: {str(ms_bench_dir/\"onnx_dump.npz\")}\"\n",
    "\n",
    "def run_map():\n",
    "    # 构造映射\n",
    "    mapper = MindAccMapper(onnx_model=mindacc_model.path, onnx_dump_file=mindacc_model.onnx_output_path, ms_dump_dir=mindacc_model.ms_output_path)\n",
    "    mindacc_model.compare_map = mapper.simple_map()\n",
    "    # 构造pd.DataFrame用于显示\n",
    "    model_map_dataframe = pd.DataFrame(mindacc_model.compare_map.items(), columns=[\"MSlite输出节点\", \"ONNX输出节点\"])\n",
    "    # 在DataFrame中添加未映射的节点\n",
    "    maped_count, all_count, map_rate, unmap_list, map_list = mapper.get_map_result()\n",
    "    unmap_df = pd.DataFrame({\"MSlite输出节点\": unmap_list, \"ONNX输出节点\": \"未映射\"})\n",
    "    model_map_dataframe = pd.concat([model_map_dataframe, unmap_df], ignore_index=True)\n",
    "    return f\"映射成功率: {maped_count}/{all_count}={map_rate}\", model_map_dataframe\n",
    "    \n",
    "\n",
    "def run_compare(analyzers, model_mapping, progress=gr.Progress()):\n",
    "    nhwc2nchw = lambda x: np.transpose(x, (0, 3, 1, 2)) if len(x.shape) == 4 else x\n",
    "    # 更新映射\n",
    "    old= mindacc_model.compare_map\n",
    "    mindacc_model.compare_map = {row[\"MSlite输出节点\"]: row[\"ONNX输出节点\"] for row in model_mapping.to_dict(orient='records') if row[\"ONNX输出节点\"] != \"未映射\" and row[\"ONNX输出节点\"] != \"\"}\n",
    "    # 运行对比\n",
    "    compare_result = pd.DataFrame()\n",
    "    onnx_dump = np.load(mindacc_model.onnx_output_path)\n",
    "    for i in progress.tqdm(mindacc_model.compare_map):\n",
    "        i_path = mindacc_model.ms_output_path/i\n",
    "        i_info = MindAccMapper.get_ms_bin_info(i_path)\n",
    "        a_shape = i_info['shape']\n",
    "        a_dtype = i_info['data_type']\n",
    "        a = np.fromfile(i_path, dtype=a_dtype).reshape(a_shape)\n",
    "        b = onnx_dump[mindacc_model.compare_map[i]]\n",
    "        # a, b输入维度不一致时，a转换为nchw\n",
    "        if a.shape != b.shape:\n",
    "            a = nhwc2nchw(a)\n",
    "        # print(\"a\", i_path, a.shape, a.dtype)      \n",
    "        # print(\"b\",b.shape, b.dtype)\n",
    "        for analyzer in analyzers:\n",
    "            # 设置节点名\n",
    "            compare_result.loc[i, \"节点\"] = i\n",
    "            try:\n",
    "                compare_result.loc[i, analyzer] = MindAccAnalyzer.analyzers[analyzer](a.flatten(), b.flatten())\n",
    "            except Exception as e:\n",
    "                # logger.warning(f\"Can't run analyzer {analyzer} for node {i}, error: {e}\")\n",
    "                pass\n",
    "    mindacc_model.compare_result = compare_result\n",
    "    # 保存对比结果\n",
    "    date_time = pd.Timestamp.now().strftime(\"%Y-%m-%d_%H-%M-%S\")\n",
    "    file_path = mindacc_model.ms_output_path / f\"compare_result_{date_time}.csv\"\n",
    "    mindacc_model.compare_result.to_csv(file_path)\n",
    "    return compare_result, gr.DownloadButton(label=f\"下载对比结果 {file_path.name}\", value=file_path, visible=True)\n",
    "\n",
    "\n",
    "\n",
    "with gr.Blocks(theme=gr.themes.Soft()) as mindacc:\n",
    "    # gr.Markdown(\"\"\"\n",
    "    # # MindAcc\n",
    "    #  MindAcc是一个用于对比MindSpore Lite与ONNX模型推理结果的工具，\n",
    "    \n",
    "    # 支持模型转换、随机输入生成、推理运行、对比分析等功能。\n",
    "    #             \"\"\")\n",
    "    gr.HTML(\"\"\"\n",
    "        <div style=\"text-align: center; margin-bottom: 200px;\">\n",
    "            <h1 style=\"font-size: 3em;\">MindAcc</h1>\n",
    "            <p style=\"font-size: 1.5em;\">MindAcc是一个用于对比MindSpore Lite与ONNX模型推理结果的工具，</p>\n",
    "            <p style=\"font-size: 1.5em;\">支持模型转换、随机输入生成、推理运行、对比分析等功能。</p>\n",
    "            <p style=\"font-size: 1.5em;\"><a href=\"https://gitee.com/noiatrio/mindacc\" target=\"_blank\">访问Gitee地址</a></p>\n",
    "        </div>\n",
    "    \"\"\")\n",
    "    with gr.Accordion(label=\"使用说明\"):\n",
    "        gr.HTML(\"\"\"\n",
    "            <div>\n",
    "            <p style=\"font-size: 12px;\">1. 上传ONNX模型文件，将自动转换为MSlite模型并读取信息</p>\n",
    "            <p style=\"font-size: 12px;\">2. 上传MSlite输入数据或点击随机生成输入按钮</p>\n",
    "            <p style=\"font-size: 12px;\">3. 点击运行推理按钮，将使用benchmark工具获取MSlite输出，使用onnxdumper获取ONNX输出</p>\n",
    "            <p style=\"font-size: 12px;\">4. 点击生成映射按钮，将自动匹配MSlite输出节点与ONNX输出节点，若需修改映射可在节点映射表中直接修改</p>\n",
    "            <p style=\"font-size: 12px;\">5. 点击运行对比按钮，选择对比分析指标，获取对比结果</p>\n",
    "        </div>\n",
    "        \"\"\")\n",
    "    with gr.Row():\n",
    "        random_button = gr.Button(\"随机生成输入\", variant=\"primary\")\n",
    "        with gr.Column(scale=1):\n",
    "            random_seed_input = gr.Number(label=\"随机数种子\")\n",
    "            optimize_input = gr.Dropdown(label=\"convert-lite优化选项\", choices=[\"none\", \"general\", \"gpu_oriented\", \"ascend_oriented\"], value=\"none\")\n",
    "            model_info_output = gr.Textbox(label=\"模型信息\")\n",
    "        with gr.Column(scale=1):\n",
    "            with gr.Group():\n",
    "                # onnx_shape_input = gr.Textbox(label=\"ONNX 输入形状\")\n",
    "                # ms_shape_input = gr.Textbox(label=\"MSlite 输入形状\")\n",
    "                model_input = gr.File(label=\"上传模型\")\n",
    "                converted_model_output = gr.File(label=\"转换后的模型\", visible=False)\n",
    "        with gr.Column(scale=1):\n",
    "            with gr.Group():            \n",
    "                data_input = gr.File(label=\"上传 MSlite 输入数据, 或使用随机生成\")\n",
    "                ms_shape_input = gr.Textbox(label=\"MSlite 输入形状\")\n",
    "                \n",
    "            \n",
    "    with gr.Row():\n",
    "        run_infer_button = gr.Button(\"运行推理\", variant=\"primary\", scale=1)\n",
    "        infer_output = gr.Textbox(label=\"推理结果\", scale=2)\n",
    "    with gr.Row():\n",
    "        run_map_button = gr.Button(\"生成映射\", variant=\"primary\", scale=1)\n",
    "        mapping_outline = gr.Textbox(label=\"映射概览\", scale=2)\n",
    "    with gr.Row():\n",
    "        model_mapping = gr.DataFrame(label=\"节点映射\", headers=[\"MSlite输出节点\", \"ONNX输出节点\"], datatype=[\"str\", \"str\"])\n",
    "    with gr.Row():    \n",
    "        run_compare_button = gr.Button(\"运行对比\", variant=\"primary\", scale=1) \n",
    "        analysis_tool = gr.CheckboxGroup([x for x in MindAccAnalyzer.analyzers.keys()], label=\"对比分析指标\", scale=2)\n",
    "    compare_output = gr.DataFrame(label=\"对比结果\", headers=[\"节点\", *MindAccAnalyzer.analyzers.keys()])\n",
    "    \n",
    "    with gr.Row():\n",
    "        run_draw_button = gr.Button(\"绘制统计图\", variant=\"primary\") \n",
    "        dowload_compare_button = gr.DownloadButton(\"下载对比结果\", variant=\"primary\")\n",
    "    \n",
    "    # 渲染柱形图\n",
    "    @gr.render(triggers=[run_draw_button.click])\n",
    "    def create_plot():\n",
    "        # gr.LinePlot(pd.DataFrame(\n",
    "        #     {\"x\": [1, 2, 3], \"y\": [1, 2, 3]}), x=\"x\", y=\"y\")\n",
    "        dataframe = mindacc_model.compare_result\n",
    "\n",
    "        if '节点' not in dataframe.columns or len(dataframe['节点']) == 0:\n",
    "            print(\"No data\")\n",
    "            gr.Markdown(\"## 等待对比结果\")\n",
    "        else:\n",
    "            # 每一个指标绘制一个柱形图\n",
    "            # 获取dataframe的列名\n",
    "            analyzers = dataframe.columns[1:]\n",
    "            for analyzer in analyzers:\n",
    "                # 取出指标数据\n",
    "                data = pd.DataFrame({\"节点\": dataframe['节点'], analyzer: dataframe[analyzer]})\n",
    "                if len(data) == 0:\n",
    "                    continue\n",
    "                # 绘制柱形图\n",
    "                gr.BarPlot(data, x=\"节点\", y=analyzer)\n",
    "                \n",
    "    model_input.upload(\n",
    "        fn=process_model,\n",
    "        inputs=[model_input, optimize_input],\n",
    "        outputs=[model_info_output, converted_model_output],\n",
    "    )\n",
    "    data_input.upload(\n",
    "        fn=process_input,\n",
    "        inputs=[data_input],\n",
    "        outputs=[],\n",
    "    )\n",
    "    random_button.click(fn=random_input, inputs=random_seed_input, outputs=[data_input, ms_shape_input])\n",
    "    run_infer_button.click(fn=run_infer, inputs=None, outputs=[infer_output])\n",
    "    run_map_button.click(fn=run_map, inputs=None, outputs=[mapping_outline, model_mapping])\n",
    "    run_compare_button.click(fn=run_compare, inputs=[analysis_tool, model_mapping], outputs=[compare_output, dowload_compare_button])\n",
    "\n",
    "mindacc.launch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "onnx",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

一键复制编辑原始数据按行查看历史

提交于 2024-10-05 14:02 . feat: 添加分析错误处理

马建仓 AI 助手

尝试更多

代码解读

代码找茬

代码优化

Python

https://gitee.com/hust-openharmony-club/mindacc.git

git@gitee.com:hust-openharmony-club/mindacc.git

hust-openharmony-club

mindacc

MindAcc

master

华科OpenHarmony技术俱乐部/MindAcc

简介

发行版

贡献者

近期动态

华科OpenHarmony技术俱乐部/MindAcc .gitee-modal { width: 500px !important; }

简介

发行版

贡献者

近期动态

搜索帮助

华科OpenHarmony技术俱乐部/MindAcc