eval.ipynb · cheasim/标题段落分类

Create your Gitee Account

Explore and code with more than 12 million developers，Free private repositories ！：）

This repository doesn't specify license. Please pay attention to the specific project description and its upstream code dependency when using it.

Clone or Download

eval.ipynb 9.17 KB

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python388jvsc74a57bd04501e3efcf9c5caf904b8d89761fcebd342201584c66ffbc27e4a38fa95146b3",
   "display_name": "Python 3.8.8 64-bit ('pl': conda)"
  },
  "metadata": {
   "interpreter": {
    "hash": "4501e3efcf9c5caf904b8d89761fcebd342201584c66ffbc27e4a38fa95146b3"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import pickle\n",
    "from modeling_bert import BertConfig, BertModel\n",
    "from transformers import BertModel as huggingfaceBERT\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "outputPrepend"
    ]
   },
   "outputs": [],
   "source": [
    "path = \"./PLM/pytorch_model.bin\"\n",
    "para_dict = torch.load(path)\n",
    "print(para_dict)\n",
    "huggingface_bert = huggingfaceBERT.from_pretrained(\"/home/xx/pretrained_model/bert-base-uncased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_config = BertConfig.from_json_file(\"./PLM/bert_config.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_params = {}\n",
    "for k, v in para_dict.items():\n",
    "    new_params[k.replace(\"gamma\",\"weight\").replace(\"beta\",\"bias\")] = v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "huggingface_bert.load_state_dict(new_params, strict=False)\n",
    "huggingface_bert.save_pretrained(\".\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k, v in para_dict.items():\n",
    "    if \"embeddings\" in k:\n",
    "        print(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "txt = []\n",
    "import json\n",
    "for name in [\"train\"]:\n",
    "    with open(f\"./dataset/dialogue/{name}.json\") as file:\n",
    "        origin = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"val.txt\", \"w\", encoding=\"UTF-8\") as writer:\n",
    "    for t in txt:\n",
    "        writer.writelines(t+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert json to csv for data augmentation\n",
    "#\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "a = pd.read_csv(\"train_aug.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = [ o for o in a[\"text\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for idx,x in enumerate(origin):\n",
    "    origin[idx][0] = [t[idx]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"train.json\", \"w\", encoding=\"UTF-8\") as file:\n",
    "    json.dump(origin, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"train.json\", \"r\") as file:\n",
    "    t = json.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(origin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_data = []\n",
    "for d in data:\n",
    "    for xy in d[1]:\n",
    "        new_data.append([d[0], [xy]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[0, 14, 9, 8, 10, 8, 9, 9, 8, 10, 11, 8, 9, 8, 9, 8, 9, 9, 8, 0, 9, 8, 8, 5, 8, 8, 1, 2, 8, 8, 8, 8, 8, 0, 8, 8, 5, 8]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "with open(\"dataset/dialogue/train.json\", \"r\") as file:\n",
    "    data = json.load(file)\n",
    "new_data = []\n",
    "for d in data:\n",
    "    for xy in d[1]:\n",
    "        new_data.append([d[0], [xy]])\n",
    "\n",
    "# Minimum-including Algorithm\n",
    "# Python implement\n",
    "# data = [dict()]\n",
    "import random\n",
    "random.seed(2021)\n",
    "random.shuffle(new_data)\n",
    "\n",
    "def getS(data, shot, num_labels):\n",
    "    S = []\n",
    "    label_set = [None] * (num_labels+1)\n",
    "    for d in data:\n",
    "        for i in range(1, num_labels+1):\n",
    "            if i in d[1][0][\"rid\"]:\n",
    "                # print(d[1][\"rid\"])\n",
    "                if label_set[i] == None:\n",
    "                    label_set[i] = [d]\n",
    "                else:\n",
    "                    label_set[i].append(d)\n",
    "    \n",
    "    for i in range(1, num_labels+1):\n",
    "        # print(i, len(label_set[i]))\n",
    "        cnt = 0\n",
    "        if label_set[i] == None: continue\n",
    "        for j in range(len(label_set[i])):\n",
    "            if cnt < shot and label_set[i][j] not in S:\n",
    "                S.append(label_set[i][j])\n",
    "                cnt += 1\n",
    "    \n",
    "    each_label_num = [0] * (num_labels+1)\n",
    "    for sample in S:\n",
    "        for rid in sample[1][0][\"rid\"]:\n",
    "            each_label_num[rid] += 1\n",
    "    \n",
    "    random.shuffle(S)\n",
    "    \n",
    "    for idx, sample in enumerate(S):\n",
    "        for rid in sample[1][0][\"rid\"]:\n",
    "            each_label_num[rid] -= 1\n",
    "        flag = False\n",
    "        for i in range(1, num_labels+1):\n",
    "            if each_label_num[i] < shot and i != 19 and i != 33:\n",
    "                flag = True\n",
    "                break\n",
    "        if flag:\n",
    "            for rid in sample[1][0][\"rid\"]:\n",
    "                each_label_num[rid] += 1\n",
    "        else:\n",
    "            S.pop(idx)\n",
    "    print(each_label_num)\n",
    "    return S\n",
    "\n",
    "t = getS(new_data, 8, 37)\n",
    "with open(\"train.json\", \"w\", encoding=\"UTF-8\") as file:\n",
    "    json.dump(t, file)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"train.json\", \"w\", encoding=\"UTF-8\") as file:\n",
    "    json.dump(t, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "each_label_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get label word by relation word\n",
    "#\n",
    "#\n",
    "from transformers import AutoTokenizer\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "import re\n",
    "import torch\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"/home/xx/pretrained_model/bert-base-uncased\")\n",
    "def split_label_words(tokenizer, label_list):\n",
    "    label_word_list = []\n",
    "    for label in label_list:\n",
    "        if label == 'no_relation':\n",
    "            label_word_id = tokenizer.encode('None', add_special_tokens=False)\n",
    "            label_word_list.append(torch.tensor(label_word_id))\n",
    "        else:\n",
    "            tmps = re.split(':|_|/', label)[1:]\n",
    "            label_word_id = []\n",
    "            for tmp in tmps:\n",
    "                label_word_id += tokenizer.encode(tmp, add_special_tokens=False)\n",
    "            label_word_list.append(torch.tensor(label_word_id))\n",
    "    padded_label_word_list = pad_sequence([x  for x in label_word_list], batch_first=True, padding_value=0)\n",
    "    return padded_label_word_list\n",
    "\n",
    "with open(\"./dataset/relation.txt\", \"r\") as file:\n",
    "    label_list = []\n",
    "    for line in file.readlines():\n",
    "        label_list.append(line.replace(\"\\n\",\"\"))\n",
    "\n",
    "t = split_label_words(tokenizer, label_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"word_idx.pt\", \"wb\") as file:\n",
    "    torch.save(t, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}

Copy Edit Raw Blame History

authored 2021-04-11 13:04 . 123

马建仓 AI 助手

尝试更多

代码解读

代码找茬

代码优化

https://gitee.com/cheasim/title-context-classification.git

git@gitee.com:cheasim/title-context-classification.git

cheasim

title-context-classification

标题段落分类

master

cheasim/标题段落分类

About

Releases

Contributors

Activities