filepath.ipynb · gomico/filepath_grabber

加入 Gitee

与超过 1200万开发者一起发现、参与优秀开源项目，私有仓库也完全免费：）

文件

该仓库未声明开源许可证文件（LICENSE），使用请关注具体项目描述及其代码上游依赖。

克隆/下载

filepath.ipynb 28.47 KB

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from IPython.core.interactiveshell import InteractiveShell\n",
    "# InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Usage \n",
    "\n",
    "Environment: Python 3, Jupyter Notebook (web IDE or vscode)\n",
    "\n",
    "Run the first or second code cell below for openEuler or OpenHarmony\n",
    "\n",
    "Running time according to the number of PRs (approx. 2s/PR in test environment, affected by connection speed)\n",
    "\n",
    "**final-*date*-*time*.txt** is the final output\n",
    "\n",
    "Copy and paste content of **final-*date*-*time*.txt** to Excel"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### For openEuler issues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For openEuler issues\n",
    "import json\n",
    "import re\n",
    "from urllib import request\n",
    "from collections import Counter\n",
    "import time\n",
    "import os\n",
    "\n",
    "\n",
    "def getIssueList():\n",
    "    issue_list = open(\"issues_oe.txt\", \"a+\")  # create issue list file\n",
    "    pr_list = open(\"prs_oe.txt\", \"a+\")\n",
    "    title_counter = {'a': 1}  # counter value initialization\n",
    "    issue_page_no = 1\n",
    "    while len(title_counter) > 0:\n",
    "        issue_page_url = \"https://gitee.com/openeuler/docs/issues?issue_search=%5BAuto%5D+This+is+an+English+translation+issue.&page=\" + \\\n",
    "            str(issue_page_no)+\"&state=open\"\n",
    "        issue_page_content = request.urlopen(issue_page_url).read().decode()\n",
    "        title_repeats = re.findall(\n",
    "            r'This is an English translation issue\\.\\[[0-9]*\\]\" class=\"title\" href=\"/openeuler/docs/issues/[0-9A-Z]{6}', issue_page_content)\n",
    "        for issue_index in range(len(title_repeats)):\n",
    "            issue_title = re.search(\"[0-9]{1,6}\", title_repeats[issue_index])\n",
    "            issue_ID = re.search(\"[0-9A-Z]{6}\", title_repeats[issue_index])\n",
    "            print(issue_title.group(0)+\"\\t\"+issue_ID.group(0)+\"^^\" +\n",
    "                  \"https://gitee.com/openeuler/docs/issues/\"+issue_ID.group(0), file=issue_list)\n",
    "            print(issue_title.group(0), file=pr_list)\n",
    "            print(\"acquired issue No.\"+issue_title.group(0) +\n",
    "                  \" with ID \"+issue_ID.group(0)+\" (\"+issue_index.group(0)+\")\")\n",
    "        # title_repeats = list(map(str,title_repeats))\n",
    "        title_counter = Counter(title_repeats)\n",
    "        # print(title_counter)\n",
    "        issue_page_no = issue_page_no + 1\n",
    "    issue_list.close()\n",
    "    pr_list.close()\n",
    "\n",
    "# convert issues_oe.txt into a dict\n",
    "\n",
    "\n",
    "def fileToDict():\n",
    "    file_dict = {}\n",
    "    file_f = open(\"issues_oe.txt\")\n",
    "    for line in file_f:\n",
    "        key, value = line.split()\n",
    "        file_dict[key] = value\n",
    "    file_f.close()\n",
    "    return file_dict\n",
    "\n",
    "\n",
    "def getPRList():\n",
    "    pr_list = open(\"prs_oe.txt\", \"r\").readlines()\n",
    "    for i in range(len(pr_list)):\n",
    "        pr_list[i] = pr_list[i].rstrip('\\n')\n",
    "    print(\"=====================================================\")\n",
    "    print(\"{0} issues in total\".format(len(pr_list)))\n",
    "    print(\"=====================================================\")\n",
    "    # pr_list.sort()\n",
    "    return pr_list\n",
    "\n",
    "# Use by default\n",
    "\n",
    "\n",
    "def getFiles(list_prs):\n",
    "    for index in range(len(list_prs)):\n",
    "        pr_no = list_prs[index]\n",
    "        print(\"({0}/{1}) Retrieving issue {2}...\".format(index +\n",
    "              1, len(list_prs), pr_no))\n",
    "        f = open(\"branch_file_oe.txt\", \"a+\")\n",
    "        file_url = \"https://gitee.com/openeuler/docs/pulls/\"+pr_no+\"/files.json\"\n",
    "        pr_url = \"https://gitee.com/openeuler/docs/pulls/\"+pr_no\n",
    "        # print(pr_no, file = f)\n",
    "        # print(file_url, file = f)\n",
    "        branch_response = request.urlopen(pr_url)\n",
    "        response = request.urlopen(file_url)\n",
    "        branch = branch_response.read().decode()    # string of PR page html\n",
    "        files = response.read().decode()    # string of PR file json content\n",
    "        d_files = json.loads(files)         # string to dict\n",
    "        l_diffs = d_files['diffs']          # pick 'diffs' list from d_files\n",
    "        for i in range(len(l_diffs)):       # pick 'head' dict from l_diffs\n",
    "            d_file = l_diffs[i]\n",
    "            # search branch object\n",
    "            branchObj = re.findall(\n",
    "                '(?<=href=\"/openeuler/docs/tree/).*(?=\"><span)', branch)\n",
    "            # select branch string\n",
    "            s_branch = branchObj[len(branchObj)-1]\n",
    "            s_name = d_file[\"head\"][\"name\"]  # pick 'name' string\n",
    "            s_name = re.sub(' *[0-9]* *→ *[0-9]*$', \"\", s_name)\n",
    "            int_added_lines = d_file[\"added_lines\"]\n",
    "            int_removed_lines = d_file[\"removed_lines\"]\n",
    "            print(\"branch=\"+s_branch+\"\\tfile=\"+s_name+\"\\tlines added=\" +\n",
    "                  str(int_added_lines)+\"\\tlines removed=\"+str(int_removed_lines))\n",
    "            print(pr_no+\"\\t\"+s_branch+\"\\t\"+s_name+\"\\t\" +\n",
    "                  str(int_added_lines)+\"\\t\"+str(int_removed_lines), file=f)\n",
    "        f.close()\n",
    "        print(\"-----------------------------------------------------\")\n",
    "\n",
    "# Use to resume from breakpoints\n",
    "\n",
    "\n",
    "def getFilesResume(pr_list):\n",
    "    with open(\"old_branch_file_oe.txt\", \"r\", encoding=\"gbk\") as old_files:\n",
    "        with open(\"prs_oe.txt\", \"r\") as prs:\n",
    "            with open(\"branch_file_oe.txt\", \"a+\") as new_files:\n",
    "                list_old_files = old_files.read().splitlines()\n",
    "                list_prs = prs.read().splitlines()\n",
    "                list_old_prs = []\n",
    "                for j in range(len(list_old_files)):\n",
    "                    old_pr = list_old_files[j].split(\"\\t\")\n",
    "                    list_old_prs.append(old_pr[0])\n",
    "                for i in range(len(list_prs)):\n",
    "                    pr_no = list_prs[i]\n",
    "                    print(\n",
    "                        \"({0}/{1}) Retrieving issue {2}...\".format(i+1, len(list_prs), pr_no))\n",
    "                    if pr_no in list_old_prs:\n",
    "                        print(\"Already exists in the last run\")\n",
    "                        for j in range(len(list_old_files)):\n",
    "                            list_old_files_lines = list_old_files[j].split(\n",
    "                                \"\\t\")\n",
    "                            if list_old_files_lines[0] == pr_no:\n",
    "                                print(\"branch={0}\\tfile={1}\\tlines added={2}\\tlines removed={3}\".format(\n",
    "                                    list_old_files_lines[1], list_old_files_lines[2], list_old_files_lines[3], list_old_files_lines[4]))\n",
    "                                print(\"{0}\\t{1}\\t{2}\\t{3}\\t{4}\".format(\n",
    "                                    list_old_files_lines[0], list_old_files_lines[1], list_old_files_lines[2], list_old_files_lines[3], list_old_files_lines[4]), file=new_files)\n",
    "                    else:\n",
    "                        file_url = \"https://gitee.com/openeuler/docs/pulls/\"+pr_no+\"/files.json\"\n",
    "                        pr_url = \"https://gitee.com/openeuler/docs/pulls/\"+pr_no\n",
    "                        branch_response = request.urlopen(pr_url)\n",
    "                        response = request.urlopen(file_url)\n",
    "                        branch = branch_response.read().decode()    # string of PR page html\n",
    "                        files = response.read().decode()    # string of PR file json content\n",
    "                        d_files = json.loads(files)         # string to dict\n",
    "                        # pick 'diffs' list from d_files\n",
    "                        l_diffs = d_files['diffs']\n",
    "                        # pick 'head' dict from l_diffs\n",
    "                        for i in range(len(l_diffs)):\n",
    "                            d_file = l_diffs[i]\n",
    "                            # search branch object\n",
    "                            branchObj = re.findall(\n",
    "                                '(?<=href=\"/openeuler/docs/tree/).*(?=\"><span)', branch)\n",
    "                            # select branch string\n",
    "                            s_branch = branchObj[len(branchObj)-1]\n",
    "                            # pick 'name' string\n",
    "                            s_name = d_file[\"head\"][\"name\"]\n",
    "                            s_name = re.sub(' *[0-9]* *→ *[0-9]*$', \"\", s_name)\n",
    "                            int_added_lines = d_file[\"added_lines\"]\n",
    "                            int_removed_lines = d_file[\"removed_lines\"]\n",
    "                            print(\"branch=\"+s_branch+\"\\tfile=\"+s_name+\"\\tlines added=\" +\n",
    "                                  str(int_added_lines)+\"\\tlines removed=\"+str(int_removed_lines))\n",
    "                            print(pr_no+\"\\t\"+s_branch+\"\\t\"+s_name+\"\\t\" +\n",
    "                                  str(int_added_lines)+\"\\t\"+str(int_removed_lines), file=new_files)\n",
    "    print(\"-----------------------------------------------------\")\n",
    "\n",
    "# concatenate files\n",
    "\n",
    "\n",
    "def concatFiles(file_dict):\n",
    "    with open(\"branch_file_oe.txt\", \"r\") as input:\n",
    "        timestr = time.strftime(\"%Y%m%d-%H%M%S\")\n",
    "        with open(\"openEuler_issue_-\"+timestr+\".txt\", \"w+\") as final:\n",
    "            for line in input.readlines():\n",
    "                # issue_id = re.split(\"\\t|\\n\",line)[0]\n",
    "                # print(re.split(\"\\t|\\n\",line)[0]+\"\\t\"+re.sub(\"\\^\\^\",\"\\t\",file_dict.get(issue_id))+\"\\t\"+re.split(\"\\t|\\n\",line)[1]+\"\\t\"+re.split(\"\\t|\\n\",line)[2], file = final)\n",
    "                print(re.split(\"\\t|\\n\", line)[0]+\"\\t\"+re.sub(\"\\^\\^\", \"\\t\", file_dict.get(re.split(\"\\t|\\n\", line)[0]))+\"\\t\"+re.split(\n",
    "                    \"\\t|\\n\", line)[1]+\"\\t\"+re.split(\"\\t|\\n\", line)[2]+\"\\t\"+re.split(\"\\t|\\n\", line)[3]+\"\\t\"+re.split(\"\\t|\\n\", line)[4], file=final)\n",
    "                # print(issue_id+\"\\t\"+re.sub(\"\\^\\^\",\"\\t\",file_dict.get(issue_id))+\"\\t\"+line.split(\"\\t\")[1]+\"\\t\"+line.split(\"\\t\")[2], file = final)\n",
    "\n",
    "\n",
    "if os.path.exists(\"issues_oe.txt\"):\n",
    "    try:\n",
    "        os.rename(\"issues_oe.txt\", \"old_issues_oe.txt\")\n",
    "    except FileExistsError:\n",
    "        os.remove(\"old_issues_oe.txt\")\n",
    "        os.rename(\"issues_oe.txt\", \"old_issues_oe.txt\")\n",
    "    print(\"issues_oe.txt exists, renaming to old_issues_oe.txt\")\n",
    "if os.path.exists(\"prs_oe.txt\"):\n",
    "    try:\n",
    "        os.rename(\"prs_oe.txt\", \"old_prs_oe.txt\")\n",
    "    except FileExistsError:\n",
    "        os.remove(\"old_prs_oe.txt\")\n",
    "        os.rename(\"prs_oe.txt\", \"old_prs_oe.txt\")\n",
    "        print(\"prs_oe.txt exists, renaming to old_prs_oe.txt\")\n",
    "if os.path.exists(\"branch_file_oe.txt\"):\n",
    "    try:\n",
    "        os.rename(\"branch_file_oe.txt\", \"old_branch_file_oe.txt\")\n",
    "    except FileExistsError:\n",
    "        os.remove(\"old_branch_file_oe.txt\")\n",
    "        os.rename(\"branch_file_oe.txt\", \"old_branch_file_oe.txt\")\n",
    "        print(\"branch_file_oe.txt exists, renaming to old_branch_file_oe.txt\")\n",
    "getIssueList()\n",
    "fileToDict()\n",
    "if os.path.exists(\"old_branch_file_oe.txt\"):\n",
    "    getFilesResume(getPRList())\n",
    "else:\n",
    "    getFiles(getPRList())\n",
    "concatFiles(fileToDict())\n",
    "os.remove(\"issues_oe.txt\")\n",
    "print(\"removed issues_oe.txt\")\n",
    "os.remove(\"prs_oe.txt\")\n",
    "print(\"removed prs_oe.txt\")\n",
    "os.remove(\"branch_file_oe.txt\")\n",
    "print(\"removed branch_file_oe.txt\")\n",
    "if os.path.exists(\"old_issues_oe.txt\"):\n",
    "    os.remove(\"old_issues_oe.txt\")\n",
    "    print(\"removed old_issues_oe.txt\")\n",
    "if os.path.exists(\"old_prs_oe.txt\"):\n",
    "    os.remove(\"old_prs_oe.txt\")\n",
    "    print(\"removed old_prs_oe.txt\")\n",
    "if os.path.exists(\"old_branch_file_oe.txt\"):\n",
    "    os.remove(\"old_branch_file_oe.txt\")\n",
    "    print(\"removed old_branch_file_oe.txt\")\n",
    "print(\"See openEuler_issue_*.txt\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### For OpenHarmony issues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For OpenHarmony issues\n",
    "import json\n",
    "import re\n",
    "from urllib import request\n",
    "from collections import Counter\n",
    "import time\n",
    "import os\n",
    "\n",
    "\n",
    "def getOpenIssueList():\n",
    "    issue_list = open(\"issues_oh.txt\", \"a+\")  # create issue list file\n",
    "    pr_list = open(\"prs_oh.txt\", \"a+\")\n",
    "    title_counter = {'a': 1}  # counter value initialization\n",
    "    issue_page_no = 1\n",
    "    while len(title_counter) > 0:\n",
    "        issue_page_url = \"https://gitee.com/openharmony/docs/issues?issue_search=%5BAuto%5D+This+is+an+English+translation+task.&page=\" + \\\n",
    "            str(issue_page_no)+\"&state=open\"\n",
    "        issue_page_content = request.urlopen(issue_page_url).read().decode()\n",
    "        title_repeats = re.findall(\n",
    "            r'This is an English translation task\\.\\[[0-9]*\\]\" class=\"title\" href=\"/openharmony/docs/issues/[0-9A-Z]{6}', issue_page_content)\n",
    "        for issue_index in range(len(title_repeats)):\n",
    "            issue_number = re.search(\"[0-9]{1,6}\", title_repeats[issue_index])\n",
    "            issue_ID = re.search(\"[0-9A-Z]{6}\", title_repeats[issue_index])\n",
    "            print(issue_number.group(0)+\"\\t\"+issue_ID.group(0)+\"^^\" +\n",
    "                  \"https://gitee.com/openharmony/docs/issues/\"+issue_ID.group(0)+\"^^\"+\"NotAssigned\", file=issue_list)\n",
    "            print(issue_number.group(0), file=pr_list)\n",
    "            print(\"acquired issue No.\"+issue_number.group(0) +\n",
    "                  \" with ID \"+issue_ID.group(0)+\", not assigned\")\n",
    "        # title_repeats = list(map(str,title_repeats))\n",
    "        title_counter = Counter(title_repeats)\n",
    "        # print(title_counter)\n",
    "        issue_page_no = issue_page_no + 1\n",
    "    issue_list.close()\n",
    "    pr_list.close()\n",
    "\n",
    "\n",
    "def getProgressingIssueList():\n",
    "    issue_list = open(\"issues_oh.txt\", \"a+\")  # create issue list file\n",
    "    pr_list = open(\"prs_oh.txt\", \"a+\")\n",
    "    title_counter = {'a': 1}  # counter value initialization\n",
    "    issue_page_no = 1\n",
    "    while len(title_counter) > 0:\n",
    "        issue_page_url = \"https://gitee.com/openharmony/docs/issues?issue_search=%5BAuto%5D+This+is+an+English+translation+task.&page=\" + \\\n",
    "            str(issue_page_no)+\"&state=progressing\"\n",
    "        issue_page_content = request.urlopen(issue_page_url).read().decode()\n",
    "        title_repeats = re.findall(\n",
    "            r'This is an English translation task\\.\\[[0-9]*\\]\" class=\"title\" href=\"/openharmony/docs/issues/[0-9A-Z]{6}', issue_page_content)\n",
    "        assignee_repeats = re.findall(\n",
    "            r'(?<=target=\"_blank\" data-username=\")[^\"]+', issue_page_content)\n",
    "        # for issue_index in range(len(title_repeats)):\n",
    "        for issue_index in range(len(assignee_repeats)):\n",
    "            issue_number = re.search(\"[0-9]{1,6}\", title_repeats[issue_index])\n",
    "            issue_ID = re.search(\"[0-9A-Z]{6}\", title_repeats[issue_index])\n",
    "            assignee_ID = assignee_repeats[issue_index]\n",
    "            print(issue_number.group(0)+\"\\t\"+issue_ID.group(0)+\"^^\" +\n",
    "                  \"https://gitee.com/openharmony/docs/issues/\"+issue_ID.group(0)+\"^^\"+assignee_ID, file=issue_list)\n",
    "            # assignee_reponse = request.urlopen(\"https://gitee.com/openharmony/docs/issues/\"+issue_ID.group(0))\n",
    "            # assignee = assignee_reponse.read().decode()\n",
    "            print(issue_number.group(0), file=pr_list)\n",
    "            print(\"acquired issue No.\"+issue_number.group(0) +\n",
    "                  \" with ID \"+issue_ID.group(0)+\" assigned to \"+assignee_ID)\n",
    "        # title_repeats = list(map(str,title_repeats))\n",
    "        title_counter = Counter(title_repeats)\n",
    "        # print(title_counter)\n",
    "        issue_page_no = issue_page_no + 1\n",
    "    issue_list.close()\n",
    "    pr_list.close()\n",
    "\n",
    "# convert issues_oh.txt into a dict\n",
    "\n",
    "\n",
    "def fileToDict():\n",
    "    file_dict = {}\n",
    "    file_f = open(\"issues_oh.txt\")\n",
    "    for line in file_f:\n",
    "        key, value = line.split()\n",
    "        file_dict[key] = value\n",
    "    file_f.close()\n",
    "    return file_dict\n",
    "\n",
    "\n",
    "def getPRList():\n",
    "    pr_list = open(\"prs_oh.txt\", \"r\").readlines()\n",
    "    for i in range(len(pr_list)):\n",
    "        pr_list[i] = pr_list[i].rstrip('\\n')\n",
    "    print(\"=====================================================\")\n",
    "    print(\"{0} issues in total\".format(len(pr_list)))\n",
    "    print(\"=====================================================\")\n",
    "    # pr_list.sort()\n",
    "    return pr_list\n",
    "\n",
    "# Use by default\n",
    "\n",
    "\n",
    "def getFiles(list_prs):\n",
    "    for index in range(len(list_prs)):\n",
    "        pr_no = list_prs[index]\n",
    "        print(\"({0}/{1}) Retrieving issue {2}...\".format(index +\n",
    "              1, len(list_prs), pr_no))\n",
    "        f = open(\"branch_file_oh.txt\", \"a+\")\n",
    "        file_url = \"https://gitee.com/openharmony/docs/pulls/\"+pr_no+\"/files.json\"\n",
    "        pr_url = \"https://gitee.com/openharmony/docs/pulls/\"+pr_no\n",
    "        # print(pr_no, file = f)\n",
    "        # print(file_url, file = f)\n",
    "        branch_response = request.urlopen(pr_url)\n",
    "        response = request.urlopen(file_url)\n",
    "        branch = branch_response.read().decode()    # string of PR page html\n",
    "        files = response.read().decode()    # string of PR file json content\n",
    "        d_files = json.loads(files)         # string to dict\n",
    "        l_diffs = d_files['diffs']          # pick 'diffs' list from d_files\n",
    "        for i in range(len(l_diffs)):       # pick 'head' dict from l_diffs\n",
    "            d_file = l_diffs[i]\n",
    "            # search branch object\n",
    "            branchObj = re.findall(\n",
    "                '(?<=href=\"/openharmony/docs/tree/).*(?=\"><span)', branch)\n",
    "            # select branch string\n",
    "            s_branch = branchObj[len(branchObj)-1]\n",
    "            s_name = d_file[\"head\"][\"name\"]  # pick 'name' string\n",
    "            s_name = re.sub(' *[0-9]* *→ *[0-9]*$', \"\", s_name)\n",
    "            int_added_lines = d_file[\"added_lines\"]\n",
    "            int_removed_lines = d_file[\"removed_lines\"]\n",
    "            print(\"branch=\"+s_branch+\"\\tfile=\"+s_name+\"\\tlines added=\" +\n",
    "                  str(int_added_lines)+\"\\tlines removed=\"+str(int_removed_lines))\n",
    "            print(pr_no+\"\\t\"+s_branch+\"\\t\"+s_name+\"\\t\" +\n",
    "                  str(int_added_lines)+\"\\t\"+str(int_removed_lines), file=f)\n",
    "        f.close()\n",
    "        print(\"-----------------------------------------------------\")\n",
    "\n",
    "# Use to resume from breakpoints\n",
    "\n",
    "\n",
    "def getFilesResume(pr_list):\n",
    "    with open(\"old_branch_file_oh.txt\", \"r\", encoding=\"gbk\") as old_files:\n",
    "        with open(\"prs_oh.txt\", \"r\") as prs:\n",
    "            with open(\"branch_file_oh.txt\", \"a+\") as new_files:\n",
    "                list_old_files = old_files.read().splitlines()\n",
    "                list_prs = prs.read().splitlines()\n",
    "                list_old_prs = []\n",
    "                for j in range(len(list_old_files)):\n",
    "                    old_pr = list_old_files[j].split(\"\\t\")\n",
    "                    list_old_prs.append(old_pr[0])\n",
    "                for i in range(len(list_prs)):\n",
    "                    pr_no = list_prs[i]\n",
    "                    print(\n",
    "                        \"({0}/{1}) Retrieving issue {2}...\".format(i+1, len(list_prs), pr_no))\n",
    "                    if pr_no in list_old_prs:\n",
    "                        print(\"Already exists in the last run\")\n",
    "                        for j in range(len(list_old_files)):\n",
    "                            list_old_files_lines = list_old_files[j].split(\n",
    "                                \"\\t\")\n",
    "                            if list_old_files_lines[0] == pr_no:\n",
    "                                print(\"branch={0}\\tfile={1}\\tlines added={2}\\tlines removed={3}\".format(\n",
    "                                    list_old_files_lines[1], list_old_files_lines[2], list_old_files_lines[3], list_old_files_lines[4]))\n",
    "                                print(\"{0}\\t{1}\\t{2}\\t{3}\\t{4}\".format(\n",
    "                                    list_old_files_lines[0], list_old_files_lines[1], list_old_files_lines[2], list_old_files_lines[3], list_old_files_lines[4]), file=new_files)\n",
    "                    else:\n",
    "                        file_url = \"https://gitee.com/openharmony/docs/pulls/\"+pr_no+\"/files.json\"\n",
    "                        pr_url = \"https://gitee.com/openharmony/docs/pulls/\"+pr_no\n",
    "                        branch_response = request.urlopen(pr_url)\n",
    "                        response = request.urlopen(file_url)\n",
    "                        branch = branch_response.read().decode()    # string of PR page html\n",
    "                        files = response.read().decode()    # string of PR file json content\n",
    "                        d_files = json.loads(files)         # string to dict\n",
    "                        # pick 'diffs' list from d_files\n",
    "                        l_diffs = d_files['diffs']\n",
    "                        # pick 'head' dict from l_diffs\n",
    "                        for i in range(len(l_diffs)):\n",
    "                            d_file = l_diffs[i]\n",
    "                            # search branch object\n",
    "                            branchObj = re.findall(\n",
    "                                '(?<=href=\"/openharmony/docs/tree/).*(?=\"><span)', branch)\n",
    "                            # select branch string\n",
    "                            s_branch = branchObj[len(branchObj)-1]\n",
    "                            # pick 'name' string\n",
    "                            s_name = d_file[\"head\"][\"name\"]\n",
    "                            s_name = re.sub(' *[0-9]* *→ *[0-9]*$', \"\", s_name)\n",
    "                            int_added_lines = d_file[\"added_lines\"]\n",
    "                            int_removed_lines = d_file[\"removed_lines\"]\n",
    "                            print(\"branch=\"+s_branch+\"\\tfile=\"+s_name+\"\\tlines added=\" +\n",
    "                                  str(int_added_lines)+\"\\tlines removed=\"+str(int_removed_lines))\n",
    "                            print(pr_no+\"\\t\"+s_branch+\"\\t\"+s_name+\"\\t\" +\n",
    "                                  str(int_added_lines)+\"\\t\"+str(int_removed_lines), file=new_files)\n",
    "    print(\"-----------------------------------------------------\")\n",
    "\n",
    "# concatenate files\n",
    "\n",
    "\n",
    "def concatFiles(file_dict):\n",
    "    with open(\"branch_file_oh.txt\", \"r\") as input:\n",
    "        timestr = time.strftime(\"%Y%m%d-%H%M%S\")\n",
    "        with open(\"OpenHarmony_issue_-\"+timestr+\".txt\", \"w+\") as final:\n",
    "            for line in input.readlines():\n",
    "                # issue_id = re.split(\"\\t|\\n\",line)[0]\n",
    "                # print(re.split(\"\\t|\\n\",line)[0]+\"\\t\"+re.sub(\"\\^\\^\",\"\\t\",file_dict.get(issue_id))+\"\\t\"+re.split(\"\\t|\\n\",line)[1]+\"\\t\"+re.split(\"\\t|\\n\",line)[2], file = final)\n",
    "                print(re.split(\"\\t|\\n\", line)[0]+\"\\t\"+re.sub(\"\\^\\^\", \"\\t\", file_dict.get(re.split(\"\\t|\\n\", line)[0]))+\"\\t\"+re.split(\n",
    "                    \"\\t|\\n\", line)[1]+\"\\t\"+re.split(\"\\t|\\n\", line)[2]+\"\\t\"+re.split(\"\\t|\\n\", line)[3]+\"\\t\"+re.split(\"\\t|\\n\", line)[4], file=final)\n",
    "                # print(issue_id+\"\\t\"+re.sub(\"\\^\\^\",\"\\t\",file_dict.get(issue_id))+\"\\t\"+line.split(\"\\t\")[1]+\"\\t\"+line.split(\"\\t\")[2], file = final)\n",
    "\n",
    "\n",
    "if os.path.exists(\"issues_oh.txt\"):\n",
    "    try:\n",
    "        os.rename(\"issues_oh.txt\", \"old_issues_oh.txt\")\n",
    "    except FileExistsError:\n",
    "        os.remove(\"old_issues_oh.txt\")\n",
    "        os.rename(\"issues_oh.txt\", \"old_issues_oh.txt\")\n",
    "    print(\"issues_oh.txt exists, renaming to old_issues_oh.txt\")\n",
    "if os.path.exists(\"prs_oh.txt\"):\n",
    "    try:\n",
    "        os.rename(\"prs_oh.txt\", \"old_prs_oh.txt\")\n",
    "    except FileExistsError:\n",
    "        os.remove(\"old_prs_oh.txt\")\n",
    "        os.rename(\"prs_oh.txt\", \"old_prs_oh.txt\")\n",
    "        print(\"prs_oh.txt exists, renaming to old_prs_oh.txt\")\n",
    "if os.path.exists(\"branch_file_oh.txt\"):\n",
    "    try:\n",
    "        os.rename(\"branch_file_oh.txt\", \"old_branch_file_oh.txt\")\n",
    "    except FileExistsError:\n",
    "        os.remove(\"old_branch_file_oh.txt\")\n",
    "        os.rename(\"branch_file_oh.txt\", \"old_branch_file_oh.txt\")\n",
    "        print(\"branch_file_oh.txt exists, renaming to old_branch_file_oh.txt\")\n",
    "getOpenIssueList()\n",
    "getProgressingIssueList()\n",
    "fileToDict()\n",
    "if os.path.exists(\"old_branch_file_oh.txt\"):\n",
    "    getFilesResume(getPRList())\n",
    "else:\n",
    "    getFiles(getPRList())\n",
    "concatFiles(fileToDict())\n",
    "os.remove(\"issues_oh.txt\")\n",
    "print(\"removed issues_oh.txt\")\n",
    "os.remove(\"prs_oh.txt\")\n",
    "print(\"removed prs_oh.txt\")\n",
    "os.remove(\"branch_file_oh.txt\")\n",
    "print(\"removed branch_file_oh.txt\")\n",
    "if os.path.exists(\"old_issues_oh.txt\"):\n",
    "    os.remove(\"old_issues_oh.txt\")\n",
    "    print(\"removed old_issues_oh.txt\")\n",
    "if os.path.exists(\"old_prs_oh.txt\"):\n",
    "    os.remove(\"old_prs_oh.txt\")\n",
    "    print(\"removed old_prs_oh.txt\")\n",
    "if os.path.exists(\"old_branch_file_oh.txt\"):\n",
    "    os.remove(\"old_branch_file_oh.txt\")\n",
    "    print(\"removed old_branch_file_oh.txt\")\n",
    "print(\"See OpenHarmony_issue_*.txt\")\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### For test only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.7 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6f933b35030d5caf06cfdb3911180350fd622a3cd42ee69c19e77294781c1200"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

一键复制编辑原始数据按行查看历史

提交于 2023-04-25 17:29 . ✨ feat: Resume from breakpoints

马建仓 AI 助手

尝试更多

代码解读

代码找茬

代码优化

https://gitee.com/gomico/filepath_grabber.git

git@gitee.com:gomico/filepath_grabber.git

gomico

filepath_grabber

master

gomico/filepath_grabber

简介

发行版 (7)

贡献者

近期动态

gomico/filepath_grabber .gitee-modal { width: 500px !important; }

简介

发行版 (7)

贡献者

近期动态

搜索帮助

gomico/filepath_grabber