webmining_cnki_final_DM.ipynb · 连丹琪/final_webmining_cnki

9月25日晚8点，Gitee 吐槽大会正式开幕，吐槽火力全开，Gitee 接招不慌！

加入 Gitee

与超过 1200万开发者一起发现、参与优秀开源项目，私有仓库也完全免费：）

该仓库未声明开源许可证文件（LICENSE），使用请关注具体项目描述及其代码上游依赖。

克隆/下载

webmining_cnki_final_DM.ipynb 35.19 KB

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2412d499",
   "metadata": {},
   "source": [
    "# 准备工作"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "6216679b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import requests\n",
    "import requests_html\n",
    "from requests_html import HTMLSession\n",
    "from lxml import etree\n",
    "from lxml.html import fromstring\n",
    "from urllib.parse import urljoin\n",
    "from random import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import base64\n",
    "import json\n",
    "from PIL import Image\n",
    "\n",
    "import selenium\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.desired_capabilities import DesiredCapabilities\n",
    "from selenium.webdriver.common.by import By"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "869f3b99",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:13: DeprecationWarning: use options instead of chrome_options\n",
      "  del sys.path[0]\n"
     ]
    }
   ],
   "source": [
    "#caps=dict()\n",
    "#caps[\"pageLoadStrategy\"] = \"none\"   # Do not wait for full page load\n",
    "\n",
    "opts = webdriver.ChromeOptions()\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "\n",
    "out_path = r'C:\\Users\\19108\\Desktop\\Internet _New Media\\WEB_Mining\\homework\\finalproject\\ldq_webwmning_cnki\\download'  # 是你想指定的路径\n",
    "prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': out_path}\n",
    "opts.add_experimental_option('prefs', prefs)\n",
    "driver = webdriver.Chrome(executable_path=r'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver.exe',chrome_options=opts)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90594946",
   "metadata": {},
   "source": [
    "# 打开知网高级检索页面"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "5d640700",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://www.cnki.net/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "6436fd3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('//*[@id=\"highSearch\"]').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7738f8f",
   "metadata": {},
   "source": [
    "# 登陆中大南方账号"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "c14bcfc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('//*[@id=\"Ecp_top_login\"]/a').click()\n",
    "driver.find_element_by_xpath('//*[@id=\"Ecp_Button2\"]').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04d7a2db",
   "metadata": {},
   "source": [
    "# 检查知网登陆情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "ba54ddfc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'中山大学南...'"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "driver.find_element_by_id('Ecp_loginShowName1').get_attribute('innerHTML')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49e9bada",
   "metadata": {},
   "source": [
    "# 窗口信息检查"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "7cadefe9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CDwindow-3083B085DA757E971766B3B6071EF0E8',\n",
       " 'CDwindow-855BC8A656FA743097400F6C1DB63F38']"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "driver.window_handles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "ed3c6260",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: DeprecationWarning: use driver.switch_to.window instead\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "driver.switch_to_window(driver.window_handles[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "465212a4",
   "metadata": {},
   "source": [
    "# 点击学术期刊"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "3b2c5dfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('/html/body/div[3]/div[1]/div/ul[1]/li[1]/a').click()\n",
    "time.sleep(0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b693415",
   "metadata": {},
   "source": [
    "# 点击专业检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "13d113e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_name('majorSearch').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e128dd4a",
   "metadata": {},
   "source": [
    "# 设置query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "dd705a63",
   "metadata": {},
   "outputs": [],
   "source": [
    "query =\"SU %= '可持续发展'  AND SU %= '数字' OR SU %= '绿色营销'\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d6866e4",
   "metadata": {},
   "source": [
    "# 输入检索内容"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "5a14d161",
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/textarea')\n",
    "element.clear()\n",
    "element.send_keys(query)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5ebc477",
   "metadata": {},
   "source": [
    "# 选择SCI,EI,北大核心,CSSCI,CSCD期刊"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "fb61683d",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[2]/input').click()\n",
    "driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[3]/input').click()\n",
    "driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[4]/input').click()\n",
    "driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[5]/input').click()\n",
    "driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[1]/div[3]/div/label[6]/input').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8927500f",
   "metadata": {},
   "source": [
    "# 点击检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "22b375fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[1]/div[1]/div[2]/div[2]/input').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8f9fd0c",
   "metadata": {},
   "source": [
    "# 按照被引数量文章排序"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "7040dac0",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('//*[@id=\"orderList\"]/li[3]').click()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eccd2e7d",
   "metadata": {},
   "source": [
    "# 检查检索信息，更换页面文章数量50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "60d2d863",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'共找到<em>1,843</em>条结果'"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "element = element = driver.find_element_by_xpath('//span[@class=\"pagerTitleCell\"]')\n",
    "element.get_attribute('innerHTML')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "b1a6b6c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 更换每页文章数量为50\n",
    "element = driver.find_element_by_id('perPageDiv')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "f8c0e720",
   "metadata": {},
   "outputs": [],
   "source": [
    "element = driver.find_element_by_xpath('//li[@data-val=\"50\"]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "3167c82d",
   "metadata": {},
   "outputs": [],
   "source": [
    "HOST = \"https://kns.cnki.net/\"\n",
    "html = driver.page_source\n",
    "soup = etree.HTML(html)\n",
    "tr_list= soup.xpath('//div[@id=\"gridTable\"]/table/tbody/tr')\n",
    "headers = {\n",
    "    \"Cookie\":'Ecp_ClientId=3200615112602316607; cnkiUserKey=03369524-5653-0508-82c6-2fdbcdb21fe4; RsPerPage=20; _pk_ref=%5B%22%22%2C%22%22%2C1607047617%2C%22https%3A%2F%2Fwww.cnki.net%2F%22%5D; Ecp_ClientIp=202.116.81.140; UM_distinctid=178689abc586d3-0e68ae30691535-5771133-1fa400-178689abc597f5; Ecp_loginuserjf=15014134753; Ecp_session=1; ASP.NET_SessionId=gt3afeursglwft12mavvuvzf; SID_kns8=123111; SID_kns_new=kns123106; CurrSortFieldType=desc; SID_kcms=124103; SID_kxreader_new=011121; Hm_lvt_6e967eb120601ea41b9d312166416aa6=1621474773; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22179876d894d472-0e2dc24144c1fc-2363163-2073600-179876d894e760%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%7D%2C%22%24device_id%22%3A%22179876d894d472-0e2dc24144c1fc-2363163-2073600-179876d894e760%22%7D; Hm_lvt_ba7af201fc75865e9846f701ccb53e6b=1621474773; SID_kns=025123113; SID_klogin=125144; Hm_lpvt_6e967eb120601ea41b9d312166416aa6=1621475961; Hm_lpvt_ba7af201fc75865e9846f701ccb53e6b=1621475961; Ecp_loginuserbk=GDZSDX; knsLeftGroupSelectItem=null5%3B9%3B; CurrSortField=%e8%a2%ab%e5%bc%95%2f(%e8%a2%ab%e5%bc%95%e9%a2%91%e6%ac%a1%2c%27integer%27); _pk_ref=%5B%22%22%2C%22%22%2C1622444666%2C%22https%3A%2F%2Fwww.cnki.net%2F%22%5D; _pk_ses=*; _pk_id=90adc29b-dc70-48ff-8dcc-60e442d5a980.1619271339.5.1622444668.1622444666.; Ecp_LoginStuts={\"IsAutoLogin\":false,\"UserName\":\"GZ0513\",\"ShowName\":\"%e4%b8%ad%e5%b1%b1%e5%a4%a7%e5%ad%a6%e5%8d%97%e6%96%b9%e5%ad%a6%e9%99%a2\",\"UserType\":\"bk\",\"BUserName\":\"\",\"BShowName\":\"\",\"BUserType\":\"\",\"r\":\"4SVIml\"}; LID=WEEvREcwSlJHSldSdmVqM1BLVW9SQVdSTUtBeko5WUtLcE8xUldvbWhxRT0=$9A4hF_YAuvQ5obgVAqNKPCYcEjKensW4IQMovwHtwkF4VYPoHbKxJw!!; c_m_LinID=LinID=WEEvREcwSlJHSldSdmVqM1BLVW9SQVdSTUtBeko5WUtLcE8xUldvbWhxRT0=$9A4hF_YAuvQ5obgVAqNKPCYcEjKensW4IQMovwHtwkF4VYPoHbKxJw!!&ot=05/31/2021 15:24:33; c_m_expire=2021-05-31 15:24:33',\n",
    "    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36'\n",
    "}\n",
    "def get_data(href):   #一个封装好的完善详细页链接的函数\n",
    "    DbCode = re.findall(\"DbCode=(.*?)&\",href)[0]\n",
    "    dbname = re.findall(\"dbname=(.*?)&\",href)[0]\n",
    "    filename = re.findall(\"filename=(.*?)&\",href)[0]\n",
    "    href = f\"https://kns.cnki.net/kcms/detail/detail.aspx?dbcode={DbCode}&dbname={dbname}&filename={filename}\"\n",
    "    return href\n",
    "\n",
    "def download(href):  #一个封装好的获取PDF下载链接的函数\n",
    "    page_source = requests.get(url=href,headers=headers).text\n",
    "    page_source_soup = etree.HTML(page_source)\n",
    "    download_url = page_source_soup.xpath('//li[@class=\"btn-dlpdf\"]/a/@href')\n",
    "    if download_url:\n",
    "        return urljoin(HOST,download_url[0])\n",
    "    else:\n",
    "        return \"无下载链接\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "2cfc80ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_pages =18\n",
    "\n",
    "title_list = []\n",
    "author_list = []\n",
    "href_list = []\n",
    "download_url_list = []\n",
    "\n",
    "\n",
    "def main():\n",
    "    now_page_count = 1\n",
    "    while now_page_count < num_pages:\n",
    "        html = driver.page_source\n",
    "        soup = etree.HTML(html)\n",
    "        tr_list= soup.xpath('//div[@id=\"gridTable\"]/table/tbody/tr')\n",
    "        for tr in tr_list:\n",
    "            title = \"\".join(tr.xpath('td[@class=\"name\"]//text()')).strip().replace(\"\\n\",\"\").replace(\"                              网络首发\",\"\")\n",
    "            author = \",\".join(tr.xpath('td[@class=\"author\"]/a//text()')).strip().replace(\"\\n\",\"\")\n",
    "            href = tr.xpath('td[@class=\"name\"]/a/@href')[0]\n",
    "            href = get_data(href)\n",
    "            download_url = download(href)\n",
    "            title_list.append(title)\n",
    "            author_list.append(author)\n",
    "            href_list.append(href)\n",
    "            download_url_list.append(download_url)\n",
    "        now_page_count = now_page_count + 1\n",
    "        driver.find_element_by_xpath('//*[@id=\"PageNext\"]').click()\n",
    "        time.sleep(0.5)\n",
    "    data = {\"标题\":title_list,\"作者\":author_list,\"详细页链接\":href_list,\"PDF下载链接\":download_url_list}\n",
    "    data = pd.DataFrame(data)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "76e44e2a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>作者</th>\n",
       "      <th>详细页链接</th>\n",
       "      <th>PDF下载链接</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>论21世纪的矿山——数字矿山</td>\n",
       "      <td>吴立新,殷作如,邓智毅,齐安文,杨可明</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>未来图书馆的新模式——智慧图书馆</td>\n",
       "      <td>王世伟</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>智慧城市的愿景与架构</td>\n",
       "      <td>许庆瑞,吴志岩,陈力田</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>消费者创新性对绿色消费行为的影响机制研究</td>\n",
       "      <td>劳可夫</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>对我国消费者绿色消费观念和行为的实证研究</td>\n",
       "      <td>司林胜</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>845</th>\n",
       "      <td>跨越式发展  2008年中国电影产业综述</td>\n",
       "      <td>陆绍阳</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>846</th>\n",
       "      <td>基于系统视角的企业绿色营销研究</td>\n",
       "      <td>孟庆春,丁雪,赵炳新</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>847</th>\n",
       "      <td>浅析我国饭店业的绿色营销</td>\n",
       "      <td>赵海燕</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>848</th>\n",
       "      <td>消费者品牌转换行为促成机制构建</td>\n",
       "      <td>陶宇红,井绍平</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>849</th>\n",
       "      <td>青海绿色食品品牌营销问题及其对策研究</td>\n",
       "      <td>蔡守琴</td>\n",
       "      <td>https://kns.cnki.net/kcms/detail/detail.aspx?d...</td>\n",
       "      <td>https://kns.cnki.net/kcms/download.aspx?filena...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>850 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       标题                   作者  \\\n",
       "0          论21世纪的矿山——数字矿山  吴立新,殷作如,邓智毅,齐安文,杨可明   \n",
       "1        未来图书馆的新模式——智慧图书馆                  王世伟   \n",
       "2              智慧城市的愿景与架构          许庆瑞,吴志岩,陈力田   \n",
       "3    消费者创新性对绿色消费行为的影响机制研究                  劳可夫   \n",
       "4    对我国消费者绿色消费观念和行为的实证研究                  司林胜   \n",
       "..                    ...                  ...   \n",
       "845  跨越式发展  2008年中国电影产业综述                  陆绍阳   \n",
       "846       基于系统视角的企业绿色营销研究           孟庆春,丁雪,赵炳新   \n",
       "847          浅析我国饭店业的绿色营销                  赵海燕   \n",
       "848       消费者品牌转换行为促成机制构建              陶宇红,井绍平   \n",
       "849    青海绿色食品品牌营销问题及其对策研究                  蔡守琴   \n",
       "\n",
       "                                                 详细页链接  \\\n",
       "0    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "1    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "2    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "3    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "4    https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "..                                                 ...   \n",
       "845  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "846  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "847  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "848  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "849  https://kns.cnki.net/kcms/detail/detail.aspx?d...   \n",
       "\n",
       "                                               PDF下载链接  \n",
       "0    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "1    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "2    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "3    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "4    https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "..                                                 ...  \n",
       "845  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "846  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "847  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "848  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "849  https://kns.cnki.net/kcms/download.aspx?filena...  \n",
       "\n",
       "[850 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = main()\n",
    "display(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "c7aa4abc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#把数据导入excel\n",
    "data.to_excel('cnki_webmining_DM.xlsx',sheet_name=\"18pages_857\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a351692c",
   "metadata": {},
   "source": [
    "# PDF档下载及解决验证码(.pdf)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f37d6352",
   "metadata": {},
   "source": [
    "## 调用图鉴API"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9425b5ed",
   "metadata": {},
   "source": [
    "[python开发文档实例](http://www.ttshitu.com/docs/python.html#pageTitle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "47517367",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a8t4\n"
     ]
    }
   ],
   "source": [
    "import base64\n",
    "import json\n",
    "import requests\n",
    "\n",
    "def base64_api(uname, pwd, img, typeid):\n",
    "    with open(img, 'rb') as f:\n",
    "        base64_data = base64.b64encode(f.read())\n",
    "        b64 = base64_data.decode()\n",
    "    data = {\"username\": 'lindaankei', \"password\": '03280328', \"typeid\": typeid, \"image\": b64}\n",
    "    result = json.loads(requests.post(\"http://api.ttshitu.com/predict\", json=data).text)\n",
    "    if result['success']:\n",
    "        return result[\"data\"][\"result\"]\n",
    "    else:\n",
    "        return result[\"message\"]\n",
    "    return \"\"\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    img_path = 'C:/Users/19108/Desktop/Internet _New Media/WEB_Mining/homework/finalproject/ldq_webwmning_cnki/code.png'\n",
    "    result = base64_api(uname='lindaankei', pwd='03280328', img=img_path, typeid=3)\n",
    "    print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe689c96",
   "metadata": {},
   "source": [
    "## 下载PDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "d41f125d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out_path =r\"C:/Users/19108/Desktop/Internet _New Media/WEB_Mining/homework/finalproject/ldq_webwmning_cnki/download\"\n",
    "for i in os.listdir(out_path):\n",
    "    exist = len(os.listdir(out_path))\n",
    "exist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "ec517709",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:6: DeprecationWarning: use options instead of chrome_options\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第 1 篇下载成功\n",
      "第 2 篇下载成功\n",
      "第 3 篇下载成功\n",
      "第 4 篇下载成功\n",
      "第 5 篇下载成功\n",
      "第 6 篇下载成功\n",
      "第 7 篇下载成功\n",
      "第 8 篇下载成功\n",
      "第 9 篇下载成功\n",
      "第 10 篇下载成功\n",
      "第 11 篇下载成功\n",
      "第 12 篇下载成功\n",
      "第 13 篇下载成功\n"
     ]
    }
   ],
   "source": [
    "#用新打开的视窗来下载，依然自己设定默认下载路径\n",
    "options = webdriver.ChromeOptions()\n",
    "out_path = r'C:\\Users\\19108\\Desktop\\Internet _New Media\\WEB_Mining\\homework\\finalproject\\ldq_webwmning_cnki\\download'  # 是你想指定的路径\n",
    "prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': out_path}\n",
    "opts.add_experimental_option('prefs', prefs)\n",
    "driver = webdriver.Chrome(executable_path=r'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver.exe',chrome_options=opts)\n",
    "#刚刚已经把pdf下载链接存到了excel表里所以直接用\n",
    "data = pd.read_excel(\"cnki_webmining_DM.xlsx\")\n",
    "base = \"https://kns.cnki.net/\"\n",
    "download_url_list = data[\"PDF下载链接\"].to_list()\n",
    "for url in download_url_list[0:13]:\n",
    "    try:\n",
    "        driver.get(url)\n",
    "        time.sleep(6)\n",
    "        if driver.find_element_by_xpath('//*[@id=\"vImg\"]'):\n",
    "            driver.save_screenshot('code.png')\n",
    "            img = driver.find_element_by_xpath('//*[@id=\"vImg\"]')\n",
    "            left = img.location['x']+50\n",
    "            top = img.location['y']+70\n",
    "            right = img.location['x'] + img.size['width']+200\n",
    "            bottom = img.location['y'] + img.size['height']+80\n",
    "            photo = Image.open('code.png')\n",
    "            photo = photo.crop((left, top, right, bottom))\n",
    "            photo.save('code.png')\n",
    "            img_path = \"C:/Users/19108/Desktop/Internet _New Media/WEB_Mining/homework/finalproject/ldq_webwmning_cnki/code.png\"\n",
    "            result = base64_api(uname='lindaankei',pwd='03280328',img=img_path,typeid=3)\n",
    "            driver.find_element_by_id('vcode').send_keys(result) # 将验证码填入\n",
    "            time.sleep(0.5)\n",
    "            driver.find_element_by_xpath('/html/body/div/form/dl/dd/button').click() # 点击提交进行下载\n",
    "            #再设定一个下载后再数一次文件数量的值，用以判断\n",
    "            for i in os.listdir(out_path):\n",
    "                new_exist=len(os.listdir(out_path))\n",
    "#                 print(\"new_exist\")\n",
    "            print('第',download_url_list.index(url)+1,'篇下载成功')\n",
    "    except:\n",
    "            if new_exist == exist:\n",
    "                print('第',download_url_list.index(url)+1,'篇文件没有下载成功')\n",
    "            else:\n",
    "                print('第',download_url_list.index(url)+1,'篇下载成功')\n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06c7c347",
   "metadata": {},
   "source": [
    "# 导出refworks文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "8597f2df",
   "metadata": {},
   "outputs": [],
   "source": [
    "#回到第一页\n",
    "element = driver.find_element_by_xpath('//*[@id=\"total\"]')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "0da4250f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#每页50篇文章，下载大于800篇至少需要翻17页，但知网规定一次最多选取500篇，所以这里设置（0，10）\n",
    "for i in range(0,10):\n",
    "    driver.find_element_by_xpath('//*[@id=\"selectCheckAll1\"]').click()\n",
    "    driver.find_element_by_id('PageNext').click()\n",
    "    time.sleep(5)\n",
    "    display(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "1f196008",
   "metadata": {},
   "outputs": [],
   "source": [
    "#定位到要悬停的元素\n",
    "from selenium.webdriver.common.action_chains import ActionChains\n",
    "move = driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/a')\n",
    "#对定位到的元素执行悬停操作\n",
    "ActionChains(driver).move_to_element(move).perform()\n",
    "move = driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/a')\n",
    "ActionChains(driver).move_to_element(move).perform()\n",
    "driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/ul/li[8]/a').click()#导出 refworks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "dbb14738",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CDwindow-3083B085DA757E971766B3B6071EF0E8',\n",
       " 'CDwindow-855BC8A656FA743097400F6C1DB63F38',\n",
       " 'CDwindow-9A80B60A4070D525AAE6DB198463A65A']"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#窗口信息检查\n",
    "driver.window_handles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "a856872e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "#窗口切换\n",
    "driver.switch_to_window(driver.window_handles[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "3342c5c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#导出 refworks\n",
    "element=driver.find_element_by_xpath('//*[@id=\"litotxt\"]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "f84b36ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "#后300篇导出同理，先回到检索窗口\n",
    "driver.switch_to_window(driver.window_handles[1])\n",
    "#消除原先选定\n",
    "element=driver.find_element_by_xpath('//*[@id=\"gridTable\"]/div[1]/div[2]/div[1]/a')\n",
    "element.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "b0a798a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(0,6):\n",
    "    driver.find_element_by_xpath('//*[@id=\"selectCheckAll1\"]').click()\n",
    "    driver.find_element_by_id('PageNext').click()\n",
    "    time.sleep(5)\n",
    "    display(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "25dfffac",
   "metadata": {},
   "outputs": [],
   "source": [
    "#定位到要悬停的元素\n",
    "from selenium.webdriver.common.action_chains import ActionChains\n",
    "move = driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/a')\n",
    "#对定位到的元素执行悬停操作\n",
    "ActionChains(driver).move_to_element(move).perform()\n",
    "move = driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/a')\n",
    "ActionChains(driver).move_to_element(move).perform()\n",
    "driver.find_element_by_xpath('//*[@id=\"batchOpsBox\"]/li[2]/ul/li[1]/ul/li[8]/a').click()#导出 refworks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "cb5e3aef",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: DeprecationWarning: use driver.switch_to.window instead\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "#窗口切换\n",
    "driver.switch_to_window(driver.window_handles[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "bb3006fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "#导出 refworks\n",
    "element=driver.find_element_by_xpath('//*[@id=\"litotxt\"]/a')\n",
    "element.click()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}

一键复制编辑原始数据按行查看历史

提交于 2021-07-04 22:11 . 上传源文件（除pdf外）

马建仓 AI 助手

尝试更多

代码解读

代码找茬

代码优化

https://gitee.com/lian_dan_qi/final_webmining_cnki.git

git@gitee.com:lian_dan_qi/final_webmining_cnki.git

lian_dan_qi

final_webmining_cnki

master

连丹琪/final_webmining_cnki

简介

发行版

贡献者

近期动态

连丹琪/final_webmining_cnki .gitee-modal { width: 500px !important; }

简介

发行版

贡献者

近期动态

搜索帮助

连丹琪/final_webmining_cnki