1 Star 0 Fork 2

july/doc_downloader_web

forked from weder/doc_downloader_web 
加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
该仓库未声明开源许可证文件(LICENSE),使用请关注具体项目描述及其代码上游依赖。
克隆/下载
taodocs.py 3.35 KB
一键复制 编辑 原始数据 按行查看 历史
豆腐 提交于 2020-02-10 17:09 . init commit
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import NoSuchElementException, JavascriptException, StaleElementReferenceException
import base64
import time
import sys
import os
from tqdm import trange
from img2pdf import conpdf
import urllib.request
import shutil
def download(url, callback):
option = webdriver.ChromeOptions()
option.add_argument('headless')
option.add_argument('log-level=3')
driver = webdriver.Chrome(chrome_options=option)
title = "output"
try:
driver.set_page_load_timeout(15)
driver.get(url)
title = driver.title[:-8]
except:
return False, "下载失败,超时"
print(f'淘豆网: 《{title}》')
if os.path.exists(f'./output/{title}.pdf'):
return True, title
time.sleep(5)
while True:
try:
driver.execute_script("window.scrollBy(0,10000)")
time.sleep(1)
# 展开全部
elem_cont_button = driver.find_element_by_class_name(
"banner-more-btn")
elem_cont_button = elem_cont_button.find_element_by_tag_name(
'span')
driver.execute_script(
"arguments[0].scrollIntoView(true);", elem_cont_button)
actions = ActionChains(driver)
actions.move_to_element(elem_cont_button).perform()
time.sleep(0.5)
driver.execute_script("arguments[0].click();", elem_cont_button)
# break
except NoSuchElementException:
break
except StaleElementReferenceException:
break
except JavascriptException:
continue
try:
# 获取页数
num_of_pages = driver.find_element_by_id(
'docPage').get_attribute('innerHTML')
num_of_pages = int(num_of_pages)
if os.path.exists(f'./temp/{title}'):
shutil.rmtree(f'./temp/{title}')
os.makedirs(f'./temp/{title}')
for pages in trange(num_of_pages):
callback(pages, num_of_pages, "正在下载:%s" % title)
try:
time.sleep(0.5)
element = driver.find_element_by_id(f"page{pages + 1}")
driver.execute_script("arguments[0].scrollIntoView(true);", element)
actions = ActionChains(driver)
actions.move_to_element(element).perform()
time.sleep(0.5)
imgElement = element.find_element_by_tag_name('img')
imgUrl = imgElement.get_attribute('src')
response = urllib.request.urlopen(imgUrl)
html = response.read()
with open(f'./temp/{title}/{pages}.jpg', 'wb') as f:
f.write(html)
except Exception as e:
print('下载中断,信息:\n%r' % e)
break
driver.quit()
print('下载完毕,正在转码')
callback(99, 100, "正在转码")
conpdf(f'output/{title}.pdf', f'temp/{title}', '.jpg')
return True, title
except Exception as e:
return False, e
if __name__ == "__main__":
download('https://www.taodocs.com/p-292564682.html')
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
1
https://gitee.com/julianbarros/doc_downloader_web.git
git@gitee.com:julianbarros/doc_downloader_web.git
julianbarros
doc_downloader_web
doc_downloader_web
master

搜索帮助

0d507c66 1850385 C8b1a773 1850385