# 获取目录，用于进行文件下载和文件完整性的校验
# 采用Seluem获取页面数据

import lxml
from lxml import etree
from selenium import webdriver
from time import sleep
from selenium.webdriver.chrome.options import Options
import os

import requests
def downFile(urlpath,savepath):
    r = requests.get(urlpath)
    with open(savepath, "wb") as code:
      code.write(r.content)

# 设置谷歌无界面浏览器
chrome_options = Options()
# chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')  # 禁用GPU加速
#设置为无图模式
chrome_options.add_argument('blink-settings=imagesEnabled=false')

# webdriver位置
path = r'C:/soft/chromedriver.exe'
# 浏览器参数设置
browser = webdriver.Chrome(
    executable_path=path, options=chrome_options)

# 写入日志
def WiteLog(list):
    with open('log.txt','a',encoding='utf-8') as f:
        newlist=[]
        for item in list:
            newlist.append(str(item))
        f.write('\n'.join(newlist)+'\n\n')

# 处理一个页面
def GetOnePage(url='https://unpkg.com/browse/element-ui@2.15.3/',rootPath='./'):
    browser.get(url)
    html=browser.page_source
        
    # xpath进行提取
    selector = etree.HTML(html)
    list = selector.xpath('//tbody//tr')
    newlist=[]
    # ['lib', '-', '-']
    # ['packages', '-', '-']
    for item in list:
        dd=item.xpath('./td//text()')
        newlist.append(dd)

    # 写入日志
    WiteLog(newlist)

    logStr=''
    # 数据处理
    for item in newlist:
        if(len(item)>1):
            rootPath=url.replace('https://unpkg.com/browse/element-ui@2.15.3/','')

            if(item[1]=='-'):
                print('进入文件夹：'+item[0])
                logStr= logStr + rootPath+item[0]+'\n'

                #进入文件夹
                sleep(3)
                GetOnePage(url+item[0]+'/',rootPath+item[0])
            else:
                #下载文件
                logStr=logStr + rootPath + item[0]+'\n'
                print('下载完成：'+item[0])

    with open('文件目录.txt','a',encoding='utf-8') as f:
         f.write(logStr)

GetOnePage()

print('处理完成')
browser.quit()