"""
0、普通单线程执行 default_run
1、线程池提升爬虫速度 thread_run
2、进程池提升爬虫速度 process_run

大型爬虫，结构复杂一些
可以使用ThreadPoolExecutor, ProcessPoolExecutor 可能会报错

#之前的作业回顾：
1）多线程中的线程任务，是否可以开启新的线程或进程
2）多进程中的进程任务，是否可以开启新的线程或进程

"""
import os
import time
from functools import wraps
import requests
import pymongo
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor

client = pymongo.MongoClient(host='localhost', port=27017)
db = client.wzw
collection = db.tera

url = "http://42.194.197.95:8001/base"
urls = [url] * 10  # 模拟10页内容

cookies = {
    'session': '.eJyrViotTi1SsqpWyiyOT0zJzcxTsjLQUcrJTwexSopKU3WUcvOTMnNSlayUDM3gQEkHrDE-M0XJyhjCzkvMBSmKKTU3NbKIKTUzMjZXqq0FAN1MHbY.ZO7riw.LBko0twqJe6CWmoXuqgeYYZTZWQ'
}


def quick_get(_url):
    r = requests.get(_url, cookies=cookies)

    soup = BeautifulSoup(r.text, 'lxml')
    movie_list = soup.find('div', class_='movie-list').find_all('a')
    datas = []

    for movie in movie_list:
        img_url = movie.find('img').attrs.get('src')
        title = movie.find('h5').get_text()
        desc = movie.find('p').get_text()
        score = movie.find('small').get_text()
        desc2 = movie.find('small').find_next_sibling().get_text()

        datas.append({
            'img_url': img_url,
            'title': title,
            'desc': desc,
            'score': score,
            'desc2': desc2
        })
    #time.sleep(1)  #可以使用sleep来模拟耗时工作
    collection.insert_many(datas)


def run_time(func):
    # 添加装饰器用来计算时间
    @wraps(func)
    def wrapper(*args, **kwargs):
        start_time = time.perf_counter()    #启动时间(官方推荐的计时方式)
        r = func(*args, **kwargs)
        end_time = time.perf_counter()
        print(f'PID:{os.getpid()} . function_module: {func.__module__} .function_name: {func.__name__}, run_time: {end_time - start_time}')
        return r
    return wrapper


@run_time
def default_run():
    for _url in urls:
        quick_get(_url)


@run_time
def thread_run():
    #GIL -> 单线程；
    #请求网络，阻塞IO -> 丢给操作系统；阻塞时，线程池立刻去做其它事情
    with ThreadPoolExecutor(20) as thread_executor:
        thread_executor.submit(quick_get, url)

@run_time
def process_run():
    # cpu+1
    #进程有独立的空间，进程间切换会有较大的资源开销
    #多进程可以提升爬虫速度，但不明显，因为进程更适合CPU密集型程序
    #多进程，多核心并行 复杂程序
    with ProcessPoolExecutor(10) as process_executor:
        process_executor.submit(quick_get, url)


if __name__ == '__main__':
    default_run()
    thread_run()
    process_run()