#! /usr/bin/env python3
# coding=utf-8
import Addmethod
import deal_urls
import html_parse
import shuchu
import urls_download


class Spider():
	def __init__(self):
		self.urls = deal_urls.Urls()
		self.download = urls_download.Download()
		self.parse = html_parse.Parse()
		self.output = shuchu.Output()
		self.add_method = Addmethod.Jiexi()

	def zhixing(self, root_url):
		self.urls.add_one_url(root_url)
		new_url = self.urls.get_one_url()
		print("现在正在爬取主页，网址url为%s" % (new_url))
		html = self.download.xiazai(new_url)
		new_urls = self.parse.Parser(new_url, html)
		print("恭喜您，主页爬取成功，获得以下URL")
		for url in new_urls:
			print(url)

		for url in new_urls:
			htm = self.download.xiazai(url)
			new_data = self.add_method.get_data(url, htm)
			print(new_data)
			'''这里可以调用一个方法，用字典存在数组里'''
			self.output.collect(new_data)

		self.output.add_redis()

if __name__ == "__main__":
	root_url = "https://yande.re/post"
	obj = Spider()
	obj.zhixing(root_url)