1 Star 1 Fork 0

zhanjun520/python_spider

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
克隆/下载
获取贴吧n页数据.py 2.18 KB
一键复制 编辑 原始数据 按行查看 历史
zhanjun520 提交于 2022-02-16 17:45 . 小修改
# coding=gbk
# ^...^*:詹峻
# 开发时间:2022/1/10 14:11
import requests
class Tieba_spider(object):
def __init__(self, text):
self.text = text
self.ur1 = 'https://tieba.baidu.com/f?kw=' + text + '&pn={}'
self.headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36 SLBrowser/7.0.0.12151 SLBChan/103"}
def get_ur1_list(self):
"""生成ur1列表"""
ur1_list = [self.ur1.format(i * 50) for i in range(5)]
return ur1_list
def get_data_fromurl(self, ur1):
"""从服务器获取数据,并且解码返回"""
# 这个请求头不能获得完整的HTML内容,可以删除。
response = requests.get(ur1, headers=self.headers)
# 此方法可以获得请求浏览器的请求头
# print(response.request.headers)
return response.content.decode()
def save_html(self, html_str, num):
"""保存到本地"""
file_name = "贴吧_" + text + '第{}页'.format(num) + '.html'
with open(file_name, 'w', encoding='utf-8') as f:
f.write(html_str)
def run(self):
ur1_list = self.get_ur1_list()
for item_ur1 in ur1_list:
html_str = self.get_data_fromurl(item_ur1)
self.save_html(html_str, ur1_list.index(item_ur1) + 1)
if __name__ == '__main__':
text = input("请输入贴吧的名字:")
spider = Tieba_spider(text)
spider.run()
"""ur1 = 'https://tieba.baidu.com/f?kw={}&pn={}'
text = input("请输入贴吧的名字:")
ur1_list = [ur1.format(text, i * 50) for i in range(5)]
headers = {
"User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Mobile Safari/537.36 Edg/97.0.1072.55"}
for item_ur1 in ur1_list:
response = requests.get(item_ur1, headers=headers)
file_name = "贴吧_" + text + '第{}页'.format(ur1_list.index(item_ur1) + 1) + '.html'
with open(file_name, 'w', encoding='utf-8') as f:
f.write(response.content.decode())"""
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
Python
1
https://gitee.com/zhanjun520/zhanjun_one.git
git@gitee.com:zhanjun520/zhanjun_one.git
zhanjun520
zhanjun_one
python_spider
master

搜索帮助

0d507c66 1850385 C8b1a773 1850385