"""
抓取民政部官网最新行政区划代码

2024年已经不一样了，所以运行不过

这个案例主要是提醒，如果返回结果是JS，则像这样处理
因为如果返回JS，网页能识别，自动跳转的真实链接，但是爬虫不能，所以结果会报错

"""
import requests
from lxml import etree
import re

class MzbSpider:
    def __init__(self):
        self.url = 'https://www.mca.gov.cn/n156/n2679/index.html'
        self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'}

    def get_html(self, url):
        """获取 Html"""
        html = requests.get(url=url,headers=self.headers)
        print(html)
        return html

    def xpath_func(self, html, xpath_bds):
        """ xpath 解析提取数据 """
        p = etree.HTML(html)
        r_list = p.xpath(xpath_bds)

        return r_list

    def parse_html(self):
        """爬虫逻辑函数"""
        # 1 向主页发请求，提取最新月份的链接
        # 2 向最新月份链接发请求，来提取具体的数据
        one_html = self.get_html(url=self.url)
        return
        one_xpath = '//table/tr[2]/td/[2]/a/@href'
        href_list = self.xpath_func(html=one_html, xpath_bds=one_xpath)
        if href_list:
            one_url = href_list[0]
            print(one_url)
            self.detail_page(one_url)
        else:
            print('提取最新链接失败')

    def detail_page(self, one_url):
        """获取具体数据"""
        two_html = self.get_html(url=one_url)
        # 响应内容中嵌入了JS，进行了URL地址跳转
        # 从响应内容two_html中提取真实返回数据的链接
        true_url = self.get_true_url(two_html)
        # 从真实链接中提取具体数据
        true_html = self.get_html(url=true_url)
        two_xpath = '//tr[@height="19"]'
        tr_list = self.xpath_func(html=two_html,xpath_bds=two_xpath)
        item = {}
        for tr in tr_list:
            item['name'] = tr.xpath('./td[3]/text()')[0].strip()
            item['code'] = tr.xpath('./td[2]/text() | ./td[2]/span/text()')[0].strip()
            print(item)


    def get_true_url(self, two_html):
        """从响应内容中提取真实数据的URL地址"""
        regex = 'window.location.href="(.*?)"'
        pattern = re.compile(regex,re.S)

        return pattern.findall(two_html)[0]

    def run(self):
        self.parse_html()

if __name__ == '__main__':
    spider = MzbSpider()
    spider.run()