import datetime
import json
import os
import re
import time
from random import choice
import requests
from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter
from retry import retry

from ClassTool import ClassTool

baseTool = ClassTool()

from BaseCore import BaseCore

baseCore = BaseCore()
log = baseCore.getLogger()


@retry(tries=5, delay=3)
def getSoup(url):
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
        'Accept-Encoding': 'gzip, deflate, br, zstd',
        'Accept-Language': 'zh-CN,zh-TW;q=0.9,zh;q=0.8',
        'Connection': 'keep-alive',
        # 'Cookie': 'wdcid=043e7d9c8a90fbd9; wdcid=45e06544d2ddbc06; __auc=a67cddac18991603628dac4003a; wdses=14387868be342942; SERVERID=adb2d3a906b8c5e3f02ddd9c20949df0|1714008937|1714008813; wdlast=1714008941',
        'Host': 'www.gov.cn',
        # 'Referer': 'https://www.gov.cn/zhengce/wenjian/zhongyang/home_1.htm',
        'Sec-Fetch-Dest': 'document',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-Site': 'same-origin',
        'Sec-Fetch-User': '?1',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0',
        'sec-ch-ua': '"Chromium";v="124", "Microsoft Edge";v="124", "Not-A.Brand";v="99"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
    }
    req = requests.get(url, headers=headers, proxies=baseCore.get_proxy())
    req.encoding = req.apparent_encoding
    soup = BeautifulSoup(req.text, 'html.parser')
    req.close()
    return soup


def getTotal(soup):
    total = soup.find('div', class_='news_box').text
    print(total)
    totalPage = re.findall('nPageCount = (\d+);', total)[0]
    return int(totalPage)


def getDic(title, url):
    soup = getSoup(url)
    soup = baseTool.paserUrl(str(soup), url)
    dateAndOrigin = soup.find('div', class_='pages-date').text.strip()
    publishDate = dateAndOrigin.split('来源：')[0].strip()
    publishDate = datetime.datetime.strptime(publishDate, '%Y-%m-%d %H:%M').strftime('%Y-%m-%d %H:%M:%S')
    pub_org = dateAndOrigin.split('来源：')[1].split('字号')[0].strip()
    contentWithTag = soup.find('div', class_='pages_content')
    content = contentWithTag.text
    contentWithTag = str(contentWithTag)
    id_list = []
    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    dic = {
        'attachmentIds': id_list,  # 附件id
        'author': '',  # 作者
        'content': content,  # 正文不带标签
        'contentWithTag': contentWithTag,  # 正文带标签
        'createDate': time_now,  # 创建时间
        'deleteFlag': 0,  # 是否删除(0为默认，1为删除)
        'id': '',  #
        'labels': [{'relationId': "1783325599872438274", 'relationName': "中央有关文件", 'labelMark': "policy"}],
        # 关联标签id  关联标签名称  关联标签标识
        'origin': '中华人民共和国中央人民政府',  # 政策发布机关
        'organ': pub_org,  # 政策发文机关
        'topicClassification': '',  # 政策文件分类
        'issuedNumber': '',  # 发文字号
        'publishDate': publishDate,  # 发布时间
        'writtenDate': None,  # 成文时间
        'sid': '1697458829758697473',  # 信息源id
        'sourceAddress': url,  # 原文链接
        'summary': '',  # 摘要
        'title': title  # 标题
    }
    flag = baseTool.sendKafka(dic)
    if flag:
        baseTool.save_data(dic)
    return flag


def doJob():
    start_time = time.time()
    num = 0
    flg = False
    url = 'https://www.gov.cn/zhengce/wenjian/zhongyang/home.htm'
    soup = getSoup(url)
    # totalPage = getTotal(soup)
    totalPage = 1
    for i in range(totalPage):
        if flg:
            break
        if i == 1:
            url = url.replace('home', 'home_1')
            soup = getSoup(url)
        elif i > 1:
            url = url.replace(f'home_{i - 1}', f'home_{i}')
            soup = getSoup(url)
        liList = soup.find('div', class_='news_box').find_all('li')
        for li in liList:
            title = li.find('a').text.strip()
            href = li.find('a').get('href')
            if 'https://www.gov.cn/' not in href:
                href = href.replace('../../', 'https://www.gov.cn/zhengce/')
            is_href = baseTool.db_storage.find_one({'网址': href})
            if is_href:
                flg = True
                break
            dateFlg = li.find('span').text.strip()
            if dateFlg < '2020-01-01':
                flg = True
                break
            if getDic(title, href):
                num += 1
        time.sleep(2)
    end_time = time.time()
    log.info(f'共采集{num}条中央有关文件,耗时{end_time - start_time}')


if __name__ == '__main__':
    doJob()
    baseCore.close()
