import datetime
import json
import random
import re
import time

import pymysql
import requests
from bs4 import BeautifulSoup
import sys

sys.path.append('D:\\建材')
import BaseCore
from kafka import KafkaProducer
from requests.adapters import HTTPAdapter

baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()
headers = {
    #              Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
    'Accept-Language': 'zh-CN,zh-TW;q=0.9,zh;q=0.8',
}
session = requests.session()
session.mount('https://', HTTPAdapter(max_retries=3))
session.mount('http://', HTTPAdapter(max_retries=3))


# 8月以后的
# 三个月
# TIMEBEGIN_SHOW=2023-08-24&TIMEEND_SHOW=2023-09-02&TIMEBEGIN=2023-08-24&TIMEEND=2023-09-02&SOURCE_TYPE=2&DEAL_TIME=05&DEAL_CLASSIFY=01&DEAL_STAGE=0100&DEAL_PROVINCE=0&DEAL_CITY=0&DEAL_PLATFORM=0&BID_PLATFORM=0&DEAL_TRADE=0&isShowAll=1&PAGENUMBER=1&FINDTXT=
# TIMEBEGIN_SHOW=2023-08-24&TIMEEND_SHOW=2023-09-02&TIMEBEGIN=2023-08-24&TIMEEND=2023-09-02&SOURCE_TYPE=1&DEAL_TIME=05&DEAL_CLASSIFY=00&DEAL_STAGE=0000&DEAL_PROVINCE=0&DEAL_CITY=0&DEAL_PLATFORM=0&BID_PLATFORM=0&DEAL_TRADE=0&isShowAll=1&PAGENUMBER=1&FINDTXT=
# 当天
# TIMEBEGIN_SHOW=2023-08-24&TIMEEND_SHOW=2023-09-02&TIMEBEGIN=2023-08-24&TIMEEND=2023-09-02&SOURCE_TYPE=1&DEAL_TIME=01&DEAL_CLASSIFY=00&DEAL_STAGE=0000&DEAL_PROVINCE=0&DEAL_CITY=0&DEAL_PLATFORM=0&BID_PLATFORM=0&DEAL_TRADE=0&isShowAll=1&PAGENUMBER=1&FINDTXT=
# TIMEBEGIN_SHOW=2023-08-24&TIMEEND_SHOW=2023-09-02&TIMEBEGIN=2023-08-24&TIMEEND=2023-09-02&SOURCE_TYPE=2&DEAL_TIME=01&DEAL_CLASSIFY=01&DEAL_STAGE=0100&DEAL_PROVINCE=0&DEAL_CITY=0&DEAL_PLATFORM=0&BID_PLATFORM=0&DEAL_TRADE=0&isShowAll=1&PAGENUMBER=1&FINDTXT=

# 发送kafka
def sendKafka(dic_news):
    try:
        producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
        kafka_result = producer.send("tenderClusterData",
                                     json.dumps(dic_news, ensure_ascii=False).encode('utf8'))

        print(kafka_result.get(timeout=10))

        log.info(f"{dic_news['title']}.....{dic_news['subtitle']}.....{dic_news['sourceAddress']}传输成功")
    except Exception as e:
        log.error(f'{e}')
        log.error(f"{dic_news['title']}.....{dic_news['subtitle']}.....{dic_news['sourceAddress']}传输失败")


# 获取总页数
def getTotal(url, params):
    try:
        req_ = session.post(url, params=params, timeout=30)
        req_.encoding = req_.apparent_encoding
        res = json.loads(req_.text)
        total = res['ttlpage']
    except:
        session.close()
        url_ = 'http://www.ggzy.gov.cn/'
        session.get(url=url_, headers=headers, timeout=30)
        req_ = session.post(url, params=params, timeout=30)
        req_.encoding = req_.apparent_encoding
        res = json.loads(req_.text)
        total = res['ttlpage']
    return total


# 获取一也所有数据
def getDatas(url, params):
    try:
        req_ = session.get(url, params=params, timeout=30)
        req_.encoding = req_.apparent_encoding
        res = json.loads(req_.text)
        datas = res['data']
    except:
        session.close()
        url_ = 'http://www.ggzy.gov.cn/'
        session.get(url=url_, headers=headers, timeout=30)
        req_ = session.get(url, params=params, timeout=30)
        req_.encoding = req_.apparent_encoding
        res = json.loads(req_.text)
        datas = res['data']
    return datas


# 获取编号
def getTaN(href):
    try:
        req_content = session.get(href, timeout=30)
        req_content.encoding = req_content.apparent_encoding
        soup = BeautifulSoup(req_content.text, 'html.parser')
        if '编号：' not in req_content.text:
            contentNo = ''
        else:
            contentNo = \
                soup.select('body > div.fully > p')[0].text.split('信息来源：')[0].split(
                    '编号：')[1]
        title = soup.select('body > div.fully > h4')[0].text
    except:
        session.close()
        url_ = 'http://www.ggzy.gov.cn/'
        session.get(url=url_, headers=headers, timeout=30)
        req_content = session.get(href, timeout=30)
        req_content.encoding = req_content.apparent_encoding
        soup = BeautifulSoup(req_content.text, 'html.parser')
        if '编号：' not in req_content.text:
            contentNo = ''
        else:
            contentNo = \
                soup.select('body > div.fully > p')[0].text.split('信息来源：')[0].split(
                    '编号：')[1]
        title = soup.select('body > div.fully > h4')[0].text
    return contentNo, title


# 解析数据
def getData(data):
    # pub_time = data['timeShow']
    province = data['districtShow']
    href = data['url']
    href_ = href.replace('/information/html/a', '/html/b')
    infoType = data['stageShow']
    businessType = data['classifyShow']
    origin = data['platformName']
    trade = data['tradeShow']
    contentNo, title_1 = getTaN(href)
    req_content = session.get(href_, timeout=30)
    req_content.encoding = req_content.apparent_encoding
    soup = BeautifulSoup(req_content.text, 'lxml')
    pub_time = soup.select('body > div > p > span')[0].text.split('：')[1].lstrip().strip()
    title_2 = soup.select('body > div > h4')[0].text.replace('\n', '').replace('\r', '')
    contentWithTag = soup.select('body > div')[0]
    content = contentWithTag.text
    data = {
        'businessType': businessType,  # 业务类型
        'infoType': infoType,  # 信息类型
        'trade': trade,  # 行业
        'province': province,  # 省份
        'origin': origin,  # 来源网站
        'isAbroad': '1',  # 1、国内 2、国外
        'contentNo': contentNo,  # 编号
        'content': content,  # 内容(不带标签)
        'contentWithTag': str(contentWithTag),  # 内容(带标签)
        'sid': '1699606095238131714',  # 信息源id
        'publishDate': pub_time,  # 发布时间
        'sourceAddress': href,  # 原文链接
        'title': title_1,  # 标题
        'subtitle': title_2  # 二级标题
    }
    return data


def zb():
    now = str(datetime.date.today())
    past = str(datetime.date.today() - datetime.timedelta(days=50))
    num = 0
    url = 'http://www.ggzy.gov.cn/'
    req = session.get(url=url, headers=headers, timeout=30)
    if req.status_code != 200:
        log.error('网站连接失败')
        return
    url_ = 'http://deal.ggzy.gov.cn/ds/deal/dealList_find.jsp'
    # 当天
    params_s = [
        f'TIMEBEGIN_SHOW=2023-10-01&TIMEEND_SHOW={now}&TIMEBEGIN=2023-10-01&TIMEEND={now}&SOURCE_TYPE=1&DEAL_TIME=06&DEAL_CLASSIFY=00&DEAL_STAGE=0000&DEAL_PROVINCE=0&DEAL_CITY=0&DEAL_PLATFORM=0&BID_PLATFORM=0&DEAL_TRADE=0&isShowAll=1&PAGENUMBER=1&FINDTXT=',
        f'TIMEBEGIN_SHOW=2023-10-01&TIMEEND_SHOW={now}&TIMEBEGIN=2023-10-01&TIMEEND={now}&SOURCE_TYPE=2&DEAL_TIME=06&DEAL_CLASSIFY=00&DEAL_STAGE=0000&DEAL_PROVINCE=0&DEAL_CITY=0&DEAL_PLATFORM=0&BID_PLATFORM=0&DEAL_TRADE=0&isShowAll=1&PAGENUMBER=1&FINDTXT=']
    for params_ in params_s:
        total = getTotal(url_, params_)
        for page in range(total):
            datas = getDatas(url_, params_)
            for data_ in datas:
                try:
                    data = getData(data_)
                    # print(data['contentWithTag'])
                    sendKafka(data)
                    # print(data)
                    num += 1
                except:
                    session.close()
                    session.get(url=url, headers=headers, timeout=30)
                    data = getData(data_)
                    # print(data['contentWithTag'])
                    sendKafka(data)
                    num += 1
                time.sleep(0.5)
            params_ = params_.replace(f'PAGENUMBER={page + 1}', f'PAGENUMBER={page + 2}')
    log.info(f"共采集{num}条数据")


if __name__ == '__main__':
    zb()
    # current_time = datetime.datetime.now()
    # midnight_time = current_time.replace(hour=0, minute=0, second=0, microsecond=0) + datetime.timedelta(days=1)
    # sleep_seconds = (midnight_time - current_time).total_seconds()
    # time.sleep(sleep_seconds)
    baseCore.close()
