# -*- coding: utf-8 -*-
'''
成功100 发送数据失败200  请求失败400  文章内容为空500 处理style标签失败700
'''
import datetime
import re, sys

import pymongo
import requests, time, random, json, pymysql, redis

import urllib3
from bs4 import BeautifulSoup

from obs import ObsClient
from kafka import KafkaProducer
from retry import retry

sys.path.append('D:\\zzsn_spider\\base')
import BaseCore

# todo 连接mongo
db_storage = pymongo.MongoClient('mongodb://114.115.221.202:27017/', username='admin', password='ZZsn@9988').ZZSN[
    '机器人分会']


baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()
cnx_ = baseCore.cnx
cursor_ = baseCore.cursor
# cnx = pymysql.connect(host="114.116.44.11", user="root", password="f7s0&7qqtK", db="clb_project", charset="utf8mb4")
# cursor = cnx.cursor()
r = baseCore.r
urllib3.disable_warnings()

rMonitor = redis.Redis(host='114.116.90.53', port=6380, password='clbzzsn', db=15)


@retry(tries=2, delay=5)
def sendMonitorKafka(dic_news):
    producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'], api_version=(2, 7, 0))
    kafka_result = producer.send("data_lifecycle_log_data_crawler",
                                 json.dumps(dic_news, ensure_ascii=False).encode('utf-8'))
    log.info('数据监控Kafka发送成功')


@retry(tries=3, delay=5)
def sendKafka(result_dict):
    producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'], api_version=(2, 7, 0))
    kafka_result = producer.send("crawlerInfo",
                                 json.dumps(result_dict, ensure_ascii=False).encode('utf8'))
    log.info(f"{result_dict['title']}===发送kafka成功！")


# 更新状态表
def updatewxLink(link, info_source_code, state):
    updateSuccess = f"update wx_link set state= {state} where link='{link}' and info_source_code='{info_source_code}' "
    cursor_.execute(updateSuccess)
    cnx_.commit()


def getjsonInfo():
    # todo:从redis中获取一条
    linkid = baseCore.redicPullData('WeiXinGZH:linkid_yx')
    if linkid:
        pass
    else:
        log.info('-----没有数据了-----')
        return False, False
    # 从数据库中获取信息 一条
    select_sql = f"select * from wx_link  where state=0 and id= '{linkid}'"
    cursor_.execute(select_sql)
    row = cursor_.fetchone()
    cnx_.commit()
    if row:
        pass
    else:
        log.info('-----没有数据了-----')
        return False, False
    dict_json = {
        'sid': row[1],
        'site_uri': row[2],
        'site_name': row[3],
        'info_source_code': row[4],
        'title': row[5],
        'publish_time': row[6],
        'link': row[7]
    }
    # 拿到一条数据 更新状态
    update_sql = f"update wx_link set state=1 where link='{row[7]}' and info_source_code='{row[4]}' "
    cursor_.execute(update_sql)
    cnx_.commit()
    return dict_json, linkid


@retry(tries=20, delay=2)
def getrequest(url_news):
    ip = baseCore.get_proxy()
    res_news = requests.get(url_news, proxies=ip, timeout=20)
    if res_news.status_code != 200:
        raise
    return res_news


def get_info(dict_json, linkid):
    # list_all_info = []
    # num_caiji = 0
    kaishi_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    obsClient = ObsClient(
        access_key_id='VEHN7D0TJ9316H8AHCAV',  # 你的华为云的ak码
        secret_access_key='heR353lvSWVPNU8pe2QxDtd8GDsO5L6PGH5eUoQY',  # 你的华为云的sk
        server='https://obs.cn-north-1.myhuaweicloud.com'  # 你的桶的地址
    )

    news_title = dict_json['title']
    sid = dict_json['sid']
    news_date = dict_json['publish_time']
    origin = dict_json['site_name']
    url_news = dict_json['link']
    info_source_code = dict_json['info_source_code']

    # while True:
    #     try:
    #         ip = baseCore.get_proxy()
    #         res_news = requests.get(url_news, proxies=ip, timeout=20)
    #         break
    #     except:
    #         time.sleep(3)
    # 400请求失败
    # updatewxLink(url_news, info_source_code, 400)
    # return False

    # 修改请求方法,retry 3次
    try:
        res_news = getrequest(url_news)
        # print(res_news)
    except:
        # 修改回原状态，重新放入redis
        updatewxLink(url_news, info_source_code, 0)
        log.info(f'{origin}---{news_date}--{news_title}---请求失败-- 重新放入redis')
        baseCore.rePutIntoR('WeiXinGZH:linkid_yx', linkid)
        # try:
        #     res_news = requests.get(url_news, timeout=20)
        # except:
        #     # 400请求失败
        #     updatewxLink(url_news, info_source_code, 400)
        return False
    soup_news = BeautifulSoup(res_news.content, 'html.parser')
    if '无法查看' in soup_news.text or '该页面不存在' in soup_news.text or '该内容已被发布者删除' in soup_news.text or '此内容因违规无法查看' in soup_news.text or '该公众号已迁移' in soup_news.text or '此内容被多人投诉' in soup_news.text:
        log.info(f'--errorCode:800--{origin}---{news_date}---{news_title}----此内容发送失败无法查看')
        updatewxLink(url_news, info_source_code, 800)
        return False
    try:
        news_html = soup_news.find('div', {'id': 'js_content'})
        news_html['style'] = 'width: 814px ; margin: 0 auto;'

        # del news_html['style']
        news_html = rm_style_attr(news_html)
        del news_html['id']
        del news_html['class']
    except Exception as e:
        log.info(f'--errorCode:700--{url_news}-----------{e}')
        # log.error(f'{url_news}-----{info_source_code}')
        updatewxLink(url_news, info_source_code, 0)
        log.info(f'{origin}---{news_date}--{news_title}---style标签解析失败---重新放入redis')
        baseCore.rePutIntoR('WeiXinGZH:linkid_yx', linkid)
        return False
    try:
        news_content = news_html.text
    except:
        log.info(f'--------内容为空--------{url_news}--------')
        time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        false = [
            news_title,
            url_news,
            news_html,
            '文章内容为空',
            time_now
        ]
        insertSql = f"insert into WeixinGZH (site_name,site_url,json_error_info,error_type,create_time) values (%s,%s,%s,%s,%s)"
        cursor_.execute(insertSql, tuple(false))
        cnx_.commit()
        updatewxLink(url_news, info_source_code, 500)
        return False

    list_img = news_html.find_all('img')
    for num_img in range(len(list_img)):
        img_one = list_img[num_img]

        url_src = img_one.get('data-src')
        # print(url_src)
        if url_src and 'gif' in url_src:
            url_img = ''
            img_one.extract()
        else:
            try:
                try:
                    name_img = url_src.split('/')[-2] + '.' + url_src.split('wx_fmt=')[1]
                except:
                    img_one.extract()
                    continue
                try:
                    res = requests.get(url_src, timeout=20)
                except:
                    img_one.extract()
                    continue
                resp = None
                for i in range(10):
                    try:
                        resp = obsClient.putContent('zzsn', name_img, content=res.content)
                        break
                    except:
                        time.sleep(2)
                if resp:
                    pass
                else:
                    img_one.extract()
                    continue
                try:
                    # log.info(f'resp:{resp}')
                    url_img = resp['body']['objectUrl']
                    str_url_img = f'<img src="{url_img}">'
                except Exception as e:
                    log.info(f'--errorCode:300--{url_news}-----------{e}')
                    updatewxLink(url_news, info_source_code, 300)
                    return False
                try:
                    img_one.replace_with(BeautifulSoup(str_url_img, 'lxml').img)
                except Exception as e:
                    log.info(f'--errorCode:300--{url_news}-----------{e}')
                    updatewxLink(url_news, info_source_code, 300)
                    return False
            except Exception as e:
                log.info(f'--errorCode:600--{url_news}-----------{e}')
                updatewxLink(url_news, info_source_code, 600)
                return False

    for tag in news_html.descendants:
        try:
            del tag['style']
        except:
            pass

    list_section = news_html.find_all('section')
    for section in list_section:
        section.name = 'div'
    uniqueCode = baseCore.getUniqueCode('WX', '231')
    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    dic_info = {
        'sid': sid,
        'title': news_title,
        'content': news_content,
        'contentWithtag': str(news_html),
        'summary': '',
        'author': '',
        'origin': origin,
        'publishDate': news_date,
        'sourceAddress': url_news,
        'source': '11',
        'createDate': time_now,
        'uniqueCode': uniqueCode,
    }
    try:
        #todo:数据不发送kafka,直接存入mongo
        db_storage.insert_one(dic_info)
    except:
        log.info('------数据存储失败------')
        updatewxLink(url_news, info_source_code, 200)
    updatewxLink(url_news, info_source_code, 100)
    return True


def rm_style_attr(soup):
    # 查找所有含有style属性的标签
    style_tags = soup.find_all(style=True)
    # 遍历每个style标签
    for style_tag in style_tags:
        try:
            # 使用正则表达式替换
            styleattr = style_tag['style']
            styleattr = re.sub(r'visibility:(?s).{1,}?;', '', styleattr)
            styleattr = re.sub(r'font-family:(?s).{1,}?;', '', styleattr)
            styleattr = re.sub(r'color:(?s).{1,}?;', '', styleattr)
            styleattr = re.sub(r'font-size:(?s).{1,}?;', '', styleattr)
            style_tag['style'] = styleattr
        except:
            continue

    # first_div = soup.select('div[id="js_content"]')
    # # 设置style属性
    # first_div['style'] = 'width: 814px ; margin: 0 auto;'

    first_div = soup.select('div[id="js_content"]')
    if first_div:
        first_div = first_div[0]  # 获取第一个匹配的元素
        first_div['style'] = 'width: 814px ; margin: 0 auto;'  # 设置style属性

    return soup


if __name__ == "__main__":
    num_caiji = 0
    list_all_info = []
    while True:
        # 一次拿取一篇文章
        # todo: 从redis拿数据 更新mysql状态

        dict_json, linkid = getjsonInfo()
        try:
            if dict_json:
                if get_info(dict_json, linkid):
                    num_caiji = num_caiji + 1
                    log.info(f'-----已采集{num_caiji}篇文章---来源{dict_json["site_name"]}----')
            else:
                time.sleep(20)
                continue
        except Exception as e:
            baseCore.rePutIntoR('WeiXinGZH:linkid_yx', linkid)
            log.info(f'-----{linkid}--{e}失败---重新塞入redis----')
    baseCore.close()
