# -*- coding: utf-8 -*-
'''
成功100 发送数据失败200  请求失败400  文章内容为空500
'''
import re

import requests, time, random, json, pymysql, redis

import urllib3
from bs4 import BeautifulSoup

from obs import ObsClient
from kafka import KafkaProducer
from retry import retry

from base.BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
cnx_ = baseCore.cnx
cursor_ = baseCore.cursor
# cnx = pymysql.connect(host="114.116.44.11", user="root", password="f7s0&7qqtK", db="clb_project", charset="utf8mb4")
# cursor = cnx.cursor()
r = baseCore.r
urllib3.disable_warnings()

def rePutIntoR(item):
    r.rpush('WeiXinGZH:linkid', item)

def updatewxLink(link,info_source_code,state):
    updateSuccess = f"update wx_link set state= {state} where link='{link}' and info_source_code='{info_source_code}' "
    cursor_.execute(updateSuccess)
    cnx_.commit()


def getjsonInfo():
    # todo:从redis中获取一条
    linkid = baseCore.redicPullData('WeiXinGZH:linkid')
    if linkid:
        pass
    else:
        log.info('-----没有数据了-----')
        return False, False
    #从数据库中获取信息 一条
    select_sql = f"select * from wx_link  where state=0 and id= '{linkid}'"
    cursor_.execute(select_sql)
    row = cursor_.fetchone()
    cnx_.commit()
    if row:
        pass
    else:
        log.info('-----没有数据了-----')
        return False, False
    dict_json = {
        'sid':row[1],
        'site_uri':row[2],
        'site_name':row[3],
        'info_source_code':row[4],
        'title':row[5],
        'publish_time':row[6],
        'link':row[7]
    }
    # 拿到一条数据 更新状态
    update_sql = f"update wx_link set state=1 where link='{row[7]}' and info_source_code='{row[4]}' "
    cursor_.execute(update_sql)
    cnx_.commit()
    return dict_json, linkid

@retry(tries=3,delay=2)
def getrequest(url_news):

    ip = baseCore.get_proxy()
    res_news = requests.get(url_news, proxies=ip, timeout=20)
    if res_news.status_code != 200:
        raise
    return res_news

def get_info(dict_json):
    # list_all_info = []
    # num_caiji = 0
    kaishi_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    obsClient = ObsClient(
        access_key_id='VEHN7D0TJ9316H8AHCAV',  # 你的华为云的ak码
        secret_access_key='heR353lvSWVPNU8pe2QxDtd8GDsO5L6PGH5eUoQY',  # 你的华为云的sk
        server='https://obs.cn-north-1.myhuaweicloud.com'  # 你的桶的地址
    )

    news_title = dict_json['title']
    sid = dict_json['sid']
    news_date = dict_json['publish_time']
    origin = dict_json['site_name']
    url_news = dict_json['link']
    info_source_code = dict_json['info_source_code']

    # try:
    #     ip = baseCore.get_proxy()
    #     res_news = requests.get(url_news, proxies=ip, timeout=20)
    # except:
    #     #400请求失败
    #     updatewxLink(url_news, info_source_code, 400)
    #     return False

    # 修改请求方法,retry 3次
    try:
        res_news = getrequest(url_news)
        # print(res_news)
    except:
        try:
            res_news = requests.get(url_news, timeout=20)
        except:
            # 400请求失败
            updatewxLink(url_news, info_source_code, 400)
            return False
    soup_news = BeautifulSoup(res_news.content, 'html.parser')
    try:
        news_html = soup_news.find('div', {'id': 'js_content'})
        news_html['style'] = 'width: 814px ; margin: 0 auto;'

        #del news_html['style']
        news_html = rm_style_attr(news_html)
        del news_html['id']
        del news_html['class']
    except Exception as e:
        log.error(f'{url_news}-----{info_source_code}')
        return False
    try:
        news_content = news_html.text
    except:
        log.info(f'--------内容为空--------{url_news}--------')
        time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        false = [
            news_title,
            url_news,
            news_html,
            '文章内容为空',
            time_now
        ]
        insertSql = f"insert into WeixinGZH (site_name,site_url,json_error_info,error_type,create_time) values (%s,%s,%s,%s,%s)"
        cursor_.execute(insertSql, tuple(false))
        cnx_.commit()
        updatewxLink(url_news,info_source_code,500)
        return False

    list_img = news_html.find_all('img')
    for num_img in range(len(list_img)):
        img_one = list_img[num_img]

        url_src = img_one.get('data-src')
        # print(url_src)
        if 'gif' in url_src:
            url_img = ''
            img_one.extract()
        else:
            try:
                try:
                    name_img = url_src.split('/')[-2] + '.' + url_src.split('wx_fmt=')[1]
                except:
                    img_one.extract()
                    continue
                try:
                    res = requests.get(url_src, timeout=20)
                except:
                    img_one.extract()
                    continue
                resp = obsClient.putContent('zzsn', name_img, content=res.content)
                try:
                    url_img = resp['body']['objectUrl']
                    str_url_img = f'<img src="{url_img}">'
                except Exception as e:
                    log.info(f'--error--{url_news}-----------{e}')
                    updatewxLink(url_news, info_source_code, 300)
                    return False
                try:
                    img_one.replace_with(BeautifulSoup(str_url_img, 'lxml').img)
                except Exception as e:
                    log.info(f'--error--{url_news}-----------{e}')
                    updatewxLink(url_news, info_source_code, 300)
                    return False
            except Exception as e:
                log.info(f'--error--{url_news}-----------{e}')
                updatewxLink(url_news, info_source_code, 600)
                return False

    for tag in news_html.descendants:
        try:
            del tag['style']
        except:
            pass

    list_section = news_html.find_all('section')
    for section in list_section:
        section.name = 'div'

    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    dic_info = {
        'sid': sid,
        'title': news_title,
        'content': news_content,
        'contentWithtag': str(news_html),
        'summary': '',
        'author': '',
        'origin': origin,
        'publishDate': news_date,
        'sourceAddress': url_news,
        'source': '11',
        'createDate': time_now
    }
    for nnn in range(0, 3):
        try:
            producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
            kafka_result = producer.send("crawlerInfo", json.dumps(dic_info, ensure_ascii=False).encode('utf8'))
            kafka_time_out = kafka_result.get(timeout=10)
            # add_url(sid, url_news)
            break
        except:
            time.sleep(5)
            log.info('------数据发送kafka失败------')
            updatewxLink(url_news,info_source_code,200)
            continue

    list_all_info.append(dic_info)
    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    dic_info2 = {
        'infoSourceId': sid,
        'code': info_source_code,
        'num': num_caiji,
        'collectTime': kaishi_time,
        'dispatcherTime': time_now,
        'dispatcherStatus': '1',
        'source': '1',
    }
    for nnn2 in range(0, 3):
        try:
            producer2 = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
            kafka_result2 = producer2.send("collectionAndDispatcherInfo",
                                           json.dumps(dic_info2, ensure_ascii=False).encode('utf8'))
            break
        except:
            time.sleep(5)
            continue
    updatewxLink(url_news,info_source_code,100)
    return True

def rm_style_attr(soup):
    # 查找所有含有style属性的标签
    style_tags = soup.find_all(style=True)
    # 遍历每个style标签
    for style_tag in style_tags:
        try:
            # 使用正则表达式替换
            styleattr = style_tag['style']
            styleattr = re.sub(r'visibility:(?s).{1,}?;', '', styleattr)
            styleattr = re.sub(r'font-family:(?s).{1,}?;', '', styleattr)
            styleattr = re.sub(r'color:(?s).{1,}?;', '', styleattr)
            styleattr = re.sub(r'font-size:(?s).{1,}?;', '', styleattr)
            style_tag['style'] = styleattr
        except:
            continue

    # first_div = soup.select('div[id="js_content"]')
    # # 设置style属性
    # first_div['style'] = 'width: 814px ; margin: 0 auto;'

    first_div = soup.select('div[id="js_content"]')
    if first_div:
        first_div = first_div[0]  # 获取第一个匹配的元素
        first_div['style'] = 'width: 814px ; margin: 0 auto;'  # 设置style属性

    return soup

if __name__=="__main__":
    num_caiji = 0
    list_all_info = []
    while True:
        #一次拿取一篇文章
        # todo: 从redis拿数据 更新mysql状态

        dict_json, linkid =getjsonInfo()
        try:
            if dict_json:
                if get_info(dict_json):
                    num_caiji = num_caiji + 1
                    log.info(f'-----已采集{num_caiji}篇文章---来源{dict_json["site_name"]}----')
            else:
                continue
        except:
            rePutIntoR(linkid)

    baseCore.close()