# -*- coding: utf-8 -*-
'''
成功100 发送数据失败200  请求失败400  文章内容为空500
'''
import re

import requests, time, random, json, pymysql, redis

import urllib3
from bs4 import BeautifulSoup

from obs import ObsClient
from kafka import KafkaProducer

from base.BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
cnx_ = baseCore.cnx
cursor_ = baseCore.cursor
# cnx = pymysql.connect(host="114.116.44.11", user="root", password="f7s0&7qqtK", db="clb_project", charset="utf8mb4")
# cursor = cnx.cursor()
r = baseCore.r
urllib3.disable_warnings()

def rePutIntoR(item):
    r.rpush('WeiXinGZH:infoSourceCode', item)

def updatewxLink(link,info_source_code,state):
    updateSuccess = f"update wx_link set state= {state} where link='{link}' and info_source_code='{info_source_code}' "
    cursor_.execute(updateSuccess)
    cnx_.commit()


def getjsonInfo():
    #从数据库中获取信息 一条
    select_sql = "select * from wx_link  where state=0 order by id asc limit 1"
    cursor_.execute(select_sql)
    row = cursor_.fetchone()
    cnx_.commit()
    if row:
        pass
    else:
        log.info('-----没有数据了-----')
        return False
    dict_json = {
        'sid':row[1],
        'site_uri':row[2],
        'site_name':row[3],
        'info_source_code':row[4],
        'title':row[5],
        'publish_time':row[6],
        'link':row[7]
    }
    # 拿到一条数据 更新状态
    update_sql = f"update wx_link set state=1 where link='{row[7]}' and info_source_code='{row[4]}' "
    cursor_.execute(update_sql)
    cnx_.commit()
    return dict_json

def get_info(dict_json):
    # list_all_info = []
    # num_caiji = 0
    kaishi_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    obsClient = ObsClient(
        access_key_id='VEHN7D0TJ9316H8AHCAV',  # 你的华为云的ak码
        secret_access_key='heR353lvSWVPNU8pe2QxDtd8GDsO5L6PGH5eUoQY',  # 你的华为云的sk
        server='https://obs.cn-north-1.myhuaweicloud.com'  # 你的桶的地址
    )

    news_title = dict_json['title']
    sid = dict_json['sid']
    news_date = dict_json['publish_time']
    origin = dict_json['site_name']
    url_news = dict_json['link']
    info_source_code = dict_json['info_source_code']
    # url_ft = check_url(sid, url_news)
    # if url_ft:
    #     return list_all_info,num_caiji
    try:
        ip = baseCore.get_proxy()
        res_news = requests.get(url_news, proxies=ip,timeout=20)
    except:
        #400请求失败
        updatewxLink(url_news,info_source_code,400)
        return False
    soup_news = BeautifulSoup(res_news.content, 'html.parser')

    news_html = soup_news.find('div', {'id': 'js_content'})
    news_html['style'] = 'width: 814px ; margin: 0 auto;'
    try:
        #del news_html['style']
        news_html=rm_style_attr(news_html)
        del news_html['id']
        del news_html['class']
    except:
        pass
    try:
        news_content = news_html.text
    except:
        log.info(f'--------内容为空--------{url_news}--------')
        time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        false = [
            news_title,
            url_news,
            news_html,
            '文章内容为空',
            time_now
        ]
        insertSql = f"insert into WeixinGZH (site_name,site_url,json_error_info,error_type,create_time) values (%s,%s,%s,%s,%s)"
        cursor_.execute(insertSql, tuple(false))
        cnx_.commit()
        updatewxLink(url_news,info_source_code,500)
        return False

    list_img = news_html.find_all('img')
    for num_img in range(len(list_img)):
        img_one = list_img[num_img]

        url_src = img_one.get('data-src')
        # print(url_src)
        if 'gif' in url_src:
            url_img = ''
            img_one.extract()
        else:
            try:
                try:
                    name_img = url_src.split('/')[-2] + '.' + url_src.split('wx_fmt=')[1]
                except:
                    img_one.extract()
                    continue
                try:
                    res = requests.get(url_src, timeout=20)
                except:
                    img_one.extract()
                    continue
                resp = obsClient.putContent('zzsn', name_img, content=res.content)
                try:
                    url_img = resp['body']['objectUrl']
                    str_url_img = f'<img src="{url_img}">'
                except Exception as e:
                    log.info(f'--error--{url_news}-----------{e}')
                    updatewxLink(url_news, info_source_code, 300)
                    return False
                try:
                    img_one.replace_with(BeautifulSoup(str_url_img, 'lxml').img)
                except Exception as e:
                    log.info(f'--error--{url_news}-----------{e}')
                    updatewxLink(url_news, info_source_code, 300)
                    return False
            except Exception as e:
                log.info(f'--error--{url_news}-----------{e}')
                updatewxLink(url_news, info_source_code, 600)
                return False

    for tag in news_html.descendants:
        try:
            del tag['style']
        except:
            pass

    list_section = news_html.find_all('section')
    for section in list_section:
        section.name = 'div'

    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    dic_info = {
        'sid': sid,
        'title': news_title,
        'content': news_content,
        'contentWithtag': str(news_html),
        'summary': '',
        'author': '',
        'origin': origin,
        'publishDate': news_date,
        'sourceAddress': url_news,
        'source': '11',
        'createDate': time_now
    }
    for nnn in range(0, 3):
        try:
            producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
            kafka_result = producer.send("crawlerInfo", json.dumps(dic_info, ensure_ascii=False).encode('utf8'))
            kafka_time_out = kafka_result.get(timeout=10)
            # add_url(sid, url_news)
            break
        except:
            time.sleep(5)
            log.info('------数据发送kafka失败------')
            updatewxLink(url_news,info_source_code,200)
            continue

    list_all_info.append(dic_info)
    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    dic_info2 = {
        'infoSourceId': sid,
        'code': info_source_code,
        'num': num_caiji,
        'collectTime': kaishi_time,
        'dispatcherTime': time_now,
        'dispatcherStatus': '1',
        'source': '1',
    }
    for nnn2 in range(0, 3):
        try:
            producer2 = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
            kafka_result2 = producer2.send("collectionAndDispatcherInfo",
                                           json.dumps(dic_info2, ensure_ascii=False).encode('utf8'))
            break
        except:
            time.sleep(5)
            continue
    updatewxLink(url_news,info_source_code,100)
    return True

def rm_style_attr(soup):
    # 查找所有含有style属性的标签
    style_tags = soup.find_all(style=True)
    # 遍历每个style标签
    for style_tag in style_tags:
        try:
            # 使用正则表达式替换
            styleattr = style_tag['style']
            styleattr = re.sub(r'visibility:(?s).{1,}?;', '', styleattr)
            styleattr = re.sub(r'font-family:(?s).{1,}?;', '', styleattr)
            styleattr = re.sub(r'color:(?s).{1,}?;', '', styleattr)
            styleattr = re.sub(r'font-size:(?s).{1,}?;', '', styleattr)
            style_tag['style'] = styleattr
        except:
            continue

    first_div = soup.select('div[id="js_content"]')
    # 设置style属性
    first_div['style'] = 'width: 814px ; margin: 0 auto;'
    return soup

if __name__=="__main__":
    num_caiji = 0
    list_all_info = []
    while True:
        #一次拿取一篇文章
        dict_json  =getjsonInfo()
        if dict_json:
            if get_info(dict_json):
                num_caiji = num_caiji + 1
                log.info(f'-----已采集{num_caiji}篇文章---来源{dict_json["site_name"]}----')
        else:
            break

    baseCore.close()