"""
国外智库-欧盟 经合组织
"""
import json
import time

import pymongo
from bs4 import BeautifulSoup
import requests
from datetime import datetime

from kafka import KafkaProducer
from retry import retry

import BaseCore

baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()
db_storage = pymongo.MongoClient('mongodb://114.115.221.202:27017', username='admin', password='ZZsn@9988').ZZSN[
    '国外智库']

@retry(tries=2, delay=5)
def sendKafka(dic):
    producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'], max_request_size=1024 * 1024 * 20)
    kafka_result = producer.send("research_center_fourth",
                                 json.dumps(dic, ensure_ascii=False).encode('utf8'))
    log.info(f'{dic["sourceAddress"]}传输成功')

def secrchATT(item_id, retData, type_id, order_by):
    sel_sql = '''select id from clb_sys_attachment where item_id = %s and path = %s and type_id=%s and order_by=%s '''
    baseCore.cursor_.execute(sel_sql, (item_id, retData['path'], type_id, order_by))
    selects = baseCore.cursor_.fetchone()
    return selects

# 插入到att表 返回附件id
def tableUpdate(retData, file_name, num, publishDate,origin):
    item_id = retData['item_id']
    type_id = retData['type_id']
    group_name = retData['group_name']
    path = retData['path']
    full_path = retData['full_path']
    category = retData['category']
    file_size = retData['file_size']
    status = retData['status']
    create_by = retData['create_by']
    page_size = retData['page_size']
    create_time = retData['create_time']
    order_by = num
    object_key = full_path.split('https://zzsn.obs.cn-north-1.myhuaweicloud.com/')[1]

    Upsql = '''insert into clb_sys_attachment(name,type_id,item_id,group_name,path,full_path,category,file_size,order_by,status,create_by,create_time,object_key,bucket_name,publish_time,source) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''

    values = (
        file_name+'.pdf', type_id, item_id, group_name, path, full_path, category, file_size, order_by,
        status, create_by,
        create_time, object_key, 'zzsn', publishDate,origin)

    baseCore.cursor_.execute(Upsql, values)  # 插入
    baseCore.cnx_.commit()  # 提交
    baseCore.getLogger().info("更新完成:{}".format(Upsql))
    selects = secrchATT(item_id, retData, type_id, order_by)
    id = selects[0]
    return id

def save_data(dic_news):
    aaa_dic = {
        '附件id': dic_news['attachmentIds'],
        '网址': dic_news['sourceAddress'],
        'tid': '',
        '来源': f"经济合作与发展组织",
        '创建时间': dic_news['createDate'],
        '带标签内容': dic_news['contentWithTag'][:100],
        '发布时间': dic_news['publishDate'],
        '标题': dic_news['title']
    }
    db_storage.insert_one(aaa_dic)

@retry(tries=2, delay=5)
def translate(title, contentWithTag):
    headers = {
        'Content-Type': 'application/json',
    }
    dic_info = {
        'title': title,
        # 'summary': '<div>apple</div>',
        'contentWithTag': contentWithTag
    }
    dic_info = json.dumps(dic_info)
    req = requests.post('http://117.78.23.14:5001/translate', data=dic_info, headers=headers)
    dataJson = req.json()
    if dataJson['status'] == 'failed':
        raise
    titleRaw = dataJson['title']
    contentWithTagRaw = dataJson['contentWithTag']
    titleRaw = BeautifulSoup(titleRaw,'html.parser')
    titleRaw = titleRaw.text
    contentWithTagRaw = BeautifulSoup(contentWithTagRaw,'html.parser')
    return titleRaw, contentWithTagRaw

def doJob():
    num = 1
    url = 'https://www.oecd-ilibrary.org/economics/oecd-policy-responses-on-the-impacts-of-the-war-in-ukraine_dc825602-en?page=1'
    headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Cache-Control': 'max-age=0',
    'Cookie': 'JSESSIONID=BHezogPwi8NJVECsKXCXqijdQ00-yMJHw_gR8wiC.ip-10-240-5-121; __cf_bm=c2byUypnSjXPS_UFDM7BMRGDxN6AQEkNVUjzw9HuSq8-1707054653-1-AbbI7JWWkfWKVGi8SKI06f0jGEjPdk5kvHAIRRpBHSSSnmxj1IcvGUT8+/O6R0U2RLZJECZdUzZIXAwFuEz5lPo=; _gcl_au=1.1.201344533.1707054655; _gid=GA1.2.557164000.1707054655; cb-enabled=enabled; cf_clearance=6tK6.WKHJbXXoV4NTgbyHRhetRxMdWPZofwlv01F65Y-1707054656-1-AfrYlWnLLZFC1sKxeFVQintPrZnjvjoJSZwRRhAYwqRHGdWbU5IFZQDJZJM21l20Tj6gk4JxNobWT0wGzp1Dgjw=; _ce.irv=new; cebs=1; _ce.clock_event=1; _ce.clock_data=72%2C123.149.3.159%2C1%2C9c1ce27f08b16479d2e17743062b28ed; custom_cookie_AB=1; AWSALB=I/eGQ0glcxuROskD1JKEl/dqsqElpmo/MnwLboJZJB2QthQFFWnLA3gzuJTskEaZxJD7VuWEEsqjhLVvhq4q2Wt0RebuRhukeHpKvgmGMelxpn/RiDmehyvxTOiS; AWSALBCORS=I/eGQ0glcxuROskD1JKEl/dqsqElpmo/MnwLboJZJB2QthQFFWnLA3gzuJTskEaZxJD7VuWEEsqjhLVvhq4q2Wt0RebuRhukeHpKvgmGMelxpn/RiDmehyvxTOiS; _gat_UA-1887794-2=1; _dc_gtm_UA-136634323-1=1; _ga_F5XZ540Q4V=GS1.1.1707054655.1.1.1707055119.7.0.0; _ga=GA1.1.1014316406.1707054655; _ga_F7KSNTXTRX=GS1.1.1707054655.1.1.1707055119.0.0.0; cebsp_=5; _ce.s=v~212f033193b9432855ae8335d6d3969cc1f8b751~lcw~1707055134688~lva~1707054658247~vpv~0~v11.fhb~1707054659602~v11.lhb~1707055126493~v11.cs~325107~v11.s~6d7ba630-c364-11ee-aba8-136dbbf9a447~v11.sla~1707055134688~v11.send~1707055135439~lcw~1707055135439',
    'Referer': 'https://www.oecd-ilibrary.org/economics/oecd-policy-responses-on-the-impacts-of-the-war-in-ukraine_dc825602-en?page=2',
    'Sec-Ch-Ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
    'Sec-Ch-Ua-Mobile': '?0',
    'Sec-Ch-Ua-Platform': '"Windows"',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-User': '?1',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
    }
    req = requests.get(url=url, headers=headers)
    soup = BeautifulSoup(req.content, 'html.parser')
    div_part = soup.find_all('div', class_='col-xs-12 body-section')[1]
    div_list = div_part.find_all('div', class_='row panel')
    for div in div_list:
        start_time = time.time()
        title = div.find('div', class_='col-lg-7 col-xs-12 resume-item').find('p', class_='intro-item').find('strong', class_='book-title').text
        href = 'https://www.oecd-ilibrary.org' + div.find('div', class_='col-lg-7 col-xs-12 resume-item').find('p', class_='intro-item').find('a')['href']
        is_href = db_storage.find_one({'网址': href})
        if is_href:
            log.info(f'{href}===已采集')
            continue
        pubtime_ = div.find('div', class_='col-lg-7 col-xs-12 resume-item').find('p', class_='intro-item').find('strong', class_='book-title gray').text
        # 定义原始时间的格式
        time_format = "%d %b %Y"
        # 转换为标准时间
        standard_time = datetime.strptime(pubtime_, time_format).strftime("%Y-%m-%d")
        if standard_time > '2023-01-30':
            pass
        else:
            break
        year = standard_time[:4]
        pdf_part = div.find('div', class_='col-lg-5 col-xs-12 actions-item').find('ul', class_='actions').find_all('li')[1].find('a').get('href')
        pdf_url = 'https://www.oecd-ilibrary.org' + pdf_part

        req_news = requests.get(url=href, headers=headers)
        soup_news = BeautifulSoup(req_news.content, 'html.parser')
        # print(title, standard_time, pdf_url, href)
        contentWithTag = soup_news.find('div', class_='description js-desc-fade show-all')
        content = contentWithTag.get_text()
        # todo:翻译
        try:
            titleRaw, contentWithTagRaw = translate(str(title), str(contentWithTag))
            log.info(f'{href}===翻译成功')
        except Exception as e:
            log.error(f'{href}===翻译失败==={e}')
            continue
        retData = baseCore.uptoOBS(pdf_url, title, 15, '', pathType, taskType, start_time, create_by)
        num += 1
        id_list = []
        if retData['state']:
            att_id = tableUpdate(retData, title, num, standard_time, '经济合作与发展组织')
            if att_id:
                id_list.append(att_id)
                now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                lang = baseCore.detect_language(content)

                contentRaw = contentWithTagRaw.text
                contentWithTagRaw = str(contentWithTagRaw)
                dic = {
                    'id': f'1620244462491893761{int(time.time())}',
                    'subjectId': '1620244462491893761',
                    'checkStatus': 1,
                    'deleteFlag': 0,
                    'topNum': 0,
                    'content': content,
                    'contentRaw': contentRaw,
                    'contentWithTag': str(contentWithTag),
                    'contentWithTagRaw': contentWithTagRaw,
                    'createDate': now,
                    'labels': [
                        {'labelMark': 'organization', 'relationId': '1619903523269271554', 'relationName': '经济合作与发展组织'}],
                    'lang': lang,
                    'origin': '经济合作与发展组织',
                    'publishDate': standard_time,
                    'sourceAddress': href,
                    'title': title,
                    'titleRaw': titleRaw,
                    'updateDate': now,
                    'attachmentIds':id_list
                }
                sendKafka(dic)
                try:
                    save_data(dic)
                except:
                    log.error(f'{href}===数据库保存失败')
        # break

if __name__ == "__main__":
    pathType = 'PolicyDocuments/'
    taskType = '国外智库-经合组织'
    create_by = 'XueLingKun'

    doJob()

