# -*- coding: utf-8 -*-
# @Author: MENG
# @Time  : 2022-3-18
import redis
import requests
from langid import langid
from pyquery import PyQuery as pq
import time
import json
import pymongo
from kafka import KafkaProducer
from requests.packages import urllib3
urllib3.disable_warnings()
db_storage = pymongo.MongoClient('mongodb://114.115.221.202:27017', username='admin', password='zzsn@9988').caiji['人民网-习讲话数据库_copy']

def newsdata(art_content_dict,art_type_dict,dic_lables):
    for key, value in art_content_dict.items():
        labels = []
        tags = art_type_dict.get(key)
        if tags is None:
            tags = []
        value_new = value
        value_new['tags'] = tags
        # todo:lables映射
        for tag in tags:
            labelRemarks = tag['type']
            relationName = tag['name']
            item = labelRemarks + "|" + relationName
            item_value = dic_lables[item]
            labelMark = item_value.split("|")[0]
            relationId = item_value.split("|")[1]
            label = {
                "labelMark": labelMark,
                "labelRemarks": labelRemarks,
                "relationId": relationId,
                "relationName": relationName
            }
            labels.append(label)
        value_new['labels'] = labels
        value_new['subjectId'] = "1534423014825668610"
        time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        value_new['createDate'] = time_now
        value_new['checkStatus'] = "1"
        value_new['deleteFlag'] = "0"
        value_new['topNum'] = "0"
        value_new['summary'] = ""
        post_dict = value_new
        for i in range(5):
            try:
                db_storage.update_one({'id': post_dict['id']}, {'$set': {'tags': tags}})
                break
            except:
                time.sleep(2)
                continue
        if post_dict['is_repeat'] == '1':
            continue

        try:
            del post_dict['is_repeat']
            del post_dict['tags']
            # 发送kafka
            producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'], max_request_size=1024 * 1024 * 20)
            kafka_result = producer.send("research_center_fourth",
                                         json.dumps(post_dict, ensure_ascii=False).encode('utf8'))

            print(kafka_result.get(timeout=10))

            dic_result = {
                'success': 'ture',
                'message': '操作成功',
                'code': '200',
            }
            print(dic_result)
            old_dic = post_dict
            try:
                del old_dic['contentWithTag']
                del old_dic['lang']
                del old_dic['labels']
                del old_dic['createDate']
                del old_dic['checkStatus']
                del old_dic['deleteFlag']
                del old_dic['topNum']
                del old_dic['summary']
                # post_url = 'http://114.116.19.92:8088/api/reptile/autoSaveXJPSpeak'
                # headers = {'Content-Type': 'application/json'}
                # resp_json = requests.post(url=post_url, headers=headers, verify=False, data=json.dumps(old_dic)).json()
                # print('推送：', resp_json['msg'])
            except:
                print('数据传接口失败，正在重试！')

        except Exception as e:
            dic_result = {
                'success': 'false',
                'message': '操作失败',
                'code': '204',
                'e': e
            }
            print(dic_result)
            time.sleep(5)
            db_storage.delete_one({'id': post_dict['id']})
            continue

# 习讲话数据库 新增数据
def get_content():
    art_content_dict = {}
    article_id_list = []
    headers = {
        'Proxy-Connection': 'keep-alive',
        'Accept': '*/*',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
        'X-Requested-With': 'XMLHttpRequest',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Cookie': 'sfr=1; sso_c=0; __jsluid_h=5b9f09f6fdae46fadb89e1e02dca3238; wdcid=04fccdf5121158c0; wdses=72d07de4316a36a5; ci_session=4irerhndk5mc48dq7ldebjn5m47fqptg; wdlast=1646734820; ci_session=4irerhndk5mc48dq7ldebjn5m47fqptg'
    }
    for page in range(9, 0, -1):
        url = f"http://jhsjk.people.cn/testnew/result?keywords=&isFuzzy=0&searchArea=0&year=0&form=0&type=0&page={page}&origin=%E5%85%A8%E9%83%A8&source=2"
        try:
            resp_json = requests.request("GET", url, headers=headers, verify=False).json()
            time.sleep(0.2)
            data_list = resp_json['list']
        except:
            print('请求错误1')
            continue
        for data_dict in data_list[::-1]:
            article_id = data_dict['article_id']
            is_article_id = db_storage.find_one({'id': article_id})
            if is_article_id:
                continue
            title = data_dict['title']
            pub_time = data_dict['input_date']
            if pub_time <= '2023-11-06':
                continue
            title_dict_list = db_storage.find({'title': title, 'is_repeat': ''})
            is_repeat = ''
            for title_dict in title_dict_list:
                pub_time1 = title_dict['publishDate']
                time_ = int(int(time.mktime(time.strptime(pub_time, "%Y-%m-%d %H:%M:%S"))) - int(time.mktime(time.strptime(pub_time1, "%Y-%m-%d %H:%M:%S"))))
                if -259200 < time_ < 259200:
                    is_repeat = '1'
                    break
            article_id_list.append(article_id)
            href = 'http://jhsjk.people.cn/article/' + article_id
            try:
                href_text = requests.request("GET", href, headers=headers, verify=False).content
                time.sleep(0.2)
                doc_href = pq(href_text)
                content_html1 = str(doc_href('.d2txt_con.clearfix'))
                content_html2 = str(doc_href('.editor.clearfix'))
            except:
                print('请求错误2')
                continue
            content_html = content_html1 + '\n' + content_html2
            content = pq(content_html).text()
            lang = langid.classify(content)
            if lang == '':
                lang = 'cn'
            if lang[0] == '':
                lang = 'cn'
            else:
                lang = lang[0]
            if content.strip() == '':
                print(href, '内容为空')
                continue
            origin = data_dict['origin_name']
            a_dict = {
                'id': "1534423014825668610" + article_id,
                'title': title,
                'author': '',
                'origin': origin,
                'contentWithTag': content_html,
                'content': content,
                'publishDate': pub_time,
                'sourceAddress': href,
                'tags': [],
                'lang': lang,
                'is_repeat': is_repeat
            }
            art_content_dict[article_id] = a_dict
            db_a_dict = a_dict.copy()
            db_storage.insert_one(db_a_dict)
            if is_repeat == '':
                print(href)
    if article_id_list is []:
        return
    art_type_dict = {}
    result_lists = [
        ['类型', '讲话', '706', '69'], ['类型', '会议', '701', '178'], ['类型', '活动', '702', '63'], ['类型', '考察', '703', '72'],
        ['类型', '会见', '704', '174'], ['类型', '出访', '705', '188'], ['类型', '函电', '707', '194'], ['类型', '其他', '708', '203'],
        ['时间', '2023', '2023', '11'], ['时间', '2022', '2022', '10'], ['时间', '2021', '2021', '9'],
        ['时间', '2019', '2019', '8'],
        ['时间', '2018', '2018', '7'], ['时间', '2017', '2017', '6'], ['时间', '2016', '2016', '5'],
        ['时间', '2015', '2015', '4'],
        ['时间', '2014', '2014', '3'], ['时间', '2013', '2013', '2'], ['时间', '2012', '2012', '1'],
        ['领域', '经济', '101', '18'], ['领域', '政治', '102', '21'], ['领域', '文化', '103', '14'], ['领域', '社会', '104', '15'],
        ['领域', '生态', '105', '7'], ['领域', '党建', '106', '9'], ['领域', '国防', '107', '6'], ['领域', '外交', '108', '50'],

    ]
    dic_lables = {
        "类型|讲话": "important_speech_type|1700334917807710209",
        "类型|会议": "important_speech_type|1700334936166178818",
        "类型|活动": "important_speech_type|1700334960560250881",
        "类型|考察": "important_speech_type|1700334978285379585",
        "类型|会见": "important_speech_type|1700335044605714433",
        "类型|出访": "important_speech_type|1700335078852206593",
        "类型|函电": "important_speech_type|1700335099689508866",
        "类型|其他": "important_speech_type|1700335118056366082",
        "时间|2012": "important_speech_time|1700334545970077697",
        "时间|2013": "important_speech_time|1700334647757447170",
        "时间|2014": "important_speech_time|1700334667915272194",
        "时间|2015": "important_speech_time|1700334686550564865",
        "时间|2016": "important_speech_time|1700334704925810689",
        "时间|2017": "important_speech_time|1700334722529304578",
        "时间|2018": "important_speech_time|1700334738320859137",
        "时间|2019": "important_speech_time|1700334758302523393",
        "时间|2020": "important_speech_time|1700334777827008514",
        "时间|2021": "important_speech_time|1700334797477322753",
        "时间|2022": "important_speech_time|1700334814468448258",
        "时间|2023": "important_speech_time|1700334832495566850",
        "领域|经济": "important_speech_area|1700335225803841537",
        "领域|政治": "important_speech_area|1700335248096567297",
        "领域|文化": "important_speech_area|1700335379638329345",
        "领域|社会": "important_speech_area|1700335412873994242",
        "领域|生态": "important_speech_area|1700335541211308033",
        "领域|党建": "important_speech_area|1700335587780665346",
        "领域|国防": "important_speech_area|1700335615895085058",
        "领域|外交": "important_speech_area|1700335820430319618",

    }

    for result_list in result_lists:
        sort = result_list[0]
        sort_text = result_list[1]
        print(sort_text)
        if sort == '类型':
            form = result_list[2]
            type_ = '0'
            year = '0'
        elif sort == '时间':
            form = '0'
            type_ = '0'
            year = result_list[2]
        else:
            form = '0'
            type_ = result_list[2]
            year = '0'
        # total_page = result_list[3]
        total_page = 10
        for page in range(1, int(total_page)):
            url = f"http://jhsjk.people.cn/testnew/result?keywords=&isFuzzy=0&searchArea=0&year={year}&form={form}&type={type_}&page={page}&origin=%E5%85%A8%E9%83%A8&source=2"
            payload = {}
            try:
                resp_json = requests.request("GET", url, headers=headers, verify=False, data=payload).json()
                data_list = resp_json['list']
            except:
                print('请求错误3')
                continue
            for data_dict in data_list:
                article_id = data_dict['article_id']
                if article_id in article_id_list:
                    type_dict = {
                        "type": sort,
                        "name": sort_text,
                    }
                    type_list = art_type_dict.get(article_id)
                    if type_list is None:
                        type_lists = [type_dict]
                    else:
                        type_lists = type_list + [type_dict]
                    new_lst = []

                    # 遍历原列表的字典元素
                    for item in type_lists:
                        # 如果字典元素不在新列表中，则添加到新列表中
                        if item not in new_lst:
                            new_lst.append(item)

                    art_type_dict[article_id] = new_lst

    newsdata(art_content_dict, art_type_dict, dic_lables)


if __name__ == '__main__':
    try:
        get_content()
    except Exception as e:
        print(e)
        pass





"""
#本地访问地址 http://192.168.1.190:8008/api/reptile/autoSaveXJPSpeak
#服务器地址   http://114.116.19.92:8088/api/reptile/autoSaveXJPSpeak

#请求方式： post


#请求体数据,JSON格式
{
	"id": "32396175",
	"title": "习近平：只有攥紧中国种子，才能端稳中国饭碗",
	"author": "",
	"origin": "人民日报客户端",
	"content": "带标签",
	"publishDate": "习近平：只有攥紧中国种子，才能端稳中国饭碗",
	"sourceAddress": "http://jhsjk.people.cn/article/32396175",
	"tags": [{
		"type": "类型",
		"name": "考察"
	}, {
		"type": "领域",
		"name": "文化"
	}]
}}
"""