
"""
天眼查动态采集流程，
1.根据信用代码获取天眼查的公司id
2.获取对应的动态数量
3.进行列表翻页采集
4.对动态详情进行请求和抽取
5.将相关信息发送到kafka和日志记录

"""
import json
import random
import time

import pymysql
import requests
from kafka import KafkaProducer

from BaseCore import BaseCore
from getTycId import getTycIdByXYDM
import smart_extractor

class Tycdt(object):

    def __init__(self):
        self.baseCore=BaseCore()
        self.log = self.baseCore.getLogger()
        self.cnx_ = self.baseCore.cnx
        self.cursor_ = self.baseCore.cursor
        self.taskType = '企业动态/天眼查/补采20W+'
        # 初始化，设置中文分词
        self.smart =smart_extractor.SmartExtractor('cn')
        self.cnx = pymysql.connect(host='114.116.44.11', user='caiji', password='f7s0&7qqtK', db='dbScore', charset='utf8mb4')
        self.cursor = self.cnx.cursor()
        self.pageSize = 10
        self.headers = {
        'Accept': 'application/json, text/plain, */*',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Connection': 'keep-alive',
        'Content-Type': 'application/json',
        'Host': 'capi.tianyancha.com',
        'Origin': 'https://www.tianyancha.com',
        'Referer': 'https://www.tianyancha.com/',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-site',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36',
        'X-AUTH-TOKEN': 'eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiIxODcwMzc1MjYwMCIsImlhdCI6MTY5NzE5MDMwMywiZXhwIjoxNjk5NzgyMzAzfQ.awXuS-59RzK35r0aUJq4Rj83JzyAOvsdUfL_ojp66CVQMjlLv_ZDD9g5gCoZKE21LN1JYRMLNZhuWsHhxapROw',
        'X-TYCID': '6f6298905d3011ee96146793e725899d',
        'sec-ch-ua': '"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
        'version': 'TYC-Web'
    }

    # 日志信息保存至现已创建好数据库中,因此并没有再对此前保存日志信息数据库进行保存
    def doJob(self):
        while True:
            # 根据从Redis中拿到的社会信用代码,在数据库中获取对应基本信息
            social_code = self.baseCore.redicPullData('NewsEnterprise:gnqybc_socialCode')
            # social_code = '913205002517479347'
            # 判断 如果Redis中已经没有数据，则等待
            if social_code == None:
                time.sleep(20)
                continue
            start = time.time()
            try:
                data = self.baseCore.getInfomation(social_code)
                if len(data) != 0:
                    pass
                else:
                    #数据重新塞入redis
                    self.baseCore.rePutIntoR('NewsEnterprise:gnqybc_socialCode',social_code)
                    continue
                id = data[0]
                xydm = data[2]
                tycid = data[11]
                if tycid == None or tycid == '':
                    try:
                        retData = getTycIdByXYDM(xydm)
                        if retData['tycData'] and retData['reput']:
                            tycid = retData['tycData']['id']
                            # todo:写入数据库
                            updateSql = f"update EnterpriseInfo set TYCID = '{tycid}' where SocialCode = '{xydm}'"
                            self.cursor_.execute(updateSql)
                            self.cnx_.commit()
                        elif not retData['tycData'] and retData['reput']:
                            state = 0
                            takeTime = self.baseCore.getTimeCost(start, time.time())
                            self.baseCore.recordLog(social_code, self.taskType, state, takeTime, '', '获取天眼查id失败')
                            self.log.info(f'======={social_code}====重新放入redis====')
                            self.baseCore.rePutIntoR('NewsEnterprise:gnqybc_socialCode', social_code)
                            continue
                        elif not retData['reput'] and not retData['tycData']:
                            continue
                    except:
                        state = 0
                        takeTime = self.baseCore.getTimeCost(start, time.time())
                        self.baseCore.recordLog(social_code, self.taskType, state, takeTime, '', '获取天眼查id失败')
                        self.baseCore.rePutIntoR('NewsEnterprise:gnqybc_socialCode', social_code)
                        continue
                count = data[17]
                self.log.info(f"{id}---{xydm}----{tycid}----开始处理")
                start_time = time.time()
                # 开始采集企业动态
                retData = self.beinWork(tycid, xydm,start_time)
                # 信息采集完成后将该企业的采集次数更新
                runType = 'NewsRunCount'
                count += 1
                self.baseCore.updateRun(social_code, runType, count)
                total = retData['total']
                up_okCount = retData['up_okCount']
                up_errorCount = retData['up_errorCount']
                up_repetCount = retData['up_repetCount']
                self.log.info(
                    f"{id}---{xydm}----{tycid}----结束处理，耗时{self.baseCore.getTimeCost(start_time, time.time())}---总数:{total}---成功数:{up_okCount}----失败数:{up_errorCount}--重复数:{up_repetCount}")
            except Exception as e:
                self.log.info(f'==={social_code}=====获取企业信息失败====')
                #重新塞入redis
                self.baseCore.rePutIntoR('NewsEnterprise:gnqybc_socialCode',social_code)
                state = 0
                takeTime = self.baseCore.getTimeCost(start, time.time())
                self.baseCore.recordLog(social_code, self.taskType, state, takeTime, '', f'获取企业信息失败--{e}')
                time.sleep(5)
            #break

        self.cursor.close()
        self.cnx.close()
        # 释放资源
        self.baseCore.close()

    def beinWork(self,tyc_code, social_code,start_time):

        time.sleep(3)
        # retData={'up_state':False,'total':0,'up_okCount':0,'up_errorCount':0,'up_repetCount':0}
        retData = {'total': 0, 'up_okCount': 0, 'up_errorCount': 0, 'up_repetCount': 0}
        t = time.time()
        url = f'https://capi.tianyancha.com/cloud-yq-news/company/detail/publicmsg/news/webSimple?_={t}&id={tyc_code}&ps={self.pageSize}&pn=1&emotion=-100&event=-100'
        try:
            for m in range(0, 3):
                ip = self.baseCore.get_proxy()
                self.headers['User-Agent'] = self.baseCore.getRandomUserAgent()
                response = requests.get(url=url, headers=self.headers, proxies=ip, verify=False,timeout=10)
                # time.sleep(random.randint(3, 5))
                break
            if (response.status_code == 200):
                pass
        except Exception as e:
            self.log.error(f"{tyc_code}-----获取总数接口失败")
            error = '获取总数接口失败'
            state = 0
            takeTime = self.baseCore.getTimeCost(start_time, time.time())
            self.baseCore.recordLog(social_code, self.taskType, state, takeTime, url, f'{error}----{e}')
            return retData
        try:
            json_1 = json.loads(response.content.decode('utf-8'))
            total = json_1['data']['total']
        except:
            self.log.error(f"{tyc_code}-----获取总数失败")
            e = '获取总数失败'
            state = 0
            takeTime = self.baseCore.getTimeCost(start_time, time.time())
            self.baseCore.recordLog(social_code, self.taskType, state, takeTime, url, e)
            return retData

        if (total > 0):
            if (total % self.pageSize == 0):
                totalPage = total // self.pageSize
            else:
                totalPage = total // self.pageSize + 1
        else:
            self.log.error(f"{tyc_code}--------总数为0")
            retData['state'] = True
            return retData
        self.log.info(f"{tyc_code}-------总数：{total}----总页数:{totalPage}")

        retData['total'] = total
        up_okCount = 0
        up_errorCount = 0
        up_repetCount = 0
        for num in range(1, totalPage + 1):
            # time.sleep(3)
            self.log.info(f"获取分页数据--{tyc_code}----分页{num}----开始")
            start_page = time.time()
            url_page = f'https://capi.tianyancha.com/cloud-yq-news/company/detail/publicmsg/news/webSimple?_={time.time()}&id={tyc_code}&ps={self.pageSize}&pn={num}&emotion=-100&event=-100'
            for m in range(0, 3):
                try:
                    ip = self.baseCore.get_proxy()
                    self.headers['User-Agent'] = self.baseCore.getRandomUserAgent()
                    response_page = requests.get(url=url_page, headers=self.headers, proxies=ip,timeout=10, verify=False)
                    # time.sleep(1)
                    break
                except:
                    pass

            if (response_page.status_code == 200):
                pass
            else:
                self.log.error(f"{tyc_code}--{num}页---获取分页数据失败")
                e = '获取分页数据失败'
                state = 0
                takeTime = self.baseCore.getTimeCost(start_time, time.time())
                self.baseCore.recordLog(social_code, self.taskType, state, takeTime, url_page, e)
                up_errorCount = up_errorCount + self.pageSize
                continue
            try:
                json_page = json.loads(response_page.content.decode('utf-8'))
                info_list_page = json_page['data']['items']
            except:
                self.log.error(f"{tyc_code}--{num}页---获取分页数据失败")
                e = '获取分页数据失败'
                state = 0
                takeTime = self.baseCore.getTimeCost(start_time, time.time())
                self.baseCore.recordLog(social_code, self.taskType, state, takeTime, url_page, e)
                up_errorCount = up_errorCount + self.pageSize
                continue
            pageIndex = 0
            for info_page in info_list_page:
                pageIndex = pageIndex + 1
                title = info_page['title']
                source = info_page['website']
                link = info_page['uri']
                try:
                    sel_sql = '''select social_credit_code from brpa_source_article_news where source_address = %s and social_credit_code=%s and type='2' '''
                    self.cursor_.execute(sel_sql, (link, social_code))
                except Exception as e:
                    print(e)
                selects = self.cursor_.fetchone()
                if selects:
                    self.log.info(f'{tyc_code}-----{social_code}----{link}:已经存在')

                    # todo:如果该条数据存在则说明该条数据之后的都已经采集完成，就可以跳出函数，执行下一个企业

                    retData['up_okCount'] = up_okCount
                    retData['up_errorCount'] = up_errorCount
                    retData['up_repetCount'] = up_repetCount
                    #return retData
                    continue
                try:
                    time_struct = time.localtime(int(info_page['rtm'] / 1000))  # 首先把时间戳转换为结构化时间
                    time_format = time.strftime("%Y-%m-%d %H:%M:%S", time_struct)  # 把结构化时间转换为格式化时间
                except:
                    time_format = self.baseCore.getNowTime(1)
                #记录时间 对比时间
                #if time_format > '2023-09-25' and time_format < '2023-10-01':
                #pass
                #else:
                #continue
                try:
                    # 开始进行智能解析
                    # lang = baseCore.detect_language(title)
                    # smart = smart_extractor.SmartExtractor(lang)
                    html_text=self.reqDetailmsg(link)
                    article=self.smart.extract_by_html(html_text,title)
                    content=article.cleaned_text
                    contentText=article.text
                    # #带标签正文
                    # contentText = self.smart.extract_by_url(link).text
                    # #不带标签正文
                    # content = self.smart.extract_by_url(link).cleaned_text
                    # time.sleep(3)
                except Exception as e:
                    contentText = ''
                if contentText == '':
                    self.log.error(f'获取正文失败：--------{tyc_code}--------{num}--------{link}')
                    e = '获取正文失败'
                    state = 0
                    takeTime = self.baseCore.getTimeCost(start_time, time.time())
                    self.baseCore.recordLog(social_code, self.taskType, state, takeTime, link, e)
                    up_errorCount = up_errorCount + 1
                    try:
                        insert_err_sql = f"insert into dt_err(xydm,`from`,url,title,pub_date,zhaiyao,create_date,state,pageNo,pageIndex) values('{social_code}','{source}','{link}','{title}','{time_format}','{info_page['abstracts']}',now(),1,{num},{pageIndex})"
                        self.cursor.execute(insert_err_sql)
                        self.cnx.commit()
                    except:
                        pass
                    continue
                try:
                    insert_sql = '''insert into brpa_source_article_news(social_credit_code,source_address,origin,type,publish_time,content,create_time) values(%s,%s,%s,%s,%s,%s,now())'''
                    # 动态信息列表
                    up_okCount = up_okCount + 1
                    list_info = [
                        social_code,
                        link,
                        f'天眼查-{source}',
                        '2',
                        time_format,
                        content[:500]
                    ]
                    self.cursor_.execute(insert_sql, tuple(list_info))
                    self.cnx_.commit()
                    # 采集一条资讯记录一条，记录该企业采到了多少的资讯
                    self.log.info(f'{social_code}----{link}:新增一条')
                    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                    # todo:插入一条数据，并传入kafka
                    dic_news = {
                        'attachmentIds': '',
                        'author': '',
                        'content': content,
                        'contentWithTag': contentText,
                        'createDate': time_now,
                        'deleteFlag': '0',
                        'id': '',
                        'keyWords': '',
                        'lang': 'zh',
                        'origin': source,
                        'publishDate': time_format,
                        #'sid': '1684032033495392257',
                        'sid': '1714852232679067649',
                        'sourceAddress': link,  # 原文链接
                        'summary': info_page['abstracts'],
                        'title': title,
                        'type': 2,
                        'socialCreditCode': social_code,
                        'year': time_format[:4]
                    }
                except Exception as e:
                    self.log.info(f'传输失败:{social_code}----{link}')
                    error = '数据库传输失败'
                    state = 0
                    takeTime = self.baseCore.getTimeCost(start_time, time.time())
                    self.baseCore.recordLog(social_code, self.taskType, state, takeTime, link, f'{error}----{e}')
                    continue
                    # print(dic_news)
                    # 将相应字段通过kafka传输保存
                try:
                    producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
                    kafka_result = producer.send("researchReportTopic",
                                                 json.dumps(dic_news, ensure_ascii=False).encode('utf8'))

                    print(kafka_result.get(timeout=10))
                    dic_result = {
                        'success': 'ture',
                        'message': '操作成功',
                        'code': '200',
                    }
                    self.log.info(dic_result)
                    # 传输成功,写入日志中
                    state = 1
                    takeTime = self.baseCore.getTimeCost(start_time, time.time())
                    self.baseCore.recordLog(social_code, self.taskType, state, takeTime, link, '成功')
                    # return True
                except Exception as e:
                    dic_result = {
                        'success': 'false',
                        'message': '操作失败',
                        'code': '204',
                        'e': e
                    }
                    self.log.error(dic_result)
                    e = 'Kafka操作失败'
                    state = 0
                    takeTime = self.baseCore.getTimeCost(start_time, time.time())
                    self.baseCore.recordLog(social_code, self.taskType, state, takeTime, link, e)

            self.log.info(f"获取分页数据--{tyc_code}----分页{num}，耗时{self.baseCore.getTimeCost(start_page, time.time())}")
        retData['up_okCount'] = up_okCount
        retData['up_errorCount'] = up_errorCount
        retData['up_repetCount'] = up_repetCount
        return retData

    def reqDetailmsg(self,url):
        header={
            'Connection':'keep-alive',
            'sec-ch-ua':'"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117"',
            'sec-ch-ua-platform':'"Windows"',
            'Upgrade-Insecure-Requests':'1',
            'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36',
            'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'Sec-Fetch-Site':'none',
            'Sec-Fetch-Mode':'navigate',
            'Sec-Fetch-User':'?1',
            'Sec-Fetch-Dest':'document',
            'Accept-Encoding':'gzip, deflate, br',

        }
        self.headers['User-Agent'] = self.baseCore.getRandomUserAgent()
        # proxy = {'https': 'http://127.0.0.1:1080', 'http': 'http://127.0.0.1:1080'}
        proxy = self.baseCore.get_proxy()
        for i in range(0,2):
            try:
                response=requests.get(url=url,headers=header,timeout=10,proxies=proxy,verify=False)
                response.encoding = response.apparent_encoding
                htmltext=response.text
            except Exception as e:
                htmltext=''
                self.log.info(f'{url}---详情请求失败--{e}')
            if htmltext:
                self.log.info(f'{url}---详情请求成功')
                break
        return htmltext



if __name__ == '__main__':

    tydt=Tycdt()
    tydt.doJob()
