"""信用中国-严重失信名单
链接地址 :
{
    source:
    type: 严重失信主体名单
    searchState: 1
    entityType: 1
    scenes: defaultscenario
    keyword: 雷州市白金银座演艺文化实业有限公司
    tyshxydm: 91440882315032592M
    page: 1
    pageSize: 10
}
"""
import json
import time
from urllib import parse

import redis
import requests
from bs4 import BeautifulSoup
from kafka import KafkaProducer
from retry import retry

from base.BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
cnx = baseCore.cnx
cursor = baseCore.cursor
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=0)
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
taskType = '企业负面新闻'
def sendKafka(dic_news):
    start_time = time.time()
    try:  # 114.116.116.241
        producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'],max_request_size=1024*1024*20)
        kafka_result = producer.send("crawlerInfo",
                                     json.dumps(dic_news, ensure_ascii=False).encode('utf8'))

        print(kafka_result.get(timeout=10))

        dic_result = {
            'success': 'ture',
            'message': '操作成功',
            'code': '200',
        }
        log.info(dic_result)
        # 传输成功,写入日志中
        state = 1
        takeTime = baseCore.getTimeCost(start_time, time.time())
        return True

    except Exception as e:

        dic_result = {
            'success': 'false',
            'message': '操作失败',
            'code': '204',
            'e': e
        }
        state = 0
        takeTime = baseCore.getTimeCost(start_time, time.time())
        baseCore.recordLog(social_code, taskType, state, takeTime, dic_news['title'], 'Kafka操作失败')
        log.info(dic_result)
        return False

@retry(tries=3,delay=1)
def getRequest(url,headers):

    req = requests.get(url=url,headers=headers,verify=False,timeout=30)
    json_data = req.json()
    return json_data

# 严重失信
def dishonesty(headers,com_name,social_code):
    list_dishonesty = []
    param = {
        'tableName':'credit_zgf_fr_sxbzxr',
        'searchState': '1',
        'scenes': 'defaultscenario',
        'keyword': com_name,
        'tyshxydm': social_code,
        'page': '1',
        'pageSize': '10'
    }

    url = f'https://public.creditchina.gov.cn/private-api/catalogSearch?tableName=credit_zgf_fr_sxbzxr&searchState=1&scenes=defaultscenario&keyword={param["keyword"]}&tyshxydm={param["tyshxydm"]}&page=1&pageSize=10'
    json_data = getRequest(url, headers)
    # print(json_data)
    if json_data['status'] == 1:
        pass
    total_size = json_data['data']['totalSize']
    if total_size > 0:
        pass
    else:
        log.info(f'该企业{com_name}无严重失信信息')
        return url,list_dishonesty
    for page in range(1,total_size+1):
        param_page = {
            'tableName': 'credit_zgf_fr_sxbzxr',
            'searchState': '1',
            'scenes': 'defaultscenario',
            'keyword': com_name,
            'tyshxydm': social_code,
            'page': page,
            'pageSize': '10'
        }
        url_page = f'https://public.creditchina.gov.cn/private-api/catalogSearch?tableName=credit_zgf_fr_sxbzxr&searchState=1&scenes=defaultscenario&keyword={param["keyword"]}&tyshxydm={param["tyshxydm"]}&page={param_page["page"]}&pageSize=10'
        json_data = getRequest(url_page, headers)
        # print(json_data)
        if json_data['status'] == 1:
            pass
        info_list = json_data['data']['list']
        for info in info_list:
            entity = info
            iname = entity['iname']  # 失信被执行人姓名/名称
            cardnumber = entity['cardnumber']  # 组织机构代码
            court_name = entity['court_name']  # 执行法院
            area_name = entity['area_name']  # 省份
            case_code = entity['case_code']  # 执行依据文号
            reg_date = entity['reg_date']  # 立案时间
            gist_cid = entity['gist_cid']  # 案号
            gist_unit = entity['gist_unit']  # 做出执行依据单位
            duty = entity['duty']  # 生效法律文书确定的义务
            performance = entity['performance']  # 被执行人的履行情况
            disreput_type_name = entity['disreput_type_name']  # 失信被执行人行为具体情形
            publish_date = entity['publish_date']  # 发布时间
            performed_part = entity['performed_part']  # 已履行部分
            unperform_part = entity['unperform_part']  # 未履行部分
            dataSource = info['dataSource']  # 数据来源
            dic_dishonesty = {
                '失信被执行人姓名/名称': iname,
                '组织机构代码':cardnumber,
                '执行法院':court_name,
                '省份':area_name,
                '执行依据文号':case_code,
                '立案时间':reg_date,
                '案号':gist_cid,
                '做出执行依据单位':gist_unit,
                '生效法律文书确定的义务':duty,
                '被执行人的履行情况':performance,
                '失信被执行人行为具体情形':disreput_type_name,
                '发布时间':publish_date,
                '已履行部分':performed_part,
                '未履行部分':unperform_part,
                '数据来源':dataSource
            }
            list_dishonesty.append(dic_dishonesty)
            # r.sadd('dishonesty::' +social_code , )

    return url,list_dishonesty
# 行政处罚
def punish(headers,com_name,social_code):
    list_punish = []
    param = {
        'tableName':'credit_xyzx_fr_xzcf_new',
        'searchState': '1',
        'scenes': 'defaultscenario',
        'keyword': com_name,
        'tyshxydm': social_code,
        'page': '1',
        'pageSize': '10'
    }

    url = f'https://public.creditchina.gov.cn/private-api/catalogSearch?tableName=credit_xyzx_fr_xzcf_new&searchState=1&scenes=defaultscenario&keyword={param["keyword"]}&tyshxydm={param["tyshxydm"]}&page=1&pageSize=10'
    json_data = getRequest(url, headers)
    # print(json_data)
    if json_data['status'] == 1:
        pass
    #总条数
    total_size = json_data['data']['totalSize']
    if total_size > 0:
        pass
    else:
        log.info(f'该企业{com_name}无行政处罚信息')
        return url,list_punish
    for page in range(1,total_size+1):
        param_page = {
            'tableName': 'credit_xyzx_fr_xzcf_new',
            'searchState': '1',
            'scenes': 'defaultscenario',
            'keyword': com_name,
            'tyshxydm': social_code,
            'page': page,
            'pageSize': '10'
        }
        url_page = f'https://public.creditchina.gov.cn/private-api/catalogSearch?tableName=credit_xyzx_fr_xzcf_new&searchState=1&scenes=defaultscenario&keyword={param_page["keyword"]}&tyshxydm={param_page["tyshxydm"]}&page={param_page["page"]}&pageSize=10'
        json_data = getRequest(url_page, headers)
        # print(json_data)
        if json_data['status'] == 1:
            pass

        info_list = json_data['data']['list']

        for entity in info_list:
            cf_wsh = entity['cf_wsh']   # 行政处罚决定书文号
            cf_cflb = entity['cf_cflb']  # 处罚类别
            cf_jdrq = entity['cf_jdrq']     # 处罚决定日期
            cf_nr = entity['cf_nr']     # 处罚内容
            cf_nr_fk = entity['cf_nr_fk']   # 罚款金额（万元）
            cf_nr_wfff = entity['cf_nr_wfff']   # 没收违法所得、没收非法财物的金额（万元）
            cf_nr_zkdx = entity['cf_nr_zkdx']   # 暂扣或吊销证照名称及编号
            cf_wfxw = entity['cf_wfxw']     # 违法行为类型
            cf_sy = entity['cf_sy']     # 违法事实
            cf_yj = entity['cf_yj']     # 处罚依据
            cf_cfjg = entity['cf_cfjg']     # 处罚机关
            cf_cfjgdm = entity['cf_cfjgdm']     # 处罚机关统一社会信用代码
            cf_sjly = entity['cf_sjly']     # 数据来源
            cf_sjlydm = entity['cf_sjlydm']     # 数据来源单位统一社会信用代码

            dic_punish = {
                '行政处罚决定书文号':cf_wsh,
                '处罚类别':cf_cflb,
                '处罚决定日期':cf_jdrq,
                '处罚内容':cf_nr,
                '罚款金额（万元）':cf_nr_fk,
                '没收违法所得、没收非法财物的金额（万元）':cf_nr_wfff,
                '暂扣或吊销证照名称及编号':cf_nr_zkdx,
                '违法行为类型':cf_wfxw,
                '违法事实':cf_sy,
                '处罚依据':cf_yj,
                '处罚机关':cf_cfjg,
                '处罚机关统一社会信用代码':cf_cfjgdm,
                '数据来源':cf_sjly,
                '数据来源单位统一社会信用代码':cf_sjlydm
            }
            list_punish.append(dic_punish)
    return url,list_punish

# 经营异常
def abnormal(headers,com_name,social_code):
    list_abhormal = []
    param = {
        'tableName': 'credit_scjdglzj_fr_ycjyml',
        'searchState': '1',
        'scenes': 'defaultscenario',
        'keyword': com_name,
        'tyshxydm': social_code,
        'page': '1',
        'pageSize': '10'
    }

    url = f'https://public.creditchina.gov.cn/private-api/catalogSearch?tableName=credit_scjdglzj_fr_ycjyml&searchState=1&scenes=defaultscenario&keyword={param["keyword"]}&tyshxydm={param["tyshxydm"]}&page=1&pageSize=10'
    json_data = getRequest(url, headers)
    # print(json_data)
    if json_data['status'] == 1:
        pass
    # 总条数
    total_size = json_data['data']['totalSize']
    if total_size > 0:
        pass
    else:
        log.info(f'该企业{com_name}无经营异常信息')
        return url,list_abhormal
    for page in range(1, total_size+1):
        param_page = {
            'tableName': 'credit_xyzx_fr_xzcf_new',
            'searchState': '1',
            'scenes': 'defaultscenario',
            'keyword': com_name,
            'tyshxydm': social_code,
            'page': page,
            'pageSize': '10'
        }
        url = f'https://public.creditchina.gov.cn/private-api/catalogSearch?tableName=credit_scjdglzj_fr_ycjyml&searchState=1&scenes=defaultscenario&keyword={param_page["keyword"]}&tyshxydm={param_page["tyshxydm"]}&page={param_page["page"]}&pageSize=10'
        json_data = getRequest(url, headers)
        if json_data['status'] == 1:
            pass
        info_list = json_data['data']['list']
        for entity in info_list:
            entname = entity['entname']     # 企业名称
            uniscid = entity['uniscid']  # 社会统一信用代码
            lerep = entity['lerep']  # 法定代表人
            pripid = entity['pripid']  # 主体身份代码
            regno = entity['regno']  # 注册号
            specausename = entity['specausename']  # 列入经营异常名录原因类型名称
            abntime = entity['abntime']  # 设立日期
            decorgname = entity['decorgname']  # 列入决定机关名称
            dataSource = entity['dataSource']  # 数据来源
            dic_abnormal = {
                '企业名称':entname,
                '社会统一信用代码':uniscid,
                '法定代表人':lerep,
                '主体身份代码':pripid,
                '注册号':regno,
                '列入经营异常名录原因类型名称':specausename,
                '设立日期':abntime,
                '列入决定机关名称':decorgname,
                '数据来源':dataSource
            }
            list_abhormal.append(dic_abnormal)
    return url,list_abhormal

def dic_data(com_name,listData,type,detailurl):
    dic_news = {
        'title':com_name + type,
        'structuredData':listData[:1],
        'ynStructure':1,
        'content': '',
        'contentHtml': '',
        'source': '信用中国',
        'publishtime': '',
        'detailurl': detailurl,
    }
    return dic_news

def insertinto(dic_):
    sql = "INSERT INTO zhejiangfmnews (title, structuredData, source) VALUES (%s, %s, %s)"
    cursor.execute(sql, (dic_["title"], str(dic_["structuredData"]), dic_["source"]))
    cnx.commit()

if __name__=='__main__':

    headers = {
        'Referer': 'https://www.creditchina.gov.cn/',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-site',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36',
        'sec-ch-ua': '"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"'
    }
    query = "select * from zhejiang "
    cursor.execute(query)
    results = cursor.fetchall()
    for result in results:
        com_name = result[1]
        log.info(f'------------正在采集{com_name}----------------')
        social_code = result[0]
        try:
            url_dishonesty,list_dishonesty = dishonesty(headers,com_name,social_code)
        except:
            list_dishonesty = []
            log.info(f'error-------{com_name}')
            continue
        if list_dishonesty:
            dic_dishonesty = dic_data(com_name,list_dishonesty,'严重违法失信信息',url_dishonesty)
            # sendKafka(dic_dishonesty)
            insertinto(dic_dishonesty)
            log.info(f'----{com_name}---新增')


        url_punish,list_punish = punish(headers,com_name,social_code)
        if list_punish:
            dic_punish = dic_data(com_name, list_punish, '行政处罚信息', url_punish)
            insertinto(dic_punish)
            log.info(f'----{com_name}---新增')
        # sendKafka(dic_punish)

        url_abnormal,list_abnormal = abnormal(headers,com_name,social_code)
        if list_abnormal:
            dic_abnormal = dic_data(com_name, list_abnormal, '经营异常信息', url_abnormal)
            insertinto(dic_abnormal)
            log.info(f'----{com_name}---新增')
        # sendKafka(dic_abnormal)

    # 报告链接
    # url_report = f'https://public.creditchina.gov.cn/credit-check/pdf/clickDownload?companyName={com_name}&entityType=1&uuid=&tyshxydm={social_code}'
    # report_json = getRequest(url_report, headers)
    # reportNumber = report_json['data']['reportNumber']
    # pdf_url = f'https://public.creditchina.gov.cn/credit-check/pdf/clickDownloadOBS?reportNumber={reportNumber}'
    # respon = requests.get(url=pdf_url,headers=headers,verify=False,timeout=30)




