import traceback
import urllib
import uuid

from selenium import webdriver
from bs4 import BeautifulSoup
from urllib import parse
import requests, re, time, pymysql, json, redis
from kafka import KafkaProducer

import urllib3
urllib3.disable_warnings()
from obs import ObsClient
import fitz

import sys
sys.path.append('D:\\kkwork\\zzsn_spider\\base')
import BaseCore
baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()

obsClient = ObsClient(
    access_key_id='VEHN7D0TJ9316H8AHCAV',  # 你的华为云的ak码
    secret_access_key='heR353lvSWVPNU8pe2QxDtd8GDsO5L6PGH5eUoQY',  # 你的华为云的sk
    server='https://obs.cn-north-1.myhuaweicloud.com'  # 你的桶的地址
)
# tracker_conf = get_tracker_conf('./client.conf')
# client = Fdfs_client(tracker_conf)
chromedriver = 'D:/chrome/113/chromedriver.exe'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36'
}

pathType_a = 'QYResearchReport/'
pathHYType = 'HYResearchReport/'
cnx = pymysql.connect(host='114.116.44.11', user='caiji', password='f7s0&7qqtK', db='clb_project', charset='utf8mb4')


def clean_text(text):
    """
        清理多余空行
    :param text:
    :return:
    """
    soup = BeautifulSoup(text, 'html.parser')
    # log.info(soup.get_text())
    text = soup.get_text()
    # str1 = re.sub('[\n]+', '\n', 'dfadf   d\n \n\n \nfa  ds ')
    text_ = re.sub('\n+', '\n', text.replace('\t', '').replace('\r', ''))
    return text_

    # 获取文件大小


def convert_size(size_bytes):
    # 定义不同单位的转换值
    units = ['bytes', 'KB', 'MB', 'GB', 'TB']
    i = 0
    while size_bytes >= 1024 and i < len(units) - 1:
        size_bytes /= 1024
        i += 1
    return f"{size_bytes:.2f} {units[i]}"


def getuuid():
    get_timestamp_uuid = uuid.uuid1()  # 根据 时间戳生成 uuid , 保证全球唯一
    return get_timestamp_uuid


# 数据入库，返回主键id传到kafka中
def tableUpdate(year, name_pdf, type_id, item_id, group_name, path, full_path, category, file_size, order_by, status,
                create_by, create_time, come, page_size):
    with cnx.cursor() as cursor:
        Upsql = '''insert into clb_sys_attachment_copy2(year,name,type_id,item_id,group_name,path,full_path,category,file_size,order_by,status,create_by,create_time,source,page_size,object_key,bucket_name) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''

        values = (
        year, name_pdf, type_id, item_id, group_name, path, full_path, category, file_size, order_by, status, create_by,
        create_time, come, page_size, full_path.split('https://zzsn.obs.cn-north-1.myhuaweicloud.com/')[1], 'zzsn')
        # log.info(values)
        cursor.execute(Upsql, values)  # 插入
        cnx.commit()  # 提交

        querySql = '''select id from clb_sys_attachment_copy2 where type_id=4 and full_path = %s'''  # and stock_code = "01786.HK"
        cursor.execute(querySql, full_path)
        selects = cursor.fetchone()
        pdf_id = selects[0]
    # cnx.close()
    # log.info("更新完成:{}".format(pdf_id))
    return pdf_id


# redis去重
def add_check_url(article_url):
    r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn')
    # res = r.sadd(f'report_pdf_two_history', article_url,3)
    res = r.sadd(f'report_pdf_three_history_2', article_url, 3)  # 注意是 保存set的方式
    if res == 0:  # 若返回0,说明插入不成功，表示有重复
        return True
    else:
        return False


def uptoOBS(pdf_url, name_pdf, type_id, pathType):
    retData = {'state': False, 'type_id': type_id, 'group_name': '', 'path': '',
               'full_path': '',
               'category': 'pdf', 'file_size': '', 'status': 1, 'create_by': 'XueLingKun',
               'create_time': '', 'page_size': '', 'content': ''}
    for i in range(0, 3):
        try:
            response = requests.get(pdf_url, headers=headers, verify=False, timeout=20)
            file_size = int(response.headers.get('Content-Length'))
            break
        except:
            time.sleep(3)
            continue
    page_size = 0
    for i in range(0, 3):
        try:
            name = str(getuuid()) + '.pdf'
            now_time = time.strftime("%Y-%m")
            result = obsClient.putContent('zzsn', pathType + name, content=response.content)
            with fitz.open(stream=response.content, filetype='pdf') as doc:
                page_size = doc.page_count
                for page in doc.pages():
                    retData['content'] += page.get_text()
            break
        except Exception as e:
            time.sleep(3)
            continue

    if page_size < 1:
        # pdf解析失败
        # log.info(f'======pdf解析失败=====')
        return retData
    else:
        try:
            time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
            retData['state'] = True
            retData['path'] = result['body']['objectUrl'].split('.com')[1]
            retData['full_path'] = result['body']['objectUrl']
            retData['file_size'] = convert_size(file_size)
            retData['create_time'] = time_now
            retData['page_size'] = page_size
        except Exception as e:
            return retData

        return retData


# 下载pdf文件，上传至服务器
def download(data, order_by):
    url_pdf = data['url_pdf']
    name_pdf = data['title']
    if '.pdf' not in name_pdf:
        name_pdf = name_pdf + '.pdf'
    year = int(data['year'])
    type_id = data['type_id']
    item_id = data['item_id']
    category = data['category']
    create_by = data['create_by']
    publishDate = data['publishDate']
    origin = data['origin']
    if origin == '行业研报':
        pathType = pathHYType
    else:
        pathType = pathType_a
    sourceAddress = data['sourceAddress']
    content = data['content']
    summary = str(data['summary'])
    sid = data['sid']
    try:
        come = data['come']
    except:
        come = ''

    tf_url = add_check_url(sourceAddress)
    if tf_url:
        dic_result = {
            'success': 'ture',
            'message': '数据已存在',
            'code': '200',
        }
        log.info(dic_result)
        return
    if url_pdf:
        pass
    else:
        time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        dic_news = {
            'attachmentIds': '',
            'author': '',
            'content': content,
            'contentWithTag': '',
            'createDate': time_now,
            'deleteFlag': '0',
            'id': '',
            'keyWords': '',
            'lang': '',
            'origin': origin,
            'publishDate': publishDate,
            'sid': sid,
            'sourceAddress': sourceAddress,
            'summary': summary,
            'title': name_pdf,
            'type': '0'
        }
        # 将相应字段通过kafka传输保存
        producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
        try:
            kafka_result = producer.send("researchReportStudyTopic", json.dumps(dic_news, ensure_ascii=False).encode('utf8'))
            log.info(kafka_result.get(timeout=10))
        except:
            log.info(f'无pdf链接：{dic_news["title"]}、{dic_news["createDate"]}')

        dic_result = {
            'success': 'ture',
            'message': '操作成功',
            'code': '200',
        }
        log.info(dic_result)
        return

    if 'http' not in url_pdf:
        url_pdf = 'https://' + url_pdf

    # 文件上传到obs
    retData = uptoOBS(url_pdf, name_pdf, 4, pathType)
    if retData['state']:
        pass
    else:
        log.info(f'====pdf解析失败====')
        return
    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    page_size = retData['page_size']
    path = retData['path']
    full_path = retData['full_path']
    file_size = retData['file_size']

    status = 1
    create_time = time_now
    # year,name_pdf,type_id,item_id,group_name,path,full_path,category,file_size,order_by,status,create_by,create_time,come,page_size):
    att_id = tableUpdate(year, name_pdf, type_id, item_id, '', path, full_path, category, file_size, order_by, status,
                         create_by, create_time, come, page_size)
    if att_id:
        pass
    else:
        return
    # except:
    #     # log.info('下载失败')
    #     dic_result = {
    #         'success':'false',
    #         'message':'下载失败',
    #         'code':'204',
    #     }
    #     log.info(dic_result)
    #     return
    # # 将本地文件连接保存进MySQL表用于查询
    # table_id = tableUpdate(year,name_pdf,type_id,item_id,group_name,path,full_path,
    #         category,file_size,order_by,status,create_by,create_time,come,page_size)
    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    dic_news = {
        'attachmentIds': att_id,
        'author': '',
        'content': content,
        'contentWithTag': '',
        'createDate': time_now,
        'deleteFlag': '0',
        'id': '',
        'keyWords': '',
        'lang': '',
        'origin': origin,
        'publishDate': publishDate,
        'sid': sid,
        'sourceAddress': sourceAddress,
        'summary': summary,
        'title': name_pdf.split('.pdf')[0],
        'type': '0'
    }
    # log.info(dic_news)
    # 将相应字段通过kafka传输保存
    try:
        producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
        kafka_result = producer.send("researchReportStudyTopic", json.dumps(dic_news, ensure_ascii=False).encode('utf8'))

        log.info(kafka_result.get(timeout=10))
    except Exception as e:
        msg = traceback.format_exc()
        log.info(msg)
        log.info(f'传输失败：{dic_news["title"]}、{dic_news["publishDate"]}')
    dic_result = {
        'success': 'ture',
        'message': '操作成功',
        'code': '200',
    }
    log.info(dic_result)
    return



# 东方财富网
def dongfangcaifu():
    cnx2 = pymysql.connect(host='114.115.159.144', user='root', password='zzsn9988', db='clb_project',
                           charset='utf8mb4')
    list_short_name = []
    list_social_code = []

    with cnx2.cursor() as cursor:
        sel_sql = '''select securities_short_name,social_credit_code from sys_base_enterprise_ipo'''
        cursor.execute(sel_sql)
        selects = cursor.fetchall()

        for row in selects:
            short_name = row[0]
            social_code = row[1]
            if short_name:
                list_short_name.append(short_name)
                list_social_code.append(social_code)

    for key_word_num in range(0, len(list_short_name)):
        log.info(key_word_num)
        key_word = list_short_name[key_word_num]
        social_code = list_social_code[key_word_num]
        log.info(key_word)
        page = 1
        # for page in range(1,500):
        #     log.info(page)
        param = {
            "uid": "",
            "keyword": key_word,
            "type": ["researchReport"],
            "client": "web",
            "clientVersion": "curr",
            "clientType": "web",
            "param": {"researchReport": {"client": "web", "pageSize": 10, "pageIndex": page}}
        }

        param_url = parse.quote(str(param).replace(" ", ""))
        # param_url = f'%7B"uid"%3A""%2C"keyword"%3A"{key_word}"%2C"type"%3A%5B"researchReport"%5D%2C"client"%3A"web"%2C"clientVersion"%3A"curr"%2C"clientType"%3A"web"%2C"param"%3A%7B"researchReport"%3A%7B"client"%3A"web"%2C"pageSize"%3A10%2C"pageIndex"%3A{page}%7D%7D%7D'

        t = int(time.time() * 1000)
        url = f'https://search-api-web.eastmoney.com/search/jsonp?cb=&param={param_url}&_={t}'
        res = requests.get(url).text[1:-1]
        res_json = json.loads(res)
        list_all = res_json['result']['researchReport']

        if list_all:
            pass
        else:
            continue
        for one_news in list_all:
            news_title = one_news['title']
            news_title = news_title.replace('<em>', '').replace('</em>', '')
            news_date = one_news['date'][:10]
            comparison_date = "2023-12-08"
            # 比较发布日期是否小于2023-10-06
            if news_date < comparison_date:
                continue
            else:
                pass
            news_come = one_news['source']
            news_code = one_news['code']

            news_url = f'https://data.eastmoney.com/report/zw_stock.jshtml?infocode={news_code}'

            news_res = requests.get(news_url)
            news_soup = BeautifulSoup(news_res.content, 'html.parser')

            try:
                if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                    continue
            except:
                continue
            try:
                news_content = news_soup.find('div', {'class': 'newsContent'}).text.strip()
            except:
                news_content = news_soup.find('div', {'class': 'ctx-content'}).text.strip()

            try:
                news_pdf = news_soup.find('div', {'class': 'detail-header'}).find_all('a')[-1].get('href')
            except:
                news_pdf = news_soup.find('span', {'class': 'to-link'}).a.get('href')

            dic_post = {
                'title': news_title,  # 报告名称
                'url_pdf': news_pdf,  # 报告链接
                'year': news_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': social_code,  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'TangYuHang',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': news_date,  # 时间
                'origin': '东方财富网-研报中心',  # 来源
                'sourceAddress': news_url,  # 原文链接
                'content': '',  # 内容
                'summary': news_content,  # 摘要
                'sid': '1662008733005160449',  # 信息源id
                'come': news_come,
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1
        #     log.info(page,dic_post)
        # url = 'http://114.115.155.139:5002/report_download'
        # # report-list
        # res = requests.post(url, data=json.dumps(dic_post))
        # log.info(res.json())
        # dic_news = {
        #     '关键字':key_word,
        #     '标题':news_title,
        #     '时间':news_date,
        #     '来源':news_come,
        #     '摘要':news_content,
        #     '原文链接':news_url,
        #     'PDF链接':news_pdf,
        # }
        # list_all_info.append(dic_news)
        # if len(list_all) != 10:
        #     break


# 东方财富网2
def dongfangcaifu2():
    list_short_name = ['新', '的', '电', '能']

    for key_word_num in range(0, len(list_short_name)):
        log.info(key_word_num)
        key_word = list_short_name[key_word_num]
        social_code = ''
        log.info(key_word)
        page = 1
        # for page in range(1,500):
        #     log.info(page)
        param = {
            "uid": "",
            "keyword": key_word,
            "type": ["researchReport"],
            "client": "web",
            "clientVersion": "curr",
            "clientType": "web",
            "param": {"researchReport": {"client": "web", "pageSize": 10, "pageIndex": page}}
        }

        param_url = parse.quote(str(param).replace(" ", ""))
        # param_url = f'%7B"uid"%3A""%2C"keyword"%3A"{key_word}"%2C"type"%3A%5B"researchReport"%5D%2C"client"%3A"web"%2C"clientVersion"%3A"curr"%2C"clientType"%3A"web"%2C"param"%3A%7B"researchReport"%3A%7B"client"%3A"web"%2C"pageSize"%3A10%2C"pageIndex"%3A{page}%7D%7D%7D'

        t = int(time.time() * 1000)
        url = f'https://search-api-web.eastmoney.com/search/jsonp?cb=&param={param_url}&_={t}'
        res = requests.get(url).text[1:-1]
        res_json = json.loads(res)
        list_all = res_json['result']['researchReport']

        if list_all:
            pass
        else:
            continue
        for one_news in list_all:
            news_title = one_news['title']
            news_title = news_title.replace('<em>', '').replace('</em>', '')
            news_date = one_news['date'][:10]
            comparison_date = "2023-12-08"
            # 比较发布日期是否小于2023-10-06
            if news_date < comparison_date:
                continue
            else:
                pass
            news_come = one_news['source']
            news_code = one_news['code']

            news_url = f'https://data.eastmoney.com/report/zw_stock.jshtml?infocode={news_code}'

            news_res = requests.get(news_url)
            news_soup = BeautifulSoup(news_res.content, 'html.parser')

            try:
                if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                    continue
            except:
                continue
            try:
                news_content = news_soup.find('div', {'class': 'newsContent'}).text.strip()
            except:
                news_content = news_soup.find('div', {'class': 'ctx-content'}).text.strip()

            try:
                news_pdf = news_soup.find('div', {'class': 'detail-header'}).find_all('a')[-1].get('href')
            except:
                news_pdf = news_soup.find('span', {'class': 'to-link'}).a.get('href')

            dic_post = {
                'title': news_title,  # 报告名称
                'url_pdf': news_pdf,  # 报告链接
                'year': news_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': social_code,  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'TangYuHang',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': news_date,  # 时间
                'origin': '东方财富网-研报中心',  # 来源
                'sourceAddress': news_url,  # 原文链接
                'content': '',  # 内容
                'summary': news_content,  # 摘要
                'sid': '1662008733005160449',  # 信息源id
                'come': news_come,
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1


# 东方财富网3 个股研报
def dongfangcaifu3():
    t = str(int(time.time()) * 1000)
    import datetime

    now = datetime.datetime.now()

    # 将日期格式化为“%Y-%m-%d”格式
    # 当前日期
    formatted_date = now.strftime("%Y-%m-%d")
    pre_year = int(time.strftime('%Y', time.localtime(time.time()))) - 2
    month_day = time.strftime('%m-%d', time.localtime(time.time()))
    pre_date = '{}-{}'.format(pre_year, month_day)

    # log.info("格式化后的日期为：", formatted_date)
    # for i in range(1,1349):
    for i in range(1, 15):
        url = f'https://reportapi.eastmoney.com/report/list?industryCode=*&pageSize=50&industry=*&rating=&ratingChange=&beginTime={pre_date}&endTime={formatted_date}&pageNo={i}&fields=&qType=0&orgCode=&code=*&rcode=&p={i}&pageNum={i}&pageNumber={i}&_={t}'
        # url = 'https://reportapi.eastmoney.com/report/list?industryCode=*&pageSize=50&industry=*&rating=&ratingChange=&beginTime=2021-06-13&endTime=2023-06-13&pageNo=1&fields=&qType=0&orgCode=&code=*&rcode=&p=1&pageNum=1&pageNumber=1&_=1686645164397'
        res = requests.get(url).text
        # log.info(res)
        res_json = json.loads(res)

        list_all = res_json['data']
        # log.info(list_all)

        for one_news in list_all:
            news_title = one_news['title']
            # log.info(news_title)
            news_date = one_news['publishDate'][:10]
            comparison_date = "2023-12-08"
            # 比较发布日期是否小于2023-10-06
            if news_date < comparison_date:
                continue
            else:
                pass
            news_come = one_news['orgSName']

            news_href = 'https://data.eastmoney.com/report/info/' + one_news['infoCode'] + '.html'

            news_res = requests.get(news_href)
            news_soup = BeautifulSoup(news_res.content, 'html.parser')
            # log.info(news_soup)
            try:
                if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                    continue
            except:
                continue
            # todo:摘要去标签
            news_content = news_soup.find(id='ContentBody').text.replace(' ', '').strip()
            news_content = clean_text(news_content)
            news_result = news_soup.find(class_='report-infos')
            # log.info(news_result)

            news_pdf = news_result.find_all('span')[4].find('a')['href']
            # log.info(news_pdf)
            dic_post = {
                'title': news_title,  # 报告名称
                'url_pdf': news_pdf,  # 报告链接
                'year': news_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': '',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'XueLingKun',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': news_date,  # 时间
                'origin': '个股研报',  # 来源
                'sourceAddress': news_href,  # 原文链接
                'content': '',  # 内容
                'summary': news_content,  # 摘要
                # 1673152967405879298
                'sid': '1673152967405879298',  # 信息源id
                'come': news_come,
            }
            # list_quchong.append(dic_post)
            order_by = 1
            download(dic_post, order_by)
            order_by += 1
            # log.info(dic_post['title'],dic_post['publishDate'])


# 东方财富网4 行业研报
def dongfangcaifu4():
    t = str(int(time.time()) * 1000)
    import datetime

    now = datetime.datetime.now()

    # 将日期格式化为“%Y-%m-%d”格式
    # 当前日期
    formatted_date = now.strftime("%Y-%m-%d")
    pre_year = int(time.strftime('%Y', time.localtime(time.time()))) - 2
    month_day = time.strftime('%m-%d', time.localtime(time.time()))
    pre_date = '{}-{}'.format(pre_year, month_day)

    # log.info("格式化后的日期为：", formatted_date)
    for i in range(1, 15):

        url = f'https://reportapi.eastmoney.com/report/list?&industryCode=*&pageSize=50&industry=*&rating=*&ratingChange=*&beginTime={pre_date}&endTime={formatted_date}&pageNo={i}&fields=&qType=1&orgCode=&rcode=&p={i}&pageNum={i}&pageNumber={i}&_={t}'
        # url = "https://reportapi.eastmoney.com/report/list?&industryCode=*&pageSize=50&industry=*&rating=*&ratingChange=*&beginTime=2021-06-27&endTime=2023-06-27&pageNo=6&fields=&qType=1&orgCode=&rcode=&p=6&pageNum=6&pageNumber=6&_=1687831020493"
        res = requests.get(url).text
        # log.info(res)
        res_json = json.loads(res)

        list_all = res_json['data']
        # log.info(list_all)

        for one_news in list_all:
            news_title = one_news['title']
            # log.info(news_title)
            news_date = one_news['publishDate'][:10]
            news_come = one_news['orgSName']
            news_date = one_news['publishDate'][:10]
            comparison_date = "2023-12-08"
            # 比较发布日期是否小于2023-10-06
            if news_date < comparison_date:
                continue
            else:
                pass
            news_href = 'https://data.eastmoney.com/report/info/' + one_news['infoCode'] + '.html'

            news_res = requests.get(news_href)
            news_soup = BeautifulSoup(news_res.content, 'html.parser')
            # log.info(news_soup)
            try:
                if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                    continue
            except:
                continue
            # news_content = news_soup.find(id='ContentBody')
            try:
                news_content = news_soup.find(id='ContentBody').text.replace(' ', '').strip()
                news_content = clean_text(news_content)
            except:
                # ctx-content
                # log.info(news_href,news_title)
                news_content = news_soup.find(id='ctx-content').text.replace(' ', '').strip()
                news_content = clean_text(news_content)

            try:
                news_result = news_soup.find(class_='report-infos')
                # log.info(news_result)
                news_pdf = news_result.find_all('span')[4].find('a')['href']
                # log.info(news_pdf)
            except:
                news_pdf = news_soup.find('span', class_='to-link').find('a')['href']
                # log.info(news_soup)
            dic_post = {
                'title': news_title,  # 报告名称
                'url_pdf': news_pdf,  # 报告链接
                'year': news_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': '',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'XueLingKun',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': news_date,  # 时间
                'origin': '行业研报',  # 来源
                'sourceAddress': news_href,  # 原文链接
                'content': '',  # 内容
                'summary': news_content,  # 摘要
                'sid': '1673154805203091457',  # 信息源id
                'come': news_come,
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1

            # log.info(f'成功：{dic_post["title"]},{dic_post["publishDate"]}')


# 东方财富网5 新股研报
def dongfangcaifu5():
    t = str(int(time.time()) * 1000)
    import datetime

    now = datetime.datetime.now()

    # 将日期格式化为“%Y-%m-%d”格式
    # 当前日期
    formatted_date = now.strftime("%Y-%m-%d")
    pre_year = int(time.strftime('%Y', time.localtime(time.time()))) - 2
    month_day = time.strftime('%m-%d', time.localtime(time.time()))
    pre_date = '{}-{}'.format(pre_year, month_day)

    # log.info("格式化后的日期为：", formatted_date)

    for i in range(1, 5):
        url = f'https://reportapi.eastmoney.com/report/newStockList?pageSize=50&beginTime={pre_date}&endTime={formatted_date}&pageNo={i}&fields=&qType=4&p={i}&pageNum={i}&pageNumber={i}&_={t}'

        res = requests.get(url).text
        log.info(res)
        res_json = json.loads(res)

        list_all = res_json['data']
        # log.info(list_all)

        for one_news in list_all:
            news_title = one_news['title']
            # log.info(news_title)
            news_date = one_news['publishDate'][:10]
            news_come = one_news['orgSName']
            comparison_date = "2023-12-08"
            # 比较发布日期是否小于2023-10-06
            if news_date < comparison_date:
                continue
            else:
                pass
            news_href = 'https://data.eastmoney.com/report/info/' + one_news['infoCode'] + '.html'

            news_res = requests.get(news_href)
            news_soup = BeautifulSoup(news_res.content, 'html.parser')
            # log.info(news_soup)
            try:
                if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                    continue
            except:
                continue
            news_content = news_soup.find(id='ContentBody')
            # news_content=str(news_content)
            try:
                news_result = news_soup.find(class_='report-infos')
                # log.info(news_result)
                news_pdf = news_result.find_all('span')[4].find('a')['href']
                # log.info(news_pdf)
            except:
                news_pdf = news_soup.find('span', class_='to-link').find('a')['href']
                # log.info(news_soup)
            dic_post = {
                'title': news_title,  # 报告名称
                'url_pdf': news_pdf,  # 报告链接
                'year': news_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': '',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'XueLingKun',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': news_date,  # 时间
                'origin': '新股研报',  # 来源
                'sourceAddress': news_href,  # 原文链接
                'content': '',  # 内容
                'summary': news_content,  # 摘要
                # 1673155535028764674
                'sid': '1673155535028764674',  # 信息源id
                'come': news_come,
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1
        # log.info(f'成功：{dic_post["title"]},{dic_post["publishDate"]}')


# 东方财富网6 宏观研究
def dongfangcaifu6():
    t = str(int(time.time()) * 1000)
    import datetime

    now = datetime.datetime.now()

    # 将日期格式化为“%Y-%m-%d”格式
    # 当前日期
    formatted_date = now.strftime("%Y-%m-%d")
    pre_year = int(time.strftime('%Y', time.localtime(time.time()))) - 2
    month_day = time.strftime('%m-%d', time.localtime(time.time()))
    pre_date = '{}-{}'.format(pre_year, month_day)

    # log.info("格式化后的日期为：", formatted_date)
    for i in range(1, 15):
        url = f'https://reportapi.eastmoney.com/report/jg?pageSize=50&beginTime={pre_date}&endTime={formatted_date}&pageNo={i}&fields=&qType=3&orgCode=&author=&p={i}&pageNum={i}&pageNumber={i}&_={t}'

        res = requests.get(url).text
        # log.info(res)
        res_json = json.loads(res)

        list_all = res_json['data']
        # log.info(list_all)

        for one_news in list_all:
            news_title = one_news['title']
            # log.info(news_title)
            news_date = one_news['publishDate'][:10]
            comparison_date = "2023-12-08"
            # 比较发布日期是否小于2023-10-06
            if news_date < comparison_date:
                continue
            else:
                pass
            news_come = one_news['orgSName']
            # https://data.eastmoney.com/report/zw_macresearch.jshtml?encodeUrl=H5bb92gpAmVPNj5wTfzCzue8aSv0RJSrltfLaEk8UoQ=
            # H5bb92gpAmVPNj5wTfzCzue8aSv0RJSrltfLaEk8UoQ
            news_href = 'https://data.eastmoney.com/report/zw_macresearch.jshtml?encodeUrl=' + one_news[
                'encodeUrl'] + '='

            news_res = requests.get(news_href)
            news_soup = BeautifulSoup(news_res.content, 'html.parser')
            # log.info(news_soup)
            try:
                if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                    continue
            except:
                continue
            # news_content = news_soup.find(id='ContentBody').text.replace(' ','').strip()
            try:
                news_content = news_soup.find(id='ContentBody').text.replace(' ', '').strip()
                news_content = clean_text(news_content)
            except:
                # ctx-content
                news_content = news_soup.find(id='ctx-content').text.replace(' ', '').strip()
                news_content = clean_text(news_content)
                # log.info(news_href)
            try:
                news_result = news_soup.find(class_='report-infos')
                # log.info(news_result)
                news_pdf = news_result.find_all('span')[4].find('a')['href']
                # log.info(news_pdf)
            except:
                news_pdf = news_soup.find('span', class_='to-link').find('a')['href']
                # log.info(news_soup)
            dic_post = {
                'title': news_title,  # 报告名称
                'url_pdf': news_pdf,  # 报告链接
                'year': news_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': '',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'XueLingKun',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': news_date,  # 时间
                'origin': '宏观研究',  # 来源
                'sourceAddress': news_href,  # 原文链接
                'content': '',  # 内容
                'summary': news_content,  # 摘要
                'sid': '1673155789744652290',  # 信息源id
                'come': news_come,
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1

            # log.info(f'成功：{dic_post["title"]},{dic_post["publishDate"]}')


# 东方财富网7 策略报告
def dongfangcaifu7():
    t = str(int(time.time()) * 1000)
    import datetime

    now = datetime.datetime.now()

    # 将日期格式化为“%Y-%m-%d”格式
    # 当前日期
    formatted_date = now.strftime("%Y-%m-%d")
    pre_year = int(time.strftime('%Y', time.localtime(time.time()))) - 2
    month_day = time.strftime('%m-%d', time.localtime(time.time()))
    pre_date = '{}-{}'.format(pre_year, month_day)

    # log.info("格式化后的日期为：", formatted_date)
    for i in range(1, 3):
        url = f'https://reportapi.eastmoney.com/report/jg?pageSize=50&beginTime={pre_date}&endTime={formatted_date}&pageNo={i}&fields=&qType=2&orgCode=&author=&p={i}&pageNum={i}&pageNumber={i}&_={t}'

        res = requests.get(url).text
        # log.info(res)
        res_json = json.loads(res)

        list_all = res_json['data']
        # log.info(list_all)

        for one_news in list_all:
            news_title = one_news['title']
            # log.info(news_title)
            news_date = one_news['publishDate'][:10]
            comparison_date = "2023-12-08"
            # 比较发布日期是否小于2023-10-06
            if news_date < comparison_date:
                continue
            else:
                pass
            news_come = one_news['orgSName']
            # https://data.eastmoney.com/report/zw_macresearch.jshtml?encodeUrl=H5bb92gpAmVPNj5wTfzCzue8aSv0RJSrltfLaEk8UoQ=
            # H5bb92gpAmVPNj5wTfzCzue8aSv0RJSrltfLaEk8UoQ
            news_href = 'https://data.eastmoney.com/report/zw_macresearch.jshtml?encodeUrl=' + one_news[
                'encodeUrl'] + '='

            news_res = requests.get(news_href)
            news_soup = BeautifulSoup(news_res.content, 'html.parser')
            # log.info(news_soup)
            try:
                if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                    continue
            except:
                continue
            # news_content = news_soup.find(id='ContentBody')
            try:
                news_content = news_soup.find(id='ctx-content').text.replace(' ', '').strip()
                news_content = clean_text(news_content)
            except:
                news_content = news_soup.find(id='ContentBody').text.replace(' ', '').strip()
                news_content = clean_text(news_content)
                # log.info(news_href,news_title)
            try:
                news_result = news_soup.find(class_='report-infos')
                # log.info(news_result)
                news_pdf = news_result.find_all('span')[4].find('a')['href']
                # log.info(news_pdf)
            except:
                news_pdf = news_soup.find('span', class_='to-link').find('a')['href']
                # log.info(news_soup)
            dic_post = {
                'title': news_title,  # 报告名称
                'url_pdf': news_pdf,  # 报告链接
                'year': news_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': '',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'XueLingKun',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': news_date,  # 时间
                'origin': '策略报告',  # 来源
                'sourceAddress': news_href,  # 原文链接
                'content': '',  # 内容
                'summary': news_content,  # 摘要
                'sid': '1673155946779394050',  # 信息源id
                'come': news_come,
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1
            # log.info(f'成功：{dic_post["title"]},{dic_post["publishDate"]}')


if __name__ == '__main__':
    # try:
    #     log.info('mob')
    #     Mob()
    # except:
    #     pass
    # try:
    #     log.info('yidong_guanxiangtai')
    #     yidong_guanxiangtai()
    # except:
    #     pass
    # try:
    #     log.info('juliangsuanshu')
    #     juliangsuanshu()
    # except:
    #     pass
    # try:
    #     log.info('ke36')
    #     ke36()
    # except:
    #     pass
    # try:
    #     log.info('qianyanzhishiku')
    #     qianyanzhishiku()
    # except:
    #     pass
    # try:
    #     log.info('shijiejingjiluntan')
    #     shijiejingjiluntan()
    # except:
    #     pass
    # try:
    #     log.info('dongfangcaifu')
    #     dongfangcaifu()
    # except:
    #     pass
    # try:
    #     log.info('dongfangcaifu2')
    #     dongfangcaifu2()
    # except:
    #     pass
    #
    # try:
    #     log.info('dongfangcaifu3')
    #     dongfangcaifu3()
    # except Exception as e:
    #     pass
    #
    # try:
    #     log.info('dongfangcaifu4')
    #     dongfangcaifu4()
    # except:
    #     pass

    try:
        log.info('dongfangcaifu5')
        dongfangcaifu5()
    except:
        pass

    # try:
    #     log.info('dongfangcaifu6')
    #     dongfangcaifu6()
    # except:
    #     pass
    #
    # try:
    #     log.info('dongfangcaifu7')
    #     dongfangcaifu7()
    # except:
    #     pass
