import os
import subprocess
import traceback
import urllib
import uuid
from datetime import datetime
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium import webdriver
from bs4 import BeautifulSoup
from urllib import parse
import requests, re, time, pymysql, json, redis
from kafka import KafkaProducer

import urllib3
from selenium.webdriver.support.wait import WebDriverWait

urllib3.disable_warnings()
from obs import ObsClient
import fitz

import sys
sys.path.append('D:\\kkwork\\zzsn_spider\\base')
import BaseCore
baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn')
obsClient = ObsClient(
    access_key_id='VEHN7D0TJ9316H8AHCAV',  # 你的华为云的ak码
    secret_access_key='heR353lvSWVPNU8pe2QxDtd8GDsO5L6PGH5eUoQY',  # 你的华为云的sk
    server='https://obs.cn-north-1.myhuaweicloud.com'  # 你的桶的地址
)
# tracker_conf = get_tracker_conf('./client.conf')
# client = Fdfs_client(tracker_conf)
# chromedriver = 'D:/chrome/113/chromedriver.exe'
opt = webdriver.ChromeOptions()
opt.add_argument(
        'user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36')

opt.add_argument("--ignore-certificate-errors")
opt.add_argument("--ignore-ssl-errors")
opt.add_experimental_option("excludeSwitches", ["enable-automation"])
opt.add_experimental_option('excludeSwitches', ['enable-logging'])
opt.add_experimental_option('useAutomationExtension', False)
opt.binary_location = r'D:/Google/Chrome/Application/chrome.exe'
chromedriver = r'D:/cmd100/chromedriver.exe'

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36'
}

pathType_a = 'QYResearchReport/'
pathHYType = 'HYResearchReport/'
cnx = pymysql.connect(host='114.116.44.11', user='caiji', password='f7s0&7qqtK', db='clb_project', charset='utf8mb4')


def clean_text(text):
    """
        清理多余空行
    :param text:
    :return:
    """
    soup = BeautifulSoup(text, 'html.parser')
    # log.info(soup.get_text())
    text = soup.get_text()
    # str1 = re.sub('[\n]+', '\n', 'dfadf   d\n \n\n \nfa  ds ')
    text_ = re.sub('\n+', '\n', text.replace('\t', '').replace('\r', ''))
    return text_

    # 获取文件大小


def convert_size(size_bytes):
    # 定义不同单位的转换值
    units = ['bytes', 'KB', 'MB', 'GB', 'TB']
    i = 0
    while size_bytes >= 1024 and i < len(units) - 1:
        size_bytes /= 1024
        i += 1
    return f"{size_bytes:.2f} {units[i]}"


def getuuid():
    get_timestamp_uuid = uuid.uuid1()  # 根据 时间戳生成 uuid , 保证全球唯一
    return get_timestamp_uuid


# 数据入库，返回主键id传到kafka中
def tableUpdate(year, name_pdf, type_id, item_id, group_name, path, full_path, category, file_size, order_by, status,
                create_by, create_time, come, page_size):
    with cnx.cursor() as cursor:
        Upsql = '''insert into clb_sys_attachment(year,name,type_id,item_id,group_name,path,full_path,category,file_size,order_by,status,create_by,create_time,source,page_size,object_key,bucket_name) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''

        values = (
        year, name_pdf, type_id, item_id, group_name, path, full_path, category, file_size, order_by, status, create_by,
        create_time, come, page_size, full_path.split('https://zzsn.obs.cn-north-1.myhuaweicloud.com/')[1], 'zzsn')
        # log.info(values)
        cursor.execute(Upsql, values)  # 插入
        cnx.commit()  # 提交

        querySql = '''select id from clb_sys_attachment where type_id=4 and full_path = %s'''  # and stock_code = "01786.HK"
        cursor.execute(querySql, full_path)
        selects = cursor.fetchone()
        pdf_id = selects[0]
    # cnx.close()
    # log.info("更新完成:{}".format(pdf_id))
    return pdf_id


# redis去重
def add_check_url(article_url):

    # res = r.sadd(f'report_pdf_two_history', article_url,3)
    res = r.sadd(f'report_pdf_three_history', article_url, 3)  # 注意是 保存set的方式
    if res == 0:  # 若返回0,说明插入不成功，表示有重复
        return True
    else:
        return False

# redis上传失败删除数据
def delete_url(article_url):
    # r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn')
    res = r.srem('report_pdf_three_history', article_url)
    if res > 0:
        return True
    else:
        return False

def uptoOBS(pdf_url, name_pdf, type_id, pathType):
    retData = {'state': False, 'type_id': type_id, 'group_name': '', 'path': '',
               'full_path': '',
               'category': 'pdf', 'file_size': '', 'status': 1, 'create_by': 'XueLingKun',
               'create_time': '', 'page_size': '', 'content': ''}
    for i in range(0, 3):
        try:
            ip = baseCore.get_proxy()
            response = requests.get(pdf_url, headers=headers, verify=False, timeout=20)
            file_size = int(response.headers.get('Content-Length'))
            break
        except:
            time.sleep(3)
            continue
    page_size = 0
    for i in range(0, 3):
        try:
            name = str(getuuid()) + '.pdf'
            now_time = time.strftime("%Y-%m")
            result = obsClient.putContent('zzsn', pathType + name, content=response.content)
            with fitz.open(stream=response.content, filetype='pdf') as doc:
                page_size = doc.page_count
                for page in doc.pages():
                    retData['content'] += page.get_text()
            break
        except Exception as e:
            time.sleep(3)
            continue

    if page_size < 1:
        # pdf解析失败
        # log.info(f'======pdf解析失败=====')
        return retData
    else:
        try:
            time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
            retData['state'] = True
            retData['path'] = result['body']['objectUrl'].split('.com')[1]
            retData['full_path'] = result['body']['objectUrl']
            retData['file_size'] = convert_size(file_size)
            retData['create_time'] = time_now
            retData['page_size'] = page_size
        except Exception as e:
            return retData

        return retData


# 下载pdf文件，上传至服务器
def download(data, order_by):
    url_pdf = data['url_pdf']
    name_pdf = data['title']
    if '.pdf' not in name_pdf:
        name_pdf = name_pdf + '.pdf'
    year = int(data['year'])
    type_id = data['type_id']
    item_id = data['item_id']
    category = data['category']
    create_by = data['create_by']
    publishDate = data['publishDate']
    origin = data['origin']
    if origin == '行业研报':
        pathType = pathHYType
    else:
        pathType = pathType_a
    sourceAddress = data['sourceAddress']
    content = data['content']
    summary = str(data['summary'])
    sid = data['sid']
    try:
        come = data['come']
    except:
        come = ''

    tf_url = add_check_url(sourceAddress)
    if tf_url:
        dic_result = {
            'success': 'ture',
            'message': '数据已存在',
            'code': '200',
        }
        log.info(dic_result)
        return
    if url_pdf:
        pass
    else:
        time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        dic_news = {
            'attachmentIds': '',
            'author': '',
            'content': content,
            'contentWithTag': '',
            'createDate': time_now,
            'deleteFlag': '0',
            'id': '',
            'keyWords': '',
            'lang': '',
            'origin': origin,
            'publishDate': publishDate,
            'sid': sid,
            'sourceAddress': sourceAddress,
            'summary': summary,
            'title': name_pdf.split('.pdf')[0],
            'type': '0'
        }
        # 将相应字段通过kafka传输保存
        producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
        try:
            kafka_result = producer.send("researchReportStudyTopic", json.dumps(dic_news, ensure_ascii=False).encode('utf8'))
            log.info(kafka_result.get(timeout=10))
        except:
            log.info(f'无pdf链接：{dic_news["title"]}、{dic_news["createDate"]}')

        dic_result = {
            'success': 'ture',
            'message': '操作成功',
            'code': '200',
        }
        log.info(dic_result)
        return

    if 'http' not in url_pdf:
        url_pdf = 'https://' + url_pdf

    # 文件上传到obs
    retData = uptoOBS(url_pdf, name_pdf, 4, pathType)
    if retData['state']:
        pass
    else:
        log.info(f'====pdf解析失败====')
        delete_url(sourceAddress)
        # # 获取当前进程pid
        # current_pid = baseCore.getPID()
        # # todo: 重新启动新进程，杀死当前进程
        # subprocess.Popen([sys.executable] + sys.argv)
        # os.kill(current_pid, 9)
        return
    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    page_size = retData['page_size']
    path = retData['path']
    full_path = retData['full_path']
    file_size = retData['file_size']

    status = 1
    create_time = time_now
    # year,name_pdf,type_id,item_id,group_name,path,full_path,category,file_size,order_by,status,create_by,create_time,come,page_size):
    att_id = tableUpdate(year, name_pdf, type_id, item_id, '', path, full_path, category, file_size, order_by, status,
                         create_by, create_time, come, page_size)
    if att_id:
        pass
    else:
        return
    # except:
    #     # log.info('下载失败')
    #     dic_result = {
    #         'success':'false',
    #         'message':'下载失败',
    #         'code':'204',
    #     }
    #     log.info(dic_result)
    #     return
    # # 将本地文件连接保存进MySQL表用于查询
    # table_id = tableUpdate(year,name_pdf,type_id,item_id,group_name,path,full_path,
    #         category,file_size,order_by,status,create_by,create_time,come,page_size)
    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
    dic_news = {
        'attachmentIds': att_id,
        'author': '',
        'content': content,
        'contentWithTag': '',
        'createDate': time_now,
        'deleteFlag': '0',
        'id': '',
        'keyWords': '',
        'lang': '',
        'origin': origin,
        'publishDate': publishDate,
        'sid': sid,
        'sourceAddress': sourceAddress,
        'summary': summary,
        'title': name_pdf.split('.pdf')[0],
        'type': '0'
    }
    # log.info(dic_news)
    # 将相应字段通过kafka传输保存
    try:
        producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
        kafka_result = producer.send("researchReportStudyTopic", json.dumps(dic_news, ensure_ascii=False).encode('utf8'))

        log.info(kafka_result.get(timeout=10))
    except Exception as e:
        msg = traceback.format_exc()
        log.info(msg)
        log.info(f'传输失败：{dic_news["title"]}、{dic_news["publishDate"]}')
    dic_result = {
        'success': 'ture',
        'message': '操作成功',
        'code': '200',
    }
    log.info(dic_result)
    return

# def Mob():
#     url = 'https://www.mob.com/mobData/report'
#     res = requests.get(url=url,headers=headers).content
#     soup = BeautifulSoup(res,'html.parser')
#     max_info = soup.find('span',class_='el-pagination__total').text
#     max_info = re.findall('\d{1,4}',max_info)[0]
#     # print(type(max_info))
#     max_page = int((int(max_info)/9) + 1)
#     print(max_page)
#     i_id = 0
#     for page in range(max_page):
#         url = 'https://www.mob.com/mobdata/report?page={}'.format(page+1)
#         res = requests.get(url=url, headers=headers).content
#         soup = BeautifulSoup(res, 'html.parser')
#         result = soup.find('ul', class_='fix')
#         li_list = result.find_all('li')
#         # for id in range(1, 149):
#         id = i_id
#         for li in li_list:
#             id += 1
#             title = li.find('div',class_='title').text
#             time = li.find('div',class_='date tc').text.strip()
#             year = re.findall('\d{4}',time)[0]
#         # for id in range(29,178):
#             real_id = 178 - id
#             href = 'https://www.mob.com/mobdata/report/{}'.format(real_id)
#             # href = 'https://www.mob.com/mobdata/report/169'
#             res_href = requests.get(url=href,headers=headers).content
#             i_soup = BeautifulSoup(res_href,'html.parser')
#             url_pdf = 'https://api.os.mob.com/api/academy_report/download/' +  i_soup.find('div', class_='report-top').find('a')['href']
#             summary_list = i_soup.find(class_='picture-content htmlContent').find_all('h3')
#             fin_summary = []
#             for s in summary_list:
#                 summary = s.text
#                 fin_summary.append(summary)
#             summary = ''.join(fin_summary)
#             dic_post = {
#                 'title': title,  # 报告名称
#                 'url_pdf': url_pdf,  # 报告链接
#                 'year': year,  # 报告年份
#                 'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
#                 'item_id': 'YanBao',  # 关联记录id，如：企业信用代码
#                 'category': 'pdf',  # 文件后缀名，如：pdf
#                 'create_by': 'XueLingKun',  # 创建人，使用驼峰命名，如：TangYuHang
#                 'publishDate': time,  # 时间
#                 'origin': 'Mob研究院',  # 来源
#                 'sourceAddress': href,  # 原文链接
#                 'content': '',  # 内容
#                 'summary': summary,  # 摘要
#                 'sid': '1662008807781212161',  # 信息源id
#             }
#             order_by = 1
#             download(dic_post,order_by)
#             order_by += 1
#             # print(dic_post)
#             # url = 'http://114.115.155.139:5002/report_download'
#             # # report-list
#             # res = requests.post(url, data=json.dumps(dic_post))
#             # print(res.json())
#         i_id += 9

def Mob():
    # loginfo = baseCore.redicPullData('Mob:loginfo')
    # account = loginfo.split('|')[0]
    # password = loginfo.split('|')[1]
    # usecount = loginfo.split('|')[2]
    usecount = 0
    # 测试用
    account = '13636711746'
    password = 'Zhenghao123'

    # account = '18703752600'
    # password = 'Axlk010208!'
    # account = '13273737131'
    # password = 'liu1230...'
    # account = '15237560528'
    # password = 'xlk123456!'
    # account = '17103126138'
    # password = '171BlackOne'
    # account = '17103128590'
    # password = '171BlackTwo'
    browser = webdriver.Chrome(chrome_options=opt, executable_path=chromedriver)
    f_url = 'https://www.mob.com/developer/login'
    browser.get(f_url)
    browser.find_element(By.CLASS_NAME, 's1').click()
    browser.find_element(By.CSS_SELECTOR, 'input[type="text"]').send_keys(f'{account}')
    browser.find_element(By.CSS_SELECTOR, 'input[type="password"]').send_keys(f'{password}')
    browser.find_element(By.XPATH, '//*[@id="app"]/section/div/div[2]/div/div[2]/section/div[3]/div/form/div[3]/div/button/span').click()
    if usecount < 5:
        pass
    else:
        return Mob()
    # 获取登录的信息

    # url = browser.current_url
    # print(url)
    url = 'https://www.mob.com/mobdata/report'
    browser.get(url)

    # tags = browser.find_elements(By.CLASS_NAME, 'main-title')
    # for tag in tags:
    #     if 'Mob研究院' in tag.text:
    #         tag.click()
    #     else:
    #         continue
    #     # try:
    #     #     web = tag.find_element(By.CLASS_NAME, "")
    #     #     web.click()
    #     #     break
    #     # except:
    #     #     continue
    cookies_list = browser.get_cookies()
    cookies = {}
    # 获取cookie中的name和value,转化成requests可以使用的形式
    for cookie in cookies_list:
        cookies[cookie['name']] = cookie['value']
    # cookies_ = json.loads('{' + re.findall("{(.*?)}", str(cookies).replace("\'", "\""))[0] + '}')
    # cookies_ = json.dumps(cookies)
    session = requests.session()
    session.cookies.update(cookies)

    for i in range(5):
        url = f'https://api.os.mob.com/api/academy_report/list?limit=18&page={i}&keyword=&year='
        req = session.get(url=url, headers=headers)
        data_json = req.json()
        news_list = data_json['data']['list']
        for info in news_list:
            title = info['title']
            publishDate = info['effective_date']
            year = publishDate[:4]
            report_id = info['report_id']
            href = 'https://www.mob.com/mobdata/report/{}'.format(report_id)
            # tf_url = add_check_url(href)
            is_member = r.sismember('report_pdf_three_history', href)
            if is_member:
                continue

            res_href = session.get(url=href, headers=headers).content
            i_soup = BeautifulSoup(res_href,'html.parser')
            summary_list = i_soup.find(class_='picture-content htmlContent').find_all('h3')
            news_url = f'https://api.os.mob.com/api/academy_report/download/{report_id}'

            headers['token'] = '05bc441a-b09b-40cb-ab65-8d9e63e5c529'

            news_req = session.get(url=news_url,headers=headers)
            pdf_url = news_req.json()['data']
            fin_summary = []
            for s in summary_list:
                summary = s.text
                fin_summary.append(summary)
            summary = ''.join(fin_summary)
            dic_post = {
                'title': title,  # 报告名称
                'url_pdf': pdf_url,  # 报告链接
                'year': year,  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': 'YanBao',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'XueLingKun',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': publishDate,  # 时间
                'origin': 'Mob研究院',  # 来源
                'sourceAddress': href,  # 原文链接
                'content': '',  # 内容
                'summary': summary,  # 摘要
                'sid': '1662008807781212161',  # 信息源id
            }
            order_by = 1
            download(dic_post,order_by)
            order_by += 1


def yidong_guanxiangtai():

    url = 'http://mi.talkingdata.com/reports.html?category=all&tag=all&page=1'
    res = requests.get(url=url,headers=headers).content
    soup = BeautifulSoup(res,'html.parser')
    max_page = soup.find(class_='results-page')
    max_page = max_page.find_all('li',class_='trans')[-2].text
    # print((max_page))
    for page in range(int(max_page)):
        url = 'http://mi.talkingdata.com/reports.html?category=all&tag=all&page={}'.format(page)
        res = requests.get(url=url,headers=headers).content
        soup = BeautifulSoup(res, 'html.parser')
        result = soup.find(class_='content-data-report clearfix')
        div_list = result.find_all(class_='reports-list clearfix')
        for div in div_list:
            info_list = div.find_all(class_='download-book')
            for info in info_list:
                href = info.find('b').find('a')['href']
                title = info.find('b').find('a')['title'].strip()
                time = info.find(class_='operate-book').find('p').text
                year = re.findall('\d{4}', time)[0]
                # print(href,title,time)
                res_href = requests.get(url=href,headers=headers).content
                i_soup = BeautifulSoup(res_href,'html.parser')
                i_result = i_soup.find(class_='report-content l')
                p_list = []
                plist = i_result.find_all('p')
                for p in plist:
                    if 'img' in str(p) or 'TalkingData' in str(p) or 'http' in str(p) or '请填写相关信息' in str(p):
                        continue
                    else:
                        p = p.text.strip()
                        p_list.append(p)
                # print(p_list)
                summary = ''.join(p_list).replace('\n','')
                # print(summary)
                url_pdf = i_soup.find(class_='operate-verify').find('button')['data-url']
                # print(url_pdf)
                dic_post = {
                    'title': title,  # 报告名称
                    'url_pdf': url_pdf,  # 报告链接
                    'year': year,  # 报告年份
                    'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                    'item_id': 'YanBao',  # 关联记录id，如：企业信用代码
                    'category': 'pdf',  # 文件后缀名，如：pdf
                    'create_by': 'XueLingKun',  # 创建人，使用驼峰命名，如：TangYuHang
                    'publishDate': time,  # 时间
                    'origin': '移动观象台',  # 来源
                    'sourceAddress': href,  # 原文链接
                    'content': '',  # 内容
                    'summary': summary,  # 摘要
                    'sid': '1662008276140597250',  # 信息源id
                }
                order_by = 1
                download(dic_post, order_by)
                order_by += 1
                # print(page,dic_post)
                # url = 'http://114.115.155.139:5002/report_download'
                # # report-list
                # res = requests.post(url, data=json.dumps(dic_post))
                # print(res.json())


# # 巨量算数
# def juliangsuanshu():
#     # browser = webdriver.Chrome(chromedriver)
#     browser = webdriver.Chrome(chrome_options=opt, executable_path=chromedriver)
#
#     url = 'https://trendinsight.oceanengine.com/arithmetic-report'
#     browser.get(url)#跳到指定页面
#
#     page_source = browser.page_source#获取页面信息
#     soup = BeautifulSoup(page_source, 'html.parser')
#
#     list_all = soup.find('div',{'class':'index-module__reportList--nit0R'}).find_all('div',{'class':'card-module__cardContent--GDAoy index-module__cardContent--vRJI_'})
#     for one_info in list_all:
#         info_title = one_info.a.text.strip()
#         info_date = one_info.find('div',{'class':'card-module__releaseTime--MbbUa'}).text.split('：')[1]
#         info_href = one_info.a.get('href')
#         info_url = 'https://trendinsight.oceanengine.com'+info_href
#
#         res_info = requests.get(info_url)
#         soup_info = BeautifulSoup(res_info.content,'html.parser')
#         list_script = soup_info.find_all('script')
#         for script in list_script:
#             if 'window._SSR_DATA' in script.text:
#                 json_str = script.text
#         info_json = json.loads(json_str.replace('window._SSR_DATA = ',''))
#
#         info_zhaiyao = info_json['data']['storeState']['report_detail']['report_info']['introduction']
#         info_pdf = info_json['data']['storeState']['report_detail']['report_info']['post_files'][0]['file_url']
#
#         dic_post = {
#             'title': info_title,  # 报告名称
#             'url_pdf': info_pdf,  # 报告链接
#             'year': info_date[:4],  # 报告年份
#             'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
#             'item_id': 'YanBao',  # 关联记录id，如：企业信用代码
#             'category': 'pdf',  # 文件后缀名，如：pdf
#             'create_by': 'TangYuHang',  # 创建人，使用驼峰命名，如：TangYuHang
#             'publishDate': info_date,  # 时间
#             'origin': '巨量算数',  # 来源
#             'sourceAddress': info_url,  # 原文链接
#             'content': '',  # 内容
#             'summary': info_zhaiyao,  # 摘要
#             'sid': '1662008524476948481',  # 信息源id
#         }
#         order_by = 1
#         download(dic_post, order_by)
#         order_by += 1
#     #     print(page,dic_post)
#         # url = 'http://114.115.155.139:5002/report_download'
#         # # report-list
#         # res = requests.post(url, data=json.dumps(dic_post))
#         # print(res.json())
#         time.sleep(2)
#     browser.quit()

# 巨量算数

def getnews(browser):
    page_source = browser.page_source  # 获取页面信息
    soup = BeautifulSoup(page_source, 'html.parser')

    list_all = soup.find('div', {'class': 'byted-loading byted-loading-block'}).find_all('div', {
        'class': 'commonCardContainer-TMfUEr hoverShadow-oVbBH0 reportListCard-EhYynV'})
    for one_info in list_all:
        try:
            info_title = one_info.a.text.strip()
            info_date = one_info.find('div', {'class': 'releaseTime-MbbUaH'}).text.split('：')[1]
            info_href = one_info.a.get('href')
            info_url = 'https://trendinsight.oceanengine.com' + info_href

            res_info = requests.get(info_url)
            soup_info = BeautifulSoup(res_info.content, 'html.parser')
            list_script = soup_info.find_all('script')
            for script in list_script:
                if 'window._SSR_DATA' in script.text:
                    json_str = script.text
            info_json = json.loads(json_str.replace('window._SSR_DATA = ', ''))

            info_zhaiyao = info_json['data']['storeState']['report_detail']['report_info']['introduction']
            info_pdf = info_json['data']['storeState']['report_detail']['report_info']['post_files'][0]['file_url']

            dic_post = {
                'title': info_title,  # 报告名称
                'url_pdf': info_pdf,  # 报告链接
                'year': info_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': 'YanBao',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'TangYuHang',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': info_date,  # 时间
                'origin': '巨量算数',  # 来源
                'sourceAddress': info_url,  # 原文链接
                'content': '',  # 内容
                'summary': info_zhaiyao,  # 摘要
                'sid': '1662008524476948481',  # 信息源id
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1
            #     print(page,dic_post)
            # url = 'http://114.115.155.139:5002/report_download'
            # # report-list
            # res = requests.post(url, data=json.dumps(dic_post))
            # print(res.json())
            time.sleep(2)
        except Exception as e:
            continue
    # todo:点击下一页
    # wait = WebDriverWait(browser, 30)
    # wait.until(EC.presence_of_element_located((By.CLASS_NAME, "byted-pager-item-group")))
    # try:
    #     browser.find_element(By.XPATH, '//ul[@class="byted-pager-item-group"]/li[last()]').click()
    # except:
    #     time.sleep(1)
    #     browser.find_element(By.XPATH, '//ul[@class="byted-pager-item-group"]/li[last()]').click()
    # return getnews(browser)


def juliangsuanshu():
    # browser = webdriver.Chrome(chromedriver)
    browser = webdriver.Chrome(chrome_options=opt, executable_path=chromedriver)

    url = 'https://trendinsight.oceanengine.com/arithmetic-report'
    browser.get(url)#跳到指定页面
    getnews(browser)
    browser.quit()

def ke36switch(browser,info_url):
    try:
        browser.get(info_url)  # 跳到指定页面

        page_source = browser.page_source  # 获取页面信息
        soup_info = BeautifulSoup(page_source, 'html.parser')
        info_date = soup_info.find('meta', {'property': 'article:published_time'}).get('content')[:10]
        return soup_info
    except:
        browser.quit()
        proxy = baseCore.get_proxy()
        # proxy = {
        #     'http': '222.90.4.73:40018',
        #     'httpS': '222.90.4.73:40018'
        # }
        opt.add_argument('--proxy-server=' + proxy['http'].split('://')[1])
        # opt.add_argument('--proxy-server=' + proxy['http'])
        browser = webdriver.Chrome(chrome_options=opt, executable_path=chromedriver)
        browser.refresh()
        ke36switch(browser,info_url)

# 36氪
def ke36():
    # browser = webdriver.Chrome(chromedriver)
    proxy = baseCore.get_proxy()
    opt.add_argument('--proxy-server=' + proxy['http'].split('://')[1])
    # opt.add_argument('--proxy-server=' + proxy['http'])
    browser = webdriver.Chrome(chrome_options=opt, executable_path=chromedriver)
    url = 'https://36kr.com/academe'
    browser.get(url)#跳到指定页面
    time.sleep(3)
    for i in range(10):
        try:
            wait = WebDriverWait(browser, 10)
            wait.until(EC.presence_of_element_located((By.CLASS_NAME, 'show-more')))
            js = "var q=document.documentElement.scrollTop=3000"
            browser.execute_script(js)
            time.sleep(2)
            browser.find_element(By.CLASS_NAME, 'show-more').click()
        except:
            break
    wait = WebDriverWait(browser, 10)
    wait.until(EC.presence_of_element_located((By.TAG_NAME, 'body')))
    page_source = browser.page_source#获取页面信息
    soup = BeautifulSoup(page_source, 'html.parser')

    list_all = soup.find('div',{'class':'report-list-wrapper'}).find_all('div',{'class':'report-card type-4'})
    for one_info in list_all[::-1]:
        info_title = one_info.find('div',{'class':'title'}).text
        info_zhaiyao = one_info.find('div',{'class':'desc'}).text
        info_url = one_info.a.get('href')
        # is_member = r.sismember('report_pdf_three_history', info_url)
        # if is_member:
        #     continue
        soup_info = ke36switch(browser,info_url)

        info_date = soup_info.find('meta', {'property': 'article:published_time'}).get('content')[:10]
        if info_date < '2023-05-10':
            pass
        else:
            time.sleep(1)
            continue
        try:
            info_content = soup_info.find('div',{'class':'common-width margin-bottom-20'}).text
        except:
            proxy = baseCore.get_proxy()
            opt.add_argument('--proxy-server=' + proxy['http'])
            browser = webdriver.Chrome(chrome_options=opt, executable_path=chromedriver)
            ke36switch(browser, info_url)
        dic_post = {
            'title': info_title,  # 报告名称
            'url_pdf': '',  # 报告链接
            'year': info_date[:4],  # 报告年份
            'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
            'item_id': 'YanBao',  # 关联记录id，如：企业信用代码
            'category': '',  # 文件后缀名，如：pdf
            'create_by': 'TangYuHang',  # 创建人，使用驼峰命名，如：TangYuHang
            'publishDate': info_date,  # 时间
            'origin': '36氪研究院',  # 来源
            'sourceAddress': info_url,  # 原文链接
            'content': info_content,  # 内容
            'summary': info_zhaiyao,  # 摘要
            'sid': '1662008421217378306',  # 信息源id
        }
        order_by = 1
        # download(dic_post, order_by)
        order_by += 1
    #     print(page,dic_post)
        # url = 'http://114.115.155.139:5002/report_download'
        # # report-list
        # res = requests.post(url, data=json.dumps(dic_post))
        # print(res.json())
        time.sleep(2)

    browser.quit()


# 前沿知识库
def qianyanzhishiku():
    for i in range(40,60):
        log.info(f'====第{i}页====')
        url = f'https://wk.askci.com/Periodical/quality/index_{i}.shtml'

        res = requests.get(url)
        soup = BeautifulSoup(res.content,'html.parser')

        # list_all = soup.find('div',{'class':'quality_report pt-20 pb-40'}).find_all('li')
        list_all = soup.find('div',{'class':'show_report_list'}).find_all('li')
        for one_info in list_all:
            info_title = one_info.a.get('title')
            info_date = one_info.find('div',{'class':'time'}).text.replace('年','-').replace('月','-01')
            info_href = one_info.a.get('href')
            info_url = 'https://wk.askci.com'+info_href

            res_info = requests.get(info_url)
            soup_info = BeautifulSoup(res_info.content,'html.parser')
            info_pdf_url = soup_info.find('iframe',{'scrolling':'auto'}).get('src').split('pdfpath=')[1]
            info_pdf = urllib.parse.unquote(info_pdf_url)

            dic_post = {
                'title': info_title,  # 报告名称
                'url_pdf': info_pdf,  # 报告链接
                'year': info_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': 'YanBao',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'TangYuHang',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': info_date,  # 时间
                'origin': '前沿知识库',  # 来源
                'sourceAddress': info_url,  # 原文链接
                'content': '',  # 内容
                'summary': '',  # 摘要
                'sid': '1662008620631367682',  # 信息源id
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1
        #     print(page,dic_post)
            # url = 'http://114.115.155.139:5002/report_download'
            # # report-list
            # res = requests.post(url, data=json.dumps(dic_post))
            # print(res.json())
            time.sleep(2)


# # 世界经济论坛
# def shijiejingjiluntan():
#     allnum = {'一': '01', '二': '02', '三': '03', '四': '04', '五': '05', '六': '06', '七': '07', '八': '08', '九': '09', '十': '10', '十一': '11', '十二': '12'}
#
#     url = f'https://cn.weforum.org/reports'
#
#     res = requests.get(url)
#     soup = BeautifulSoup(res.content,'html.parser')
#
#     list_all = soup.find('div',{'class':'collection-group collection-group--custom js-scroll'}).find_all('div',{'class':'report-listing-tout__content'})
#
#     for one_info in list_all:
#         info_title = one_info.find('h4').text.strip()
#         info_date = one_info.find('div',{'class':'report-listing-tout__date'}).text.strip()
#         try:
#             info_pdf = one_info.find('a').get('href')
#         except:
#             info_pdf = ''
#         list_date = info_date.replace('月','').split(' ')
#         info_date = list_date[2]+'-'+allnum[list_date[1]]+'-'+list_date[0]
#
#         info_href = one_info.find('a',{'class':'report-listing-tout__cta'}).get('href')
#         info_url = 'https://cn.weforum.org'+info_href
#
#         res_info = requests.get(info_url)
#         soup_info = BeautifulSoup(res_info.content,'html.parser')
#
#         info_zhaiyao = soup_info.find('div',{'class':'report__intro'}).text.strip()
#         info_content = soup_info.find('div',{'class':'small-12 medium-8 columns'}).text.strip()
#
#         dic_post = {
#             'title': info_title,  # 报告名称
#             'url_pdf': info_pdf,  # 报告链接
#             'year': info_date[:4],  # 报告年份
#             'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
#             'item_id': 'YanBao',  # 关联记录id，如：企业信用代码
#             'category': 'pdf',  # 文件后缀名，如：pdf
#             'create_by': 'TangYuHang',  # 创建人，使用驼峰命名，如：TangYuHang
#             'publishDate': info_date,  # 时间
#             'origin': '世界经济论坛',  # 来源
#             'sourceAddress': info_url,  # 原文链接
#             'content': info_zhaiyao,  # 内容
#             'summary': info_content,  # 摘要
#             'sid': '1662008019231088642',  # 信息源id
#         }
#         order_by = 1
#         download(dic_post, order_by)
#         order_by += 1
#     #     print(page,dic_post)
#         # url = 'http://114.115.155.139:5002/report_download'
#         # # report-list
#         # res = requests.post(url, data=json.dumps(dic_post))
#         # print(res.json())
#         time.sleep(2)



# 世界经济论坛
def shijiejingjiluntan():
    allnum = {'一': '01', '二': '02', '三': '03', '四': '04', '五': '05', '六': '06', '七': '07', '八': '08', '九': '09', '十': '10', '十一': '11', '十二': '12'}

    for i in range(1, 2):

        # res = requests.get(url)
        # soup = BeautifulSoup(res.content,'html.parser')
        browser = webdriver.Chrome(chrome_options=opt, executable_path=chromedriver)

        url = f'https://cn.weforum.org/publications/?page={i}'
        browser.get(url)  # 跳到指定页面
        time.sleep(5)
        wait = WebDriverWait(browser, 30)
        wait.until(EC.presence_of_element_located((By.CLASS_NAME, "wef-184hs11")))
        page_source = browser.page_source  # 获取页面信息
        soup = BeautifulSoup(page_source, 'html.parser')
        time.sleep(2)
        list_all = soup.find('div', {'class':'wef-qrllub'}).find_all('div',{'class':'wef-184hs11'})
        time.sleep(2)
        for one_info in list_all:
            tag = one_info.find('div', class_='wef-wx6hgt').find_all('div',class_='wef-0')[1]
            info_title = tag.find('a').text.strip()
            info_date = one_info.find('div',{'class':'wef-1nvfeoy'}).find('time')['datetime']
            datetime_obj = datetime.strptime(info_date, '%Y-%m-%dT%H:%M:%SZ')
            info_date = datetime_obj.strftime('%Y-%m-%d')
            # if info_date >= '2022-07-21':
            #     continue
            try:
                info_zhaiyao = one_info.find('div', {'class': 'wef-8xl60i'}).text.strip()
            except:
                info_zhaiyao = ''
            try:
                info_pdf = one_info.find('div',{'class':'wef-1nvfeoy'}).find('a').get('href')
            except:
                info_pdf = ''

            info_href = tag.find('a').get('href')


            res_info = requests.get(info_href)
            soup_info = BeautifulSoup(res_info.content,'html.parser')


            info_content = soup_info.find('div',{'class':'small-12 medium-8 columns'}).text.strip()

            dic_post = {
                'title': info_title,  # 报告名称
                'url_pdf': info_pdf,  # 报告链接
                'year': info_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': 'YanBao',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'TangYuHang',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': info_date,  # 时间
                'origin': '世界经济论坛',  # 来源
                'sourceAddress': info_href,  # 原文链接
                'content': info_content,  # 内容
                'summary': info_zhaiyao,  # 摘要
                'sid': '1662008019231088642',  # 信息源id
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1
    #     print(page,dic_post)
        # url = 'http://114.115.155.139:5002/report_download'
        # # report-list
        # res = requests.post(url, data=json.dumps(dic_post))
        # print(res.json())
            time.sleep(2)
        browser.quit()

def get_json(key_word,page,headers):
    param = {
        "uid": "",
        "keyword": key_word,
        "type": ["researchReport"],
        "client": "web",
        "clientVersion": "curr",
        "clientType": "web",
        "param": {"researchReport": {"client": "web", "pageSize": 10, "pageIndex": page}}
    }

    param_url = parse.quote(str(param).replace(" ", ""))
    # param_url = parse.quote(str(param))
    # param_url = f'%7B"uid"%3A""%2C"keyword"%3A"{key_word}"%2C"type"%3A%5B"researchReport"%5D%2C"client"%3A"web"%2C"clientVersion"%3A"curr"%2C"clientType"%3A"web"%2C"param"%3A%7B"researchReport"%3A%7B"client"%3A"web"%2C"pageSize"%3A10%2C"pageIndex"%3A{page}%7D%7D%7D'

    t = int(time.time() * 1000)
    url = f'https://search-api-web.eastmoney.com/search/jsonp?cb=&param={param_url}&_={t}'
    # url = 'https://search-api-web.eastmoney.com/search/jsonp?cb=jQuery35103326233792363984_1702455623969&param=%7B%22uid%22%3A%22%22%2C%22keyword%22%3A%22%E7%A7%91%E8%BE%BE%E8%87%AA%E6%8E%A7%22%2C%22type%22%3A%5B%22researchReport%22%5D%2C%22client%22%3A%22web%22%2C%22clientVersion%22%3A%22curr%22%2C%22clientType%22%3A%22web%22%2C%22param%22%3A%7B%22researchReport%22%3A%7B%22client%22%3A%22web%22%2C%22pageSize%22%3A10%2C%22pageIndex%22%3A1%7D%7D%7D&_=1702455623970'

    res = requests.get(url=url, headers=headers).text[1:-1]
    res_json = json.loads(res)
    return res_json
# 东方财富网
def dongfangcaifu():
    headers = {
        'Accept': '*/*',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh-TW;q=0.9,zh;q=0.8',
        'Cache-Control': 'no-cache',
        'Connection': 'keep-alive',
        'Host': 'search-api-web.eastmoney.com',
        'Pragma': 'no-cache',
        'Sec-Fetch-Dest': 'script',
        'Sec-Fetch-Mode': 'no-cors',
        'Sec-Fetch-Site': 'same-site',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
        'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Microsoft Edge";v="120"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"'
    }
    cnx2 = pymysql.connect(host='114.116.44.11', user='caiji', password='f7s0&7qqtK', db='clb_project',
                                    charset='utf8mb4')
    list_short_name = []
    list_social_code = []

    with cnx2.cursor() as cursor:
        sel_sql = '''select securities_short_name,social_credit_code from sys_base_enterprise_ipo where category = 1'''
        cursor.execute(sel_sql)
        selects = cursor.fetchall()

        for row in selects:
            short_name = row[0]
            social_code = row[1]
            if short_name:
                list_short_name.append(short_name)
                list_social_code.append(social_code)

    for key_word_num in range(0, len(list_short_name)):
        log.info(key_word_num)
        key_word = list_short_name[key_word_num]
        social_code = list_social_code[key_word_num]
        log.info(key_word)
        page = 1
        # for page in range(1,500):
        #     log.info(page)
        res_json_ = get_json(key_word, page, headers)
        # 添加页数
        total = res_json_['hitsTotal']
        page = (total/10) + 1
        for page_ in range(1,page+1):
            res_json = get_json(key_word,page_,headers)
            list_all = res_json['result']['researchReport']

            if list_all:
                pass
            else:
                continue
            for one_news in list_all:
                news_title = one_news['title']
                news_title = news_title.replace('<em>', '').replace('</em>', '')
                news_date = one_news['date'][:10]
                comparison_date = "2023-12-08"
                # 比较发布日期是否小于2023-10-06
                if news_date < comparison_date:
                    continue
                else:
                    pass
                news_come = one_news['source']
                news_code = one_news['code']

                news_url = f'https://data.eastmoney.com/report/zw_stock.jshtml?infocode={news_code}'

                news_res = requests.get(news_url)
                news_soup = BeautifulSoup(news_res.content, 'html.parser')

                try:
                    if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                        continue
                except:
                    continue
                try:
                    news_content = news_soup.find('div', {'class': 'newsContent'}).text.strip()
                except:
                    news_content = news_soup.find('div', {'class': 'ctx-content'}).text.strip()

                try:
                    news_pdf = news_soup.find('div', {'class': 'detail-header'}).find_all('a')[-1].get('href')
                except:
                    news_pdf = news_soup.find('span', {'class': 'to-link'}).a.get('href')

                dic_post = {
                    'title': news_title,  # 报告名称
                    'url_pdf': news_pdf,  # 报告链接
                    'year': news_date[:4],  # 报告年份
                    'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                    'item_id': social_code,  # 关联记录id，如：企业信用代码
                    'category': 'pdf',  # 文件后缀名，如：pdf
                    'create_by': 'TangYuHang',  # 创建人，使用驼峰命名，如：TangYuHang
                    'publishDate': news_date,  # 时间
                    'origin': '东方财富网-研报中心',  # 来源
                    'sourceAddress': news_url,  # 原文链接
                    'content': '',  # 内容
                    'summary': news_content,  # 摘要
                    'sid': '1662008733005160449',  # 信息源id
                    'come': news_come,
                }
                order_by = 1
                download(dic_post, order_by)
                order_by += 1

# 东方财富网2
def dongfangcaifu2():
    list_short_name = ['新', '的', '电', '能']

    for key_word_num in range(0, len(list_short_name)):
        log.info(key_word_num)
        key_word = list_short_name[key_word_num]
        social_code = ''
        log.info(key_word)
        page = 1
        # for page in range(1,500):
        #     log.info(page)
        param = {
            "uid": "",
            "keyword": key_word,
            "type": ["researchReport"],
            "client": "web",
            "clientVersion": "curr",
            "clientType": "web",
            "param": {"researchReport": {"client": "web", "pageSize": 10, "pageIndex": page}}
        }

        param_url = parse.quote(str(param).replace(" ", ""))
        # param_url = f'%7B"uid"%3A""%2C"keyword"%3A"{key_word}"%2C"type"%3A%5B"researchReport"%5D%2C"client"%3A"web"%2C"clientVersion"%3A"curr"%2C"clientType"%3A"web"%2C"param"%3A%7B"researchReport"%3A%7B"client"%3A"web"%2C"pageSize"%3A10%2C"pageIndex"%3A{page}%7D%7D%7D'

        t = int(time.time() * 1000)
        url = f'https://search-api-web.eastmoney.com/search/jsonp?cb=&param={param_url}&_={t}'
        res = requests.get(url).text[1:-1]
        res_json = json.loads(res)
        list_all = res_json['result']['researchReport']

        if list_all:
            pass
        else:
            continue
        for one_news in list_all:
            news_title = one_news['title']
            news_title = news_title.replace('<em>', '').replace('</em>', '')
            news_date = one_news['date'][:10]
            comparison_date = "2023-12-08"
            # 比较发布日期是否小于2023-10-06
            if news_date < comparison_date:
                continue
            else:
                pass
            news_come = one_news['source']
            news_code = one_news['code']

            news_url = f'https://data.eastmoney.com/report/zw_stock.jshtml?infocode={news_code}'

            news_res = requests.get(news_url)
            news_soup = BeautifulSoup(news_res.content, 'html.parser')

            try:
                if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                    continue
            except:
                continue
            try:
                news_content = news_soup.find('div', {'class': 'newsContent'}).text.strip()
            except:
                news_content = news_soup.find('div', {'class': 'ctx-content'}).text.strip()

            try:
                news_pdf = news_soup.find('div', {'class': 'detail-header'}).find_all('a')[-1].get('href')
            except:
                news_pdf = news_soup.find('span', {'class': 'to-link'}).a.get('href')

            dic_post = {
                'title': news_title,  # 报告名称
                'url_pdf': news_pdf,  # 报告链接
                'year': news_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': social_code,  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'TangYuHang',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': news_date,  # 时间
                'origin': '东方财富网-研报中心',  # 来源
                'sourceAddress': news_url,  # 原文链接
                'content': '',  # 内容
                'summary': news_content,  # 摘要
                'sid': '1662008733005160449',  # 信息源id
                'come': news_come,
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1


# 东方财富网3 个股研报
def dongfangcaifu3():
    t = str(int(time.time()) * 1000)
    import datetime

    now = datetime.datetime.now()

    # 将日期格式化为“%Y-%m-%d”格式
    # 当前日期
    formatted_date = now.strftime("%Y-%m-%d")
    pre_year = int(time.strftime('%Y', time.localtime(time.time()))) - 2
    month_day = time.strftime('%m-%d', time.localtime(time.time()))
    pre_date = '{}-{}'.format(pre_year, month_day)

    # log.info("格式化后的日期为：", formatted_date)
    # for i in range(1,1349):
    for i in range(1, 15):
        # ip = baseCore.get_proxy()
        url = f'https://reportapi.eastmoney.com/report/list?industryCode=*&pageSize=50&industry=*&rating=&ratingChange=&beginTime={pre_date}&endTime={formatted_date}&pageNo={i}&fields=&qType=0&orgCode=&code=*&rcode=&p={i}&pageNum={i}&pageNumber={i}&_={t}'
        # url = 'https://reportapi.eastmoney.com/report/list?industryCode=*&pageSize=50&industry=*&rating=&ratingChange=&beginTime=2021-06-13&endTime=2023-06-13&pageNo=1&fields=&qType=0&orgCode=&code=*&rcode=&p=1&pageNum=1&pageNumber=1&_=1686645164397'
        res = requests.get(url=url, headers=headers, verify=False).text
        # log.info(res)
        res_json = json.loads(res)

        list_all = res_json['data']
        # log.info(list_all)

        for one_news in list_all:
            news_title = one_news['title']
            # log.info(news_title)
            news_date = one_news['publishDate'][:10]
            comparison_date = "2023-12-14"
            # 比较发布日期是否小于2023-10-06
            if news_date < comparison_date:
                continue
            else:
                pass
            news_come = one_news['orgSName']

            news_href = 'https://data.eastmoney.com/report/info/' + one_news['infoCode'] + '.html'

            news_res = requests.get(url=news_href,headers=headers,verify=False)
            news_soup = BeautifulSoup(news_res.content, 'html.parser')
            # log.info(news_soup)
            try:
                if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                    continue
            except:
                continue
            # todo:摘要去标签
            news_content = news_soup.find(id='ContentBody').text.replace(' ', '').strip()
            news_content = clean_text(news_content)
            news_result = news_soup.find(class_='report-infos')
            # log.info(news_result)

            news_pdf = news_result.find_all('span')[4].find('a')['href']
            # log.info(news_pdf)
            dic_post = {
                'title': news_title,  # 报告名称
                'url_pdf': news_pdf,  # 报告链接
                'year': news_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': '',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'XueLingKun',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': news_date,  # 时间
                'origin': '个股研报',  # 来源
                'sourceAddress': news_href,  # 原文链接
                'content': '',  # 内容
                'summary': news_content,  # 摘要
                # 1673152967405879298
                'sid': '1673152967405879298',  # 信息源id
                'come': news_come,
            }
            # list_quchong.append(dic_post)
            order_by = 1
            download(dic_post, order_by)
            order_by += 1
            # log.info(dic_post['title'],dic_post['publishDate'])


# 东方财富网4 行业研报
def dongfangcaifu4():
    t = str(int(time.time()) * 1000)
    import datetime

    now = datetime.datetime.now()

    # 将日期格式化为“%Y-%m-%d”格式
    # 当前日期
    formatted_date = now.strftime("%Y-%m-%d")
    pre_year = int(time.strftime('%Y', time.localtime(time.time()))) - 2
    month_day = time.strftime('%m-%d', time.localtime(time.time()))
    pre_date = '{}-{}'.format(pre_year, month_day)

    # log.info("格式化后的日期为：", formatted_date)
    for i in range(1, 15):
        # ip = baseCore.get_proxy()
        url = f'https://reportapi.eastmoney.com/report/list?&industryCode=*&pageSize=50&industry=*&rating=*&ratingChange=*&beginTime={pre_date}&endTime={formatted_date}&pageNo={i}&fields=&qType=1&orgCode=&rcode=&p={i}&pageNum={i}&pageNumber={i}&_={t}'
        # url = "https://reportapi.eastmoney.com/report/list?&industryCode=*&pageSize=50&industry=*&rating=*&ratingChange=*&beginTime=2021-06-27&endTime=2023-06-27&pageNo=6&fields=&qType=1&orgCode=&rcode=&p=6&pageNum=6&pageNumber=6&_=1687831020493"
        for i in range(0,3):
            try:
                res = requests.get(url=url,headers=headers,verify=False).text
                break
            except:
                continue
        # log.info(res)
        res_json = json.loads(res)

        list_all = res_json['data']
        # log.info(list_all)

        for one_news in list_all:
            news_title = one_news['title']
            # log.info(news_title)
            news_date = one_news['publishDate'][:10]
            news_come = one_news['orgSName']
            news_date = one_news['publishDate'][:10]
            comparison_date = "2023-12-14"
            # 比较发布日期是否小于2023-10-06
            if news_date < comparison_date:
                continue
            else:
                pass
            news_href = 'https://data.eastmoney.com/report/info/' + one_news['infoCode'] + '.html'

            news_res = requests.get(url=news_href,headers=headers,verify=False)
            news_soup = BeautifulSoup(news_res.content, 'html.parser')
            # log.info(news_soup)
            try:
                if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                    continue
            except:
                continue
            # news_content = news_soup.find(id='ContentBody')
            try:
                news_content = news_soup.find(id='ContentBody').text.replace(' ', '').strip()
                news_content = clean_text(news_content)
            except:
                # ctx-content
                # log.info(news_href,news_title)
                news_content = news_soup.find(id='ctx-content').text.replace(' ', '').strip()
                news_content = clean_text(news_content)

            try:
                news_result = news_soup.find(class_='report-infos')
                # log.info(news_result)
                news_pdf = news_result.find_all('span')[4].find('a')['href']
                # log.info(news_pdf)
            except:
                news_pdf = news_soup.find('span', class_='to-link').find('a')['href']
                # log.info(news_soup)
            dic_post = {
                'title': news_title,  # 报告名称
                'url_pdf': news_pdf,  # 报告链接
                'year': news_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': '',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'XueLingKun',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': news_date,  # 时间
                'origin': '行业研报',  # 来源
                'sourceAddress': news_href,  # 原文链接
                'content': '',  # 内容
                'summary': news_content,  # 摘要
                'sid': '1673154805203091457',  # 信息源id
                'come': news_come,
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1

            # log.info(f'成功：{dic_post["title"]},{dic_post["publishDate"]}')


# 东方财富网5 新股研报
def dongfangcaifu5():
    t = str(int(time.time()) * 1000)
    import datetime

    now = datetime.datetime.now()

    # 将日期格式化为“%Y-%m-%d”格式
    # 当前日期
    formatted_date = now.strftime("%Y-%m-%d")
    pre_year = int(time.strftime('%Y', time.localtime(time.time()))) - 2
    month_day = time.strftime('%m-%d', time.localtime(time.time()))
    pre_date = '{}-{}'.format(pre_year, month_day)

    # log.info("格式化后的日期为：", formatted_date)

    for i in range(1, 10):
        # ip = baseCore.get_proxy()
        url = f'https://reportapi.eastmoney.com/report/newStockList?pageSize=50&beginTime={pre_date}&endTime={formatted_date}&pageNo={i}&fields=&qType=4&p={i}&pageNum={i}&pageNumber={i}&_={t}'

        res = requests.get(url=url,headers=headers,verify=False).text
        log.info(res)
        res_json = json.loads(res)

        list_all = res_json['data']
        # log.info(list_all)

        for one_news in list_all:
            news_title = one_news['title']
            # log.info(news_title)
            news_date = one_news['publishDate'][:10]
            news_come = one_news['orgSName']
            comparison_date = "2023-12-14"
            # 比较发布日期是否小于2023-10-06
            if news_date < comparison_date:
                continue
            else:
                pass
            news_href = 'https://data.eastmoney.com/report/info/' + one_news['infoCode'] + '.html'

            news_res = requests.get(url=news_href,headers=headers,verify=False)
            news_soup = BeautifulSoup(news_res.content, 'html.parser')
            # log.info(news_soup)
            try:
                if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                    continue
            except:
                continue
            news_content = news_soup.find(id='ContentBody')
            # news_content=str(news_content)
            try:
                news_result = news_soup.find(class_='report-infos')
                # log.info(news_result)
                news_pdf = news_result.find_all('span')[4].find('a')['href']
                # log.info(news_pdf)
            except:
                news_pdf = news_soup.find('span', class_='to-link').find('a')['href']
                # log.info(news_soup)
            dic_post = {
                'title': news_title,  # 报告名称
                'url_pdf': news_pdf,  # 报告链接
                'year': news_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': '',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'XueLingKun',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': news_date,  # 时间
                'origin': '新股研报',  # 来源
                'sourceAddress': news_href,  # 原文链接
                'content': '',  # 内容
                'summary': news_content,  # 摘要
                # 1673155535028764674
                'sid': '1673155535028764674',  # 信息源id
                'come': news_come,
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1
        # log.info(f'成功：{dic_post["title"]},{dic_post["publishDate"]}')


# 东方财富网6 宏观研究
def dongfangcaifu6():
    t = str(int(time.time()) * 1000)
    import datetime

    now = datetime.datetime.now()

    # 将日期格式化为“%Y-%m-%d”格式
    # 当前日期
    formatted_date = now.strftime("%Y-%m-%d")
    pre_year = int(time.strftime('%Y', time.localtime(time.time()))) - 2
    month_day = time.strftime('%m-%d', time.localtime(time.time()))
    pre_date = '{}-{}'.format(pre_year, month_day)

    # log.info("格式化后的日期为：", formatted_date)
    for i in range(1, 15):
        # ip = baseCore.get_proxy()
        url = f'https://reportapi.eastmoney.com/report/jg?pageSize=50&beginTime={pre_date}&endTime={formatted_date}&pageNo={i}&fields=&qType=3&orgCode=&author=&p={i}&pageNum={i}&pageNumber={i}&_={t}'

        for i in range(0, 3):
            try:
                res = requests.get(url=url, headers=headers, verify=False).text
                break
            except:
                continue
        # log.info(res)
        res_json = json.loads(res)

        list_all = res_json['data']
        # log.info(list_all)

        for one_news in list_all:
            news_title = one_news['title']
            # log.info(news_title)
            news_date = one_news['publishDate'][:10]
            comparison_date = "2023-12-14"
            # 比较发布日期是否小于2023-10-06
            if news_date < comparison_date:
                continue
            else:
                pass
            news_come = one_news['orgSName']
            # https://data.eastmoney.com/report/zw_macresearch.jshtml?encodeUrl=H5bb92gpAmVPNj5wTfzCzue8aSv0RJSrltfLaEk8UoQ=
            # H5bb92gpAmVPNj5wTfzCzue8aSv0RJSrltfLaEk8UoQ
            news_href = 'https://data.eastmoney.com/report/zw_macresearch.jshtml?encodeUrl=' + one_news[
                'encodeUrl'] + '='

            news_res = requests.get(url=news_href,headers=headers,verify=False)
            news_soup = BeautifulSoup(news_res.content, 'html.parser')
            # log.info(news_soup)
            try:
                if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                    continue
            except:
                continue
            # news_content = news_soup.find(id='ContentBody').text.replace(' ','').strip()
            try:
                news_content = news_soup.find(id='ContentBody').text.replace(' ', '').strip()
                news_content = clean_text(news_content)
            except:
                # ctx-content
                news_content = news_soup.find(id='ctx-content').text.replace(' ', '').strip()
                news_content = clean_text(news_content)
                # log.info(news_href)
            try:
                news_result = news_soup.find(class_='report-infos')
                # log.info(news_result)
                news_pdf = news_result.find_all('span')[4].find('a')['href']
                # log.info(news_pdf)
            except:
                news_pdf = news_soup.find('span', class_='to-link').find('a')['href']
                # log.info(news_soup)
            dic_post = {
                'title': news_title,  # 报告名称
                'url_pdf': news_pdf,  # 报告链接
                'year': news_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': '',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'XueLingKun',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': news_date,  # 时间
                'origin': '宏观研究',  # 来源
                'sourceAddress': news_href,  # 原文链接
                'content': '',  # 内容
                'summary': news_content,  # 摘要
                'sid': '1673155789744652290',  # 信息源id
                'come': news_come,
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1

            # log.info(f'成功：{dic_post["title"]},{dic_post["publishDate"]}')


# 东方财富网7 策略报告
def dongfangcaifu7():
    t = str(int(time.time()) * 1000)
    import datetime

    now = datetime.datetime.now()

    # 将日期格式化为“%Y-%m-%d”格式
    # 当前日期
    formatted_date = now.strftime("%Y-%m-%d")
    pre_year = int(time.strftime('%Y', time.localtime(time.time()))) - 2
    month_day = time.strftime('%m-%d', time.localtime(time.time()))
    pre_date = '{}-{}'.format(pre_year, month_day)

    # log.info("格式化后的日期为：", formatted_date)
    for i in range(1, 3):
        # ip = baseCore.get_proxy()
        url = f'https://reportapi.eastmoney.com/report/jg?pageSize=50&beginTime={pre_date}&endTime={formatted_date}&pageNo={i}&fields=&qType=2&orgCode=&author=&p={i}&pageNum={i}&pageNumber={i}&_={t}'

        res = requests.get(url=url,headers=headers,verify=False).text
        # log.info(res)
        res_json = json.loads(res)

        list_all = res_json['data']
        # log.info(list_all)

        for one_news in list_all:
            news_title = one_news['title']
            # log.info(news_title)
            news_date = one_news['publishDate'][:10]
            comparison_date = "2023-12-14"
            # 比较发布日期是否小于2023-10-06
            if news_date < comparison_date:
                continue
            else:
                pass
            news_come = one_news['orgSName']
            # https://data.eastmoney.com/report/zw_macresearch.jshtml?encodeUrl=H5bb92gpAmVPNj5wTfzCzue8aSv0RJSrltfLaEk8UoQ=
            # H5bb92gpAmVPNj5wTfzCzue8aSv0RJSrltfLaEk8UoQ
            news_href = 'https://data.eastmoney.com/report/zw_macresearch.jshtml?encodeUrl=' + one_news[
                'encodeUrl'] + '='

            news_res = requests.get(url=news_href,headers=headers,verify=False)
            news_soup = BeautifulSoup(news_res.content, 'html.parser')
            # log.info(news_soup)
            try:
                if '抱歉，您访问的页面不存在或已删除！' in news_soup.title.text:
                    continue
            except:
                continue
            # news_content = news_soup.find(id='ContentBody')
            try:
                news_content = news_soup.find(id='ctx-content').text.replace(' ', '').strip()
                news_content = clean_text(news_content)
            except:
                news_content = news_soup.find(id='ContentBody').text.replace(' ', '').strip()
                news_content = clean_text(news_content)
                # log.info(news_href,news_title)
            try:
                news_result = news_soup.find(class_='report-infos')
                # log.info(news_result)
                news_pdf = news_result.find_all('span')[4].find('a')['href']
                # log.info(news_pdf)
            except:
                news_pdf = news_soup.find('span', class_='to-link').find('a')['href']
                # log.info(news_soup)
            dic_post = {
                'title': news_title,  # 报告名称
                'url_pdf': news_pdf,  # 报告链接
                'year': news_date[:4],  # 报告年份
                'type_id': '4',  # 报告种类，（年报：1，季报：2，月报：3，研报：4）
                'item_id': '',  # 关联记录id，如：企业信用代码
                'category': 'pdf',  # 文件后缀名，如：pdf
                'create_by': 'XueLingKun',  # 创建人，使用驼峰命名，如：TangYuHang
                'publishDate': news_date,  # 时间
                'origin': '策略报告',  # 来源
                'sourceAddress': news_href,  # 原文链接
                'content': '',  # 内容
                'summary': news_content,  # 摘要
                'sid': '1673155946779394050',  # 信息源id
                'come': news_come,
            }
            order_by = 1
            download(dic_post, order_by)
            order_by += 1
            # log.info(f'成功：{dic_post["title"]},{dic_post["publishDate"]}')


if __name__ == '__main__':
    # try:
    #     log.info('mob')
    #     Mob()
    # except Exception as e:
    #     pass
    # try:
    #     log.info('yidong_guanxiangtai')
    #     yidong_guanxiangtai()
    # except:
    #     pass
    # try:
    #     log.info('juliangsuanshu')
    #     juliangsuanshu()
    # except Exception as e:
    #     pass
    # try:
    #     log.info('ke36')
    #     ke36()
    # except Exception as e:
    #     ke36()
    #     pass
    # try:
    #     log.info('qianyanzhishiku')
    #     qianyanzhishiku()
    # except Exception as e:
    #     pass
    try:
        log.info('shijiejingjiluntan')
        shijiejingjiluntan()
    except Exception as e:
        log.info(e)
        pass
    # try:
    #     log.info('dongfangcaifu')
    #     dongfangcaifu()
    # except:
    #     pass
    # try:
    #     log.info('dongfangcaifu2')
    #     dongfangcaifu2()
    # except:
    #     pass

    # try:
    #     log.info('dongfangcaifu3')
    #     dongfangcaifu3()
    # except Exception as e:
    #     log.info(e)
    #     pass
    #
    # try:
    #     log.info('dongfangcaifu4')
    #     dongfangcaifu4()
    # except Exception as e:
    #     log.info(e)
    #     pass
    #
    # try:
    #     log.info('dongfangcaifu5')
    #     dongfangcaifu5()
    # except Exception as e:
    #     log.info(e)
    #     pass
    #
    # try:
    #     log.info('dongfangcaifu6')
    #     dongfangcaifu6()
    # except Exception as e:
    #     log.info(e)
    #     pass
    #
    # try:
    #     log.info('dongfangcaifu7')
    #     dongfangcaifu7()
    # except Exception as e:
    #     log.info(e)
    #     pass
