import pymysql
import redis
import requests
import pandas as pd
from bs4 import BeautifulSoup
import re
import time
import numpy as np
import json
import datetime
import BaseCore
baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()
import concurrent.futures
import random
from datetime import datetime
cnx = pymysql.connect(host='114.116.44.11', user='caiji', password='f7s0&7qqtK', db='clb_project', charset='utf8mb4')
cursor= cnx.cursor()

# log = baseCore.getLogger()

# 将采集后的股票代码对应的报告期保存进redis
# def add_date(com_code,i):
#     r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3)
#     # json_cwsj = json.dumps(dic_cwsj)
#     res = r.sadd('com_jingyingfenxi_code::'+com_code,  i)  # 注意是 保存set的方式
#
# def get_i():
#     r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=3)
#     i = r.get('com_jingyingfenxi_code::'+com_code)

def operate_analysis(i,com_code,social_code):

    com_start = time.time()

    list_code = []  # 股票代码
    list_time = []  # 数据时间
    list_zygc = []  # 主营构成
    list_income = []  # 主营收入
    list_income_sca = []  # 收入比例
    list_chengben = []  # 主营成本
    list_chengben_sca = []  # 成本比例
    list_lirun = []  # 主营利润
    list_lirun_sca = []  # 利润比例
    list_maolilv = []  # 毛利率
    list_category = []  # 类别
    url_com = 'https://emweb.eastmoney.com/PC_HSF10/BusinessAnalysis/PageAjax?code={}'.format(com_code)

    json_com = requests.get(url_com).json()
    try:
        list_all_info = json_com['zygcfx']
    except:
        list_all_info = ''
        log.info(f'{com_code}---->{url_com}')
    if list_all_info:
        for one_info in list_all_info:
            if "2017" in one_info['REPORT_DATE']:  # 只要19年之后的数据
                break

            if one_info['MAINOP_TYPE'] == '1':
                list_category.append('按行业分类')
            if one_info['MAINOP_TYPE'] == '2':
                list_category.append('按产品分类')
            if one_info['MAINOP_TYPE'] == '3':
                list_category.append('按地区分类')
            select_sql = f'''select stock_code from operation_analysis_copy1 where stock_code = %s and date = '{one_info['REPORT_DATE'][:10]}' '''
            y = cursor.execute(select_sql, com_code[2:])
            if y:
                i += 1
                log.info(f'{com_code}--{one_info["REPORT_DATE"][:10]}--已入库')
                # log.info('=========================')
                continue
            else:
                log.info(f'{com_code}--{one_info["REPORT_DATE"][:10]}--开始采集')
            try:
                list_code.append(com_code[2:])
            except:
                list_code.append("--")
            try:
                list_time.append(one_info['REPORT_DATE'][:10])
            except:
                list_time.append("--")
            try:
                list_zygc.append(one_info['ITEM_NAME'])
            except:
                list_zygc.append("--")
            try:
                list_income.append(str(one_info['MAIN_BUSINESS_INCOME']) + "元")
            except:
                list_income.append("--")
            try:
                list_income_sca.append(str(str(round(one_info['MBI_RATIO'] * 100, 2)) + "%"))
            except:
                list_income_sca.append("--")
            try:
                list_chengben.append(str(one_info['MAIN_BUSINESS_COST']) + "元")
            except:
                list_chengben.append("--")
            try:
                list_chengben_sca.append(str(str(round(one_info['MBC_RATIO'] * 100, 2)) + "%"))
            except:
                list_chengben_sca.append("--")
            try:
                list_lirun.append(str(one_info['MAIN_BUSINESS_RPOFIT']) + "元")
            except:
                list_lirun.append("--")
            try:
                list_lirun_sca.append(str(str(round(one_info['MBR_RATIO'] * 100, 2)) + "%"))
            except:
                list_lirun_sca.append("--")
            try:
                list_maolilv.append(str(str(round(one_info['GROSS_RPOFIT_RATIO'] * 100, 2)) + "%"))
            except:
                list_maolilv.append("--")

        com_end = time.time()
        log.info(f'{com_code} 企业采集用时：{com_end-com_start}')
        list_updata = []

        send_start = time.time()
        for code, zygc, income, income_sca, chengben, chengben_sca, lirun, lirun_sca, maolilv, time_, category in zip(
                list_code, list_zygc, list_income,
                list_income_sca, list_chengben, list_chengben_sca, list_lirun, list_lirun_sca, list_maolilv, list_time,
                list_category):
            dic_cwsj = {
                "stockCode": code,
                "mainComposition": zygc,
                "mainIncome": income,
                "incomeRatio": income_sca,
                "mainCost": chengben,
                "costRatio": chengben_sca,
                "mainProfit": lirun,
                "profitRatio": lirun_sca,
                "grossProfitMargin": maolilv,
                "date": time_,
                "category": category,
            }
            # for nnn in range(0, 3):
            #     try:
            #         add_date(com_code,dic_cwsj)
            #         break
            #     except Exception as e:
            #         log.error(e)
            #         time.sleep(1)
            list_updata.append(dic_cwsj)
        print(list_updata)

        print("{}---{}:获取完成".format(i,com_code))
        time.sleep(1)
        json_updata = json.dumps(list_updata)
        #192.168.1.77  114.115.236.206
        try:
            response = requests.post('http://114.115.236.206:9988/datapull/sync/operationAnalysis', data=json_updata,
                                     timeout=300,
                                     verify=False)
            time.sleep(3)
            print(f'{com_code}',response.text)
        except Exception as e:
            print(e)

        send_end = time.time()
        log.info(f'{com_code} 传入接口用时：{send_end-send_start}')
        return com_code,list_code, list_zygc, list_income,list_income_sca, list_chengben, list_chengben_sca, list_lirun, list_lirun_sca, list_maolilv, list_time,list_category
    else:
        return
df_all = pd.read_excel('D:/kkwork/zzsn_spider/data/上市企业(有财务数据)清单.xlsx',dtype=str)
list_non = []


i = 1
while True:
    if i>=len(df_all):
        log.info('采集已完成')
        break
    start_time = time.time()
    com_code_ = df_all['股票代码'][i]
    social_code = df_all['信用代码'][i]
    num = len(com_code_)
    if 'HK' in com_code_:
        # com_code = com_code_.split('.')[0]
        i+=1
        continue
    else:
        if num != 6:
            com_code1 = '0' * (6 - num) + com_code_
        else:
            com_code1 = com_code_
        # 股票代码0、2、3开头的为深圳交易所，6、9开头的为上海交易所，8开头的为北京交易所
        if com_code1[0] == '2' or com_code1[0] == '0' or com_code1[0] == '3':
            com_code = 'sz' + com_code1
        elif com_code1[0] == '9' or com_code1[0] == '6':
            com_code = 'sh' + com_code1
        elif com_code1[0] == '8' or com_code1[0] == '4':
            com_code = 'bj' + com_code1
        else:
            log.error(f'{com_code_},{social_code}：无法获取正确的股票代码！')
    list_updata = []
    # # 判断该条信息是否已经采集过
    # select_sql = '''select stock_code from operation_analysis_copy1 where stock_code = %s'''
    # y = cursor.execute(select_sql, com_code[2:])
    # if y:
    #     i += 1
    #     log.info(f'{com_code}----已入库')
    #     log.info('=========================')
    #     continue
    #开始采集
    try:
        com_code,list_code, list_zygc, list_income,list_income_sca, list_chengben, list_chengben_sca, list_lirun, list_lirun_sca, list_maolilv, list_time,list_category = operate_analysis(i,com_code,social_code)
    except Exception as e:
        log.info(f'{com_code}----该企业无信息')
        dic_non = {
            '股票代码': com_code,
            '信用代码': social_code
        }
        list_non.append(dic_non)
        i += 1
        continue
    k=1
    # for code, zygc, income, income_sca, chengben, chengben_sca, lirun, lirun_sca, maolilv, time_, category in zip(
    #         list_code, list_zygc, list_income,
    #         list_income_sca, list_chengben, list_chengben_sca, list_lirun, list_lirun_sca, list_maolilv, list_time,
    #         list_category):
    #     timestamp = time.time()
    #     create_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp))
    #     current_time = datetime.now()
    #     # random_number = random.randint(100,999)
    #     if k<10:
    #
    #         id = f"{current_time.strftime('%Y%m%d%H%M%S%f')}{'00'}{k}"
    #     elif k<100:
    #         id = f"{current_time.strftime('%Y%m%d%H%M%S%f')}{'0'}{k}"
    #     else:
    #         id = f"{current_time.strftime('%Y%m%d%H%M%S%f')}{k}"
    #     # milliseconds = 100
    #     # seconds = milliseconds / 1000
    #     # time.sleep(seconds)
    #     k+=1
    #     dic_cwsj = {
    #         "id":id,
    #         "stockCode": code,
    #         "mainComposition": zygc,
    #         "mainIncome": income,
    #         "incomeRatio": income_sca,
    #         "mainCost": chengben,
    #         "costRatio": chengben_sca,
    #         "mainProfit": lirun,
    #         "profitRatio": lirun_sca,
    #         "grossProfitMargin": maolilv,
    #         "date": time_,
    #         "category": category,
    #         "create_time":create_time
    #     }
    #     # dic_cwsj = [
    #     #     id,
    #     #     code,
    #     #     zygc,
    #     #     income,
    #     #     income_sca,
    #     #     chengben,
    #     #     chengben_sca,
    #     #     lirun,
    #     #     lirun_sca,
    #     #     maolilv,
    #     #     time_,
    #     #     category,
    #     #     create_time
    #     # ]
    #     #
    #     list_updata.append(dic_cwsj)
    # log.info(f'当前数据量-----------{len(list_updata)}')
    # # if len(list_updata)==0:
    # #     dic_non = {
    # #         '股票代码':com_code,
    # #         '信用代码':social_code
    # #     }
    # #     list_non.append(dic_non)
    # # send_start = time.time()
    # #插入数据
    #
    # # for info in list_updata:
    # #     try:
    # #         insert_sql = '''insert into operation_analysis (id,stock_code,main_composition,main_income,income_ratio,main_cost,cost_ratio,main_profit,profit_ratio,gross_profit_margin,date,category,create_time) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''
    # #         cursor.execute(insert_sql, info)
    # #         cnx.commit()
    # #     except Exception as e:
    # #         current_time = datetime.now()
    # #         id = f"{current_time.strftime('%Y%m%d%H%M%S%f')}{'000'}"
    # #         insert_sql = '''insert into operation_analysis (id,stock_code,main_composition,main_income,income_ratio,main_cost,cost_ratio,main_profit,profit_ratio,gross_profit_margin,date,category,create_time) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''
    # #         cursor.execute(insert_sql, info)
    # #         cnx.commit()
    # #     # print(e)
    # # print("{}---{}:获取完成".format(i, com_code))
    # # time.sleep(1)
    # json_updata = json.dumps(list_updata)
    # # 192.168.1.77  114.115.236.206
    # send_start = time.time()
    # for nnn in range(0,5):
    #     try:
    #         response = requests.post('http://114.115.236.206:9988/datapull/sync/operationAnalysis', data=json_updata)
    #         break
    #     except Exception as e:
    #         time.sleep(60)
    # print(f'{com_code}', response.text)
    #
    # send_end = time.time()
    # log.info(f'{com_code} 传入接口用时：{send_end - send_start}')
    # # log.info(f'{com_code} 插入数据库用时：{send_end - send_start}')
    # log.info('==========================')
    # i += 1
    # end_time = time.time()

    # log.info(f'企业经营分析采集用时：{end_time-start_time}')
    break
# df_non = pd.DataFrame(list_non)
# df_non.to_excel('无经营分析数据企业.xlsx',index=False)
