经营分析

comData/caiwushuju/经营分析.py

+import pymysql
+import pymysql
+import redis
+import requests
+import pandas as pd
+from bs4 import BeautifulSoup
+import re
+import time
+import numpy as np
+import json
+import datetime
+import BaseCore
+baseCore = BaseCore.BaseCore()
+log = baseCore.getLogger()
+import concurrent.futures
+import random
+from datetime import datetime
+cnx = pymysql.connect(host='114.116.44.11', user='caiji', password='f7s0&7qqtK', db='clb_project', charset='utf8mb4')
+cursor= cnx.cursor()
+
+# log = baseCore.getLogger()
+
+# 将采集后的股票代码对应的报告期保存进redis
+# def add_date(com_code,i):
+#     r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3)
+#     # json_cwsj = json.dumps(dic_cwsj)
+#     res = r.sadd('com_jingyingfenxi_code::'+com_code,  i)  # 注意是 保存set的方式
+#
+# def get_i():
+#     r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=3)
+#     i = r.get('com_jingyingfenxi_code::'+com_code)
+
+def operate_analysis(i,com_code,social_code):
+
+    com_start = time.time()
+
+    list_code = []  # 股票代码
+    list_time = []  # 数据时间
+    list_zygc = []  # 主营构成
+    list_income = []  # 主营收入
+    list_income_sca = []  # 收入比例
+    list_chengben = []  # 主营成本
+    list_chengben_sca = []  # 成本比例
+    list_lirun = []  # 主营利润
+    list_lirun_sca = []  # 利润比例
+    list_maolilv = []  # 毛利率
+    list_category = []  # 类别
+    url_com = 'https://emweb.eastmoney.com/PC_HSF10/BusinessAnalysis/PageAjax?code={}'.format(com_code)
+
+    json_com = requests.get(url_com).json()
+    try:
+        list_all_info = json_com['zygcfx']
+    except:
+        list_all_info = ''
+        log.info(f'{com_code}---->{url_com}')
+    if list_all_info:
+        for one_info in list_all_info:
+            if "2017" in one_info['REPORT_DATE']:  # 只要19年之后的数据
+                break
+
+            if one_info['MAINOP_TYPE'] == '1':
+                list_category.append('按行业分类')
+            if one_info['MAINOP_TYPE'] == '2':
+                list_category.append('按产品分类')
+            if one_info['MAINOP_TYPE'] == '3':
+                list_category.append('按地区分类')
+            select_sql = f'''select stock_code from operation_analysis_copy1 where stock_code = %s and date = '{one_info['REPORT_DATE'][:10]}' '''
+            y = cursor.execute(select_sql, com_code[2:])
+            if y:
+                i += 1
+                log.info(f'{com_code}--{one_info["REPORT_DATE"][:10]}--已入库')
+                # log.info('=========================')
+                continue
+            else:
+                log.info(f'{com_code}--{one_info["REPORT_DATE"][:10]}--开始采集')
+            try:
+                list_code.append(com_code[2:])
+            except:
+                list_code.append("--")
+            try:
+                list_time.append(one_info['REPORT_DATE'][:10])
+            except:
+                list_time.append("--")
+            try:
+                list_zygc.append(one_info['ITEM_NAME'])
+            except:
+                list_zygc.append("--")
+            try:
+                list_income.append(str(one_info['MAIN_BUSINESS_INCOME']) + "元")
+            except:
+                list_income.append("--")
+            try:
+                list_income_sca.append(str(str(round(one_info['MBI_RATIO'] * 100, 2)) + "%"))
+            except:
+                list_income_sca.append("--")
+            try:
+                list_chengben.append(str(one_info['MAIN_BUSINESS_COST']) + "元")
+            except:
+                list_chengben.append("--")
+            try:
+                list_chengben_sca.append(str(str(round(one_info['MBC_RATIO'] * 100, 2)) + "%"))
+            except:
+                list_chengben_sca.append("--")
+            try:
+                list_lirun.append(str(one_info['MAIN_BUSINESS_RPOFIT']) + "元")
+            except:
+                list_lirun.append("--")
+            try:
+                list_lirun_sca.append(str(str(round(one_info['MBR_RATIO'] * 100, 2)) + "%"))
+            except:
+                list_lirun_sca.append("--")
+            try:
+                list_maolilv.append(str(str(round(one_info['GROSS_RPOFIT_RATIO'] * 100, 2)) + "%"))
+            except:
+                list_maolilv.append("--")
+
+        com_end = time.time()
+        log.info(f'{com_code} 企业采集用时：{com_end-com_start}')
+        list_updata = []
+
+        send_start = time.time()
+        for code, zygc, income, income_sca, chengben, chengben_sca, lirun, lirun_sca, maolilv, time_, category in zip(
+                list_code, list_zygc, list_income,
+                list_income_sca, list_chengben, list_chengben_sca, list_lirun, list_lirun_sca, list_maolilv, list_time,
+                list_category):
+            dic_cwsj = {
+                "stockCode": code,
+                "mainComposition": zygc,
+                "mainIncome": income,
+                "incomeRatio": income_sca,
+                "mainCost": chengben,
+                "costRatio": chengben_sca,
+                "mainProfit": lirun,
+                "profitRatio": lirun_sca,
+                "grossProfitMargin": maolilv,
+                "date": time_,
+                "category": category,
+            }
+            # for nnn in range(0, 3):
+            #     try:
+            #         add_date(com_code,dic_cwsj)
+            #         break
+            #     except Exception as e:
+            #         log.error(e)
+            #         time.sleep(1)
+            list_updata.append(dic_cwsj)
+        print(list_updata)
+
+        print("{}---{}:获取完成".format(i,com_code))
+        time.sleep(1)
+        json_updata = json.dumps(list_updata)
+        #192.168.1.77  114.115.236.206
+        try:
+            response = requests.post('http://114.115.236.206:9988/datapull/sync/operationAnalysis', data=json_updata,
+                                     timeout=300,
+                                     verify=False)
+            time.sleep(3)
+            print(f'{com_code}',response.text)
+        except Exception as e:
+            print(e)
+
+        send_end = time.time()
+        log.info(f'{com_code} 传入接口用时：{send_end-send_start}')
+        return com_code,list_code, list_zygc, list_income,list_income_sca, list_chengben, list_chengben_sca, list_lirun, list_lirun_sca, list_maolilv, list_time,list_category
+    else:
+        return
+df_all = pd.read_excel('D:/kkwork/zzsn_spider/data/上市企业(有财务数据)清单.xlsx',dtype=str)
+list_non = []
+
+
+i = 1
+while True:
+    if i>=len(df_all):
+        log.info('采集已完成')
+        break
+    start_time = time.time()
+    com_code_ = df_all['股票代码'][i]
+    social_code = df_all['信用代码'][i]
+    num = len(com_code_)
+    if 'HK' in com_code_:
+        # com_code = com_code_.split('.')[0]
+        i+=1
+        continue
+    else:
+        if num != 6:
+            com_code1 = '0' * (6 - num) + com_code_
+        else:
+            com_code1 = com_code_
+        # 股票代码0、2、3开头的为深圳交易所，6、9开头的为上海交易所，8开头的为北京交易所
+        if com_code1[0] == '2' or com_code1[0] == '0' or com_code1[0] == '3':
+            com_code = 'sz' + com_code1
+        elif com_code1[0] == '9' or com_code1[0] == '6':
+            com_code = 'sh' + com_code1
+        elif com_code1[0] == '8' or com_code1[0] == '4':
+            com_code = 'bj' + com_code1
+        else:
+            log.error(f'{com_code_},{social_code}：无法获取正确的股票代码！')
+    list_updata = []
+    # # 判断该条信息是否已经采集过
+    # select_sql = '''select stock_code from operation_analysis_copy1 where stock_code = %s'''
+    # y = cursor.execute(select_sql, com_code[2:])
+    # if y:
+    #     i += 1
+    #     log.info(f'{com_code}----已入库')
+    #     log.info('=========================')
+    #     continue
+    #开始采集
+    try:
+        com_code,list_code, list_zygc, list_income,list_income_sca, list_chengben, list_chengben_sca, list_lirun, list_lirun_sca, list_maolilv, list_time,list_category = operate_analysis(i,com_code,social_code)
+    except Exception as e:
+        log.info(f'{com_code}----该企业无信息')
+        dic_non = {
+            '股票代码': com_code,
+            '信用代码': social_code
+        }
+        list_non.append(dic_non)
+        i += 1
+        continue
+    k=1
+    # for code, zygc, income, income_sca, chengben, chengben_sca, lirun, lirun_sca, maolilv, time_, category in zip(
+    #         list_code, list_zygc, list_income,
+    #         list_income_sca, list_chengben, list_chengben_sca, list_lirun, list_lirun_sca, list_maolilv, list_time,
+    #         list_category):
+    #     timestamp = time.time()
+    #     create_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp))
+    #     current_time = datetime.now()
+    #     # random_number = random.randint(100,999)
+    #     if k<10:
+    #
+    #         id = f"{current_time.strftime('%Y%m%d%H%M%S%f')}{'00'}{k}"
+    #     elif k<100:
+    #         id = f"{current_time.strftime('%Y%m%d%H%M%S%f')}{'0'}{k}"
+    #     else:
+    #         id = f"{current_time.strftime('%Y%m%d%H%M%S%f')}{k}"
+    #     # milliseconds = 100
+    #     # seconds = milliseconds / 1000
+    #     # time.sleep(seconds)
+    #     k+=1
+    #     dic_cwsj = {
+    #         "id":id,
+    #         "stockCode": code,
+    #         "mainComposition": zygc,
+    #         "mainIncome": income,
+    #         "incomeRatio": income_sca,
+    #         "mainCost": chengben,
+    #         "costRatio": chengben_sca,
+    #         "mainProfit": lirun,
+    #         "profitRatio": lirun_sca,
+    #         "grossProfitMargin": maolilv,
+    #         "date": time_,
+    #         "category": category,
+    #         "create_time":create_time
+    #     }
+    #     # dic_cwsj = [
+    #     #     id,
+    #     #     code,
+    #     #     zygc,
+    #     #     income,
+    #     #     income_sca,
+    #     #     chengben,
+    #     #     chengben_sca,
+    #     #     lirun,
+    #     #     lirun_sca,
+    #     #     maolilv,
+    #     #     time_,
+    #     #     category,
+    #     #     create_time
+    #     # ]
+    #     #
+    #     list_updata.append(dic_cwsj)
+    # log.info(f'当前数据量-----------{len(list_updata)}')
+    # # if len(list_updata)==0:
+    # #     dic_non = {
+    # #         '股票代码':com_code,
+    # #         '信用代码':social_code
+    # #     }
+    # #     list_non.append(dic_non)
+    # # send_start = time.time()
+    # #插入数据
+    #
+    # # for info in list_updata:
+    # #     try:
+    # #         insert_sql = '''insert into operation_analysis (id,stock_code,main_composition,main_income,income_ratio,main_cost,cost_ratio,main_profit,profit_ratio,gross_profit_margin,date,category,create_time) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''
+    # #         cursor.execute(insert_sql, info)
+    # #         cnx.commit()
+    # #     except Exception as e:
+    # #         current_time = datetime.now()
+    # #         id = f"{current_time.strftime('%Y%m%d%H%M%S%f')}{'000'}"
+    # #         insert_sql = '''insert into operation_analysis (id,stock_code,main_composition,main_income,income_ratio,main_cost,cost_ratio,main_profit,profit_ratio,gross_profit_margin,date,category,create_time) values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''
+    # #         cursor.execute(insert_sql, info)
+    # #         cnx.commit()
+    # #     # print(e)
+    # # print("{}---{}:获取完成".format(i, com_code))
+    # # time.sleep(1)
+    # json_updata = json.dumps(list_updata)
+    # # 192.168.1.77  114.115.236.206
+    # send_start = time.time()
+    # for nnn in range(0,5):
+    #     try:
+    #         response = requests.post('http://114.115.236.206:9988/datapull/sync/operationAnalysis', data=json_updata)
+    #         break
+    #     except Exception as e:
+    #         time.sleep(60)
+    # print(f'{com_code}', response.text)
+    #
+    # send_end = time.time()
+    # log.info(f'{com_code} 传入接口用时：{send_end - send_start}')
+    # # log.info(f'{com_code} 插入数据库用时：{send_end - send_start}')
+    # log.info('==========================')
+    # i += 1
+    # end_time = time.time()
+
+    # log.info(f'企业经营分析采集用时：{end_time-start_time}')
+    break
+# df_non = pd.DataFrame(list_non)
+# df_non.to_excel('无经营分析数据企业.xlsx',index=False)