纳斯达克财务数据 10/8

comData/caiwushuju/nasdaq_caiwu.py

@@ -8,7 +8,6 @@ import pymysql
 import redis
 import requests
 from bs4 import BeautifulSoup
-from requests.adapters import HTTPAdapter
 from requests.packages import urllib3
 from retry import retry
 from base import BaseCore
@@ -20,10 +19,6 @@ cnx = pymysql.connect(host='114.115.159.144', user='caiji', password='zzsn9988',
                       charset='utf8mb4')
 cursor = cnx.cursor()
 r = baseCore.r
-URL = 'https://www.nasdaq.com/'
-session = requests.session()
-session.mount('https://', HTTPAdapter(pool_connections=20, pool_maxsize=100))
-session.mount('http://', HTTPAdapter(pool_connections=20, pool_maxsize=100))
 headers = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
 }
@@ -65,7 +60,6 @@ def add_date(com_code, date_list):
 # 数据发送端口
 def sendData(start_time, social_code, gpdm, dic_info):
     data = json.dumps(dic_info)
-    # print(data)
     url_baocun = 'http://114.115.236.206:8088/sync/finance/nsdk'
     for nnn in range(0, 3):
         try:
@@ -106,7 +100,7 @@ def getlist(table, tableName):
             else:
                 value = '-'
             date_ = years[f'value{i}']
-            if date_ :
+            if date_:
                 date = date_.split('/')[2] + '-' + date_.split('/')[0] + '-' + \
                        date_.split('/')[1]
                 list.append({f'{tableName}': name, 'value': value, 'date': date, })
@@ -139,13 +133,12 @@ def reviseData(lists, unit, tableName):
 
 
 # 获取年度财务数据
-def getYear(start_time, session, social_code, gpdm):
+def getYear(start_time, social_code, gpdm):
     ynFirst = check_code(social_code)
     date_list = []
     url = f'https://api.nasdaq.com/api/company/{gpdm}/financials?frequency=1'
     try:
-        req = session.get(url, headers=headers, verify=False)
-        req.encoding = req.apparent_encoding
+        req = requests.get(url, headers=headers, verify=False)
         data = req.json()['data']
         if data:
             unit = getUnit(gpdm)
@@ -188,8 +181,8 @@ def getYear(start_time, session, social_code, gpdm):
             state = 0
             takeTime = baseCore.getTimeCost(start_time, time.time())
             baseCore.recordLog(social_code, taskType, state, takeTime, url, f'{social_code}===无年度财务数据')
-    except:
-        log.error(f'{social_code}===年度财务数据访问失败')
+    except Exception as e:
+        r.rpush('FinanceFromNasdaq:nasdaqfinance_socialCode', social_code)
         state = 0
         takeTime = baseCore.getTimeCost(start_time, time.time())
         baseCore.recordLog(social_code, taskType, state, takeTime, url, f'{social_code}===年度财务数据访问失败')
@@ -198,13 +191,12 @@ def getYear(start_time, session, social_code, gpdm):
 
 
 # 获取季度财务数据  需要判断日期是否取与年度数据日期重合，重合需要修改类型为dateFlag字段为year
-def getQuarter(start_time, session, social_code, gpdm):
+def getQuarter(start_time, social_code, gpdm):
     ynFirst = check_code(social_code)
     date_list = []
     url = f'https://api.nasdaq.com/api/company/{gpdm}/financials?frequency=2'
     try:
-        req = session.get(url, headers=headers, verify=False)
-        req.encoding = req.apparent_encoding
+        req = requests.get(url, headers=headers, verify=False, timeout=60)
         data = req.json()['data']
         if data:
             unit = getUnit(gpdm)
@@ -250,8 +242,9 @@ def getQuarter(start_time, session, social_code, gpdm):
             state = 0
             takeTime = baseCore.getTimeCost(start_time, time.time())
             baseCore.recordLog(social_code, taskType, state, takeTime, url, f'{social_code}===无季度财务数据')
-    except:
-        log.error(f'{social_code}===季度财务数据访问失败')
+    except Exception as e:
+        r.rpush('FinanceFromNasdaq:nasdaqfinance_socialCode', social_code)
+        log.error(f'{social_code}=={gpdm}===季度财务数据访问失败')
         state = 0
         takeTime = baseCore.getTimeCost(start_time, time.time())
         baseCore.recordLog(social_code, taskType, state, takeTime, url, f'{social_code}===季度财务数据访问失败')
@@ -259,8 +252,9 @@ def getQuarter(start_time, session, social_code, gpdm):
     return date_list
 
 
+# 信用代码放入redis中
 def FinanceFromNasdaq():
-    sql = "select xydm from mgzqyjwyh_list where state=2 and exchange='Nasdaq';"
+    sql = "select xydm from mgzqyjwyh_list where state=2 and exchange='Nasdaq;"
     cursor.execute(sql)
     finance = cursor.fetchall()
     finance_list = [item[0] for item in finance]
@@ -268,14 +262,15 @@ def FinanceFromNasdaq():
         r.rpush('FinanceFromNasdaq:nasdaqfinance_socialCode', item)
     print('redis放入成功')
 
+
 def getInfomation(social_code):
     sql = f"select * from mgzqyjwyh_list where state=2 and xydm='{social_code}';"
     cursor.execute(sql)
     data = cursor.fetchone()
     return data
 
+
 def doJob():
-    session.get(URL, headers=headers)
     while True:
         social_code = baseCore.redicPullData('FinanceFromNasdaq:nasdaqfinance_socialCode')
         if not social_code or social_code == None:
@@ -288,17 +283,18 @@ def doJob():
         social_code = data_enterprise[6]
         # print(gpdm,social_code)
         # 采集年度数据
-        date_list_year = getYear(start_time, session, social_code, gpdm)
+        date_list_year = getYear(start_time, social_code, gpdm)
         # 保存年度数据到redis
         add_date(social_code, date_list_year)
         # 采集季度数据
-        date_list_quarter = getQuarter(start_time, session, social_code, gpdm)
+        date_list_quarter = getQuarter(start_time, social_code, gpdm)
         # 保存季度数据到redis
         add_date(social_code, date_list_quarter)
         timeCost = baseCore.getTimeCost(start_time, time.time())
         state = 1
         baseCore.recordLog(social_code, taskType, state, timeCost, '', '')
         log.info(f'{social_code}=={gpdm}==耗时{timeCost}')
+        time.sleep(2)
 
 
 if __name__ == '__main__':