新浪财经-纽交所财务数据脚本修改

comData/caiwushuju/RedisPPData.py

@@ -14,6 +14,12 @@ def conn11():
     cursor = conn.cursor()
     return conn,cursor
 
+def conn144():
+    conn = pymysql.Connect(host='114.115.159.144', port=3306, user='caiji', passwd='zzsn9988', db='caiji',
+                           charset='utf8')
+    cursor = conn.cursor()
+    return conn,cursor
+
 #企业公告
 def shizhiCodeFromSql():
     conn,cursor=conn11()
@@ -31,6 +37,7 @@ def shizhiCodeFromSql():
     finally:
         cursor.close()
         conn.close()
+
 #企业公告
 def yahooCodeFromSql():
     conn,cursor=conn11()
@@ -49,6 +56,25 @@ def yahooCodeFromSql():
         cursor.close()
         conn.close()
 
+#新浪纽交所股票对应的代码
+def sinausstockCodeFromSql():
+    conn,cursor=conn144()
+    try:
+        gn_query = "select ticker from mgzqyjwyh_list where state=2 and exchange='NYSE'; "
+        cursor.execute(gn_query)
+        gn_result = cursor.fetchall()
+        gn_social_list = [item[0] for item in gn_result]
+        print('sinausstockCodeFromSql开始将股票代码放入redis=======')
+        for item in gn_social_list:
+            r.rpush('sina_usstock:securities_code', item)
+        print('sinausstockCodeFromSql将股票代码放入redis结束')
+    except Exception as e:
+        log.info("数据查询异常")
+    finally:
+        cursor.close()
+        conn.close()
+
+
 def yahooCode_task():
     # 实例化一个调度器
     scheduler = BlockingScheduler()
@@ -58,9 +84,12 @@ def yahooCode_task():
     scheduler.add_job(yahooCodeFromSql, 'cron', day='*/3', hour=0, minute=0)
     # 每天执行一次
     scheduler.add_job(shizhiCodeFromSql, 'cron', hour=10,minute=0)
+    # 每天执行一次
+    scheduler.add_job(sinausstockCodeFromSql, 'cron', day='*/3', hour=0, minute=0)
     try:
-        yahooCodeFromSql()  # 定时开始前执行一次
-        shizhiCodeFromSql()  # 定时开始前执行一次
+        # yahooCodeFromSql()  # 定时开始前执行一次
+        # shizhiCodeFromSql()  # 定时开始前执行一次
+        sinausstockCodeFromSql()  # 定时开始前执行一次
         scheduler.start()
     except Exception as e:
         print('定时采集异常', e)

comData/caiwushuju/YAHOO财务数据4.py

-# -*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
@@ -373,6 +373,28 @@ class YahooCaiwu(object):
             currency=''
         return currency
 
+    #对比指标计算
+    def calculateIndexReq(self):
+        get_url = 'http://114.115.236.206:8088/sync/calculateIndex'
+        try:
+            params={
+                'type':2
+            }
+            resp = requests.get(get_url,params=params)
+            print(resp.text)
+            text=json.loads(resp.text)
+            codee=text['code']
+            while codee==-200:
+                time.sleep(600)
+                resp = requests.get(get_url)
+                print(resp.text)
+                text=json.loads(resp.text)
+                codee=text['code']
+                if  codee==-200:
+                    break
+            print('调用接口成功！！')
+        except:
+            print('调用失败！')
 if __name__ == '__main__':
     # parse_excel()
     #get_content1()
@@ -383,8 +405,11 @@ if __name__ == '__main__':
             securitiescode=yahoo.getCodeFromRedis()
             yahoo.get_content2(securitiescode)
         except Exception as e:
+            print('没有数据暂停5分钟')
+            yahoo.calculateIndexReq()
             if securitiescode:
                 yahoo.r.rpush('NoticeEnterprise:securities_code',securitiescode)
             else:
                 time.sleep(300)
+                print('没有数据暂停5分钟')
 

comData/caiwushuju/sina_usstock财务.py

-import configparser
+import configparser
@@ -20,6 +20,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 from operator import itemgetter
 from itertools import groupby
 import datetime
+from decimal import Decimal
 
 class SinaUsstock(object):
 
@@ -54,13 +55,19 @@ class SinaUsstock(object):
             seriesValue=tddoc.find('td').text().split(' ')
             for i in range(0,len(pdate)):
                 value=seriesValue[i]
-                if '亿' in value:
-                    value = value.replace("亿", "*100000000")
-                    value = eval(value)
-                elif '万' in value:
-                    value = value.replace("万", "*10000")
-                    value = eval(value)
-                vvla=str(value)
+                try:
+                    if '亿' in value:
+                        value = value.replace("亿", "").replace(",", "")
+                        value = Decimal(value) * Decimal('100000000')
+                        # value = eval(value)
+                    elif '万' in value:
+                        value = value.replace("万", "").replace(",", "")
+                        value = Decimal(value) * Decimal('10000')
+                        # value = eval(value)
+                except Exception as e:
+                    print(e)
+                    print(value)
+                vvla=str(value).replace(",", "")
                 serisemsg={
                     'name':seriesName,
                     'value':vvla,
@@ -71,6 +78,31 @@ class SinaUsstock(object):
 
         return seriesList
 
+    # 判断股票代码是否存在
+    def check_code(self,com_code):
+        r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3)
+        res = r.exists('com_sinacaiwushuju_code::'+com_code)
+        #如果key存在 则不是第一次采集该企业， res = 1
+        if res:
+            return False  #表示不是第一次采集
+        else:
+            return True #表示是第一次采集
+
+    def check_date(self,com_code,info_date):
+        r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=3)
+        res = r.sismember('com_sinacaiwushuju_code::'+com_code, info_date)  # 注意是 保存set的方式
+        if res:
+            return True
+        else:
+            return False
+
+    # 将采集后的股票代码对应的报告期保存进redis
+    def add_date(self,com_code,date_list):
+        r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3)
+        #遍历date_list 放入redis
+        for date in date_list:
+            res = r.sadd('com_sinacaiwushuju_code::'+com_code,date)
+
     def getCodeFromRedis(self):
         securitiescode=self.r.lpop('sina_usstock:securities_code')
         securitiescode = securitiescode.decode('utf-8')
@@ -209,7 +241,7 @@ class SinaUsstock(object):
 
                     #转换数据格式发送接口
                 annualzb=zbl1+zbl3+zbl5
-                annualzb=self.groupZbData(annualzb,stock,social_credit_code,'annual')
+                annualzb=self.groupZbData(annualzb,stock,social_credit_code,'year')
                 self.sendToFinance(annualzb)
                 quarterzb=zbl2+zbl4+zbl6
                 quarterzb=self.groupZbData(quarterzb,stock,social_credit_code,'quarter')
@@ -228,15 +260,26 @@ class SinaUsstock(object):
 
     def sendToFinance(self,zbmsg):
             for zbb in zbmsg:
+                com_code=zbb['securitiesCode']
+                com_date=zbb['date']
+                #判断股票代码是否采集过
+                if self.check_code(com_code):
+                    zbb['ynFirst']=True
+
                 if len(zbb) != 0:
                     # 调凯歌接口存储数据
                     data = json.dumps(zbb)
                     #暂无接口
-                    url_baocun = ''
+                    url_baocun = 'http://114.115.236.206:8088/sync/finance/sina'
                     # url_baocun = 'http://114.115.236.206:8088/sync/finance/df'
                     for nnn in range(0, 3):
                         try:
                             res_baocun = requests.post(url_baocun, data=data)
+                            #将采集到的股票代码和日期进行记录用来标记是否采集过
+                            com_date_list=[]
+                            com_date_list.append(com_date)
+                            self.add_date(com_code,com_date)
+
                             self.logger.info(res_baocun.text)
                             break
                         except:
@@ -309,7 +352,7 @@ class SinaUsstock(object):
 if __name__ == '__main__':
     sinaUsstock=SinaUsstock()
     # securitiescode= sinaUsstock.r.lpop('sina_usstock:securities_code')
-    securitiescode= sinaUsstock.getCodeFromRedis()
+    # securitiescode= sinaUsstock.getCodeFromRedis()
     securitiescode='AAPL'
     try:
         sinaUsstock.get_content2(securitiescode)