Skip to content

Z
zzsn_spider
提交 09d9ab42 作者: 刘伟刚
操作

新浪财经-纽交所财务数据脚本修改

上级 5007e9f7
正在显示 包含 109 行增加 和 12 行删除
comData/caiwushuju/RedisPPData.py
......@@ -14,6 +14,12 @@ def conn11():
cursor = conn.cursor()
return conn,cursor
def conn144():
conn = pymysql.Connect(host='114.115.159.144', port=3306, user='caiji', passwd='zzsn9988', db='caiji',
charset='utf8')
cursor = conn.cursor()
return conn,cursor
#企业公告
def shizhiCodeFromSql():
conn,cursor=conn11()
......@@ -31,6 +37,7 @@ def shizhiCodeFromSql():
finally:
cursor.close()
conn.close()
#企业公告
def yahooCodeFromSql():
conn,cursor=conn11()
......@@ -49,6 +56,25 @@ def yahooCodeFromSql():
cursor.close()
conn.close()
#新浪纽交所股票对应的代码
def sinausstockCodeFromSql():
conn,cursor=conn144()
try:
gn_query = "select ticker from mgzqyjwyh_list where state=2 and exchange='NYSE'; "
cursor.execute(gn_query)
gn_result = cursor.fetchall()
gn_social_list = [item[0] for item in gn_result]
print('sinausstockCodeFromSql开始将股票代码放入redis=======')
for item in gn_social_list:
r.rpush('sina_usstock:securities_code', item)
print('sinausstockCodeFromSql将股票代码放入redis结束')
except Exception as e:
log.info("数据查询异常")
finally:
cursor.close()
conn.close()
def yahooCode_task():
# 实例化一个调度器
scheduler = BlockingScheduler()
......@@ -58,9 +84,12 @@ def yahooCode_task():
scheduler.add_job(yahooCodeFromSql, 'cron', day='*/3', hour=0, minute=0)
# 每天执行一次
scheduler.add_job(shizhiCodeFromSql, 'cron', hour=10,minute=0)
# 每天执行一次
scheduler.add_job(sinausstockCodeFromSql, 'cron', day='*/3', hour=0, minute=0)
try:
yahooCodeFromSql() # 定时开始前执行一次
shizhiCodeFromSql() # 定时开始前执行一次
# yahooCodeFromSql() # 定时开始前执行一次
# shizhiCodeFromSql() # 定时开始前执行一次
sinausstockCodeFromSql() # 定时开始前执行一次
scheduler.start()
except Exception as e:
print('定时采集异常', e)
......
comData/caiwushuju/YAHOO财务数据4.py
# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-
......@@ -373,6 +373,28 @@ class YahooCaiwu(object):
currency=''
return currency
#对比指标计算
def calculateIndexReq(self):
get_url = 'http://114.115.236.206:8088/sync/calculateIndex'
try:
params={
'type':2
}
resp = requests.get(get_url,params=params)
print(resp.text)
text=json.loads(resp.text)
codee=text['code']
while codee==-200:
time.sleep(600)
resp = requests.get(get_url)
print(resp.text)
text=json.loads(resp.text)
codee=text['code']
if codee==-200:
break
print('调用接口成功！！')
except:
print('调用失败！')
if __name__ == '__main__':
# parse_excel()
#get_content1()
......@@ -383,8 +405,11 @@ if __name__ == '__main__':
securitiescode=yahoo.getCodeFromRedis()
yahoo.get_content2(securitiescode)
except Exception as e:
print('没有数据暂停5分钟')
yahoo.calculateIndexReq()
if securitiescode:
yahoo.r.rpush('NoticeEnterprise:securities_code',securitiescode)
else:
time.sleep(300)
print('没有数据暂停5分钟')
comData/caiwushuju/sina_usstock财务.py
import configparser
import configparser
......@@ -20,6 +20,7 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
from operator import itemgetter
from itertools import groupby
import datetime
from decimal import Decimal
class SinaUsstock(object):
......@@ -54,13 +55,19 @@ class SinaUsstock(object):
seriesValue=tddoc.find('td').text().split(' ')
for i in range(0,len(pdate)):
value=seriesValue[i]
try:
if '亿' in value:
value = value.replace("亿", "*100000000")
value = eval(value)
value = value.replace("亿", "").replace(",", "")
value = Decimal(value) * Decimal('100000000')
# value = eval(value)
elif '万' in value:
value = value.replace("万", "*10000")
value = eval(value)
vvla=str(value)
value = value.replace("万", "").replace(",", "")
value = Decimal(value) * Decimal('10000')
# value = eval(value)
except Exception as e:
print(e)
print(value)
vvla=str(value).replace(",", "")
serisemsg={
'name':seriesName,
'value':vvla,
......@@ -71,6 +78,31 @@ class SinaUsstock(object):
return seriesList
# 判断股票代码是否存在
def check_code(self,com_code):
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3)
res = r.exists('com_sinacaiwushuju_code::'+com_code)
#如果key存在 则不是第一次采集该企业， res = 1
if res:
return False #表示不是第一次采集
else:
return True #表示是第一次采集
def check_date(self,com_code,info_date):
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=3)
res = r.sismember('com_sinacaiwushuju_code::'+com_code, info_date) # 注意是 保存set的方式
if res:
return True
else:
return False
# 将采集后的股票代码对应的报告期保存进redis
def add_date(self,com_code,date_list):
r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn',db=3)
#遍历date_list 放入redis
for date in date_list:
res = r.sadd('com_sinacaiwushuju_code::'+com_code,date)
def getCodeFromRedis(self):
securitiescode=self.r.lpop('sina_usstock:securities_code')
securitiescode = securitiescode.decode('utf-8')
......@@ -209,7 +241,7 @@ class SinaUsstock(object):
#转换数据格式发送接口
annualzb=zbl1+zbl3+zbl5
annualzb=self.groupZbData(annualzb,stock,social_credit_code,'annual')
annualzb=self.groupZbData(annualzb,stock,social_credit_code,'year')
self.sendToFinance(annualzb)
quarterzb=zbl2+zbl4+zbl6
quarterzb=self.groupZbData(quarterzb,stock,social_credit_code,'quarter')
......@@ -228,15 +260,26 @@ class SinaUsstock(object):
def sendToFinance(self,zbmsg):
for zbb in zbmsg:
com_code=zbb['securitiesCode']
com_date=zbb['date']
#判断股票代码是否采集过
if self.check_code(com_code):
zbb['ynFirst']=True
if len(zbb) != 0:
# 调凯歌接口存储数据
data = json.dumps(zbb)
#暂无接口
url_baocun = ''
url_baocun = 'http://114.115.236.206:8088/sync/finance/sina'
# url_baocun = 'http://114.115.236.206:8088/sync/finance/df'
for nnn in range(0, 3):
try:
res_baocun = requests.post(url_baocun, data=data)
#将采集到的股票代码和日期进行记录用来标记是否采集过
com_date_list=[]
com_date_list.append(com_date)
self.add_date(com_code,com_date)
self.logger.info(res_baocun.text)
break
except:
......@@ -309,7 +352,7 @@ class SinaUsstock(object):
if __name__ == '__main__':
sinaUsstock=SinaUsstock()
# securitiescode= sinaUsstock.r.lpop('sina_usstock:securities_code')
securitiescode= sinaUsstock.getCodeFromRedis()
# securitiescode= sinaUsstock.getCodeFromRedis()
securitiescode='AAPL'
try:
sinaUsstock.get_content2(securitiescode)
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论