Skip to content

Z
zzsn_spider
提交 6ab56371 作者: 薛凌堃
操作

9/13

上级 1de3a9f5
正在显示 包含 90 行增加 和 94 行删除
comData/caiwushuju/东方财富网财务数据.py
"""
"""
......@@ -51,16 +51,16 @@ def get_info(social_code, com_code,info_date,delist_all,info_date_list,taskType)
except:
time.sleep(1)
#判断该报告期是否已采过
for nnn in range(0, 3):
try:
panduan = check_date(com_code,info_date)
if panduan:
return dic_info
else:
pass
break
except:
time.sleep(1)
# for nnn in range(0, 3):
# try:
# panduan = check_date(com_code,info_date)
# if panduan:
# return dic_info
# else:
# pass
# break
# except:
# time.sleep(1)
# 页面url，用于采集字段名称
url_name = f'https://emweb.eastmoney.com/PC_HSF10/NewFinanceAnalysis/Index?type=web&code={com_code}'
......@@ -455,94 +455,90 @@ def getReportTime():
list_date.append(date)
return list_date
def job(taskType):
# 将上市企业库中的全部A股代码存入list
# 需要提供股票代码、企业信用代码
def job(taskType):
# 将上市企业库中的全部A股代码存入list
# 需要提供股票代码、企业信用代码
while True:
# 从redis中获取企业信用代码
social_code = baseCore.redicPullData('FinanceFromEast:finance_socialCode')
# social_code = '91420300178856869P'
# 判断 如果Redis中已经没有数据，则等待
log.info(f'==========正在采集{social_code}============')
if social_code == None:
time.sleep(20)
continue
sql_sel = f'''select securities_code,exchange from sys_base_enterprise_ipo where category = '1' and social_credit_code='{social_code}' '''
cursor.execute(sql_sel)
row = cursor.fetchone()
try:
securities_code = row[0]
pass
except:
log.info(f'======{social_code}没有股票代码======')
continue
exchange = row[1]
# for code in list_code:
# social_code = rows[0]
# exchange = rows[2]
# if code==rows[1]:
# securities_code = code
# else:
# continue
if exchange == 1:
com_code = 'bj' + securities_code
if exchange == 2:
com_code = 'sh' + securities_code
if exchange == 3:
com_code = 'sz' + securities_code
# if com_code=='sz002163':
list_date = getReportTime()
delist = [] # 记录该企业所有无数据的报告期
date_list = [] # 记录该企业所有数据的报告期
start_time = time.time()
# 分别对每个报告期进行采集
for info_date in list_date:
delist_all = []
info_date_list = []
dic_info = get_info(social_code, com_code, info_date, delist_all, info_date_list, taskType)
# print(dic_info)
# 将采集后的报告期存入redis
if len(info_date_list) != 0:
for date in info_date_list:
date_list.append(date)
if len(dic_info) != 0:
# 调凯歌接口存储数据
data = json.dumps(dic_info)
# print(data)
url_baocun = 'http://114.115.236.206:8088/sync/finance/df'
for nnn in range(0, 3):
try:
res_baocun = requests.post(url_baocun, data=data)
break
except:
time.sleep(1)
print(res_baocun.text)
for nnn in range(0, 3):
try:
add_date(com_code, date_list)
break
except:
time.sleep(1)
# if len(info_date_list) != 0:
# for date in info_date_list:
# date_list.append(date)
log.info(date_list)
# date_list = str(date_list)
end_time = time.time()
log.info(f'===={com_code}====该企业耗时{end_time - start_time}===')
cnx.close()
cursor.close()
baseCore.close()
while True:
# 从redis中获取企业信用代码
social_code = baseCore.redicPullData('FinanceFromEast:finance_socialCode')
# social_code = '91420300178856869P'
# 判断 如果Redis中已经没有数据，则等待
log.info(f'==========正在采集{social_code}============')
if social_code == None:
time.sleep(20)
continue
sql_sel = f'''select securities_code,exchange from sys_base_enterprise_ipo where category = '1' and social_credit_code='{social_code}' '''
cursor.execute(sql_sel)
row = cursor.fetchone()
try:
securities_code = row[0]
pass
except:
log.info(f'======{social_code}没有股票代码======')
continue
exchange = row[1]
# for code in list_code:
# social_code = rows[0]
# exchange = rows[2]
# if code==rows[1]:
# securities_code = code
# else:
# continue
if exchange == 1:
com_code = 'bj' + securities_code
if exchange == 2:
com_code = 'sh' + securities_code
if exchange == 3:
com_code = 'sz' + securities_code
# if com_code=='sz002163':
list_date = getReportTime()
delist = [] # 记录该企业所有无数据的报告期
date_list = [] # 记录该企业所有数据的报告期
start_time = time.time()
# 分别对每个报告期进行采集
for info_date in list_date:
delist_all = []
info_date_list = []
dic_info = get_info(social_code, com_code, info_date, delist_all, info_date_list, taskType)
print(dic_info)
# 将采集后的报告期存入redis
# if len(info_date_list) != 0:
# for date in info_date_list:
# date_list.append(date)
# if len(dic_info) != 0:
# # 调凯歌接口存储数据
# data = json.dumps(dic_info)
# # print(data)
# url_baocun = 'http://114.115.236.206:8088/sync/finance/df'
# for nnn in range(0, 3):
# try:
# res_baocun = requests.post(url_baocun, data=data)
# break
# except:
# time.sleep(1)
# print(res_baocun.text)
#
# for nnn in range(0, 3):
# try:
# add_date(com_code, date_list)
# break
# except:
# time.sleep(1)
# # if len(info_date_list) != 0:
# # for date in info_date_list:
# # date_list.append(date)
log.info(date_list)
# date_list = str(date_list)
end_time = time.time()
log.info(f'===={com_code}====该企业耗时{end_time - start_time}===')
cnx.close()
cursor.close()
baseCore.close()
if __name__=='__main__':
task_type = '财务数据/东方财富网'
job(task_type)
......
comData/dfcfwGpdm/NQenterprise/NQgetid.py
......@@ -327,7 +327,7 @@ if __name__ == '__main__':
#从redis里拿数据
while True:
# TODO:需要隔两个小时左右抓包修改,token从数据库中获得
token = baseCore.GetToken()
token = '83a9a9be4e9ecf3a8f8a20364227dc5d'
list_weicha = []
list_all_info = []
name_list = []
......
comData/weixin_solo/get_tokenCookies.py
......@@ -57,7 +57,7 @@ if __name__=="__main__":
url = "https://mp.weixin.qq.com/"
browser.get(url)
# 可改动
time.sleep(60)
time.sleep(20)
s = requests.session()
#获取到token和cookies
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论