Skip to content

Z
zzsn_spider
提交 6ab56371 作者: 薛凌堃
操作

9/13

上级 1de3a9f5
正在显示 包含 90 行增加 和 94 行删除
comData/caiwushuju/东方财富网财务数据.py
""" """
...@@ -51,16 +51,16 @@ def get_info(social_code, com_code,info_date,delist_all,info_date_list,taskType) ...@@ -51,16 +51,16 @@ def get_info(social_code, com_code,info_date,delist_all,info_date_list,taskType)
except: except:
time.sleep(1) time.sleep(1)
#判断该报告期是否已采过 #判断该报告期是否已采过
for nnn in range(0, 3): # for nnn in range(0, 3):
try: # try:
panduan = check_date(com_code,info_date) # panduan = check_date(com_code,info_date)
if panduan: # if panduan:
return dic_info # return dic_info
else: # else:
pass # pass
break # break
except: # except:
time.sleep(1) # time.sleep(1)
# 页面url，用于采集字段名称 # 页面url，用于采集字段名称
url_name = f'https://emweb.eastmoney.com/PC_HSF10/NewFinanceAnalysis/Index?type=web&code={com_code}' url_name = f'https://emweb.eastmoney.com/PC_HSF10/NewFinanceAnalysis/Index?type=web&code={com_code}'
...@@ -455,94 +455,90 @@ def getReportTime(): ...@@ -455,94 +455,90 @@ def getReportTime():
list_date.append(date) list_date.append(date)
return list_date return list_date
def job(taskType): def job(taskType):
# 将上市企业库中的全部A股代码存入list # 将上市企业库中的全部A股代码存入list
# 需要提供股票代码、企业信用代码 # 需要提供股票代码、企业信用代码
def job(taskType): while True:
# 将上市企业库中的全部A股代码存入list # 从redis中获取企业信用代码
# 需要提供股票代码、企业信用代码 social_code = baseCore.redicPullData('FinanceFromEast:finance_socialCode')
while True: # social_code = '91420300178856869P'
# 从redis中获取企业信用代码 # 判断 如果Redis中已经没有数据，则等待
social_code = baseCore.redicPullData('FinanceFromEast:finance_socialCode') log.info(f'==========正在采集{social_code}============')
# social_code = '91420300178856869P' if social_code == None:
# 判断 如果Redis中已经没有数据，则等待 time.sleep(20)
log.info(f'==========正在采集{social_code}============') continue
if social_code == None:
time.sleep(20) sql_sel = f'''select securities_code,exchange from sys_base_enterprise_ipo where category = '1' and social_credit_code='{social_code}' '''
continue cursor.execute(sql_sel)
row = cursor.fetchone()
sql_sel = f'''select securities_code,exchange from sys_base_enterprise_ipo where category = '1' and social_credit_code='{social_code}' ''' try:
cursor.execute(sql_sel) securities_code = row[0]
row = cursor.fetchone() pass
try: except:
securities_code = row[0] log.info(f'======{social_code}没有股票代码======')
pass continue
except: exchange = row[1]
log.info(f'======{social_code}没有股票代码======') # for code in list_code:
continue # social_code = rows[0]
exchange = row[1] # exchange = rows[2]
# for code in list_code: # if code==rows[1]:
# social_code = rows[0] # securities_code = code
# exchange = rows[2] # else:
# if code==rows[1]: # continue
# securities_code = code if exchange == 1:
# else: com_code = 'bj' + securities_code
# continue if exchange == 2:
if exchange == 1: com_code = 'sh' + securities_code
com_code = 'bj' + securities_code if exchange == 3:
if exchange == 2: com_code = 'sz' + securities_code
com_code = 'sh' + securities_code # if com_code=='sz002163':
if exchange == 3:
com_code = 'sz' + securities_code list_date = getReportTime()
# if com_code=='sz002163': delist = [] # 记录该企业所有无数据的报告期
date_list = [] # 记录该企业所有数据的报告期
list_date = getReportTime() start_time = time.time()
delist = [] # 记录该企业所有无数据的报告期 # 分别对每个报告期进行采集
date_list = [] # 记录该企业所有数据的报告期 for info_date in list_date:
start_time = time.time() delist_all = []
# 分别对每个报告期进行采集 info_date_list = []
for info_date in list_date: dic_info = get_info(social_code, com_code, info_date, delist_all, info_date_list, taskType)
delist_all = [] print(dic_info)
info_date_list = [] # 将采集后的报告期存入redis
dic_info = get_info(social_code, com_code, info_date, delist_all, info_date_list, taskType) # if len(info_date_list) != 0:
# print(dic_info) # for date in info_date_list:
# 将采集后的报告期存入redis # date_list.append(date)
if len(info_date_list) != 0: # if len(dic_info) != 0:
for date in info_date_list: # # 调凯歌接口存储数据
date_list.append(date) # data = json.dumps(dic_info)
if len(dic_info) != 0: # # print(data)
# 调凯歌接口存储数据 # url_baocun = 'http://114.115.236.206:8088/sync/finance/df'
data = json.dumps(dic_info) # for nnn in range(0, 3):
# print(data) # try:
url_baocun = 'http://114.115.236.206:8088/sync/finance/df' # res_baocun = requests.post(url_baocun, data=data)
for nnn in range(0, 3): # break
try: # except:
res_baocun = requests.post(url_baocun, data=data) # time.sleep(1)
break # print(res_baocun.text)
except: #
time.sleep(1) # for nnn in range(0, 3):
print(res_baocun.text) # try:
# add_date(com_code, date_list)
for nnn in range(0, 3): # break
try: # except:
add_date(com_code, date_list) # time.sleep(1)
break # # if len(info_date_list) != 0:
except: # # for date in info_date_list:
time.sleep(1) # # date_list.append(date)
# if len(info_date_list) != 0: log.info(date_list)
# for date in info_date_list: # date_list = str(date_list)
# date_list.append(date) end_time = time.time()
log.info(date_list) log.info(f'===={com_code}====该企业耗时{end_time - start_time}===')
# date_list = str(date_list)
end_time = time.time()
log.info(f'===={com_code}====该企业耗时{end_time - start_time}===')
cnx.close()
cursor.close()
baseCore.close()
cnx.close() cnx.close()
cursor.close() cursor.close()
baseCore.close() baseCore.close()
if __name__=='__main__': if __name__=='__main__':
task_type = '财务数据/东方财富网' task_type = '财务数据/东方财富网'
job(task_type) job(task_type)
......
comData/dfcfwGpdm/NQenterprise/NQgetid.py
...@@ -327,7 +327,7 @@ if __name__ == '__main__': ...@@ -327,7 +327,7 @@ if __name__ == '__main__':
#从redis里拿数据 #从redis里拿数据
while True: while True:
# TODO:需要隔两个小时左右抓包修改,token从数据库中获得 # TODO:需要隔两个小时左右抓包修改,token从数据库中获得
token = baseCore.GetToken() token = '83a9a9be4e9ecf3a8f8a20364227dc5d'
list_weicha = [] list_weicha = []
list_all_info = [] list_all_info = []
name_list = [] name_list = []
......
comData/weixin_solo/get_tokenCookies.py
...@@ -57,7 +57,7 @@ if __name__=="__main__": ...@@ -57,7 +57,7 @@ if __name__=="__main__":
url = "https://mp.weixin.qq.com/" url = "https://mp.weixin.qq.com/"
browser.get(url) browser.get(url)
# 可改动 # 可改动
time.sleep(60) time.sleep(20)
s = requests.session() s = requests.session()
#获取到token和cookies #获取到token和cookies
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论