提交 f79efef3 作者: 薛凌堃

证监会公告封ip处理

上级 a78391e7
...@@ -370,9 +370,12 @@ def SpiderByZJH(url, payload, dic_info, start_time,num): # dic_info 数据库 ...@@ -370,9 +370,12 @@ def SpiderByZJH(url, payload, dic_info, start_time,num): # dic_info 数据库
except: except:
# 请求失败,输出错误信息 # 请求失败,输出错误信息
log.error(f'请求失败:{url}') log.error(f'请求失败:{url}')
state = 0 #重新放入redis
takeTime = baseCore.getTimeCost(start_time, time.time()) baseCore.rePutIntoR('NoticeEnterprise:gnqy_socialCode_add', social_code)
baseCore.recordLog(social_code, taskType, state, takeTime, url, '请求失败') time.sleep(random.randint(60, 120))
# state = 0
# takeTime = baseCore.getTimeCost(start_time, time.time())
# baseCore.recordLog(social_code, taskType, state, takeTime, url, '请求失败')
soup = '' soup = ''
if soup == '': if soup == '':
return return
...@@ -414,11 +417,14 @@ def SpiderByZJH(url, payload, dic_info, start_time,num): # dic_info 数据库 ...@@ -414,11 +417,14 @@ def SpiderByZJH(url, payload, dic_info, start_time,num): # dic_info 数据库
soup = RequestUrl(href, payload, social_code, start_time) soup = RequestUrl(href, payload, social_code, start_time)
except: except:
# 请求失败,输出错误信息 # 请求失败,输出错误信息
log.error('请求失败:', url) log.error(f'请求失败:{url}')
state = 0 # 重新放入redis
takeTime = baseCore.getTimeCost(start_time, time.time()) baseCore.rePutIntoR('NoticeEnterprise:gnqy_socialCode_add', social_code)
baseCore.recordLog(social_code, taskType, state, takeTime, url, '请求失败') time.sleep(random.randint(60, 120))
soup = '' # state = 0
# takeTime = baseCore.getTimeCost(start_time, time.time())
# baseCore.recordLog(social_code, taskType, state, takeTime, url, '请求失败')
# soup = ''
if soup == '': if soup == '':
continue continue
tr_list = soup.find('div', id='txt').find_all('tr') tr_list = soup.find('div', id='txt').find_all('tr')
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论