证监会公告封ip处理

comData/noticeReport/证监会-公告.py

-
+
@@ -370,9 +370,12 @@ def SpiderByZJH(url, payload, dic_info, start_time,num):  # dic_info 数据库
     except:
         # 请求失败，输出错误信息
         log.error(f'请求失败:{url}')
-        state = 0
-        takeTime = baseCore.getTimeCost(start_time, time.time())
-        baseCore.recordLog(social_code, taskType, state, takeTime, url, '请求失败')
+        #重新放入redis
+        baseCore.rePutIntoR('NoticeEnterprise:gnqy_socialCode_add', social_code)
+        time.sleep(random.randint(60, 120))
+        # state = 0
+        # takeTime = baseCore.getTimeCost(start_time, time.time())
+        # baseCore.recordLog(social_code, taskType, state, takeTime, url, '请求失败')
         soup = ''
     if soup == '':
         return
@@ -414,11 +417,14 @@ def SpiderByZJH(url, payload, dic_info, start_time,num):  # dic_info 数据库
             soup = RequestUrl(href, payload, social_code, start_time)
         except:
             # 请求失败，输出错误信息
-            log.error('请求失败:', url)
-            state = 0
-            takeTime = baseCore.getTimeCost(start_time, time.time())
-            baseCore.recordLog(social_code, taskType, state, takeTime, url, '请求失败')
-            soup = ''
+            log.error(f'请求失败:{url}')
+            # 重新放入redis
+            baseCore.rePutIntoR('NoticeEnterprise:gnqy_socialCode_add', social_code)
+            time.sleep(random.randint(60, 120))
+            # state = 0
+            # takeTime = baseCore.getTimeCost(start_time, time.time())
+            # baseCore.recordLog(social_code, taskType, state, takeTime, url, '请求失败')
+            # soup = ''
         if soup == '':
             continue
         tr_list = soup.find('div', id='txt').find_all('tr')