天眼查企业动态维护

comData/Tyc/tyc_qydt_add.py

 
 import json
+import random
+
 import requests, time, pymysql
 import jieba
 import sys
@@ -45,24 +47,21 @@ def beinWork(tyc_code, social_code,start_time):
     retData = {'total': 0, 'up_okCount': 0, 'up_errorCount': 0, 'up_repetCount': 0}
     t = time.time()
     url = f'https://capi.tianyancha.com/cloud-yq-news/company/detail/publicmsg/news/webSimple?_={t}&id={tyc_code}&ps={pageSize}&pn=1&emotion=-100&event=-100'
-    for m in range(0, 3):
-        try:
+    try:
+        for m in range(0, 3):
             ip = baseCore.get_proxy()
             headers['User-Agent'] = baseCore.getRandomUserAgent()
             response = requests.get(url=url, headers=headers, proxies=ip, verify=False)
-            # time.sleep(random.randint(3, 5))
+            time.sleep(random.randint(3, 5))
             break
-        except Exception as e:
+        if (response.status_code == 200):
             pass
-
-    if (response.status_code == 200):
-        pass
-    else:
+    except Exception as e:
         log.error(f"{tyc_code}-----获取总数接口失败")
-        e = '获取总数接口失败'
+        error = '获取总数接口失败'
         state = 0
         takeTime = baseCore.getTimeCost(start_time, time.time())
-        baseCore.recordLog(social_code, taskType, state, takeTime, url, e)
+        baseCore.recordLog(social_code, taskType, state, takeTime, url, f'{error}----{e}')
         return retData
     try:
         json_1 = json.loads(response.content.decode('utf-8'))
@@ -177,7 +176,7 @@ def beinWork(tyc_code, social_code,start_time):
                     pass
                 continue
             try:
-                insert_sql = '''insert into brpa_source_article(social_credit_code,source_address,origin,type,create_time) values(%s,%s,%s,%s,now())'''
+                insert_sql = '''insert into brpa_source_article(social_credit_code,source_address,origin,type,publish_time,create_time) values(%s,%s,%s,%s,%s,now())'''
                 # 动态信息列表
                 up_okCount = up_okCount + 1
                 list_info = [
@@ -185,6 +184,7 @@ def beinWork(tyc_code, social_code,start_time):
                     link,
                     '天眼查',
                     '2',
+                    time_format
                 ]
                 cursor_.execute(insert_sql, tuple(list_info))
                 cnx_.commit()
@@ -214,10 +214,10 @@ def beinWork(tyc_code, social_code,start_time):
                 }
             except Exception as e:
                 log.info(f'传输失败:{social_code}----{link}')
-                e = '数据库传输失败'
+                error = '数据库传输失败'
                 state = 0
                 takeTime = baseCore.getTimeCost(start_time, time.time())
-                baseCore.recordLog(social_code, taskType, state, takeTime, link, e)
+                baseCore.recordLog(social_code, taskType, state, takeTime, link, f'{error}----{e}')
                 continue
                 # print(dic_news)
                 # 将相应字段通过kafka传输保存