11/24

comData/BaseInfo_qcc/baseinfo1113.py

@@ -540,19 +540,20 @@ if __name__ == '__main__':
 
     while True:
         nowtime = baseCore.getNowTime(1).replace('-', '')[:8]
-        file_name = f'./data/国内企业基本信息采集情况_{nowtime}.xlsx'
+        file_name = f'./data/国内企业基本信息采集情况.xlsx'
         file.createFile(file_name)
 
         cookieinfo = token.getToken()
-        id_cookie = cookieinfo[0]
-        cookie_ = json.loads(cookieinfo[1])
-        # print(type(cookies))
-        if cookie_:
+        if cookieinfo:
             pass
         else:
             log.info('==========已无cookies==========')
             time.sleep(30)
             continue
+        id_cookie = cookieinfo[0]
+        cookie_ = json.loads(cookieinfo[1])
+        # print(type(cookies))
+
         # cookie_ = json.loads(cookies)
         # print(type(cookie_))
         log.info(f"获取cookie到----{cookie_}")
@@ -621,7 +622,7 @@ if __name__ == '__main__':
         # exchange = ''
 
         count = redaytowork(com_name, social_code, securitiesCode, securitiesShortName, listingDate, category, exchange,ynDomestic, countryName, file_name)
-        time.sleep(40)
+        time.sleep(2)
         # break
         # baseCore.r.close()
         # baseCore.sendEmail(file_name)

comData/noticeReport/东方财富网-公告.py

-import os
+import os
@@ -90,8 +90,8 @@ def getOBSres(pathType,name, response):
     return result
 
 def secrchATT(item_id, retData, type_id):
-    sel_sql = f"select id from clb_sys_attachment where item_id = '{item_id}' and path = '{retData['path']}' and type_id={type_id} "
-    cursor_.execute(sel_sql)
+    sel_sql = '''select id from clb_sys_attachment where item_id = %s and path = %s and type_id=%s '''
+    cursor_.execute(sel_sql, (item_id, retData['path'], type_id))
     selects = cursor_.fetchone()
     return selects
 
@@ -129,7 +129,7 @@ def tableUpdate(retData, com_name, year, pdf_name, num):
         print(e)
     log.info(f"更新完成:{item_id}===={pdf_name+category}")
     try:
-        selects = secrchATT(item_id, pdf_name, type_id)
+        selects = secrchATT(item_id, retData, type_id)
     except Exception as e:
         log.info(e)
     id = selects[0]
@@ -217,8 +217,8 @@ def GetContent(pdf_url,info_url, pdf_name, social_code, year, pub_time, start_ti
     # print(dic_news)
     # 将相应字段通过kafka传输保存
     try:
-        producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
-        kafka_result = producer.send("researchReportTopicaaaas",
+        producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'],max_request_size=1024*1024*20)
+        kafka_result = producer.send("researchReportTopic",
                                      json.dumps(dic_news, ensure_ascii=False).encode('utf8'))
 
         print(kafka_result.get(timeout=10))
@@ -342,7 +342,7 @@ def gonggao_info(dic_info):
                         baseCore.recordLog(social_code, taskType, state, takeTime, pdf_url, '成功')
 
                         # 发送kafka成功之后 再插入数据库
-                        insert = InsterInto(social_code, pdf_url, info_date, title)
+                        insert = InsterInto(social_code, info_url, info_date, title)
                         if insert:
                             log.info(f'===={social_code}========{title}=====插入库成功')
                         pass
@@ -362,8 +362,8 @@ if __name__ =='__main__':
     while True:
         start_time = time.time()
         # 获取企业信息
-        # social_code = baseCore.redicPullData('NoticeEnterpriseEasteFinance:gnshqy_socialCode')
-        social_code = 'ZZSN23030900000316'
+        social_code = baseCore.redicPullData('NoticeEnterprise:mgqy_socialCode_add')
+        # social_code = 'ZZSN23030900000316'
         if not social_code:
             time.sleep(20)
             continue
@@ -378,8 +378,11 @@ if __name__ =='__main__':
         code = dic_info[3]
         com_name = dic_info[1]
         log.info(f'-----开始处理{com_name}----{social_code}------')
-        gonggao_info(dic_info)
-        break
+        try:
+            gonggao_info(dic_info)
+        except:
+            log.info(f'-----error:{com_name}----{social_code}------')
+