11/16

comData/noticeReport/公告补采.py

-"""
+"""
@@ -250,6 +250,7 @@ if __name__ == '__main__':
         # print(msglist)
         num = 0
         for mms in msglist:
+            start_time = time.time()
             id = mms['_id']
             title = mms['_source']['title']
             sourceAddress = mms['_source']['sourceAddress']
@@ -261,6 +262,9 @@ if __name__ == '__main__':
             att_id = upload(sourceAddress,num)
             u_attid = att_id
             esMethod.updateaunn(esMethod.index_name, str(id), u_attid)
+            end_time = time.time()
+            elapsed_time = end_time - start_time
+            log.info(f'******更新一条数据耗时{elapsed_time}秒******')
         page+=1
 
 

comData/noticeReport/公告补采1.py

-"""
+"""
@@ -24,8 +24,8 @@ baseCore = BaseCore.BaseCore()
 # cnx_ = baseCore.pool_11.connection()
 # cursor_ = cnx_.cursor()
 
-# cnx_ = baseCore.cnx_
-# cursor_ = cnx_.cursor()
+cnx_ = baseCore.cnx_
+cursor_ = cnx_.cursor()
 
 lock = threading.Lock()
 pathType = 'QYNotice/'
@@ -178,17 +178,17 @@ def getOBSres(pathType,name, response):
     # resp = obsClient.putFile('zzsn', pathType + name, file_path='要上传的那个文件的本地路径')
     return result
 
-def secrchATT(item_id, retData, type_id,order_by,cnx_,cursor_):
+def secrchATT(item_id, retData, type_id,order_by):
 
     sel_sql = '''select id from clb_sys_attachment where item_id = %s and path = %s and type_id=%s and order_by=%s '''
-    # lock.acquire()
+    lock.acquire()
     cursor_.execute(sel_sql, (item_id, retData['path'], type_id,order_by))
     selects = cursor_.fetchone()
-    # lock.release()
+    lock.release()
     return selects
 
 # 插入到att表 返回附件id
-def tableUpdate(retData, year, pdf_name, num,pub_time,origin,cnx_,cursor_):
+def tableUpdate(retData, year, pdf_name, num,pub_time,origin):
     item_id = retData['item_id']
     type_id = retData['type_id']
     group_name = retData['group_name']
@@ -216,19 +216,19 @@ def tableUpdate(retData, year, pdf_name, num,pub_time,origin,cnx_,cursor_):
             status, create_by,
             create_time, page_size, full_path.split('https://zzsn.obs.cn-north-1.myhuaweicloud.com/')[1], 'zzsn',
             pub_time, origin)
-        # lock.acquire()
+        lock.acquire()
         cursor_.execute(Upsql, values)  # 插入
         cnx_.commit()  # 提交
-        # lock.release()
+        lock.release()
     except Exception as e:
         log.info(e)
         return ''
     log.info(f"更新完成:{item_id}===={pdf_name}")
-    selects = secrchATT(item_id, retData, type_id,order_by,cnx_,cursor_)
+    selects = secrchATT(item_id, retData, type_id,order_by)
     id = selects[0]
     return id
 
-def upload(sourceAddress,num,title,social_code,year,publishDate,createDate,cnx_,cursor_):
+def upload(sourceAddress,num,title,social_code,year,publishDate,createDate):
     # todo:链接上传obs
     retData = uptoOBS(sourceAddress, title + '.pdf', 8, social_code,createDate)
     # 附件插入att数据库
@@ -239,7 +239,7 @@ def upload(sourceAddress,num,title,social_code,year,publishDate,createDate,cnx_,
         return None
     num = num + 1
     origin = '证监会'
-    att_id = tableUpdate(retData, year, title + '.pdf', num, publishDate, origin,cnx_,cursor_)
+    att_id = tableUpdate(retData, year, title + '.pdf', num, publishDate, origin)
     if att_id:
         return att_id
     else:
@@ -248,8 +248,6 @@ def upload(sourceAddress,num,title,social_code,year,publishDate,createDate,cnx_,
 def main(page,p,esMethod):
     # esMethod = EsMethod()
     # esMethod.getFileds(index_name=esMethod.index_name)
-    cnx_ = baseCore.cnx_
-    cursor_ = cnx_.cursor()
     result = esMethod.queryatt(index_name=esMethod.index_name,pnum=p)
     total = result['hits']['total']['value']
     if total==0:
@@ -269,7 +267,7 @@ def main(page,p,esMethod):
         publishDate = mms['_source']['publishDate']
         createDate = mms['_source']['createDate']
         log.info(f'{id}---{title}--{sourceAddress}---{social_code}')
-        att_id = upload(sourceAddress,num,title,social_code,year,publishDate,createDate,cnx_,cursor_)
+        att_id = upload(sourceAddress,num,title,social_code,year,publishDate,createDate)
         u_attid = att_id
         esMethod.updateaunn(esMethod.index_name, str(id), u_attid)
         end_time = time.time()

comData/noticeReport/公告补采2.py

-"""
+"""
@@ -62,7 +62,7 @@ class EsMethod(object):
            "sort": [
                {
                    "createDate": {
-                       "order": "asc"
+                       "order": "desc"
                    }
                }
            ],