12/07

base/RedisPPData.py

@@ -127,7 +127,7 @@ def NoticeDF():
     cnx, cursor = connectSql()
     # 获取美股企业
     # # mg_query = "SELECT a.SocialCode From EnterpriseInfo a ,EnterpriseType b WHERE a.SocialCode = b.SocialCode AND b.type=2 AND a.Place=0 AND SecuritiesCode is not null AND SecuritiesCode not LIKE '%.%'"
-    # mg_query = "SELECT a.SocialCode From EnterpriseInfo a ,EnterpriseType b WHERE a.SocialCode = b.SocialCode AND b.type=3 AND a.Place=0 AND SecuritiesCode is not null AND SecuritiesCode not LIKE '%.%'"
+    # mg_query = "SELECT a.SocialCode From EnterpriseInfo a ,EnterpriseType b WHERE a.SocialCode = b.SocialCode AND b.type=6 AND a.Place=0 AND SecuritiesCode is not null AND SecuritiesCode not LIKE '%.%'"
     # cursor.execute(mg_query)
     # cnx.commit()
     # mg_result = cursor.fetchall()
@@ -141,7 +141,7 @@ def NoticeDF():
     #     # r.rpush('NoticeEnterprise:mgqy_socialCode_add', item)
 
     # 获取港股企业
-    gg_query = "SELECT a.SocialCode From EnterpriseInfo a ,EnterpriseType b WHERE a.SocialCode = b.SocialCode AND b.type=2 And SecuritiesCode like '%.HK'"
+    gg_query = "SELECT a.SocialCode From EnterpriseInfo a ,EnterpriseType b WHERE a.SocialCode = b.SocialCode AND b.type=6 And SecuritiesCode like '%.HK'"
     cursor.execute(gg_query)
     cnx.commit()
     gg_result = cursor.fetchall()
@@ -588,6 +588,15 @@ def global100():
     for item in com_namelist:
         r.rpush('global100:baseinfo', item)
 
+def ipo_code():
+    cnx, cursor = connectSql()
+    query = "SELECT gpdm FROM gpdm1"
+    cursor.execute(query)
+    result = cursor.fetchall()
+    cnx.commit()
+    com_namelist = [item[0] for item in result]
+    for item in com_namelist:
+        r.rpush('gpdm:info', item)
 
 if __name__ == "__main__":
     start = time.time()
@@ -597,7 +606,7 @@ if __name__ == "__main__":
     # shuangbaiqiye()
     # zhuangjingtexind()
     # NoticeEnterprise()
-    NoticeDF()
+    # NoticeDF()
     # AnnualEnterpriseIPO()
     # AnnualEnterprise()
     # BaseInfoEnterprise()
@@ -621,5 +630,6 @@ if __name__ == "__main__":
     # NoticeEnterprise_task()
     # AnnualEnterprise_task()
     # FinanceFromEast()
+    ipo_code()
     log.info(f'====={basecore.getNowTime(1)}=====添加数据成功======耗时：{basecore.getTimeCost(start,time.time())}===')
 

comData/YanBao/deletebyid.py

@@ -47,7 +47,7 @@ if __name__ == "__main__":
     redis_conn = redis.Redis(connection_pool=pool)
     while True:
         # 从redis中读取数据，去附件表中根据title查询，更新查到的附件id
-        item = redis_conn.lpop('YanBao:up')
+        item = redis_conn.lpop('YanBao:aa')
         if item:
             log.info(item)
             id = item.decode()

comData/YanBao/get_doc_id.py

@@ -52,8 +52,8 @@ class EsMethod(object):
                        {
                            "range": {
                                "createDate": {
-                                   "gte": "2023-11-28T10:00:00",
-                                   "lte": "2023-11-29T10:00:00"
+                                   "gte": "2023-11-29T10:00:00",
+                                   "lte": "2023-12-01T10:00:00"
                                }
                            }
                        }
@@ -85,7 +85,11 @@ def main(page, p, esMethod):
     if total == 0:
         log.info('++++已没有数据+++++')
         return
-    msglist = result['hits']['hits']
+    try:
+        msglist = result['hits']['hits']
+    except:
+        log.info(f'error-----{result}')
+        return
     log.info(f'---第{page}页{len(msglist)}条数据----共{total}条数据----')
 
     for mms in msglist:
@@ -118,7 +122,7 @@ def run_threads(num_threads,esMethod,j):
 
 if __name__ == "__main__":
     j = 0
-    for i in range(5):
+    for i in range(2):
         esMethod = EsMethod()
         # result = esMethod.queryatt(index_name=esMethod.index_name, pnum=p)
         # total = result['hits']['total']['value']

comData/annualReport/fbs_annualreport_1.py

+import json
+
+from kafka import KafkaProducer
+
+import requests, time,  fitz
+import urllib3
+from base import BaseCore
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+baseCore = BaseCore.BaseCore()
+
+cnx_ = baseCore.cnx_
+cursor_ = cnx_.cursor()
+
+cnx = baseCore.cnx
+cursor = baseCore.cursor
+
+def sendKafka(dic_news):
+    try:  # 114.116.116.241
+        producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'],max_request_size=1024*1024*20)
+        kafka_result = producer.send("researchReportTopic",
+                                     json.dumps(dic_news, ensure_ascii=False).encode('utf8'))
+
+        print(kafka_result.get(timeout=10))
+
+        dic_result = {
+            'success': 'ture',
+            'message': '操作成功',
+            'code': '200',
+        }
+        log.info(dic_result)
+        return True
+    except Exception as e:
+        dic_result = {
+            'success': 'false',
+            'message': '操作失败',
+            'code': '204',
+            'e': e
+        }
+        log.info(dic_result)
+        return False
+#state1
+
+def secrchATT(type_id, xydm, att_id):
+
+    sel_sql = '''select * from clb_sys_attachment where item_id=%s and type_id=%s and id=%s'''
+    cursor_.execute(sel_sql, (xydm, type_id, att_id))
+    selects = cursor_.fetchall()
+    return selects
+
+
+if __name__ == '__main__':
+    header = {
+        'Connection': 'keep-alive',
+        'Cache-Control': 'max-age=0',
+        'sec-ch-ua': '"Chromium";v="112", "Microsoft Edge";v="112", "Not:A-Brand";v="99"',
+        'sec-ch-ua-mobile': '?0',
+        'sec-ch-ua-platform': '"Windows"',
+        'Upgrade-Insecure-Requests': '1',
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36 Edg/112.0.1722.64',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+        'Sec-Fetch-Site': 'none',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-User': '?1',
+        'Sec-Fetch-Dest': 'document',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
+    }
+    info_list = [
+        # 'ZZSN230824151202408|1724390910695550977|2017',
+        'ZZSN230912210754179|18703841781|2022'
+    ]
+    for info in info_list:
+        xydm = info.split('|')[0]
+        att_id = info.split('|')[1]
+        year = info.split('|')[2]
+        selects = secrchATT('1', xydm, att_id)
+        if len(selects) > 1:
+            pass
+        elif len(selects) == 1:
+            select = selects[0]
+            file_name = select[1]
+            origin = select[18]
+            create_time = select[13]
+            publishDate = select[21]
+            if year == '2023':
+                publishDate = '2023-08-31'
+            full_path = 'http://zzsn.luyuen.com/' + str(select[19])
+            time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+            create_time = time_now
+            content = ''
+            for i in range(0, 3):
+                try:
+                    response = requests.get(url=full_path, headers=header, timeout=30)
+                    break
+                except Exception as e:
+                    time.sleep(3)
+                    continue
+            with fitz.open(stream=response.content, filetype='pdf') as doc:
+                page_size = doc.page_count
+                log = baseCore.getLogger()
+                log.info(f'当前页码----{page_size}')
+                for page in doc.pages():
+                    content += page.get_text()
+            detect_language = baseCore.detect_language(content)
+            dic_news = {
+                'attachmentIds': att_id,
+                'author': '',
+                'content': content,
+                'contentWithTag': '',
+                'createDate': str(create_time),
+                'deleteFlag': '0',
+                'id': '',
+                'keyWords': '',
+                'lang': detect_language,
+                'origin': origin,
+                # 'origin': '雪球网',
+                'publishDate': publishDate,
+                'sid': '1684032033495392257',
+                'sourceAddress': '',  # 原文链接
+                'summary': '',
+                'title': file_name.replace('.pdf','').replace('.PDF',''),
+                'type': 1,
+                'socialCreditCode': xydm,
+                'year': year
+            }
+            if sendKafka(dic_news):
+                # 100表示成功
+                log.info(f'==========={xydm}成功============')
+
+    cnx.close()
+    cursor_.close()
+    baseCore.close()
+
+
+

comData/annualReport1014/report.py

@@ -29,7 +29,7 @@ type_id = 1
 create_by = 'XueLingKun'
 taskType = '企业年报'
 
-file_path = 'D:\\年报\\福布斯2000年报PDF下载-207'
+file_path = 'D:\\年报\\'
 log.info(f'=============当前pid为{baseCore.getPID()}==============')
 
 def sendKafka(dic_news):
@@ -133,22 +133,26 @@ if __name__=='__main__':
         start_time = time.time()
         pdf_path = file_path + '/'+file
 
-        file_rank = int(file.split('-')[0])
-        file_year = file.split('-')[1]
-        if file_year== '2023':
-            print(pdf_path)
-            continue
+        # file_rank = int(file.split('-')[0])
+        # file_year = file.split('-')[1]
+        file_year = '2021'
+        # if file_year== '2023':
+        #     print(pdf_path)
+        #     continue
         #file_rank 对应上企业信用代码
-        selectsql = f"select * from rankandcode where id = {file_rank}"
-        cursor.execute(selectsql)
-        data = cursor.fetchone()
-        cnx.commit()
-        social_code = data[1]
-        ename = data[2]
-        cname = data[3]
-        file_name = cname + ':' + file_year + '年年度报告' + '.pdf'
+        # selectsql = f"select * from rankandcode where id = {file_rank}"
+        # cursor.execute(selectsql)
+        # data = cursor.fetchone()
+        # cnx.commit()
+        # social_code = data[1]
+        # ename = data[2]
+        # cname = data[3]
+        # file_name = cname + ':' + file_year + '年年度报告' + '.pdf'
         content = ''
-        origin = cname + '官网'
+        # origin = cname + '官网'
+        social_code = 'ZZSN22080900000177'
+        file_name = '好事达:2021年年度报告.pdf'
+        origin = '好事达官网'
         #解析文件页数和内容
         log.info(f"-----------正在处理{file_name}--------------")
         with open(pdf_path, 'rb') as file:
@@ -163,7 +167,7 @@ if __name__=='__main__':
                         content += page.get_text()
                     # print(content)
             except Exception as e:
-                log.info(f'文件已损坏:{ename}')
+                # log.info(f'文件已损坏:{ename}')
                 continue
         #解析文件大小
         file_size = os.path.getsize(pdf_path)

comData/ipoInfo/getipocom.py

+"""
+采集上市信息企业信用代码，跟数据库对比
+"""
+
+import json
+import time
+import requests
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+import urllib3
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+from base.BaseCore import BaseCore
+baseCore = BaseCore()
+cnx_ = baseCore.cnx
+cursor_ = baseCore.cursor
+
+taskType = '上市信息/东方财富网'
+
+log = baseCore.getLogger()
+error_list = []
+list_all_info = []
+
+def createDriver():
+    chrome_driver = r'D:\cmd100\chromedriver.exe'
+    path = Service(chrome_driver)
+    chrome_options = webdriver.ChromeOptions()
+    chrome_options.binary_location = r'D:\Google\Chrome\Application\chrome.exe'
+    # 设置代理
+    # proxy = "127.0.0.1:8080"  # 代理地址和端口
+    # chrome_options.add_argument('--proxy-server=http://' + proxy)
+    driver = webdriver.Chrome(service=path, chrome_options=chrome_options)
+    return driver
+# 需要提供股票代码、企业信用代码
+while True:
+    start = time.time()
+    com_code1 = baseCore.redicPullData('gpdm:info')
+    # com_code1 = '837592'
+    if com_code1:
+        pass
+    else:
+        log.info('已没有数据')
+        break
+    # 股票代码0、2、3开头的为深圳交易所，6、9开头的为上海交易所，8开头的为北京交易所
+    if com_code1[0] == '2' or com_code1[0] == '0' or com_code1[0] == '3':
+        com_code = 'sz' + com_code1
+    if com_code1[0] == '9' or com_code1[0] == '6':
+        com_code = 'sh' + com_code1
+    if com_code1[0] == '8' or com_code1[0] == '4':
+        com_code = 'bj' + com_code1
+    if com_code1[0] == 'A':
+        com_code = ''
+    log.info(f'======开始采集{com_code1}======')
+    if 'bj' in com_code:
+        url = f'https://quote.eastmoney.com/bj/{com_code1}.html'
+    else:
+        url = f'https://quote.eastmoney.com/{com_code}.html'
+
+    url_1 = f'https://emweb.eastmoney.com/PC_HSF10/CompanySurvey/PageAjax?code={com_code}'
+    url_2 = f'https://emweb.eastmoney.com/PC_HSF10/BusinessAnalysis/PageAjax?code={com_code}'
+
+    browser = createDriver()
+    browser.get(url)
+    time.sleep(8)
+    page_source = browser.page_source
+    soup_t = BeautifulSoup(page_source, 'html.parser')
+    try:
+        result = soup_t.find('div',class_='quote_quotenums').text
+        # print(f'result:{result}')
+        # if result=='未上市'or result=='已退市':
+        if result == '未上市' :
+            tag = 2
+        if result == '已退市':
+            tag = 0
+        else:
+            tag = 1
+    except Exception as e:
+        error_list.append(com_code)
+        log.info(f'={com_code}===解析上市状态失败=====')
+        state = 0
+        takeTime = baseCore.getTimeCost(start, time.time())
+        baseCore.recordLog('', taskType, state, takeTime, '', f'{com_code}解析上市状态失败--e:{e}')
+        print('error')
+        continue
+
+    requests.adapters.DEFAULT_RETRIES = 5
+
+    json_1 = requests.get(url_1,verify=False).json()
+    json_2 = requests.get(url_2,verify=False).json()
+
+    zhengquan_type = json_1['jbzl'][0]['SECURITY_TYPE']
+    # print(zhengquan_type)
+    if 'A' in zhengquan_type:
+        # print(zhengquan_type)
+        category = '1'
+    if 'B' in zhengquan_type:
+        category = '2'
+    if '新三板' in zhengquan_type:
+        category = '3'
+    if 'H' in zhengquan_type:
+        category = '4'
+    id_code = json_1['jbzl'][0]['REG_NUM']
+
+    zhuyingfanwei = json_2['zyfw'][0]['BUSINESS_SCOPE']
+
+    dic_cwsj = {
+
+        "category": category,  # 股票类型(1-A股;2-B股;3-新三板;4-H股)
+        'listed':tag,
+        "securitiesCode": com_code[2:],
+        "securitiesType": zhengquan_type,
+        "socialCreditCode": id_code,
+
+    }
+
+    list_all_info.append(dic_cwsj)
+    log.info(f'======{com_code}====采集成功=====')
+
+    updateSql = f"update gpdm1 set xydm = '{id_code}',tag = '{tag}',category = '{category}' where gpdm = '{com_code1}'"
+    cursor_.execute(updateSql)
+    cnx_.commit()

comData/ipoInfo/gpdm.py

+#!/usr/bin/env python
+# coding=utf-8
+
 # 东方财富网沪深A股 股票代码采集
 # http://quote.eastmoney.com/center/gridlist.html?st=ChangePercent&sortType=C&sortRule=-1#hs_a_board
 import json
 import random
+import re
 import time
 
 import pymysql
@@ -29,25 +33,27 @@ class Gpdm(object):
         total=0
         for num in range(3):
             try:
-                url = f"http://17.push2.eastmoney.com/api/qt/clist/get?cb=jQuery1124020359136113854692_1688967721474&pn=1&pz={pageSize}&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&wbp2u=|0|0|0|web&fid=f3&fs=m:0+t:6,m:0+t:80,m:1+t:2,m:1+t:23,m:0+t:81+s:2048&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_={self.baseCore.getNowTime(3)}"
-                ip = self.baseCore.get_proxy()
+                url = f"https://41.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112401100239027529557_1701920107041&pn=1&pz={pageSize}&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&wbp2u=|0|0|0|web&fid=f3&fs=m:0+t:6,m:0+t:80,m:1+t:2,m:1+t:23,m:0+t:81+s:2048&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_={self.baseCore.getNowTime(3)}"
+                # ip = self.baseCore.get_proxy()
                 self.headers['User-Agent'] = self.baseCore.getRandomUserAgent()
-                response = requests.get(url, headers=self.headers, verify=False, proxies=ip)
+                response = requests.get(url, headers=self.headers, verify=False)#, proxies=ip)
                 time.sleep(random.randint(3, 5))
                 # jQuery1124020359136113854692_1688967721474({"rc":0,"rt":6,"svr":182993358,"lt":1,"full":1,"dlmkts":"","data":{"total":5488,"diff":[{"f1":2,"f2":35.37,"f3":130.87,"f4":20.05,"f5":505082,"f6":1561753667.0,"f7":72.85,"f8":73.63,"f9":79.87,"f10":"-","f11":-0.34,"f12":"603119","f13":1,"f14":"N\xe6\xb5\x99\xe8\x8d\xa3","f15":37.54,"f16":26.38,"f17":28.88,"f18":15.32,"f20":9903600000,"f21":2426214099,"f22":-0.03,"f23":6.46,"f24":130.87,"f25":130.87,"f62":503279629.0,"f115":70.77,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":70.7,"f3":26.98,"f4":15.02,"f5":278191,"f6":2015432017.69,"f7":19.83,"f8":73.92,"f9":44.38,"f10":"-","f11":0.41,"f12":"301371","f13":0,"f14":"N\xe6\x95\xb7\xe5\xb0\x94\xe4\xbd\xb3","f15":80.04,"f16":69.0,"f17":80.0,"f18":55.68,"f20":28285656000,"f21":2660599297,"f22":0.11,"f23":5.64,"f24":26.98,"f25":26.98,"f62":476657031.0,"f115":33.47,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":27.6,"f3":20.0,"f4":4.6,"f5":135775,"f6":348360366.27,"f7":21.04,"f8":33.94,"f9":212.8,"f10":3.1,"f11":0.0,"f12":"301316","f13":0,"f14":"\xe6\x85\xa7\xe5\x8d\x9a\xe4\xba\x91\xe9\x80\x9a","f15":27.6,"f16":22.76,"f17":23.11,"f18":23.0,"f20":11040276000,"f21":1104274261,"f22":0.0,"f23":11.68,"f24":18.1,"f25":44.43,"f62":107348086.0,"f115":124.43,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":43.62,"f3":20.0,"f4":7.27,"f5":75204,"f6":311935188.44,"f7":21.79,"f8":29.67,"f9":56.11,"f10":13.27,"f11":0.0,"f12":"301289","f13":0,"f14":"\xe5\x9b\xbd\xe7\xbc\x86\xe6\xa3\x80\xe6\xb5\x8b","f15":43.62,"f16":35.7,"f17":36.61,"f18":36.35,"f20":3402360000,"f21":1105762682,"f22":0.0,"f23":3.86,"f24":28.26,"f25":35.55,"f62":80534335.0,"f115":47.25,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":40.98,"f3":20.0,"f4":6.83,"f5":118733,"f6":464542197.42,"f7":20.73,"f8":40.73,"f9":56.02,"f10":2.57,"f11":0.0,"f12":"300881","f13":0,"f14":"\xe7\x9b\x9b\xe5\xbe\xb7\xe9\x91\xab\xe6\xb3\xb0","f15":40.98,"f16":33.9,"f17":33.9,"f18":34.15,"f20":4507800000,"f21":1194567000,"f22":0.0,"f23":5.48,"f24":23.81,"f25":42.05,"f62":16802132.0,"f115":56.01,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":21.0,"f3":19.45,"f4":3.42,"f5":50301,"f6":97244231.42,"f7":16.1,"f8":16.87,"f9":46.64,"f10":1.95,"f11":1.35,"f12":"873576","f13":0,"f14":"\xe5\xa4\xa9\xe5\x8a\x9b\xe5\xa4\x8d\xe5\x90\x88","f15":21.0,"f16":18.17,"f17":18.18,"f18":17.58,"f20":2247000000,"f21":626162250,"f22":0.72,"f23":5.16,"f24":50.21,"f25":50.21,"f62":11286257.0,"f115":29.96,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":76.8,"f3":16.21,"f4":10.71,"f5":153518,"f6":1100431330.98,"f7":23.24,"f8":73.58,"f9":190.79,"f10":1.6,"f11":0.27,"f12":"301315","f13":0,"f14":"\xe5\xa8\x81\xe5\xa3\xab\xe9\xa1\xbf","f15":79.31,"f16":63.95,"f17":63.95,"f18":66.09,"f20":6758400000,"f21":1602347750,"f22":0.17,"f23":7.03,"f24":137.84,"f25":137.84,"f62":112419255.0,"f115":102.68,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":72.99,"f3":16.17,"f4":10.16,"f5":106236,"f6":714127513.24,"f7":23.68,"f8":52.41,"f9":123.41,"f10":1.71,"f11":0.4,"f12":"301141","f13":0,"f14":"\xe4\xb8\xad\xe7\xa7\x91\xe7\xa3\x81\xe4\xb8\x9a","f15":74.88,"f16":60.0,"f17":62.85,"f18":62.83,"f20":6466528467,"f21":1479619267,"f22":0.07,"f23":3.14,"f24":96.74,"f25":78.02,"f62":-26422445.0,"f115":87.31,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":27.3,"f3":12.81,"f4":3.1,"f5":171865,"f6":442577004.48,"f7":15.25,"f8":7.3,"f9":-156.2,"f10":0.94,"f11":-0.15,"f12":"300551","f13":0,"f14":"\xe5\x8f\xa4\xe9\xb3\x8c\xe7\xa7\x91\xe6\x8a\x80","f15":27.55,"f16":23.86,"f17":24.2,"f18":24.2,"f20":9439055235,"f21":6427896275,"f22":-0.11,"f23":8.93,"f24":48.37,"f25":133.73,"f62":16013778.0,"f115":-126.12,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":84.3,"f3":12.18,"f4":9.15,"f5":124022,"f6":989104033.4,"f7":17.33,"f8":64.35,"f9":99.53,"f10":1.15,"f11":0.19,"f12":"301398","f13":0,"f14":"\xe6\x98\x9f\xe6\xba\x90\xe5\x8d\x93\xe9\x95\x81","f15":86.5,"f16":73.48,"f17":75.48,"f18":75.15,"f20":6744000000,"f21":1624735481,"f22":-0.04,"f23":6.81,"f24":157.88,"f25":173.35,"f62":-26812467.0,"f115":105.29,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":34.85,"f3":10.95,"f4":3.44,"f5":27626,"f6":95746251.0,"f7":9.87,"f8":7.18,"f9":-37.27,"f10":9.74,"f11":-0.03,"f12":"688622","f13":1,"f14":"\xe7\xa6\xbe\xe4\xbf\xa1\xe4\xbb\xaa\xe5\x99\xa8","f15":36.0,"f16":32.9,"f17":35.0,"f18":31.41,"f20":2439416569,"f21":1341637317,"f22":-0.03,"f23":4.74,"f24":-5.76,"f25":7.23,"f62":18152096.0,"f115":-36.22,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":87.8,"f3":10.66,"f4":8.46,"f5":22037,"f6":184811228.0,"f7":11.33,"f8":6.52,"f9":116.36,"f10":4.84,"f11":1.09,"f12":"688776","f13":1,"f14":"\xe5\x9b\xbd\xe5\x85\x89\xe7\x94\xb5\xe6\xb0\x94","f15":87.99,"f16":79.0,"f17":79.0,"f18":79.34,"f20":9516064188,"f21":2968587801,"f22":-0.22,"f23":5.39,"f24":-5.88,"f25":-29.79,"f62":2907315.0,"f115":65.69,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":2.05,"f3":10.22,"f4":0.19,"f5":3258788,"f6":657251653.18,"f7":9.68,"f8":6.48,"f9":-12.82,"f10":3.95,"f11":0.0,"f12":"000413","f13":0,"f14":"\xe4\xb8\x9c\xe6\x97\xad\xe5\x85\x89\xe7\x94\xb5","f15":2.05,"f16":1.87,"f17":1.87,"f18":1.86,"f20":11547137393,"f21":10310048690,"f22":0.0,"f23":0.52,"f24":17.82,"f25":15.82,"f62":213263692.0,"f115":-8.55,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":2.7,"f3":10.2,"f4":0.25,"f5":1107878,"f6":291343381.08,"f7":11.84,"f8":7.94,"f9":-19.65,"f10":2.01,"f11":0.0,"f12":"002256","f13":0,"f14":"\xe5\x85\x86\xe6\x96\xb0\xe8\x82\xa1\xe4\xbb\xbd","f15":2.7,"f16":2.41,"f17":2.44,"f18":2.45,"f20":5082512054,"f21":3769280384,"f22":0.0,"f23":4.31,"f24":11.11,"f25":12.97,"f62":96164236.0,"f115":-99.3,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":2.92,"f3":10.19,"f4":0.27,"f5":1178068,"f6":333498626.0,"f7":9.06,"f8":7.34,"f9":7.63,"f10":1.4,"f11":0.0,"f12":"600239","f13":1,"f14":"\xe4\xba\x91\xe5\x8d\x97\xe5\x9f\x8e\xe6\x8a\x95","f15":2.92,"f16":2.68,"f17":2.69,"f18":2.65,"f20":4688605774,"f21":4688605774,"f22":0.0,"f23":2.89,"f24":28.07,"f25":51.3,"f62":27795948.0,"f115":-16.59,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":3.15,"f3":10.14,"f4":0.29,"f5":2973491,"f6":920586623.66,"f7":8.74,"f8":28.9,"f9":-7.07,"f10":4.18,"f11":0.0,"f12":"002630","f13":0,"f14":"\xe5\x8d\x8e\xe8\xa5\xbf\xe8\x83\xbd\xe6\xba\x90","f15":3.15,"f16":2.9,"f17":2.95,"f18":2.86,"f20":3719520000,"f21":3240482440,"f22":0.0,"f23":4.9,"f24":26.51,"f25":7.14,"f62":-18293260.0,"f115":-5.07,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":4.79,"f3":10.11,"f4":0.44,"f5":1857359,"f6":864538200.0,"f7":10.8,"f8":9.31,"f9":24.64,"f10":9.05,"f11":0.0,"f12":"600577","f13":1,"f14":"\xe7\xb2\xbe\xe8\xbe\xbe\xe8\x82\xa1\xe4\xbb\xbd","f15":4.79,"f16":4.32,"f17":4.35,"f18":4.35,"f20":9959122877,"f21":9559956211,"f22":0.0,"f23":2.07,"f24":14.05,"f25":16.26,"f62":161845983.0,"f115":26.21,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":4.36,"f3":10.1,"f4":0.4,"f5":617159,"f6":264661451.0,"f7":11.62,"f8":2.74,"f9":122.48,"f10":3.79,"f11":0.0,"f12":"601777","f13":1,"f14":"\xe5\x8a\x9b\xe5\xb8\x86\xe7\xa7\x91\xe6\x8a\x80","f15":4.36,"f16":3.9,"f17":3.95,"f18":3.96,"f20":19931840280,"f21":9811962000,"f22":0.0,"f23":1.94,"f24":24.22,"f25":12.95,"f62":41966291.0,"f115":137.9,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":3.27,"f3":10.1,"f4":0.3,"f5":290547,"f6":93712867.6,"f7":8.08,"f8":2.28,"f9":1394.52,"f10":1.03,"f11":0.0,"f12":"002175","f13":0,"f14":"\xe4\xb8\x9c\xe6\x96\xb9\xe6\x99\xba\xe9\x80\xa0","f15":3.27,"f16":3.03,"f17":3.04,"f18":2.97,"f20":4175072977,"f21":4175040277,"f22":0.0,"f23":8.53,"f24":13.54,"f25":-8.66,"f62":52561839.0,"f115":41.98,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2},{"f1":2,"f2":2.51,"f3":10.09,"f4":0.23,"f5":1715205,"f6":423246793.0,"f7":10.96,"f8":5.97,"f9":-4.84,"f10":2.8,"f11":0.0,"f12":"600569","f13":1,"f14":"\xe5\xae\x89\xe9\x98\xb3\xe9\x92\xa2\xe9\x93\x81","f15":2.51,"f16":2.26,"f17":2.26,"f18":2.28,"f20":7209777679,"f21":7209777679,"f22":0.0,"f23":1.02,"f24":17.84,"f25":21.26,"f62":88473646.0,"f115":-2.55,"f128":"-","f140":"-","f141":"-","f136":"-","f152":2}]}});
                 content = response.content.decode('utf-8')
-                content = self.baseCore.getSubStr(content, '{', '}')
+                content = re.findall('\((.*?)\)', content)[0]
+                # content = self.baseCore.getSubStr(content, '{', '}')
                 retJson = json.loads(content)
                 total = retJson['data']['total']
                 response.close()
                 break
             except Exception as e:
                 self.log.info(f"------第{num}次出错---{e}")
+                exception = '链接失败'
+                state = 0
+                takeTime = self.baseCore.getTimeCost(start, time.time())
+                self.baseCore.recordLog('', self.taskType, state, takeTime,'http://quote.eastmoney.com/center/gridlist.html?st=ChangePercent&sortType=C&sortRule=-1#hs_a_board',exception)
+
                 continue
-        exception = '链接失败'
-        state = 0
-        takeTime = self.baseCore.getTimeCost(start, time.time())
-        self.baseCore.recordLog('', self.taskType, state, takeTime, 'http://quote.eastmoney.com/center/gridlist.html?st=ChangePercent&sortType=C&sortRule=-1#hs_a_board', exception)
         return total
 
     def getPageDta(self,pageIndex,pageSize,totalPage,gpdmList):
@@ -56,20 +62,21 @@ class Gpdm(object):
             try:
                 start = time.time()
                 self.log.info(f"【{pageIndex}/{totalPage}】-----------begin")
-                url = f"http://17.push2.eastmoney.com/api/qt/clist/get?cb=jQuery1124020359136113854692_1688967721474&pn={pageIndex}&pz={pageSize}&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&wbp2u=|0|0|0|web&fid=f3&fs=m:0+t:6,m:0+t:80,m:1+t:2,m:1+t:23,m:0+t:81+s:2048&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_={self.baseCore.getNowTime(3)}"
-                ip = self.baseCore.get_proxy()
+                url = f"https://41.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112401100239027529557_1701920107041&pn={pageIndex}&pz={pageSize}&po=1&np=1&ut=bd1d9ddb04089700cf9c27f6f7426281&fltt=2&invt=2&wbp2u=|0|0|0|web&fid=f3&fs=m:0+t:6,m:0+t:80,m:1+t:2,m:1+t:23,m:0+t:81+s:2048&fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&_={self.baseCore.getNowTime(3)}"
+                # ip = self.baseCore.get_proxy()
                 self.headers['User-Agent'] = self.baseCore.getRandomUserAgent()
-                response = requests.get(url, headers=self.headers, verify=False, proxies=ip)
+                response = requests.get(url, headers=self.headers, verify=False)#, proxies=ip)
                 time.sleep(random.randint(3, 5))
                 content = response.content.decode('utf-8')
-                content = self.baseCore.getSubStr(content, '{', '}')
+                # content = self.baseCore.getSubStr(content, '{', '}')
+                content = re.findall('\((.*?)\)', content)[0]
                 retJson = json.loads(content)
                 dataList= retJson['data']['diff']
                 for dataIndex in range(len(dataList)):
                     gpdm=dataList[dataIndex]['f12']
                     name=dataList[dataIndex]['f14']
                     gpdmListPage.append(gpdm)
-                    selectSql = f"select count(1) from gpdm where gpdm='{gpdm}' and name='{name}'"
+                    selectSql = f"select count(1) from gpdm1 where gpdm='{gpdm}' and name='{name}'"
                     self.cursor.execute(selectSql)
                     count = self.cursor.fetchone()[0]
                     if count>0:
@@ -77,7 +84,7 @@ class Gpdm(object):
                         continue
                     else:
                         self.log.info(f"{gpdm}-------{name}---新增")
-                        insertSql= f"insert into gpdm(gpdm,name,state,create_date) values ('{gpdm}','{name}',1,now())"
+                        insertSql= f"insert into gpdm1(gpdm,name,state,create_date,xydm,tag,category) values ('{gpdm}','{name}',1,now(),'','','')"
                         self.cursor.execute(insertSql)
                         self.cnx.commit()
                 response.close()
@@ -128,6 +135,7 @@ class Gpdm(object):
         self.cnx.close()
         self.baseCore.close()
         return gpdmList
-#
-# if __name__ == '__main__':
-#     doJob()
+
+if __name__ == '__main__':
+    gpdm = Gpdm()
+    gpdm.doJob()

comData/policylaw/BaseCore.py

 # 核心工具包
 import os
 import random
-import socket
 import sys
 import time
 import uuid
 
-import fitz
 import logbook
 import logbook.more
 import pandas as pd
 import requests
 import zhconv
-import pymysql
+
 import redis
-from docx import Document
-from selenium import webdriver
-from selenium.webdriver.chrome.service import Service
-from openpyxl import Workbook
+
 import langid
 
 # 创建连接池
 import pymysql
-from pymysql import connections
-from DBUtils.PooledDB import PooledDB
-# import sys
-# sys.path.append('D://zzsn_spider//base//fdfs_client')
-
-from fdfs_client.client import get_tracker_conf, Fdfs_client
 
-tracker_conf = get_tracker_conf('D:\\zzsn_spider\\comData\\policylaw\\client.conf')
-client = Fdfs_client(tracker_conf)
+from DBUtils.PooledDB import PooledDB
 # 注意 程序退出前 调用BaseCore.close() 关闭相关资源
 
 from obs import ObsClient
 import fitz
-from urllib.parse import unquote
 
 obsClient = ObsClient(
     access_key_id='VEHN7D0TJ9316H8AHCAV',  # 你的华为云的ak码
@@ -263,6 +250,7 @@ class BaseCore:
         self.cnx_ = pymysql.connect(host='114.116.44.11', user='caiji', password='f7s0&7qqtK', db='clb_project',
                                     charset='utf8mb4')
         self.cursor_ = self.cnx_.cursor()
+
         # 连接到Redis
         self.r = redis.Redis(host="114.115.236.206", port=6379, password='clbzzsn', db=6)
 
@@ -453,82 +441,11 @@ class BaseCore:
 
     # 解析word文件页数
 
-    def doc_page(self, file_path):
-        doc = Document(file_path)
-        return len(doc.sections)
-
-    def pdf_content(self, resp_content):
-        # 解析pdf文件内容
-        content = ''
-        for i in range(0, 3):
-            try:
-                result = client.upload_by_buffer(resp_content, file_ext_name='pdf')
-                with fitz.open(stream=resp_content, filetype='pdf') as doc:
-                    # page_size = doc.page_count
-                    for page in doc.pages():
-                        content += page.get_text()
-                break
-            except:
-                time.sleep(3)
-                continue
-        return content
-
     def getuuid(self):
         get_timestamp_uuid = uuid.uuid1()  # 根据 时间戳生成 uuid , 保证全球唯一
 
         return get_timestamp_uuid
 
-    # 替换为绝对路径之后，解析出来a.href
-    def uploadToserver(self, file_href, item_id):
-        category = os.path.splitext(file_href)[1]
-        # 上传至文件服务器
-        headers = {}
-        retData = {'state': False, 'type_id': 7, 'item_id': item_id, 'group_name': 'group1', 'path': '',
-                   'full_path': '',
-                   'category': category, 'file_size': '', 'status': 1, 'create_by': 'XueLingKun',
-                   'create_time': '', 'page_size': '', 'content': ''}
-        headers['User-Agent'] = self.getRandomUserAgent()
-
-        resp_content = ''
-        for i in range(0, 3):
-            try:
-                resp_content = requests.get(file_href, headers=headers, verify=False, timeout=20).content
-                break
-            except:
-                time.sleep(3)
-                continue
-        if resp_content:
-            pass
-        else:
-            return retData
-        # page_size = 0
-        # if category == '.doc' or category == '.docx':
-        #     # page_size = self.doc_page(file_href)
-        #     return retData
-        # if category == '.pdf' or category == '.PDF':
-        #     page_size = self.pdf_page(resp_content)
-        for i in range(0, 3):
-            try:
-                result = client.upload_by_buffer(resp_content, file_ext_name=category.replace('.', ''))
-                self.getLogger().info('-------文件上传成功------')
-                break
-            except:
-                time.sleep(3)
-                continue
-        # if page_size>0:
-        #     pass
-        # else:
-        #     self.getLogger().info(f'======解析失败=====')
-        #     return retData
-        time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
-        retData['state'] = True
-        retData['path'] = bytes.decode(result['Remote file_id']).replace('group1', '')
-        retData['full_path'] = bytes.decode(result['Remote file_id'])
-        retData['file_size'] = result['Uploaded size']
-        retData['create_time'] = time_now
-        # retData['page_size'] = page_size
-        return retData
-
     def secrchATT(self, item_id, retData, type_id, order_by):
         sel_sql = '''select id from clb_sys_attachment where item_id = %s and path = %s and type_id=%s and order_by=%s '''
         self.cursor_.execute(sel_sql, (item_id, retData['path'], type_id, order_by))
@@ -549,20 +466,21 @@ class BaseCore:
         page_size = retData['page_size']
         create_time = retData['create_time']
         order_by = num
+        object_key = full_path.split('https://zzsn.obs.cn-north-1.myhuaweicloud.com/')[1]
 
         Upsql = '''insert into clb_sys_attachment(name,type_id,item_id,group_name,path,full_path,category,file_size,order_by,status,create_by,create_time,object_key,bucket_name,publish_time) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''
 
         values = (
             file_name, type_id, item_id, group_name, path, full_path, category, file_size, order_by,
             status, create_by,
-            create_time, full_path.split('https://zzsn.obs.cn-north-1.myhuaweicloud.com/')[1], 'zzsn', publishDate)
+            create_time, object_key, 'zzsn', publishDate)
 
         self.cursor_.execute(Upsql, values)  # 插入
         self.cnx_.commit()  # 提交
         self.getLogger().info("更新完成:{}".format(Upsql))
         selects = self.secrchATT(item_id, retData, type_id, order_by)
         id = selects[0]
-        return id, full_path
+        return id, object_key
 
     # 获取文件大小
     def convert_size(self, size_bytes):
@@ -600,18 +518,17 @@ class BaseCore:
             except:
                 time.sleep(3)
                 continue
-        else:
-            try:
-                time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
-                retData['state'] = True
-                retData['path'] = result['body']['objectUrl'].split('.com')[1]
-                retData['full_path'] = result['body']['objectUrl']
-                retData['file_size'] = self.convert_size(file_size)
-                retData['create_time'] = time_now
-            except Exception as e:
-                print(f'error:{e}')
-                return retData
 
+        try:
+            time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+            retData['state'] = True
+            retData['path'] = result['body']['objectUrl'].split('.com')[1]
+            retData['full_path'] = result['body']['objectUrl']
+            retData['file_size'] = self.convert_size(file_size)
+            retData['create_time'] = time_now
+            return retData
+        except Exception as e:
+            print(f'error:{e}')
             return retData
 
 
@@ -626,3 +543,5 @@ class BaseCore:
 
 
 
+
+

comData/policylaw/deletedup.py

+# 根据链接删除链接重复的数据
+import json
+import threading
+import time
+import uuid
+
+import redis
+import requests
+from retry import retry
+from elasticsearch import Elasticsearch
+from base import BaseCore
+from obs import ObsClient
+import fitz
+from urllib.parse import unquote
+baseCore = BaseCore.BaseCore()
+log = baseCore.getLogger()
+baseCore = BaseCore.BaseCore()
+
+cnx_ = baseCore.cnx_
+cursor_ = cnx_.cursor()
+
+lock = threading.Lock()
+
+pool = redis.ConnectionPool(host="114.115.236.206", port=6379, password='clbzzsn', db=6)
+class EsMethod(object):
+
+    def __init__(self):
+        # 创建Elasticsearch对象，并提供账号信息
+        self.es = Elasticsearch(['http://114.116.19.92:9700'], http_auth=('elastic', 'zzsn9988'), timeout=300)
+        self.index_name = 'policy'
+
+    def queryatt(self,index_name,pnum):
+       body = {
+                "size":0,
+                "aggs":{
+                    "duplicate_titles":{
+                        "terms":{
+                            "field":"sourceAddress.keyword",
+                            "min_doc_count":2,
+                            "size":1000
+                        },
+                        "aggs":{
+                            "duplicate_docs":{
+                                "top_hits":{
+                                    "_source":{
+                                        "includes":["id","title","subjectId","sourceAddress","createDate"]
+                                    },
+                                "size":10
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+       # filter_path = ['hits.aggregations.duplicate_titles',
+       #                'hits.total.value',
+       #                'hits.hits._source.title',
+       #                'hits.hits._source.sourceAddress',
+       #                'hits.hits._source.createDate',
+       #                ]  # 字段2
+       result = self.es.search(index=index_name
+                               , doc_type='_doc'
+                               # , filter_path=filter_path
+                               , body=body)
+       # log.info(result)
+       return result
+
+def main(page, p, esMethod):
+
+    result = esMethod.queryatt(index_name=esMethod.index_name, pnum=p)
+    total = result['hits']['total']['value']
+
+    if total == 0:
+        log.info('++++已没有数据+++++')
+        return
+    documents = result["aggregations"]["duplicate_titles"]["buckets"]
+    for bucket in documents:
+        info_list = bucket["duplicate_docs"]["hits"]["hits"]
+        for info in info_list:
+
+            att_id_list = info['_source']['attachmentIds']
+            if len(att_id_list)==0:
+                unique_document_ids = info["_id"]
+                log.info(f'==={unique_document_ids}===')
+
+    # # 删除重复的文档
+    # for doc_id in unique_document_ids:
+    #     esMethod.delete(index="policy", id=doc_id)
+
+
+
+def run_threads(num_threads,esMethod,j):
+    threads = []
+
+    for i in range(num_threads):
+        page = j + i + 1
+        p = j + i * 200
+        thread = threading.Thread(target=main, args=(page, p, esMethod))
+
+        threads.append(thread)
+        thread.start()
+
+    for thread in threads:
+        thread.join()
+
+
+
+if __name__ == "__main__":
+    j = 0
+    for i in range(5):
+        esMethod = EsMethod()
+        # result = esMethod.queryatt(index_name=esMethod.index_name, pnum=p)
+        # total = result['hits']['total']['value']
+        # if total == 0:
+        #     log.info('++++已没有数据+++++')
+        #     break
+        start = time.time()
+        num_threads = 5
+        run_threads(num_threads, esMethod, j)
+        j += 1000
+
+        log.info(f'5线程 每个处理200条数据 总耗时{time.time() - start}秒')
\ No newline at end of file

comData/policylaw/policy.py

-# _*_ coding:utf-8 _*_
+#!/usr/bin/env python
+# coding=utf-8
 
 """数据全量跑一遍，不做判重逻辑"""
 import json
@@ -106,7 +107,7 @@ def save_data(dic_news):
 def sendKafka(dic_news):
     start_time = time.time()
     try:  # 114.116.116.241
-        producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
+        producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'], max_request_size=1024*1024*20)
         kafka_result = producer.send("policy",
                                      json.dumps(dic_news, ensure_ascii=False).encode('utf8'))
 
@@ -284,7 +285,7 @@ def get_content1():
                                 id_list.append(att_id)
 
                                 # todo:将返回的地址更新到soup
-                                file['href'] = full_path
+                                file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                     except:
                         log.error(f'{title}...{href}...获取内容失败')
                         continue
@@ -449,7 +450,7 @@ def get_content2():
                             id_list.append(att_id)
 
                             # todo:将返回的地址更新到soup
-                            file['href'] = full_path
+                            file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                 except:
                     log.error(f'{title}...{href}获取内容失败')
                     continue
@@ -563,7 +564,7 @@ def get_content3():
                 id_list.append(att_id)
 
                 # todo:将返回的地址更新到soup
-                file['href'] = full_path
+                file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
         time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
         # todo:传kafka字段
         dic_news = {
@@ -785,7 +786,7 @@ def bei_jing():
                     id_list.append(att_id)
 
                     # todo:将返回的地址更新到soup
-                    file['href'] = full_path
+                    file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
             # id_ = redefid(id_list)
             time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
@@ -1066,7 +1067,7 @@ def ji_lin():
                                 id_list.append(att_id)
                                 #
                                 # # todo:将返回的地址更新到soup
-                                li.find('a')['href'] = full_path
+                                li.find('a')['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                             else:
                                 continue
                 else:
@@ -1110,7 +1111,7 @@ def ji_lin():
                             id_list.append(att_id)
                             #
                             # # todo:将返回的地址更新到soup
-                            fu_jian_href['href'] = full_path
+                            fu_jian_href['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                         else:
                             continue
 
@@ -1265,7 +1266,7 @@ def shang_hai():
                             id_list.append(att_id)
 
                             # todo:将返回的地址更新到soup
-                            a['href'] = full_path
+                            a['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                         else:
                             continue
 
@@ -1558,7 +1559,7 @@ def fu_jian():
                                                                              pub_time)
                                     id_list.append(att_id)
                                     # 将文件服务器的链接替换
-                                    fu_jian['href'] = full_path
+                                    fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                         except:
                             pub_source = ''
@@ -1796,7 +1797,7 @@ def guang_dong():
                             att_id, full_path = baseCore.tableUpdate(retData, '广东省国资委', file_name, num, pub_time)
                             id_list.append(att_id)
                             # 将文件服务器的链接替换
-                            fu_jian['href'] = full_path
+                            fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                     time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                     # todo:传kafka字段
@@ -1930,7 +1931,7 @@ def hai_nan():
                                                                              pub_time)
                                     id_list.append(att_id)
                                     # 将文件服务器的链接替换
-                                    fu_jian['href'] = full_path
+                                    fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                         except:
                             try:
                                 # print(href)
@@ -1979,7 +1980,7 @@ def hai_nan():
                                             att_id, full_path = baseCore.tableUpdate(retData, '海南省国资委', file_name, num,
                                                                                      pub_time)
                                             id_list.append(att_id)
-                                            fu_jian['href'] = full_path
+                                            fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                                 except:
                                     continue
 
@@ -2219,7 +2220,7 @@ def hai_nan():
                             att_id, full_path = baseCore.tableUpdate(retData, '海南省国资委', file_name, num, pub_time)
                             id_list.append(att_id)
                             # todo:将返回的地址更新到soup
-                            fu_jian['href'] = full_path
+                            fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                             # print(f'附件：{fu_jian_href}')
                     time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                     # todo:传kafka字段
@@ -2341,7 +2342,7 @@ def hai_nan():
                                     continue
                                 att_id, full_path = baseCore.tableUpdate(retData, '海南省国资委', file_name, num, pub_time)
                                 id_list.append(att_id)
-                                fu_jian['href'] = full_path
+                                fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                                 # print(f'----附件：{fu_jian_href}')
                     else:
                         pass
@@ -2629,7 +2630,7 @@ def si_chuan():
                                 continue
                             att_id, full_path = baseCore.tableUpdate(retData, '四川省国资委', file_name, num, pub_time)
                             id_list.append(att_id)
-                            fu_jian['href'] = full_path
+                            fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                             # fu_jian_href_list.append(fu_jian_href)
                     time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
@@ -2766,7 +2767,7 @@ def guang_xi():
                                                                          pub_time)
                                 id_list.append(att_id)
                                 # 将附件链接替换
-                                fu_jian['href'] = full_path
+                                fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                         time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                         # todo:传kafka字段
@@ -2884,7 +2885,7 @@ def gui_zhou():
                             att_id, full_path = baseCore.tableUpdate(retData, '贵州省国资委', file_name, num, pub_time)
                             id_list.append(att_id)
                             # 将附件链接替换
-                            fu_jian['href'] = full_path
+                            fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                     time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                     # todo:传kafka字段
@@ -2999,7 +3000,7 @@ def yun_nan():
                                         att_id, full_path = baseCore.tableUpdate(retData, '云南省国资委', file_name, num, '')
                                         id_list.append(att_id)
                                         # 将附件链接替换
-                                        fu_jian['href'] = full_path
+                                        fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                                     except:
                                         continue
                             href_resp.close()
@@ -3121,7 +3122,7 @@ def yun_nan():
                                                                                  pub_time)
                                         id_list.append(att_id)
                                         # 将附件链接替换
-                                        fu_jian['href'] = full_path
+                                        fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                                     except:
                                         continue
                             res_.close()
@@ -3270,7 +3271,7 @@ def chong_qing():
                                                                              pub_time)
                                     id_list.append(att_id)
                                     # 将附件链接替换
-                                    fu_jian['href'] = full_path
+                                    fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                                 except:
                                     continue
                         time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
@@ -3399,7 +3400,7 @@ def tian_jin():
                                 att_id, full_path = baseCore.tableUpdate(retData, '天津市国资委', file_name, num, publishDate)
                                 id_list.append(att_id)
                                 # todo:将返回的地址更新到soup
-                                file['href'] = full_path
+                                file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                         # id_ = redefid(id_list)
                         contentWithTag = str(soup.prettify())
                         if len(contentWithTag) < 1:
@@ -3530,7 +3531,7 @@ def tian_jin():
                                 att_id, full_path = baseCore.tableUpdate(retData, '天津市国资委', file_name, num, publishDate)
                                 id_list.append(att_id)
                                 # todo:将返回的地址更新到soup
-                                file['href'] = full_path
+                                file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                         # id_ = redefid(id_list)
                         contentWithTag = str(soup.prettify())
                         if len(contentWithTag) < 1:
@@ -3665,7 +3666,7 @@ def tian_jin():
                                 att_id, full_path = baseCore.tableUpdate(retData, '天津市国资委', file_name, num, publishDate)
                                 id_list.append(att_id)
                                 # todo:将返回的地址更新到soup
-                                file['href'] = full_path
+                                file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                         # id_ = redefid(id_list)
                         contentWithTag = str(soup.prettify())
                         if len(contentWithTag) < 1:
@@ -3777,7 +3778,7 @@ def xin_jiang():
                                                                          publishDate)
                                 id_list.append(att_id)
                                 # todo:将返回的地址更新到soup
-                                file['href'] = full_path
+                                file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                         # id_ = redefid(id_list)
                         contentWithTag = str(soup.prettify())
                         if len(contentWithTag) < 1:
@@ -3885,7 +3886,7 @@ def xin_jiang():
                                                                          publishDate)
                                 id_list.append(att_id)
                                 # todo:将返回的地址更新到soup
-                                file['href'] = full_path
+                                file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                         # id_ = redefid(id_list)
                         contentWithTag = str(soup.prettify())
                         if len(contentWithTag) < 1:
@@ -4021,7 +4022,7 @@ def shan_xi():
                             att_id, full_path = baseCore.tableUpdate(retData, '山西省国资委', file_name, num, publishDate)
                             id_list.append(att_id)
                             # todo:将返回的地址更新到soup
-                            file['href'] = full_path
+                            file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                     # id_ = redefid(id_list)
                     contentWithTag = str(soup.prettify())
                     if len(contentWithTag) < 1:
@@ -4142,7 +4143,7 @@ def liao_ning():
                             att_id, full_path = baseCore.tableUpdate(retData, '辽宁省国资委', file_name, num, publishDate)
                             id_list.append(att_id)
                             # todo:将返回的地址更新到soup
-                            file['href'] = full_path
+                            file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                     # id_ = redefid(id_list)
                     contentWithTag = str(soup.prettify())
                     if len(contentWithTag) < 1:
@@ -4256,7 +4257,7 @@ def hei_long_jiang():
                                 att_id, full_path = baseCore.tableUpdate(retData, '江苏省国资委', file_name, num, publishDate)
                                 id_list.append(att_id)
                                 # todo:将返回的地址更新到soup
-                                file['href'] = full_path
+                                file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                         contentWithTag = str(soup.prettify())
                         content = soup.text
@@ -4373,7 +4374,7 @@ def jiang_su():
                             att_id, full_path = baseCore.tableUpdate(retData, '江苏省国资委', file_name, num, publishDate)
                             id_list.append(att_id)
                             # todo:将返回的地址更新到soup
-                            file['href'] = full_path
+                            file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                     contentWithTag = str(soup.prettify())
                     content = soup.text
@@ -4486,7 +4487,7 @@ def an_hui():
                                 att_id, full_path = baseCore.tableUpdate(retData, '安徽省国资委', file_name, num, publishDate)
                                 id_list.append(att_id)
                                 # todo:将返回的地址更新到soup
-                                file['href'] = full_path
+                                file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                         contentWithTag = str(soup.prettify())
                         content = soup.text
@@ -4590,7 +4591,7 @@ def an_hui():
                                 att_id, full_path = baseCore.tableUpdate(retData, '安徽省国资委', file_name, num, publishDate)
                                 id_list.append(att_id)
                                 # todo:将返回的地址更新到soup
-                                file['href'] = full_path
+                                file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                         contentWithTag = str(soup.prettify())
                         content = soup.text
@@ -4726,7 +4727,7 @@ def jiang_xi():
                             att_id, full_path = baseCore.tableUpdate(retData, '江西省国资委', file_name, num, writtenDate)
                             id_list.append(att_id)
                             # todo:将返回的地址更新到soup
-                            file['href'] = full_path
+                            file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                     contentWithTag = str(soup.prettify())
                     content = soup.text
@@ -4831,7 +4832,7 @@ def he_nan():
                         att_id, full_path = baseCore.tableUpdate(retData, '河南省国资委', file_name, num, publishDate)
                         id_list.append(att_id)
                         # todo:将返回的地址更新到soup
-                        file['href'] = full_path
+                        file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                 contentWithTag = str(soup.prettify())
                 content = soup.text
@@ -4949,7 +4950,7 @@ def hu_nan():
                             att_id, full_path = baseCore.tableUpdate(retData, '湖南省国资委', file_name, num, publishDate)
                             id_list.append(att_id)
                             # todo:将返回的地址更新到soup
-                            file['href'] = full_path
+                            file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                     contentWithTag = str(soup.prettify())
                     content = soup.text
@@ -5086,7 +5087,7 @@ def gan_su():
                             att_id, full_path = baseCore.tableUpdate(retData, '甘肃省国资委', file_name, num, publishDate)
                             id_list.append(att_id)
                             # todo:将返回的地址更新到soup
-                            file['href'] = full_path
+                            file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                     # id_ = redefid(id_list)
                     contentWithTag = str(soup.prettify())
                     content = soup.text
@@ -5243,7 +5244,7 @@ def gan_su():
                             att_id, full_path = baseCore.tableUpdate(retData, '甘肃省国资委', file_name, num, publishDate)
                             id_list.append(att_id)
                             # todo:将返回的地址更新到soup
-                            file['href'] = full_path
+                            file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                     contentWithTag = str(soup.prettify())
                     content = soup.text
@@ -5414,7 +5415,7 @@ def gan_su():
                             att_id, full_path = baseCore.tableUpdate(retData, '甘肃省国资委', file_name, num, publishDate)
                             id_list.append(att_id)
                             # todo:将返回的地址更新到soup
-                            file['href'] = full_path
+                            file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
 
                     contentWithTag = str(soup.prettify())
                     content = soup.text
@@ -5528,7 +5529,7 @@ def ning_xia():
                             att_id, full_path = baseCore.tableUpdate(retData, '宁夏回族自治区国资委', file_name, num, publishDate)
                             id_list.append(att_id)
                             # todo:将返回的地址更新到soup
-                            file['href'] = full_path
+                            file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                     # id_ = redefid(id_list)
                     contentWithTag = str(soup.prettify())
                     content = soup.text
@@ -5597,9 +5598,9 @@ def shanxi():
             else:
                 href = 'https://sxgz.shaanxi.gov.cn/' + href
             is_href = db_storage.find_one({'网址': href})
-            if is_href:
-                num += 1
-                continue
+            # if is_href:
+            #     num += 1
+            #     continue
             try:
                 res_href = requests.get(url=href, headers=headers)
                 res_href.encoding = res_href.apparent_encoding
@@ -5639,7 +5640,7 @@ def shanxi():
                         att_id, full_path = baseCore.tableUpdate(retData, '陕西省国资委', file_name, num, publishDate)
                         id_list.append(att_id)
                         # todo:将返回的地址更新到soup
-                        file['href'] = full_path
+                        file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                 # id_ = redefid(id_list)
                 contentWithTag = str(soup.prettify())
                 content = soup.text
@@ -5675,7 +5676,7 @@ def shanxi():
                     num += 1
                     count += 1
                 res_href.close()
-            except:
+            except Exception as e:
                 pass
         res.close()
     except:
@@ -5746,7 +5747,7 @@ def xi_zang():
                             att_id, full_path = baseCore.tableUpdate(retData, '西藏自治区国资委', file_name, num, publishDate)
                             id_list.append(att_id)
                             # todo:将返回的地址更新到soup
-                            file['href'] = full_path
+                            file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                     # id_ = redefid(id_list)
                     contentWithTag = str(soup.prettify())
                     # todo:替换完成之后，将附件上传至文件服务器
@@ -5853,7 +5854,7 @@ def qing_hai():
                                 att_id, full_path = baseCore.tableUpdate(retData, '青海省国资委', file_name, num, publishDate)
                                 id_list.append(att_id)
                                 # todo:将返回的地址更新到soup
-                                file['href'] = full_path
+                                file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                         # id_ = redefid(id_list)
                         contentWithTag = str(soup.prettify())
                         # todo:替换完成之后，将附件上传至文件服务器
@@ -5981,7 +5982,7 @@ def qing_hai():
                                                                                  publishDate)
                                         id_list.append(att_id)
                                         # todo:将返回的地址更新到soup
-                                        file['href'] = full_path
+                                        file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
                                 # id_ = redefid(id_list)
                                 contentWithTag = str(soup.prettify())
                                 # todo:替换完成之后，将附件上传至文件服务器
@@ -6076,7 +6077,7 @@ def he_bei():
                     att_id, full_path = baseCore.tableUpdate(retData, '河北省国资委', file_name, num, publishDate)
                     id_list.append(att_id)
                     # todo:将返回的地址更新到soup
-                    file['href'] = full_path
+                    file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
             # id_ = redefid(id_list)
             contentWithTag = str(soup.prettify())
             if len(contentWithTag) < 1:
@@ -6202,7 +6203,7 @@ def hu_bei():
                     att_id, full_path = baseCore.tableUpdate(retData, '湖北省国资委', file_name, num, publishDate)
                     id_list.append(att_id)
                     # todo:将返回的地址更新到soup
-                    file['href'] = full_path
+                    file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
             # id_ = redefid(id_list)
             contentWithTag = str(soup.prettify())
             if len(contentWithTag) < 1:
@@ -6248,41 +6249,41 @@ def hu_bei():
 
 
 if __name__ == '__main__':
-    get_content1()
-    get_content3()
-    bei_jing()
-    nei_meng_gu()
-    ji_lin()
-    shang_hai()
-    zhe_jiang()
-    fu_jian()
-    shan_dong()
-    guang_dong()
-    hai_nan()
-    si_chuan()
-    guang_xi()
-    gui_zhou()
-    yun_nan()
-    chong_qing()
-    tian_jin()
+    # get_content1()
+    # get_content3()
+    # bei_jing()
+    # nei_meng_gu()
+    # ji_lin()
+    # shang_hai()
+    # zhe_jiang()
+    # fu_jian()
+    # shan_dong()
+    # guang_dong()
+    # hai_nan()
+    # si_chuan()
+    # guang_xi()
+    # gui_zhou()
+    # yun_nan()
+    # chong_qing()
+    # tian_jin()
     xin_jiang()
-    shan_xi()
-    liao_ning()
-    hei_long_jiang()
-    jiang_su()
-    an_hui()
-    jiang_xi()
-    he_nan()
-    hu_nan()
-    gan_su()
-    ning_xia()
-    xi_zang()
-    shanxi()
-    qing_hai()
-    he_bei()
-    qing_hai()
-    get_content2()
-    current_time = datetime.datetime.now()
-    midnight_time = current_time.replace(hour=0, minute=0, second=0, microsecond=0) + datetime.timedelta(days=1)
-    sleep_seconds = (midnight_time - current_time).total_seconds()
-    time.sleep(sleep_seconds)
\ No newline at end of file
+    # shan_xi()
+    # liao_ning()
+    # hei_long_jiang()
+    # jiang_su()
+    # an_hui()
+    # jiang_xi()
+    # he_nan()
+    # hu_nan()
+    # gan_su()
+    # ning_xia()
+    # xi_zang()
+    # shanxi()
+    # qing_hai()
+    # he_bei()
+    # qing_hai()
+    # get_content2()
+    # current_time = datetime.datetime.now()
+    # midnight_time = current_time.replace(hour=0, minute=0, second=0, microsecond=0) + datetime.timedelta(days=1)
+    # sleep_seconds = (midnight_time - current_time).total_seconds()
+    # time.sleep(sleep_seconds)
\ No newline at end of file