11.29

REITs专题数据/LawRules-shenzhen.py

-import os
+import os
@@ -2,18 +2,23 @@ import os
 import time
 from urllib.parse import urljoin
 
-import numpy as np
-import pandas as pd
 import requests
 from bs4 import BeautifulSoup
 from selenium.webdriver.common.by import By
 
-from base import BaseCore
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 
+import BaseCore
 baseCore = BaseCore.BaseCore()
 log = baseCore.getLogger()
+
+from reits import Policy
+policy = Policy()
+
+
+topic = 'policy'
+webname = '深圳证券交易所'
 headers = {
     'Accept': 'application/json, text/javascript, */*; q=0.01',
     'Accept-Encoding': 'gzip, deflate',
@@ -57,9 +62,8 @@ def getFjContent(url):
     return content
 
 
-def getContent(url, publishDate, num):
-    fjhref_list = ''
-    fjtitle_list = ''
+def getContent(url, publishDate, num, id_list):
+    num += 1
     ip = baseCore.get_proxy()
     req = requests.get(url, headers=headers, proxies=ip)
     req.encoding = req.apparent_encoding
@@ -67,27 +71,23 @@ def getContent(url, publishDate, num):
     soup = paserUrl(soup, 'http://www.szse.cn/')
     contentWithTag = soup.find('div', class_='des-content')
     a_list = contentWithTag.find_all('a')
-    num_ = 1
+
     for a in a_list:
         fj_href = a.get('href')
         if not fj_href:
             continue
-        fjhref_list += fj_href + '\n'
+
         fj_title = a.text.lstrip().strip()
         category = os.path.splitext(fj_href)[1]
         if category not in fj_title:
             fj_title = fj_title + category
-        fj_title = f'{num}-{publishDate}-{fj_title}'
-        fjcontent = getFjContent(fj_href)
-        file = f'./相关政策/深圳证券交易所/政策文件/{fj_title}'
-        if os.path.exists(file):
-            fj_title = fj_title.replace(category,f'-{num_}{category}')
-            num_ += 1
-        file = f'./相关政策/深圳证券交易所/政策文件/{fj_title}'
-        fjtitle_list += fj_title + '\n'
-        with open(file, 'wb') as f:
-            f.write(fjcontent)
-        log.info(f'{fj_title}===附件下载成功')
+
+        # 上传附件至obs
+        att_id,full_path = policy.attuributefile(fj_title, fj_href, num, publishDate)
+        if att_id:
+            id_list.append(att_id)
+            a['href'] = full_path
+
     try:
         scripts = contentWithTag.find_all('script')
         for script in scripts:
@@ -102,14 +102,14 @@ def getContent(url, publishDate, num):
         pass
     pub_hao = contentWithTag.find('p').text.lstrip().strip()
     content = contentWithTag.text.lstrip().strip()
-    return pub_hao, content, fjtitle_list, fjhref_list
+    return pub_hao, content, id_list,contentWithTag
 
 
 def doJob():
-    if not os.path.exists('./相关政策/深圳证券交易所/政策文件'):
-        os.makedirs('./相关政策/深圳证券交易所/政策文件')
+    # if not os.path.exists('./相关政策/深圳证券交易所/政策文件'):
+    #     os.makedirs('./相关政策/深圳证券交易所/政策文件')
     url = 'http://www.szse.cn/lawrules/search/index.html?rulekeyword=REITs&channelCode=%5B%22rules%22,%22csrcrules%22,%22szseBussrules%22,%22memorandumServicedirect%22,%22publicadvice%22,%22lawruleSearch%22%5D&range=content&searchtype=0'
-    driver = baseCore.buildDriver()
+    driver = policy.createDriver()
     driver.get(url)
     WebDriverWait(driver, 10).until(
         EC.presence_of_element_located((By.CLASS_NAME, 'article-item'))
@@ -118,35 +118,84 @@ def doJob():
     num = 0
     data_list = []
     for div in div_list:
+        id_list = []
         title = div.find_element(By.TAG_NAME, 'a').text.lstrip().strip()
         href = div.find_element(By.TAG_NAME, 'a').get_attribute('href')
         publishDate = div.find_element(By.CLASS_NAME, 'pull-right').text.lstrip().strip()
         writtenDate = publishDate
         origin = '深圳证券交易所'
         organ = origin
+        # 文章就为pdf
+        # 根据链接判重
+        is_member = baseCore.r.sismember('REITs::' + webname, href)
+        if is_member:
+            continue
         if '.pdf' in href:
-            content = ''
+
             summary = ''
-            fjtitle_list = title + '.pdf'
-            fjhref_list = href
             pub_hao = ''
             fjcontent = getFjContent(href)
-            file = f'./相关政策/深圳证券交易所/政策文件/{title}.pdf'
-            with open(file, 'wb') as f:
-                f.write(fjcontent)
-            log.info(f'{title}===附件下载成功')
+            time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+            dic_info = {
+                'attachmentIds': id_list,
+                'author': '',
+                'content': fjcontent,
+                'contentWithTag': '',
+                'deleteFlag': 0,
+                'id': '',
+                'title': title,
+                'publishDate': publishDate,
+                'origin': origin,
+                'sourceAddress': href,
+                'writtenDate': writtenDate,
+                'organ': organ,
+                'topicClassification': '',
+                'issuedNumber': pub_hao,
+                'summary': summary,
+                'createDate': time_now,
+                'sid': '1729029275400646658',
+            }
+            try:
+                baseCore.sendkafka(dic_info, topic)
+                baseCore.r.sadd('REITs::' + webname, href)
+                log.info(f'采集成功--{title}--{href}')
+            except:
+                for att_id in id_list:
+                    baseCore.deliteATT(att_id)
         else:
             summary = div.find_element(By.CLASS_NAME, 'item-content').text.lstrip().strip()
-            pub_hao, content, fjtitle_list, fjhref_list = getContent(href, publishDate, num)
-        data = [num, title, publishDate, origin, href, writtenDate, organ, pub_hao, summary, content, fjtitle_list,
-                fjhref_list]
-        data_list.append(data)
-        log.info(f'{title}===采集成功')
-        num += 1
+            pub_hao, content, id_list, contentWithTag = getContent(href, publishDate, num, id_list)
+            time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+            dic_info = {
+                'attachmentIds': id_list,
+                'author': '',
+                'content': content,
+                'contentWithTag': str(contentWithTag),
+                'deleteFlag': 0,
+                'id': '',
+                'title': title,
+                'publishDate': publishDate,
+                'origin': origin,
+                'sourceAddress': href,
+                'writtenDate': writtenDate,
+                'organ': organ,
+                'topicClassification': '',
+                'issuedNumber': pub_hao,
+                'summary': summary,
+                'createDate': time_now,
+                'sid': '1729029275400646658',
+            }
+            try:
+                baseCore.sendkafka(dic_info, topic)
+                baseCore.r.sadd('REITs::' + webname, href)
+                log.info(f'采集成功--{title}--{href}')
+            except:
+                for att_id in id_list:
+                    baseCore.deliteATT(att_id)
     driver.close()
-    df = pd.DataFrame(np.array(data_list))
-    df.columns = ['序号', '标题', '发布时间', '来源', '原文链接', '发文时间', '发文机构', '发文字号', '摘要', '正文', '附件名称', '附件连接']
-    df.to_excel('./相关政策/深圳证券交易所/深圳证券交易所政策文件.xlsx', index=False)
+    # df = pd.DataFrame(np.array(data_list))
+    # df.columns = ['序号', '标题', '发布时间', '来源', '原文链接', '发文时间', '发文机构', '发文字号', '摘要', '正文', '附件名称', '附件连接']
+    # df.to_excel('./相关政策/深圳证券交易所/深圳证券交易所政策文件.xlsx', index=False)
 
 
 if __name__ == '__main__':