Skip to content

Z
zzsn_spider
提交 783b7172 作者: 薛凌堃
操作

11.29

上级 fa46345c
正在显示 包含 89 行增加 和 40 行删除
REITs专题数据/LawRules-shenzhen.py
import os
import os
......@@ -2,18 +2,23 @@ import os
import time
from urllib.parse import urljoin
import numpy as np
import pandas as pd
import requests
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from base import BaseCore
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import BaseCore
baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()
from reits import Policy
policy = Policy()
topic = 'policy'
webname = '深圳证券交易所'
headers = {
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Encoding': 'gzip, deflate',
......@@ -57,9 +62,8 @@ def getFjContent(url):
return content
def getContent(url, publishDate, num):
fjhref_list = ''
fjtitle_list = ''
def getContent(url, publishDate, num, id_list):
num += 1
ip = baseCore.get_proxy()
req = requests.get(url, headers=headers, proxies=ip)
req.encoding = req.apparent_encoding
......@@ -67,27 +71,23 @@ def getContent(url, publishDate, num):
soup = paserUrl(soup, 'http://www.szse.cn/')
contentWithTag = soup.find('div', class_='des-content')
a_list = contentWithTag.find_all('a')
num_ = 1
for a in a_list:
fj_href = a.get('href')
if not fj_href:
continue
fjhref_list += fj_href + '\n'
fj_title = a.text.lstrip().strip()
category = os.path.splitext(fj_href)[1]
if category not in fj_title:
fj_title = fj_title + category
fj_title = f'{num}-{publishDate}-{fj_title}'
fjcontent = getFjContent(fj_href)
file = f'./相关政策/深圳证券交易所/政策文件/{fj_title}'
if os.path.exists(file):
fj_title = fj_title.replace(category,f'-{num_}{category}')
num_ += 1
file = f'./相关政策/深圳证券交易所/政策文件/{fj_title}'
fjtitle_list += fj_title + '\n'
with open(file, 'wb') as f:
f.write(fjcontent)
log.info(f'{fj_title}===附件下载成功')
# 上传附件至obs
att_id,full_path = policy.attuributefile(fj_title, fj_href, num, publishDate)
if att_id:
id_list.append(att_id)
a['href'] = full_path
try:
scripts = contentWithTag.find_all('script')
for script in scripts:
......@@ -102,14 +102,14 @@ def getContent(url, publishDate, num):
pass
pub_hao = contentWithTag.find('p').text.lstrip().strip()
content = contentWithTag.text.lstrip().strip()
return pub_hao, content, fjtitle_list, fjhref_list
return pub_hao, content, id_list,contentWithTag
def doJob():
if not os.path.exists('./相关政策/深圳证券交易所/政策文件'):
os.makedirs('./相关政策/深圳证券交易所/政策文件')
# if not os.path.exists('./相关政策/深圳证券交易所/政策文件'):
# os.makedirs('./相关政策/深圳证券交易所/政策文件')
url = 'http://www.szse.cn/lawrules/search/index.html?rulekeyword=REITs&channelCode=%5B%22rules%22,%22csrcrules%22,%22szseBussrules%22,%22memorandumServicedirect%22,%22publicadvice%22,%22lawruleSearch%22%5D&range=content&searchtype=0'
driver = baseCore.buildDriver()
driver = policy.createDriver()
driver.get(url)
WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.CLASS_NAME, 'article-item'))
......@@ -118,35 +118,84 @@ def doJob():
num = 0
data_list = []
for div in div_list:
id_list = []
title = div.find_element(By.TAG_NAME, 'a').text.lstrip().strip()
href = div.find_element(By.TAG_NAME, 'a').get_attribute('href')
publishDate = div.find_element(By.CLASS_NAME, 'pull-right').text.lstrip().strip()
writtenDate = publishDate
origin = '深圳证券交易所'
organ = origin
# 文章就为pdf
# 根据链接判重
is_member = baseCore.r.sismember('REITs::' + webname, href)
if is_member:
continue
if '.pdf' in href:
content = ''
summary = ''
fjtitle_list = title + '.pdf'
fjhref_list = href
pub_hao = ''
fjcontent = getFjContent(href)
file = f'./相关政策/深圳证券交易所/政策文件/{title}.pdf'
with open(file, 'wb') as f:
f.write(fjcontent)
log.info(f'{title}===附件下载成功')
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
dic_info = {
'attachmentIds': id_list,
'author': '',
'content': fjcontent,
'contentWithTag': '',
'deleteFlag': 0,
'id': '',
'title': title,
'publishDate': publishDate,
'origin': origin,
'sourceAddress': href,
'writtenDate': writtenDate,
'organ': organ,
'topicClassification': '',
'issuedNumber': pub_hao,
'summary': summary,
'createDate': time_now,
'sid': '1729029275400646658',
}
try:
baseCore.sendkafka(dic_info, topic)
baseCore.r.sadd('REITs::' + webname, href)
log.info(f'采集成功--{title}--{href}')
except:
for att_id in id_list:
baseCore.deliteATT(att_id)
else:
summary = div.find_element(By.CLASS_NAME, 'item-content').text.lstrip().strip()
pub_hao, content, fjtitle_list, fjhref_list = getContent(href, publishDate, num)
data = [num, title, publishDate, origin, href, writtenDate, organ, pub_hao, summary, content, fjtitle_list,
fjhref_list]
data_list.append(data)
log.info(f'{title}===采集成功')
num += 1
pub_hao, content, id_list, contentWithTag = getContent(href, publishDate, num, id_list)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
dic_info = {
'attachmentIds': id_list,
'author': '',
'content': content,
'contentWithTag': str(contentWithTag),
'deleteFlag': 0,
'id': '',
'title': title,
'publishDate': publishDate,
'origin': origin,
'sourceAddress': href,
'writtenDate': writtenDate,
'organ': organ,
'topicClassification': '',
'issuedNumber': pub_hao,
'summary': summary,
'createDate': time_now,
'sid': '1729029275400646658',
}
try:
baseCore.sendkafka(dic_info, topic)
baseCore.r.sadd('REITs::' + webname, href)
log.info(f'采集成功--{title}--{href}')
except:
for att_id in id_list:
baseCore.deliteATT(att_id)
driver.close()
df = pd.DataFrame(np.array(data_list))
df.columns = ['序号', '标题', '发布时间', '来源', '原文链接', '发文时间', '发文机构', '发文字号', '摘要', '正文', '附件名称', '附件连接']
df.to_excel('./相关政策/深圳证券交易所/深圳证券交易所政策文件.xlsx', index=False)
# df = pd.DataFrame(np.array(data_list))
# df.columns = ['序号', '标题', '发布时间', '来源', '原文链接', '发文时间', '发文机构', '发文字号', '摘要', '正文', '附件名称', '附件连接']
# df.to_excel('./相关政策/深圳证券交易所/深圳证券交易所政策文件.xlsx', index=False)
if __name__ == '__main__':
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论