Skip to content

Z
zzsn_spider
提交 3ad4b1e5 作者: 薛凌堃
操作

11.29

上级 5d788bc9
正在显示 包含 736 行增加 和 452 行删除
REITs专题数据/reits.py
import os import os
...@@ -234,7 +234,8 @@ def reform(): ...@@ -234,7 +234,8 @@ def reform():
try: try:
publishDate = newssoup.find('div',class_="article_con article_con_title").find_all('span')[-1].text publishDate = newssoup.find('div',class_="article_con article_con_title").find_all('span')[-1].text
except: except:
publishDate = '' publishDate = None
if publishDate:
pattern = r"\d{4}年\d{1,2}月\d{1,2}日" pattern = r"\d{4}年\d{1,2}月\d{1,2}日"
match = re.match(pattern, publishDate) match = re.match(pattern, publishDate)
if match: if match:
...@@ -242,7 +243,7 @@ def reform(): ...@@ -242,7 +243,7 @@ def reform():
publishDate = date1.strftime("%Y-%m-%d") publishDate = date1.strftime("%Y-%m-%d")
pass pass
else: else:
publishDate = '' publishDate = None
policy.deletep(contentWithTag, 3, 'div', 'style', 'text-align: center;') policy.deletep(contentWithTag, 3, 'div', 'style', 'text-align: center;')
policy.deletek(contentWithTag) policy.deletek(contentWithTag)
...@@ -313,7 +314,7 @@ def reform(): ...@@ -313,7 +314,7 @@ def reform():
# baseCore.writerToExcel(DataList, file_path, sheet_name) # baseCore.writerToExcel(DataList, file_path, sheet_name)
except Exception as e: except Exception as e:
log.info(f"error！！！{newsUrl}") log.info(f"error！！！{newsUrl}")
log.info({e}) log.info(e)
log.info(f'====第{page}页====处理结束，已采集{num}条数据=================') log.info(f'====第{page}页====处理结束，已采集{num}条数据=================')
#证券期货 https://neris.csrc.gov.cn/falvfagui/multipleFindController/indexJsp #证券期货 https://neris.csrc.gov.cn/falvfagui/multipleFindController/indexJsp
...@@ -456,7 +457,7 @@ def zhengquanqihuo(): ...@@ -456,7 +457,7 @@ def zhengquanqihuo():
#深圳交易所 http://www.szse.cn/lawrules/index.html #深圳交易所 http://www.szse.cn/lawrules/index.html
#上海交易所 http://www.sse.com.cn/home/search/index.shtml?webswd=REITs #上海交易所 http://www.sse.com.cn/home/search/index.shtml?webswd=REITs
def sse(wb,file_path): def sse():
url = 'http://query.sse.com.cn/search/getESSearchDoc.do?page=0&limit=10&publishTimeEnd=&publishTimeStart=&orderByDirection=DESC&orderByKey=score&searchMode=fuzzy&spaceId=3&keyword=REITs&siteName=sse&keywordPosition=title%2Cpaper_content&channelId=10001&channelCode=8640%2C8641%2C8642%2C8643%2C8644%2C8645%2C8646%2C8647%2C8648%2C8649%2C8650%2C8651%2C8652%2C8653%2C8654%2C8655%2C8656%2C8657%2C8658%2C8659%2C8660%2C8661%2C8685%2C9348%2C12632%2C12768%2C12769%2C12770%2C12771%2C12772%2C12773%2C12774%2C12775%2C12776%2C12777%2C12778%2C12779%2C12780%2C12781%2C12782%2C12783%2C12784%2C12785%2C12786%2C12787%2C12788%2C12789%2C12790%2C12791%2C12792%2C12793%2C12794%2C12795%2C12796%2C12797%2C12798%2C12799%2C12800%2C12801%2C12802%2C12803%2C12804%2C12805%2C12806%2C12807%2C12808%2C12809%2C12810%2C12811%2C12812%2C13061%2C13282%2C13283%2C13284%2C13285%2C13286%2C13287%2C13288%2C13289%2C13294%2C13364%2C13365%2C13366%2C13367%2C14595%2C14596%2C14597%2C14598%2C14599%2C14600%2C14601%2C14602%2C14603%2C14604%2C14605%2C14606&trackId=50619067167713018335655119683810&_=1699508921761' url = 'http://query.sse.com.cn/search/getESSearchDoc.do?page=0&limit=10&publishTimeEnd=&publishTimeStart=&orderByDirection=DESC&orderByKey=score&searchMode=fuzzy&spaceId=3&keyword=REITs&siteName=sse&keywordPosition=title%2Cpaper_content&channelId=10001&channelCode=8640%2C8641%2C8642%2C8643%2C8644%2C8645%2C8646%2C8647%2C8648%2C8649%2C8650%2C8651%2C8652%2C8653%2C8654%2C8655%2C8656%2C8657%2C8658%2C8659%2C8660%2C8661%2C8685%2C9348%2C12632%2C12768%2C12769%2C12770%2C12771%2C12772%2C12773%2C12774%2C12775%2C12776%2C12777%2C12778%2C12779%2C12780%2C12781%2C12782%2C12783%2C12784%2C12785%2C12786%2C12787%2C12788%2C12789%2C12790%2C12791%2C12792%2C12793%2C12794%2C12795%2C12796%2C12797%2C12798%2C12799%2C12800%2C12801%2C12802%2C12803%2C12804%2C12805%2C12806%2C12807%2C12808%2C12809%2C12810%2C12811%2C12812%2C13061%2C13282%2C13283%2C13284%2C13285%2C13286%2C13287%2C13288%2C13289%2C13294%2C13364%2C13365%2C13366%2C13367%2C14595%2C14596%2C14597%2C14598%2C14599%2C14600%2C14601%2C14602%2C14603%2C14604%2C14605%2C14606&trackId=50619067167713018335655119683810&_=1699508921761'
headers = { headers = {
'Accept': '*/*', 'Accept': '*/*',
...@@ -470,7 +471,7 @@ def sse(wb,file_path): ...@@ -470,7 +471,7 @@ def sse(wb,file_path):
} }
result = policy.getrequest_json(headers,url) result = policy.getrequest_json(headers,url)
total_page = result['data']['totalPage'] total_page = result['data']['totalPage']
DataList = [] # DataList = []
num = 0 num = 0
webname = '上海证券交易所' webname = '上海证券交易所'
# path = 'data/上海交易所' # path = 'data/上海交易所'
...@@ -481,8 +482,9 @@ def sse(wb,file_path): ...@@ -481,8 +482,9 @@ def sse(wb,file_path):
data = policy.getrequest_json(headers, url_page) data = policy.getrequest_json(headers, url_page)
newslist = data['data']['knowledgeList'] newslist = data['data']['knowledgeList']
# print(newslist) # print(newslist)
for news in newslist[:1]: for news in newslist:
num += 1 num += 1
id_list = []
title = news['title'].replace("<em>",'').replace('</em>','') title = news['title'].replace("<em>",'').replace('</em>','')
publishDate = news['createTime'] publishDate = news['createTime']
# print(newsUrl) # print(newsUrl)
...@@ -506,9 +508,9 @@ def sse(wb,file_path): ...@@ -506,9 +508,9 @@ def sse(wb,file_path):
is_member = baseCore.r.sismember('REITs::' + webname, newsUrl) is_member = baseCore.r.sismember('REITs::' + webname, newsUrl)
if is_member: if is_member:
continue continue
try:
if '.pdf' in newsUrl: if '.pdf' in newsUrl:
# pass
content = '' content = ''
response = requests.get(newsUrl, timeout=20) response = requests.get(newsUrl, timeout=20)
with fitz.open(stream=response.content, filetype='pdf') as doc: with fitz.open(stream=response.content, filetype='pdf') as doc:
...@@ -517,27 +519,42 @@ def sse(wb,file_path): ...@@ -517,27 +519,42 @@ def sse(wb,file_path):
file_href = newsUrl file_href = newsUrl
file_name = title file_name = title
policy.attuributefile(title, newsUrl, num, publishDate) att_id, full_path = policy.attuributefile(title, newsUrl, num, publishDate)
if att_id:
id_list.append(att_id)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
dic_info = { dic_info = {
'序号': num, 'attachmentIds':id_list,
'标题': title, 'author': '',
'发布时间': publishDate, 'content': content,
'来源': source, 'contentWithTag': '',
'原文链接': newsUrl, 'deleteFlag': 0,
'发文时间': '', 'id': '',
'发文机构': '', 'title': title,
'发文字号': '', 'publishDate': publishDate,
'摘要': summary, 'origin': source,
'正文': content, 'sourceAddress': newsUrl,
'附件名称': fu_jian_name, 'writtenDate': None,
'附件链接': fu_jian_href, 'organ': '',
'topicClassification': '',
'issuedNumber': '',
'summary': summary,
'createDate': time_now,
'sid': '1729035244826374145',
} }
DataList.append(dic_info) # DataList.append(dic_info)
try:
baseCore.sendkafka(dic_info, topic)
baseCore.r.sadd('REITs::' + webname, newsUrl)
log.info(f'采集成功--{title}--{newsUrl}')
except:
for att_id in id_list:
baseCore.deliteATT(att_id)
else: else:
newssoup = policy.getrequest_soup(header, newsUrl) newssoup = policy.getrequest_soup(header, newsUrl)
# print(newssoup) # print(newssoup)
policy.paserUrl(newssoup, newsUrl)
content_ = newssoup.find('div', class_='allZoom') content_ = newssoup.find('div', class_='allZoom')
# print(content_) # print(content_)
# # 将链接替换为绝对路径 # # 将链接替换为绝对路径
...@@ -554,8 +571,7 @@ def sse(wb,file_path): ...@@ -554,8 +571,7 @@ def sse(wb,file_path):
content = contentWithTag.text content = contentWithTag.text
fujian_list = contentWithTag.find_all('a') fujian_list = contentWithTag.find_all('a')
fu_jian_name = ''
fu_jian_href = ''
for fujian in fujian_list: for fujian in fujian_list:
try: try:
file_href = fujian['href'] file_href = fujian['href']
...@@ -567,50 +583,56 @@ def sse(wb,file_path): ...@@ -567,50 +583,56 @@ def sse(wb,file_path):
pass pass
else: else:
file_name = file_name + category file_name = file_name + category
rename_file = f'{str(num)}_{publishDate[:10]}_{file_name}'.replace('\\','').replace('/','').replace('|','').replace('>','').replace('<','').replace('*','').replace('：','').replace('？','').replace('—','').replace('-','')
fu_jian_name += rename_file + '\n' att_id, full_path = policy.attuributefile(file_name, file_href, num, publishDate)
fu_jian_href += file_href + '\n' if att_id:
try: id_list.append(att_id)
policy.downloadfile(file_href, f'{path}/{rename_file}') fujian['href'] = full_path
except:
log.info(f'--{page}-{num}======{newsUrl}') contentWithTag_str = str(contentWithTag)
continue # print(contentWithTag_str)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
dic_info = { dic_info = {
'序号': num, 'attachmentIds':id_list,
'标题': title, 'author': '',
'发布时间': publishDate, 'content': content,
'来源': source, 'contentWithTag': contentWithTag_str,
'原文链接': newsUrl, 'deleteFlag': 0,
'发文时间': '', 'id': '',
'发文机构': '', 'title': title,
'发文字号': pubHao, 'publishDate': publishDate,
'摘要': summary, 'origin': source,
'正文': content, 'sourceAddress': newsUrl,
'附件名称': fu_jian_name, 'writtenDate': None,
'附件链接': fu_jian_href, 'organ': '',
'issuedNumber': pubHao,
'summary': summary,
'createDate': time_now,
'sid': '1729035244826374145'
} }
DataList.append(dic_info) try:
baseCore.sendkafka(dic_info, topic)
sheet_name = "上海交易所" baseCore.r.sadd('REITs::' + webname, newsUrl)
if sheet_name in wb.sheetnames: log.info(f'采集成功--{title}--{newsUrl}')
log.info(f"{sheet_name}工作表已存在！") except:
else: for att_id in id_list:
# 创建新工作表 baseCore.deliteATT(att_id)
wb.create_sheet(sheet_name) except Exception as e:
print(f"{sheet_name}新工作表创建完成！") log.info(f"error！！！{newsUrl}")
# 保存Excel文件 log.info(e)
wb.save(file_path) log.info(f'====第{page}页====处理结束，已采集{num}条数据=================')
baseCore.writerToExcel(DataList, file_path, sheet_name)
# 河北省人民政府 # 河北省人民政府
def hebei(): def hebei():
path = 'data/河北省人民政府' # path = 'data/河北省人民政府'
if not os.path.exists(path): # if not os.path.exists(path):
os.makedirs(path) # os.makedirs(path)
num = 0 num = 0
webname = '河北省人民政府'
url = "https://www.hebei.gov.cn/search/pcRender?pageId=b97a38833f7343cebc31dec44544f684" url = "https://www.hebei.gov.cn/search/pcRender?pageId=b97a38833f7343cebc31dec44544f684"
appNames = ['热点专题'] appNames = ['信息公开']
for appName in appNames: for appName in appNames:
payload = {'qAnd': ' ', payload = {'qAnd': ' ',
'qOr': ' ', 'qOr': ' ',
...@@ -661,7 +683,7 @@ def hebei(): ...@@ -661,7 +683,7 @@ def hebei():
soup_ = policy.requestPost_html(headers, url, payload) soup_ = policy.requestPost_html(headers, url, payload)
# 第一次请求获取页数 # 第一次请求获取页数
pages = int(soup_.find('span',class_='default-result-tolal-records').find('span').text) pages = int(soup_.find('span',class_='default-result-tolal-records').find('span').text)
DataList = [] # DataList = []
for page in range(1, pages+1): for page in range(1, pages+1):
payload_page = { payload_page = {
'qAnd': ' ', 'qAnd': ' ',
...@@ -692,15 +714,26 @@ def hebei(): ...@@ -692,15 +714,26 @@ def hebei():
list_news = soup.find_all('div',class_='szf-data-tpl1-item') list_news = soup.find_all('div',class_='szf-data-tpl1-item')
for news in list_news: for news in list_news:
num += 1 num += 1
id_list = []
title = news.find('h3').text title = news.find('h3').text
summary = news.find('div').find('p', class_='txtCon').text summary = news.find('div').find('p', class_='txtCon').text
publishDate = news.find('div').find('p', class_='dates').text.replace('发布日期：', '').replace('\n', '') publishDate_ = news.find('div').find('p', class_='dates').text.replace('发布日期：', '').replace('\n', '')
date1 = datetime.strptime(publishDate_, "%Y年%m月%d日")
publishDate = date1.strftime("%Y-%m-%d")
news_href = news.find('div').find('p', class_='txtCon').find('a')['href'] news_href = news.find('div').find('p', class_='txtCon').find('a')['href']
# news_href = 'http://info.hebei.gov.cn//hbszfxxgk/6898876/7026469/7026511/7026506/7033297/index.html' # news_href = 'http://info.hebei.gov.cn//hbszfxxgk/6898876/7026469/7026511/7026506/7033297/index.html'
try:
# 根据链接判重
is_member = baseCore.r.sismember('REITs::' + webname, news_href)
if is_member:
continue
news_req = requests.get(news_href, headers) news_req = requests.get(news_href, headers)
news_soup = BeautifulSoup(news_req.content, 'html.parser') news_soup = BeautifulSoup(news_req.content, 'html.parser')
writeDate = '' policy.paserUrl(news_soup, news_href)
writeDate = None
pub_hao = '' pub_hao = ''
source = '' source = ''
content = '' content = ''
...@@ -739,7 +772,9 @@ def hebei(): ...@@ -739,7 +772,9 @@ def hebei():
pattern = r"\d{4}年\d{1,2}月\d{1,2}日" pattern = r"\d{4}年\d{1,2}月\d{1,2}日"
match = re.search(pattern, writeDate_) match = re.search(pattern, writeDate_)
if match: if match:
writeDate = match.group(0) writeDate1 = match.group(0)
date2 = datetime.strptime(writeDate1, "%Y年%m月%d日")
writeDate = date2.strftime("%Y-%m-%d")
break break
else: else:
continue continue
...@@ -755,11 +790,9 @@ def hebei(): ...@@ -755,11 +790,9 @@ def hebei():
except: except:
pass pass
# 附件： # 附件：
fu_jian_name = ''
fu_jian_href = ''
try: try:
fujian_href = contentWithTag.find_all('a') fujian_href = contentWithTag.find_all('a')
policy.paserUrl(contentWithTag, news_href)
for file_href_ in fujian_href: for file_href_ in fujian_href:
file_href = file_href_['href'] file_href = file_href_['href']
file_name = file_href_.text file_name = file_href_.text
...@@ -768,54 +801,55 @@ def hebei(): ...@@ -768,54 +801,55 @@ def hebei():
pass pass
else: else:
file_name = file_name + category file_name = file_name + category
rename_file = f'{str(num)}_{publishDate}_{file_name}' att_id, full_path = policy.attuributefile(file_name, file_href, num, publishDate)
fu_jian_name += rename_file + '\n' if att_id:
fu_jian_href += file_href + '\n' id_list.append(att_id)
policy.downloadfile(file_href, f'{path}/{rename_file}') file_href_['href'] = full_path
contentWithTag_str = str(contentWithTag)
except Exception as e: except Exception as e:
pass contentWithTag_str = str(contentWithTag)
if content == '': if content == '':
continue continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
dic_info = { dic_info = {
'序号': num, 'attachmentIds':id_list,
'标题': title.replace('\n', ''), 'author': '',
'发布时间': publishDate, 'content': content,
'来源': source, 'contentWithTag': contentWithTag_str,
'原文链接': news_href, 'title': title.replace('\n', ''),
'发文时间': writeDate, 'publishDate': publishDate,
'发文机构': pub_origin, 'origin': source,
'发文字号': pub_hao, 'sourceAddress': news_href,
'摘要': summary.replace('\n', ''), 'writtenDate': writeDate,
'正文': content, 'organ': pub_origin,
'附件名称': fu_jian_name, 'issuedNumber': pub_hao,
'附件链接': fu_jian_href, 'summary': summary.replace('\n', ''),
'createDate': time_now,
'sid': '1729041576348274689',
} }
print(dic_info) # print(dic_info)
DataList.append(dic_info) try:
baseCore.sendkafka(dic_info, topic)
sheet_name = appName baseCore.r.sadd('REITs::' + webname, news_href)
if sheet_name in wb.sheetnames: log.info(f'采集成功--{title}--{news_href}')
log.info(f"{sheet_name}工作表已存在！") except:
else: for att_id in id_list:
# 创建新工作表 baseCore.deliteATT(att_id)
wb.create_sheet(sheet_name) except Exception as e:
print(f"{sheet_name}新工作表创建完成！") log.info(f"error！！！{news_href}")
# 保存Excel文件 log.info(e)
wb.save(file_path) log.info(f'====第{page}页====处理结束，已采集{num}条数据=================')
baseCore.writerToExcel(DataList, file_path, sheet_name)
break
# 贵州省人民政府 # 贵州省人民政府
def guizhou(): def guizhou():
url = "https://www.guizhou.gov.cn/irs/front/search" url = "https://www.guizhou.gov.cn/irs/front/search"
num = 0 num = 0
path = 'data/贵州省人民政府' # path = 'data/贵州省人民政府'
if not os.path.exists(path): # if not os.path.exists(path):
os.makedirs(path) # os.makedirs(path)
DataList = [] # DataList = []
webname = '贵州省人民政府'
payload = "{\"tenantId\":\"186\",\"configTenantId\":\"\",\"tenantIds\":\"\",\"searchWord\":\"REITs\",\"historySearchWords\":[\"REITs\"],\"dataTypeId\":\"965\",\"orderBy\":\"related\",\"searchBy\":\"all\",\"appendixType\":\"\",\"granularity\":\"ALL\",\"beginDateTime\":\"\",\"endDateTime\":\"\",\"isSearchForced\":0,\"filters\":[],\"pageNo\":1,\"pageSize\":9}" payload = "{\"tenantId\":\"186\",\"configTenantId\":\"\",\"tenantIds\":\"\",\"searchWord\":\"REITs\",\"historySearchWords\":[\"REITs\"],\"dataTypeId\":\"965\",\"orderBy\":\"related\",\"searchBy\":\"all\",\"appendixType\":\"\",\"granularity\":\"ALL\",\"beginDateTime\":\"\",\"endDateTime\":\"\",\"isSearchForced\":0,\"filters\":[],\"pageNo\":1,\"pageSize\":9}"
headers = { headers = {
'Accept': 'application/json, text/javascript, */*; q=0.01', 'Accept': 'application/json, text/javascript, */*; q=0.01',
...@@ -841,19 +875,26 @@ def guizhou(): ...@@ -841,19 +875,26 @@ def guizhou():
result_list = jsonData['data']['middle']["list"] result_list = jsonData['data']['middle']["list"]
for datainfo in result_list: for datainfo in result_list:
num += 1 num += 1
id_list = []
title = datainfo['title'] title = datainfo['title']
publishDate = datainfo['time'] publishDate = datainfo['time']
source = datainfo['source'] source = datainfo['source']
summary = datainfo['content'] summary = datainfo['content']
newsUrl = datainfo['url'] newsUrl = datainfo['url']
# 根据链接判重
is_member = baseCore.r.sismember('REITs::' + webname, newsUrl)
if is_member:
continue
try:
soup = policy.getrequest_soup(headers, newsUrl) soup = policy.getrequest_soup(headers, newsUrl)
# print(soup) # print(soup)
policy.paserUrl(soup, newsUrl)
pub_hao_ = soup.find('head').find('title').text pub_hao_ = soup.find('head').find('title').text
start_index = pub_hao_.find("(") + 1 start_index = pub_hao_.find("(") + 1
end_index = pub_hao_.find(")") end_index = pub_hao_.find(")")
pub_hao = pub_hao_[start_index:end_index] pub_hao = pub_hao_[start_index:end_index]
print(pub_hao) # print(pub_hao)
# 删除包含特定字段的标签 # 删除包含特定字段的标签
contentWithTag = soup.find('div', class_='Zoom Box') contentWithTag = soup.find('div', class_='Zoom Box')
...@@ -862,11 +903,9 @@ def guizhou(): ...@@ -862,11 +903,9 @@ def guizhou():
content = contentWithTag.text content = contentWithTag.text
# 附件： # 附件：
fu_jian_name = ''
fu_jian_href = ''
try: try:
fujian_href = contentWithTag.find_all('a') fujian_href = contentWithTag.find_all('a')
policy.paserUrl(contentWithTag, newsUrl)
for file_href_ in fujian_href: for file_href_ in fujian_href:
file_href = file_href_['href'] file_href = file_href_['href']
file_name = file_href_.text file_name = file_href_.text
...@@ -875,43 +914,46 @@ def guizhou(): ...@@ -875,43 +914,46 @@ def guizhou():
pass pass
else: else:
file_name = file_name + category file_name = file_name + category
rename_file = f'{str(num)}_{publishDate.replace("-", "")[:8]}_{file_name}' att_id, full_path = policy.attuributefile(file_name, file_href, num, publishDate)
fu_jian_name += rename_file + '\n' if att_id:
fu_jian_href += file_href + '\n' id_list.append(att_id)
policy.downloadfile(file_href, f'{path}/{rename_file}') file_href_['href'] = full_path
except: except:
pass pass
contentWithTag_str = str(contentWithTag)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
dic_info = { dic_info = {
'序号': num, 'attachmentIds':id_list,
'标题': title.replace('\n', ''), 'author': '',
'发布时间': publishDate, 'content': content,
'来源': source, 'contentWithTag': contentWithTag_str,
'原文链接': newsUrl, 'deleteFlag': 0,
'发文时间': publishDate, 'id': '',
'发文机构': '', 'title': title.replace('\n', ''),
'发文字号': pub_hao, 'publishDate': publishDate,
'摘要': summary.replace('\n', '').replace('<em>', '').replace('</em>', ''), 'source': source,
'正文': content, 'sourceAddress': newsUrl,
'附件名称': fu_jian_name, 'writtenDate': publishDate,
'附件链接': fu_jian_href, 'organ': '',
'issuedNumber': pub_hao,
'summary': summary.replace('\n', '').replace('<em>', '').replace('</em>', ''),
'createDate': time_now,
'sid': '1729046185945182210',
} }
print(dic_info) # print(dic_info)
DataList.append(dic_info) try:
baseCore.sendkafka(dic_info, topic)
sheet_name = '贵州省人民政府政策文件' baseCore.r.sadd('REITs::' + webname, newsUrl)
if sheet_name in wb.sheetnames: log.info(f'采集成功--{title}--{newsUrl}')
log.info(f"{sheet_name}工作表已存在！") except:
else: for att_id in id_list:
# 创建新工作表 baseCore.deliteATT(att_id)
wb.create_sheet(sheet_name) except Exception as e:
print(f"{sheet_name}新工作表创建完成！") log.info(f"error！！！{newsUrl}")
# 保存Excel文件 log.info(e)
wb.save(file_path) log.info(f'====处理结束，已采集{num}条数据=================')
baseCore.writerToExcel(DataList, file_path, sheet_name)
pass
if __name__=="__main__": if __name__=="__main__":
...@@ -919,8 +961,9 @@ if __name__=="__main__": ...@@ -919,8 +961,9 @@ if __name__=="__main__":
# wb = policy.createfile(file_path) # wb = policy.createfile(file_path)
# reform() # reform()
# shenzhen() # shenzhen()
zhengquanqihuo() # zhengquanqihuo()
# sse() # sse()
# hebei() # hebei()
# guizhou() guizhou()
# zhengquanqihuo() # zhengquanqihuo()
\ No newline at end of file
comData/YanBao/deletebyid.py
...@@ -32,7 +32,7 @@ class EsMethod(object): ...@@ -32,7 +32,7 @@ class EsMethod(object):
def __init__(self): def __init__(self):
# 创建Elasticsearch对象，并提供账号信息 # 创建Elasticsearch对象，并提供账号信息
self.es = Elasticsearch(['http://114.116.19.92:9700'], http_auth=('elastic', 'zzsn9988'),timeout=300 ) self.es = Elasticsearch(['http://114.116.19.92:9700'], http_auth=('elastic', 'zzsn9988'),timeout=300 )
self.index_name='researchreportdata' self.index_name='policy'
''' '''
删除 删除
...@@ -52,7 +52,10 @@ if __name__ == "__main__": ...@@ -52,7 +52,10 @@ if __name__ == "__main__":
if item: if item:
log.info(item) log.info(item)
id = item.decode() id = item.decode()
try:
esMethod.delete(esMethod.index_name,id) esMethod.delete(esMethod.index_name,id)
except:
continue
else: else:
log.info('已删除完毕') log.info('已删除完毕')
break break
......
comData/YanBao/get_doc_id.py 0 → 100644
import json
import threading
import time
import uuid
import redis
import requests
from retry import retry
from elasticsearch import Elasticsearch
from base import BaseCore
from obs import ObsClient
import fitz
from urllib.parse import unquote
baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()
baseCore = BaseCore.BaseCore()
# 使用连接池
# cnx_ = baseCore.pool_11.connection()
# cursor_ = cnx_.cursor()
cnx_ = baseCore.cnx_
cursor_ = cnx_.cursor()
lock = threading.Lock()
pathType = 'QYNotice/'
taskType = '企业研报/东方财富网'
pool = redis.ConnectionPool(host="114.115.236.206", port=6379, password='clbzzsn', db=6)
class EsMethod(object):
def __init__(self):
# 创建Elasticsearch对象，并提供账号信息
self.es = Elasticsearch(['http://114.116.19.92:9700'], http_auth=('elastic', 'zzsn9988'), timeout=300)
self.index_name = 'policy'
def queryatt(self,index_name,pnum):
body = {
"query": {
"bool": {
"must": [
{
"term": {
"sid.keyword": {
"value": "1697458829758697473"
}
}
},
{
"range": {
"createDate": {
"gte": "2023-11-28T10:00:00",
"lte": "2023-11-29T10:00:00"
}
}
}
]
}
},
"track_total_hits": True,
"size": 200,
"from": pnum
}
filter_path = ['hits.hits._id',
'hits.total.value',
'hits.hits._source.title',
'hits.hits._source.sourceAddress',
'hits.hits._source.createDate',
] # 字段2
result = self.es.search(index=index_name
, doc_type='_doc'
, filter_path=filter_path
, body=body)
# log.info(result)
return result
def main(page, p, esMethod):
redis_conn = redis.Redis(connection_pool=pool)
result = esMethod.queryatt(index_name=esMethod.index_name, pnum=p)
total = result['hits']['total']['value']
if total == 0:
log.info('++++已没有数据+++++')
return
msglist = result['hits']['hits']
log.info(f'---第{page}页{len(msglist)}条数据----共{total}条数据----')
for mms in msglist:
id = mms['_id']
title = mms['_source']['title']
sourceAddress = mms['_source']['sourceAddress']
log.info(f'{id}--{title}--{sourceAddress}---')
if redis_conn.lrem('YanBao:id', 0, id) == 0:
redis_conn.lpush('YanBao:id', id)
else:
continue
def run_threads(num_threads,esMethod,j):
threads = []
for i in range(num_threads):
page = j + i + 1
p = j + i * 200
thread = threading.Thread(target=main, args=(page, p, esMethod))
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
if __name__ == "__main__":
j = 0
for i in range(5):
esMethod = EsMethod()
# result = esMethod.queryatt(index_name=esMethod.index_name, pnum=p)
# total = result['hits']['total']['value']
# if total == 0:
# log.info('++++已没有数据+++++')
# break
start = time.time()
num_threads = 5
run_threads(num_threads, esMethod, j)
j += 1000
log.info(f'5线程 每个处理200条数据 总耗时{time.time() - start}秒')
\ No newline at end of file
comData/annualReport1023/updateyear.py
...@@ -84,14 +84,15 @@ if __name__ == "__main__": ...@@ -84,14 +84,15 @@ if __name__ == "__main__":
id_ = redis_conn.lpop('YanBao:up') id_ = redis_conn.lpop('YanBao:up')
# id = "23112104300" # id = "23112104300"
if id: if id_:
pass pass
else: else:
log.info('已无数据') log.info('已无数据')
break
id = id_.decode() id = id_.decode()
result_ = esMethod.queryatt(index_name=esMethod.index_name, id=id) result_ = esMethod.queryatt(index_name=esMethod.index_name, id=id)
result = result_['hits']['hits'][0] result = result_['hits']['hits'][0]
num = 0 num = 0
publishDate = result['_source']['publishDate'] publishDate = result['_source']['publishDate']
u_publishDate = '2023-08-31' #+ publishDate.split('T')[1] u_publishDate = '2022-12-31' #+ publishDate.split('T')[1]
esMethod.updateaunn(esMethod.index_name, str(id), u_publishDate) esMethod.updateaunn(esMethod.index_name, str(id), u_publishDate)
comData/noticeReport/东方财富网-港股公告.py
import os import os
...@@ -323,7 +323,10 @@ def spider(browser, code, social_code, com_name): ...@@ -323,7 +323,10 @@ def spider(browser, code, social_code, com_name):
# span_tag = browser.find_element(By.CLASS_NAME,'mbox') # span_tag = browser.find_element(By.CLASS_NAME,'mbox')
span_tag = browser.find_element(By.XPATH, '//div[@class="mbox"]/span[2]') span_tag = browser.find_element(By.XPATH, '//div[@class="mbox"]/span[2]')
current_page = int(span_tag.text) current_page = int(span_tag.text)
try:
totalpage = int(soup.find_all('div', class_='mbox')[-1].find_all('a')[-1].text) totalpage = int(soup.find_all('div', class_='mbox')[-1].find_all('a')[-1].text)
except:
totalpage = int(soup.find_all('div', class_='mbox')[-1].find_all('a')[-2].text)
if current_page < totalpage: if current_page < totalpage:
# 说明还未到最后一页 # 说明还未到最后一页
span_tag.find_element(By.XPATH, './following-sibling::a[1]').click() span_tag.find_element(By.XPATH, './following-sibling::a[1]').click()
......
comData/policylaw/BaseCore.py
...@@ -508,7 +508,7 @@ class BaseCore: ...@@ -508,7 +508,7 @@ class BaseCore:
except: except:
time.sleep(3) time.sleep(3)
continue continue
page_size = 0 # page_size = 0
for i in range(0, 3): for i in range(0, 3):
try: try:
# name = file_name # name = file_name
...@@ -522,11 +522,11 @@ class BaseCore: ...@@ -522,11 +522,11 @@ class BaseCore:
time.sleep(3) time.sleep(3)
continue continue
if page_size < 1: # if page_size < 1:
# pdf解析失败 # # pdf解析失败
# print(f'======pdf解析失败=====') # # print(f'======pdf解析失败=====')
return retData # return retData
else: # else:
try: try:
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
retData['state'] = True retData['state'] = True
......
comData/policylaw/policy.py
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论