Skip to content

Z
zzsn_spider
提交 af27b7ec 作者: 薛凌堃
操作

政策法规脚本维护

上级 687dbf5e
正在显示 包含 6227 行增加 和 18 行删除
comData/policylaw/BaseCore.py
......@@ -501,25 +501,26 @@ class BaseCore:
'category': category, 'file_size': '', 'status': 1, 'create_by': 'XueLingKun',
'create_time': '', 'page_size': '', 'content': ''}
headers['User-Agent'] = self.getRandomUserAgent()
for i in range(0, 3):
try:
response = requests.get(file_href, headers=headers, verify=False, timeout=20)
file_size = int(response.headers.get('Content-Length'))
break
except:
time.sleep(3)
continue
for i in range(0, 3):
try:
name = str(self.getuuid()) + category
result = obsClient.putContent('zzsn', 'PolicyDocuments/' + name, content=response.content)
break
except:
time.sleep(3)
continue
try:
for i in range(0, 3):
try:
response = requests.get(file_href, headers=headers, verify=False, timeout=20)
file_size = int(response.headers.get('Content-Length'))
break
except:
time.sleep(3)
continue
for i in range(0, 3):
try:
name = str(self.getuuid()) + category
result = obsClient.putContent('zzsn', 'PolicyDocuments/' + name, content=response.content)
break
except:
time.sleep(3)
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
retData['state'] = True
retData['path'] = result['body']['objectUrl'].split('.com')[1]
......
comData/policylaw/ClassTool.py 0 → 100644
#!/usr/bin/env python
# coding=utf-8
import json
import time
import pymongo
from bs4 import BeautifulSoup
from kafka import KafkaProducer
from requests.packages import urllib3
from urllib.parse import urljoin
from BaseCore import BaseCore
baseCore = BaseCore()
urllib3.disable_warnings()
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
log = baseCore.getLogger()
class ClassTool():
def __init__(self):
self.taskType = '政策法规'
self.db_storage =pymongo.MongoClient('mongodb://114.115.221.202:27017', username='admin', password='ZZsn@9988').ZZSN[
'国务院_国资委_copy1']
self.driver_path = r'D:\cmd100\chromedriver.exe'
self.chromr_bin = r'D:\Google\Chrome\Application\chrome.exe'
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
}
# 将html中的相对地址转换成绝对地址
def paserUrl(self, html, listurl):
# 获取所有的<a>标签和<img>标签
if isinstance(html, str):
html = BeautifulSoup(html, 'html.parser')
links = html.find_all(['a', 'img'])
# 遍历标签，将相对地址转换为绝对地址
for link in links:
if 'href' in link.attrs:
link['href'] = urljoin(listurl, link['href'])
elif 'src' in link.attrs:
link['src'] = urljoin(listurl, link['src'])
return html
def getDriver(self):
service = Service(self.driver_path)
chrome_options = webdriver.ChromeOptions()
# chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
# chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('log-level=3')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_experimental_option('excludeSwitches', ['enable-automation']) # 屏蔽chrome自动化受控提示
chrome_options.add_argument("--disable-blink-features=AutomationControlled") # 禁用启用Blink运行时的功能去掉webdriver痕迹
chrome_options.add_experimental_option('useAutomationExtension', False)
chrome_options.add_argument('lang=zh-CN,zh,zh-TW,en-US,en')
chrome_options.binary_location = self.chromr_bin
chrome_options.add_argument(
'user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.116 Safari/537.36')
# bro = webdriver.Chrome(chrome_options=chrome_options, service=service)
bro = webdriver.Chrome(chrome_options=chrome_options, executable_path=self.driver_path)
# with open('stealth.min.js') as f:
# js = f.read()
#
# bro.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
# "source": js
# })
return bro
def save_data(self, dic_news):
aaa_dic = {
'附件id': dic_news['attachmentIds'],
'网址': dic_news['sourceAddress'],
'tid': dic_news['labels'][0]['relationId'],
'来源': dic_news['labels'][0]['relationName'],
'创建时间': dic_news['createDate'],
'带标签内容': dic_news['contentWithTag'][:100],
'发布时间': dic_news['publishDate']
}
self.db_storage.insert_one(aaa_dic)
def sendKafka(self, dic_news):
try: # 114.116.116.241
producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'], max_request_size=1024 * 1024 * 20)
kafka_result = producer.send("policy",
json.dumps(dic_news, ensure_ascii=False).encode('utf8'))
print(kafka_result.get(timeout=10))
dic_result = {
'success': 'ture',
'message': '操作成功',
'code': '200',
}
log.info(dic_result)
# 传输成功,写入日志中
return True
except Exception as e:
dic_result = {
'success': 'false',
'message': '操作失败',
'code': '204',
'e': e
}
log.error(dic_result)
return False
\ No newline at end of file
comData/policylaw/an_hui.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 安徽
def an_hui():
def an_hui1():
num = 0
count = 0
start_time = time.time()
for page in range(1, 4):
url = f'http://gzw.ah.gov.cn/site/label/8888?IsAjax=1&dataType=html&_=0.4981381464472001&labelName=publicInfoList&siteId=6788071&pageSize=15&pageIndex={page}&action=list&isDate=false&dateFormat=yyyy%E5%B9%B4MM%E6%9C%88dd%E6%97%A5&length=15&organId=7031&type=4&catIds=&catId=6717051&cId=&result=&title=&fileNum=&keyWords=&file=%2Fxxgk%2FpublicInfoList_newest2020_zc'
try:
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False).text
doc_resp = pq(resp_text)
doc_items = doc_resp('tr[class="xxgk_nav_con"]').items()
for doc_item in doc_items:
title = doc_item('a').attr('title').strip()
publishDate = doc_item('td[class="fbrq"]').text().strip()
href = doc_item('a').attr('href')
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
href_text = requests.get(url=href, headers=headers, verify=False)
href_text.encoding = href_text.apparent_encoding
# soup = BeautifulSoup(href_text.text, 'html.parser')
soup = baseTool.paserUrl(href_text.text, href)
doc = pq(str(soup))
title = doc(
'div[class="div_table_suoyin"]>table>tbody>tr:nth-child(4)>td[class="pmingcheng"]').text()
topicClassification = doc(
'div[class="div_table_suoyin"]>table>tbody>tr:nth-child(1)>td:nth-child(4)').text()
writtenDate = doc(
'div[class="div_table_suoyin"]>table>tbody>tr:nth-child(2)>td:nth-child(4)').text()
organ = doc('div[class="div_table_suoyin"]>table>tbody>tr:nth-child(3)>td:nth-child(2)').text()
pub_hao = doc(
'div[class="div_table_suoyin"]>table>tbody>tr:nth-child(3)>td:nth-child(4)').text()
origin = doc('div[class="wzfbxx"]>span[class="res"]').text().replace('信息来源：', '')
publishDate = doc('div[class="wzfbxx"]>span[class="fbsj"]').text().replace('发布日期：', '')
contentWithTag = doc('div[id="zoom"]')
soup = baseTool.paserUrl(str(contentWithTag), href)
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1688', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '安徽省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
contentWithTag = str(soup.prettify())
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1688", 'relationName': "安徽省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': organ,
'topicClassification': topicClassification,
'issuedNumber': pub_hao,
'publishDate': publishDate,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
def an_hui2():
num = 0
count = 0
start_time = time.time()
for page in range(1, 25):
url = f'http://gzw.ah.gov.cn/site/label/8888?_=0.5237800193505848&labelName=publicInfoList&siteId=6788071&pageSize=15&pageIndex={page}&action=list&isDate=false&dateFormat=yyyy%E5%B9%B4MM%E6%9C%88dd%E6%97%A5&length=15&organId=7031&type=4&catIds=43793891%2C43793901&catId=&cId=&result=&title=&fileNum=&keyWords=&file=%2Fxxgk%2FpublicInfoList_newest2020_zc'
try:
res = requests.get(url=url, headers=baseTool.headers)
res.encoding = res.apparent_encoding
res_text = res.text
soup = baseTool.paserUrl(res_text, url)
doc = pq(str(soup))
tr_list = doc('tr[class="xxgk_nav_con"]')
for tr in tr_list:
trdoc = pq(tr)
title = trdoc('td[class="info"]>a').text()
href = trdoc('td[class="info"]>a').attr('href')
pub_hao = trdoc('td[class="fwrq"]').text()
writtenDate = trdoc('td[class="cwrq"]').text()
publishDate = trdoc('td[class="fwrq"]').text()
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
continue
try:
href_res = requests.get(url=href, headers=headers, verify=False)
href_res.encoding = href_res.apparent_encoding
href_text = href_res.text
# doc_href = BeautifulSoup(href_text, 'html.parser')
soup = baseTool.paserUrl(href_text, href)
doc = pq(str(soup))
# title=doc('div[class="div_table_suoyin"]>table>tbody>tr:nth-child(4)>td[class="pmingcheng"]').text()
topicClassification = doc(
'div[class="div_table_suoyin"]>table>tbody>tr:nth-child(1)>td:nth-child(4)').text()
# writtenDate=doc('div[class="div_table_suoyin"]>table>tbody>tr:nth-child(2)>td:nth-child(4)').text()
organ = doc('div[class="div_table_suoyin"]>table>tbody>tr:nth-child(3)>td:nth-child(2)').text()
# pub_hao=doc('div[class="div_table_suoyin"]>table>tbody>tr:nth-child(3)>td:nth-child(4)').text()
origin = doc('div[class="wzfbxx"]>span[class="res"]').text().replace('信息来源：', '')
# publishDate=doc('div[class="wzfbxx"]>span[class="fbsj"]').text().replace('发布日期：','')
contentWithTag = doc('div[id="zoom"]')
soup = baseTool.paserUrl(str(contentWithTag), href)
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1688', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '安徽省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
contentWithTag = str(soup.prettify())
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1688", 'relationName': "安徽省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': organ,
'topicClassification': topicClassification,
'issuedNumber': pub_hao,
'publishDate': publishDate,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
href_res.close()
except:
pass
res.close()
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
an_hui1()
an_hui2()
if __name__ == "__main__":
an_hui()
\ No newline at end of file
comData/policylaw/bei_jing.py 0 → 100644
import os
import time
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium import webdriver
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 北京
def bei_jing():
num = 0
start_time = time.time()
# 有反爬需要使用selenium
# service = Service(r'D:/chrome/113/chromedriver.exe')
# 配置selenium
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_experimental_option(
"excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension', False)
chrome_options.add_argument('lang=zh-CN,zh,zh-TW,en-US,en')
chrome_options.add_argument('log-level=3')
chrome_options.add_argument(
'user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36')
# bro = webdriver.Chrome(chrome_options=chrome_options, executable_path=r'D:/chrome/103/chromedriver.exe')
chrome_options.binary_location = r'D:\Google\Chrome\Application\chrome.exe'
chromedriver = r'D:\cmd100\chromedriver.exe'
# bro = webdriver.Chrome(chrome_options=chrome_options, executable_path=chromedriver)
bro = webdriver.Chrome(options=chrome_options, executable_path=chromedriver)
# with open('../../base/stealth.min.js') as f:
# js = f.read()
#
# bro.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
# "source": js
# })
url = 'http://gzw.beijing.gov.cn/xxfb/zcfg/index.html'
hrefs = []
try:
bro.get(url)
time.sleep(2)
bro.execute_script('window.scrollTo(0, document.body.scrollHeight)')
time.sleep(1)
while True:
# 获取所有要爬取页面的url
ul = bro.find_element(By.CLASS_NAME, 'public_list_team')
li_list = ul.find_elements(By.TAG_NAME, 'li')
for li in li_list:
href_ = li.find_element(By.TAG_NAME, 'a').get_attribute('href')
title_ = li.find_element(By.TAG_NAME, 'a').get_attribute('title')
hrefs.append([href_, title_])
updown = bro.find_element(By.CLASS_NAME, 'fanye').find_elements(By.TAG_NAME, 'a')[-1]
if updown.get_attribute('title') != '下一页':
break
updown.click()
time.sleep(2)
log.info(f'------{len(hrefs)}条数据-------------')
num = 0
count = 0
for href in hrefs:
id_list = []
title = href[1]
# todo:测试需要 注释掉判重
# 判断是否已经爬取过
is_href = baseTool.db_storage.find_one({'网址': href[0]})
if is_href:
num += 1
log.info('已采集----------跳过')
continue
# 对获取信息页面发送请求
bro.get(href[0])
time.sleep(1)
# 获取所要信息
pub = bro.find_element(By.CLASS_NAME, 'doc-info')
topic = str(pub.text).split('[主题分类] ')[1].split('\n')[0].strip()
# 发文机构
organ = str(pub.text).split('[发文机构] ')[1].split('\n')[0].strip()
pub_time = str(pub.text).split('[发布日期] ')[1].split('[有效性] ')[0].strip().lstrip()
writtenDate = str(pub.text).split('[成文日期] ')[1].split('\n')[0].strip()
# pub_source = str(pub.text).split('[发文机构] ')[1].split('[联合发文单位] ')[0].split('[实施日期] ')[0].strip().lstrip()
pub_hao = pub.find_element(By.CLASS_NAME, 'fwzh').text.replace('[发文字号] ', '').lstrip().strip()
try:
pub_list = bro.find_elements(By.CLASS_NAME, 'article-info')
for source in pub_list:
if '来源' in source.text:
pub_source = source.text.split('来源：')[1].split('\n')[0]
# print(pub_source)
except:
pub_source = ''
# .split('来源：')[1]
if '号' not in pub_hao:
pub_hao = ''
cont = bro.find_element(By.ID, 'div_zhengwen').get_attribute('innerHTML')
soup_cont = BeautifulSoup(cont, 'lxml')
soup = baseTool.paserUrl(soup_cont, href[0])
soup.prettify()
if soup.text == '' or soup.text == 'None':
log.info(f'----{href[0]}----{title}----内容为空----')
continue
# todo:去掉扫一扫
try:
soup.find('div', id='div_div').decompose()
except:
continue
# log.info(title)
fu_jian_soup = soup.find_all('a')
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
log.info(f'---{href[0]}--------{e}-------')
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xls' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1667', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '北京市国资委', file_name, num, pub_time)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# id_ = redefid(id_list)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(soup.text),
'contentWithTag': str(soup),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1667", 'relationName': "北京市国资委", 'labelMark': "policy"}],
'origin': pub_source,
'organ': organ,
'topicClassification': topic,
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href[0],
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
bro.quit()
except Exception as e:
log.info(e)
pass
if __name__ == "__main__":
bei_jing()
\ No newline at end of file
comData/policylaw/chong_qing.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 重庆
def chong_qing():
"""
http://gzw.cq.gov.cn/zwgk_191/fdzdgknr/zcwj/xzgfxwj/ 4
http://gzw.cq.gov.cn/zwgk_191/fdzdgknr/zcwj/qtwj/ 2
"""
num = 0
count = 0
pathType = 'policy/chongqing/'
start_time = time.time()
for page in range(0, 4):
if page == 0:
url = 'http://gzw.cq.gov.cn/zwgk_191/fdzdgknr/zcwj/zcwj/index.html'
else:
url = 'http://gzw.cq.gov.cn/zwgk_191/fdzdgknr/zcwj/zcwj/index_{}.html'.format(page)
# url = 'http://gzw.cq.gov.cn/zwgk_191/fdzdgknr/zcwj/zcwj/index_3.html'
try:
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False).content
doc_resp = pq(resp_text)
doc_items = doc_resp('.zsj-fr-main').items()
for doc_item in doc_items:
id_list = []
titles = doc_item('a').items()
for title_item in titles:
title = title_item.text().strip()
href = title_item('a').attr('href')
if '../' in href:
href = url.split('zcwj/index')[0] + title_item('a').attr('href').replace('../', '')
else:
href = url.split('index')[0] + title_item('a').attr('href').replace('./', '')
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
# print(href)
# href = 'https://gzw.cq.gov.cn/zwgk_191/fdzdgknr/zcwj/zcwj/202007/t20200728_7729850.html'
href_text = requests.get(url=href, headers=baseTool.headers, verify=False).content
doc_href = pq(href_text)
try:
pub_result = doc_href('.zwxl-table').text().replace(' ', '')
pub_time = pub_result.split('[发布日期]')[1].strip() + ' 00:00:00'
pub_hao = pub_result.split('[发文字号]')[1].split('[主题分类]')[0].strip()
topicClassification = pub_result.split('[主题分类]')[1].split('[体裁分类]')[0].strip()
origin = pub_result.split('[发布机构]')[1].split('[成文日期]')[0].strip()
writtenDate = pub_result.split('[成文日期]')[1].split('[发布日期]')[0].strip()
doc_href = BeautifulSoup(str(doc_href), 'html.parser')
# 相对路径转化为绝对路径
doc_href = baseTool.paserUrl(doc_href, href)
# 去掉扫一扫
try:
doc_href.find('div', id='div_div').decompose()
# 去掉分享
doc_href.find('div', class_='bdsharebuttonbox').decompose()
except:
pass
contentWithTag = doc_href.find('div', class_='zwxl-article')
content = contentWithTag.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
except:
origin = ''
topicClassification = ''
pub_time = None
writtenDate = None
pub_hao = ''
contentWithTag = doc_href.find('div', class_='zwxl-content')
content = contentWithTag.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
fu_jian_list = contentWithTag.find_all('a')
# print(fu_jian_list)
for fu_jian in fu_jian_list:
try:
fu_jian_href = fu_jian['href']
except:
continue
file_name = fu_jian.text
if '.pdf' in fu_jian_href or '.docx' in fu_jian_href or '.doc' in fu_jian_href or 'xlsx' in fu_jian_href or '.zip' in fu_jian_href \
or '.rar' in fu_jian_href or '.ppt' in fu_jian_href or '.PDF' in fu_jian_href or '.DOC' in fu_jian_href \
or '.XLS' in fu_jian_href or '.ZIP' in fu_jian_href or '.RAR' in fu_jian_href:
try:
category = os.path.splitext(fu_jian_href)[1]
if category not in file_name:
file_name = file_name + category
# 附件上传至文件服务器
retData = baseCore.uptoOBS(fu_jian_href, '1693', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '重庆市国资委', file_name, num,
pub_time)
id_list.append(att_id)
# 将附件链接替换
fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
except:
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': content,
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1693", 'relationName': "重庆市国资委",
'labelMark': "policy"}],
'origin': origin,
'organ': '',
'topicClassification': topicClassification,
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
log.info(title)
count += 1
num += 1
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
chong_qing()
\ No newline at end of file
comData/policylaw/fu_jian.py 0 → 100644
import os
import time
import requests
from bs4 import BeautifulSoup
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 福建
def fu_jian():
error_tag = str(404)
num = 0
count = 0
start_time = time.time()
url = 'http://gzw.fujian.gov.cn/zwgk/zcfg/'
try:
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False)
resp_text.encoding = 'utf-8'
html = resp_text.text
soup = BeautifulSoup(html, 'html.parser')
# print(soup)
result = soup.find_all(class_='borbot-line')
for li_list in result:
li = li_list.find_all('li')
for a in li:
id_list = []
# print(a)
a_text = str(a)
title = a_text.split('title="')[-1].split('">')[0].replace('\n', '')
href_ = str(a.find('a').get('href')) # 网站链接
href = href_.replace('../', './').replace('./', 'http://gzw.fujian.gov.cn/zwgk/')
href_text = requests.get(url=href, headers=baseTool.headers, verify=False)
href_text.encoding = href_text.apparent_encoding
i_html = href_text.text
i_soup = BeautifulSoup(i_html, 'html.parser')
try:
error_ = str(i_soup.find('strong').text)
except:
error_ = ''
if error_ == error_tag:
href = href_.replace('../', './').replace('./', 'http://gzw.fujian.gov.cn/zwgk/zcfg/')
href_text = requests.get(url=href, headers=baseTool.headers, verify=False)
href_text.encoding = href_text.apparent_encoding
i_html = href_text.text
i_soup = BeautifulSoup(i_html, 'html.parser')
try:
error_ = str(i_soup.find('strong').text)
except:
error_ = ''
if error_ == error_tag:
href = href_.replace('../../', 'http://gzw.fujian.gov.cn/')
href_text = requests.get(url=href, headers=baseTool.headers, verify=False)
href_text.encoding = href_text.apparent_encoding
i_html = href_text.text
i_soup = BeautifulSoup(i_html, 'html.parser')
real_href = href
# real_href = 'http://gzw.fujian.gov.cn/zwgk/zcfg/201806/t20180619_3065065.htm'
# print(real_href)
is_href = baseTool.db_storage.find_one({'网址': real_href})
if is_href:
num += 1
continue
try:
# 文章是远程pdf
# 直接下载文件至服务器，解析出正文内容
if '.pdf' in real_href:
# pass
resp_content = requests.get(real_href, headers=baseTool.headers, verify=False, timeout=20).content
# 解析出pdf内容
content = baseCore.pdf_content(resp_content)
contentwithtag = ''
category = os.path.splitext(real_href)[1]
if category not in title:
file_name = title + category
# 文件上传至服务器
retData = baseCore.uptoOBS(real_href, '1673', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '福建省国资委', file_name, num, '')
id_list.append(att_id)
pub_hao = ''
pub_time = None
pub_source = ''
else:
try:
href_text = requests.get(url=real_href, headers=baseTool.headers, verify=False)
href_text.encoding = href_text.apparent_encoding
i_html = href_text.text
i_soup = BeautifulSoup(i_html, 'html.parser')
# 相对路径转化为绝对路径
i_soup = baseTool.paserUrl(i_soup, real_href)
source_ = str(i_soup.find('div', attrs={'class': 'xl_tit2_l'}).text)
pub_source = source_.split('来源：')[1].split('发布时间：')[0].strip().lstrip()
pub_time = source_.split('发布时间：')[1].split('浏览量：')[0].strip().lstrip()
contentwithtag = i_soup.find('div', attrs={'class': 'xl_con1'})
content = i_soup.find('div', attrs={'class': 'xl_con1'}).text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
pub_hao = ''
# print(real_href)
# todo:获取附件地址
try:
fu_jian_list = i_soup.find('ul', class_='clearflx myzj_xl_list').find_all('a')
except:
pass
fu_jian_list = []
for fu_jian in fu_jian_list:
try:
fj_href = fu_jian['href']
except:
continue
file_name = fu_jian.text
if '.doc' in fj_href or '.docx' in fj_href or '.xlsx' in fj_href or '.pdf' in fj_href or '.xls' in fj_href or '.zip' in fj_href \
or '.rar' in fj_href or '.ppt' in fj_href or '.PDF' in fj_href or '.DOC' in fj_href \
or '.XLS' in fj_href or '.ZIP' in fj_href or '.RAR' in fj_href:
category = os.path.splitext(fj_href)[1]
if category not in file_name:
file_name = file_name + category
print(fj_href)
# 找到附件后 上传至文件服务器
retData = baseCore.uptoOBS(fj_href, '1673', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '福建省国资委', file_name, num,
pub_time)
id_list.append(att_id)
# 将文件服务器的链接替换
fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
except:
pub_source = ''
pub_time = None
contentwithtag = i_soup.find('tabs tab_base_01 rules_con1')
content = contentwithtag.text.strip()
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
pub_hao = contentwithtag.find_all('div', class_='rules_tit1 b-free-read-leaf').text.dtrip()
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': content,
'contentWithTag': str(contentwithtag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1673", 'relationName': "福建省国资委", 'labelMark': "policy"}],
'origin': pub_source,
'organ': '',
'topicClassification': '',
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': real_href,
'summary': '',
'title': title
}
# log.info(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
log.info(title)
num += 1
count += 1
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据，共耗时{end_time - start_time}')
if __name__ == "__main__":
fu_jian()
\ No newline at end of file
comData/policylaw/gan_su.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from selenium.webdriver.common.by import By
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 甘肃
def gan_su():
pathType = 'policy/gansu/'
def gan_su1():
num = 0
count = 0
start_time = time.time()
bro = baseTool.getDriver()
urls = ['http://gzw.gansu.gov.cn/gzw/c115543/xxgk_list.shtml',
'http://gzw.gansu.gov.cn/gzw/c115544/xxgk_list.shtml',
'http://gzw.gansu.gov.cn/gzw/c115545/xxgk_list.shtml',
'http://gzw.gansu.gov.cn/gzw/c115546/xxgk_list.shtml',
'http://gzw.gansu.gov.cn/gzw/c115547/xxgk_list.shtml',
'http://gzw.gansu.gov.cn/gzw/c115548/xxgk_list.shtml',
'http://gzw.gansu.gov.cn/gzw/c115549/xxgk_list.shtml',
'http://gzw.gansu.gov.cn/gzw/c115550/xxgk_list.shtml',
'http://gzw.gansu.gov.cn/gzw/c115551/xxgk_list.shtml',
'http://gzw.gansu.gov.cn/gzw/c115554/xxgk_list.shtml']
for url in urls:
hrefs = []
try:
for i in range(0, 4):
bro.get(url)
time.sleep(2)
html = bro.page_source
if len(html) < 200:
time.sleep(5)
continue
else:
break
soup = baseTool.paserUrl(html, url)
doc = pq(str(soup))
li_list = doc('ul[class="UlTab"]>li')
for li in li_list:
lidoc = pq(li)
href_ = lidoc('td[width="400"]>a').attr('href')
publishDate = lidoc('td[width="100"]').text()
dtl = {
'href': href_,
'publishDate': publishDate
}
hrefs.append(dtl)
for dd in hrefs:
href = dd['href']
publishDate = dd['publishDate']
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
for i in range(0, 4):
bro.get(href)
time.sleep(2)
dhtml = bro.page_source
if dhtml == '<html><head></head><body></body></html>':
bro.close()
bro.quit()
bro = baseTool.getDriver()
bro.get(href)
dhtml = bro.page_source
if len(dhtml) < 200:
time.sleep(5)
continue
else:
break
title = str(bro.find_element(By.CLASS_NAME, 'links_tit').text)
links_tab = str(bro.find_element(By.CLASS_NAME, 'links_tab').text)
organ = links_tab.split('发布机构')[1].split('主题分类')[0].replace('：', '').strip().lstrip()
pub_hao = links_tab.split('文号')[1].split('浏览次数')[0].replace('：', '').strip().lstrip()
writtenDate = links_tab.split('生成日期')[1].split('关键字')[0].replace('：', '').strip().lstrip()
contentWithTag = str(bro.find_element(By.CLASS_NAME, 'links_words').get_attribute('innerHTML'))
soup = baseTool.paserUrl(contentWithTag, href)
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1696', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '甘肃省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# id_ = redefid(id_list)
contentWithTag = str(soup.prettify())
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
# t = time.strptime(publishDate, "%Y年%m月%d日")
# publishDate = time.strftime("%Y-%m-%d %H:%M:%S", t)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1696", 'relationName': "甘肃省国资委", 'labelMark': "policy"}],
'origin': '',
'organ': organ,
'topicClassification': "",
'issuedNumber': pub_hao,
'publishDate': publishDate,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except Exception as e:
print(e)
pass
bro.quit()
end_time = time.time()
log.info(f'共抓取{num}条数据,共耗时{end_time - start_time}')
def gan_su2():
num = 0
count = 0
start_time = time.time()
bro = baseTool.getDriver()
url = 'http://gzw.gansu.gov.cn/gzw/c115552/xxgk_list.shtml'
hrefs = []
try:
bro.get(url)
time.sleep(3)
bro.execute_script('window.scrollTo(0, document.body.scrollHeight)')
time.sleep(1)
html = bro.page_source
soup = baseTool.paserUrl(html, url)
doc = pq(str(soup))
li_list = doc('ul[class="UlTab"]>li')
for li in li_list:
lidoc = pq(li)
href_ = lidoc('td[width="400"]>a').attr('href')
publishDate = lidoc('td[width="100"]').text()
dtl = {
'href': href_,
'publishDate': publishDate
}
hrefs.append(dtl)
for dd in hrefs:
try:
href = dd['href']
publishDate = dd['publishDate']
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
bro.get(href)
try:
alls = bro.find_element(By.CLASS_NAME, 'alls').text
if '全文' in alls:
bro.find_element(By.CLASS_NAME, 'alls').click()
except:
pass
time.sleep(3)
html = bro.page_source
if html == '<html><head></head><body></body></html>':
bro.close()
bro.quit()
bro = baseTool.getDriver()
bro.get(href)
html = bro.page_source
doc = pq(html)
origin = ''
pub_hao = ''
topicClassification = ''
topicClassification = doc('table[class="bd1"]>tbody>tr:nth-child(1)>td:nth-child(4)').text()
organ = doc('table[class="bd1"]>tbody>tr:nth-child(2)>td:nth-child(2)').text()
writtenDate = doc('table[class="bd1"]>tbody>tr:nth-child(2)>td:nth-child(4)').text()
title = doc('table[class="bd1"]>tbody>tr:nth-child(3)>td:nth-child(2)').text()
pub_hao = doc('table[class="bd1"]>tbody>tr:nth-child(4)>td:nth-child(2)').text()
contentWithTag = doc('div[id="UCAP-CONTENT"]')
if len(title) == 0:
topicClassification = doc(
'div[class="links_tab"]>table>tbody>tr:nth-child(2)>td:nth-child(4)').text()
organ = doc('div[class="links_tab"]>table>tbody>tr:nth-child(2)>td:nth-child(2)').text()
title = doc('table[class="bd1"]>tbody>tr:nth-child(3)>td:nth-child(2)').text()
writtenDate = doc('table[class="bd1"]>tbody>tr:nth-child(4)>td:nth-child(2)').text()
pub_hao = doc('table[class="bd1"]>tbody>tr:nth-child(5)>td:nth-child(2)').text()
contentWithTag = doc('div[id="content"]')
if len(title) == 0:
title = doc('h1[id="ti"]').text()
writtenDate = doc('div[class="pages-date"]').text().split(' 来源：')[0]
origin = doc('div[class="pages-date"]>span[class="font"]').text().replace("来源：", "")
if len(origin) < 1:
origin = doc('div[class="pages-date"]>span').text().replace("来源：", "")
contentWithTag = doc('div[id="UCAP-CONTENT"]')
if len(title) == 0:
title = doc('div[class="links_tit"]').text()
writtenDate = doc('div[class="links_tab"]>table>tbody>tr:nth-child(4)>td:nth-child(2)').text()
origin = doc('div[class="links_tab"]>table>tbody>tr:nth-child(2)>td:nth-child(2)').text()
pub_hao = doc('div[class="links_tab"]>table>tbody>tr:nth-child(5)>td:nth-child(2)').text()
contentWithTag = doc('div[id="content"]')
# print(title)
soup = baseTool.paserUrl(str(contentWithTag), href)
try:
div_tag = soup.find(id="div_div")
div_tag.extract()
editor = soup.find(class_='editor')
editor.extract()
except:
pass
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
log.info(f'{file_name}---{href}--')
retData = baseCore.uptoOBS(file_href, '1696', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '甘肃省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
contentWithTag = str(soup.prettify())
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
if len(content) < 2:
continue
# t = time.strptime(publishDate, "%Y年%m月%d日")
# publishDate = time.strftime("%Y-%m-%d %H:%M:%S", t)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1696", 'relationName': "甘肃省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': organ,
'topicClassification': topicClassification,
'issuedNumber': pub_hao,
'publishDate': publishDate,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except Exception as e:
print(e)
except Exception as e:
print(e)
pass
bro.quit()
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
def gan_su3():
num = 0
count = 0
start_time = time.time()
bro = baseTool.getDriver()
url = 'http://gzw.gansu.gov.cn/gzw/c115553/xxgk_list.shtml'
hrefs = []
try:
bro.get(url)
time.sleep(2)
bro.execute_script('window.scrollTo(0, document.body.scrollHeight)')
while True:
time.sleep(1)
ul = bro.find_element(By.CLASS_NAME, 'UlTab')
li_list = ul.find_elements(By.TAG_NAME, 'li')
for li in li_list:
href_ = li.find_element(By.TAG_NAME, 'a').get_attribute('href')
try:
publishDate = li.find_elements(By.XPATH, './/td[@width="100"]')[0].text
# publishDate=li.find_element(by='width',value='100').text
except Exception as e:
publishDate = ''
pass
dtl = {
'href': href_,
'publishDate': publishDate
}
hrefs.append(dtl)
try:
# bro.find_element(By.CLASS_NAME, 'nextpage').click()
bro.find_element(By.CLASS_NAME, 'nextbtn').click()
except:
break
for dd in hrefs:
href = dd['href']
publishDate = dd['publishDate']
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
bro.get(href)
time.sleep(3)
html = bro.page_source
if html == '<html><head></head><body></body></html>':
bro.close()
bro.quit()
bro = baseTool.getDriver()
bro.get(href)
html = bro.page_source
doc = pq(html)
origin = ''
pub_hao = ''
topicClassification = ''
topicClassification = doc('div[class="contenttitle"]>ul>li:nth-child(2)>font>span').text()
organ = doc('div[class="contenttitle"]>ul>li:nth-child(3)>font>span').text()
writtenDate = doc('div[class="contenttitle"]>ul>li:nth-child(4)>font>span').text()
title = doc('div[class="contenttitle"]>ul>li:nth-child(5)>font>span').text()
pub_hao = doc('div[class="contenttitle"]>ul>li:nth-child(6)>font>span').text()
publishDate = doc('div[class="contenttitle"]>ul>li:nth-child(7)>font>span').text()
contentWithTag = doc('div[id="detailContent"]')
if len(title) == 0:
topicClassification = doc(
'div[class="links_tab"]>table>tbody>tr:nth-child(2)>td:nth-child(4)').text()
organ = doc('div[class="links_tab"]>table>tbody>tr:nth-child(2)>td:nth-child(2)').text()
title = doc('table[class="bd1"]>tbody>tr:nth-child(3)>td:nth-child(2)').text()
writtenDate = doc('table[class="bd1"]>tbody>tr:nth-child(4)>td:nth-child(2)').text()
pub_hao = doc('table[class="bd1"]>tbody>tr:nth-child(5)>td:nth-child(2)').text()
contentWithTag = doc('div[id="content"]')
if len(title) == 0:
title = doc('h1[id="ti"]').text()
writtenDate = doc('div[class="pages-date"]').text().split(' 来源：')[0]
origin = doc('div[class="pages-date"]>span[class="font"]').text().replace("来源：", "")
if len(origin) < 1:
origin = doc('div[class="pages-date"]>span').text().replace("来源：", "")
contentWithTag = doc('div[id="UCAP-CONTENT"]')
if len(title) == 0:
title = doc('div[class="links_tit"]').text()
writtenDate = doc('div[class="links_tab"]>table>tbody>tr:nth-child(4)>td:nth-child(2)').text()
origin = doc('div[class="links_tab"]>table>tbody>tr:nth-child(2)>td:nth-child(2)').text()
pub_hao = doc('div[class="links_tab"]>table>tbody>tr:nth-child(5)>td:nth-child(2)').text()
contentWithTag = doc('div[id="content"]')
# print(title)
if len(title) == 0 or contentWithTag.text() == '':
title = doc('div[class="main"]>h1').text().lstrip().strip()
writtenDate = \
doc('div[class="main"]>div[class="clearbox"]>p:nth-child(1)').text().split('日期：')[0].split(' ')[
0].lstrip().strip()
origin = doc('div[class="main"]>div[class="clearbox"]>p:nth-child(1)').text().split('来源：')[
0].lstrip().strip()
contentWithTag = doc('div[class="detailContent"]')
# print(title)
soup = baseTool.paserUrl(str(contentWithTag), href)
try:
div_tag = soup.find(id="div_div")
div_tag.extract()
editor = soup.find(class_='editor')
editor.extract()
except:
pass
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1696', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '甘肃省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
contentWithTag = str(soup.prettify())
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
# print(bro.page_source)
continue
if len(content) < 2:
continue
# t = time.strptime(publishDate, "%Y年%m月%d日")
# publishDate = time.strftime("%Y-%m-%d %H:%M:%S", t)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1696", 'relationName': "甘肃省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': organ,
'topicClassification': topicClassification,
'issuedNumber': pub_hao,
'publishDate': publishDate,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except Exception as e:
ee = e.__traceback__.tb_lineno
print(ee, e)
except:
pass
bro.quit()
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
gan_su1()
gan_su2()
gan_su3()
if __name__ == "__main__":
gan_su()
\ No newline at end of file
comData/policylaw/guang_dong.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 广东
def guang_dong():
start = time.time()
num = 0
count = 0
url = 'http://gzw.gd.gov.cn/zcfg/index.html'
try:
resp_href = requests.get(url=url, headers=baseTool.headers, verify=False)
resp_href.encoding = resp_href.apparent_encoding
doc_resp = BeautifulSoup(resp_href.text, 'html.parser')
page_items = str(doc_resp.find('div', attrs={'class': 'page'}).text)
total = page_items.split('共 ')[1].split(' 条')[0].strip().lstrip()
total = int(total)
if total % 23 != 0:
pagen = total / 23 + 1
else:
pagen = total / 23
for page in range(1, int(pagen + 1)):
if page == 1:
url = 'http://gzw.gd.gov.cn/zcfg/index.html'
else:
url = f'http://gzw.gd.gov.cn/zcfg/index_{page}.html'
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False).text
doc_resp = pq(resp_text)
doc_items = doc_resp('.list li').items()
for doc_item in doc_items:
id_list = []
title = doc_item('a').text().replace('\n', '')
href = doc_item('a').attr('href')
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
# print(href)
href_text = requests.get(url=href, headers=baseTool.headers, verify=False).text
doc_href = pq(href_text)
pub_result = doc_href('.title_info_sub').text()
pub_time = pub_result.split('文章来源：')[0].replace('发布时间：', '').strip() + ' 00:00:00'
pub_source = pub_result.split('文章来源：')[1].strip()
i_soup = BeautifulSoup(href_text, 'html.parser')
i_soup = baseTool.paserUrl(i_soup, href)
content = i_soup.find('div', attrs={'class', 'box_info'})
contentwithTag = str(content)
if content == '' or content == None:
log.info(f'{href}-----{title}----内容为空----')
continue
fu_jian_list = content.find_all('a')
for fu_jian in fu_jian_list:
try:
file_name = fu_jian.text
fj_href = fu_jian['href']
except:
continue
if '.doc' in fj_href or '.docx' in fj_href or '.pdf' in fj_href or '.xls' in fj_href or '.zip' in fj_href \
or '.rar' in fj_href or '.ppt' in fj_href or '.PDF' in fj_href or '.DOC' in fj_href \
or '.xlsx' in fj_href or '.ZIP' in fj_href or '.RAR' in fj_href:
category = os.path.splitext(fj_href)[1]
if category not in file_name:
file_name = file_name + category
# 附件上传至文件服务器
retData = baseCore.uptoOBS(fj_href, '1676', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '广东省国资委', file_name, num, pub_time)
id_list.append(att_id)
# 将文件服务器的链接替换
fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': content.text,
'contentWithTag': str(contentwithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1676", 'relationName': "广东省国资委", 'labelMark': "policy"}],
'origin': pub_source,
'organ': '',
'topicClassification': '',
'issuedNumber': '',
'publishDate': pub_time,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
log.info(title)
num = num + 1
count += 1
except:
pass
except:
pass
end = time.time()
log.info(f'共抓取{count}条数据，共耗时{end - start}')
if __name__ == "__main__":
guang_dong()
\ No newline at end of file
comData/policylaw/guang_xi.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 广西
def guang_xi():
num = 0
count = 0
start_time = time.time()
url_all = """
http://gzw.gxzf.gov.cn/wjzx/2023nwj/ 1
http://gzw.gxzf.gov.cn/wjzx/2022nwj/ 1
http://gzw.gxzf.gov.cn/wjzx/2021nwj/ 1
http://gzw.gxzf.gov.cn/wjzx/2020nwj/ 1
http://gzw.gxzf.gov.cn/wjzx/2019nwj/ 1
http://gzw.gxzf.gov.cn/wjzx/2018nwj/ 2
http://gzw.gxzf.gov.cn/wjzx/2017nwj/ 2
http://gzw.gxzf.gov.cn/wjzx/2016nwj/ 2
http://gzw.gxzf.gov.cn/wjzx/2015nwj/ 3
http://gzw.gxzf.gov.cn/wjzx/2014nwj/ 2
http://gzw.gxzf.gov.cn/wjzx/2013nwj/ 2
http://gzw.gxzf.gov.cn/wjzx/2012nwj/ 2
http://gzw.gxzf.gov.cn/wjzx/2011nwj/ 5
http://gzw.gxzf.gov.cn/wjzx/wjhbdej2008n2010n/ 1
http://gzw.gxzf.gov.cn/wjzx/wjhbdyj2004n2007n/ 1
http://gzw.gxzf.gov.cn/wjzx/gfxwjhb2004n2013n/ 1
http://gzw.gxzf.gov.cn/wjzx/jshgfxwj2004n2015n/ 1
http://gzw.gxzf.gov.cn/wjzx/gfxwjhb2004n2015n/ 1
"""
url_list = url_all.split('\n')
for url_info in url_list[1:-1]:
url_info = url_info.strip()
url_1 = url_info.split(' ')[0].strip()
for page in range(0, 1):
if page == 0:
url = f'{url_1}index.shtml'
else:
url = f'{url_1}index_{page}.shtml'
try:
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False).content
doc_resp = pq(resp_text)
doc_items = doc_resp('#morelist li').items()
for doc_item in doc_items:
id_list = []
title = doc_item('a').attr('title').strip()
href = url.split('index')[0] + doc_item('a').attr('href').replace('./', '')
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
# print(href)
href_text = requests.get(url=href, headers=baseTool.headers, verify=False).content
doc_href = pq(href_text)
pub_result = doc_href('.article-inf-left').text()
pub_hao_result = doc_href('.article-h2').text()
if '﹝' in pub_hao_result and '﹞' in pub_hao_result:
pub_hao = pub_hao_result.replace('﹝', '〔').replace('﹞', '〕')
elif '〔' in pub_hao_result and '〕' in pub_hao_result:
pub_hao = pub_hao_result
else:
pub_hao = ''
pub_time = pub_result.split('来源：')[0].strip() + ':00'
try:
pub_source = pub_result.split('来源：')[1].split('作者：')[0].strip()
except:
pub_source = pub_result.split('来源：')[1].strip()
contentWithTag = doc_href('.article-con div:first-child')
# 相对路径转化为绝对路径
contentWithTag = BeautifulSoup(str(contentWithTag), 'html.parser')
contentWithTag = baseTool.paserUrl(contentWithTag, href)
content = contentWithTag.text.strip()
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
fu_jian_list = contentWithTag.find_all('a')
for fu_jian in fu_jian_list:
try:
fu_jian_href = fu_jian['href']
except:
continue
file_name = fu_jian.text.strip()
if '.docx' in fu_jian_href or '.pdf' in fu_jian_href or '.xlsx' in fu_jian_href or '.zip' in fu_jian_href \
or '.rar' in fu_jian_href or '.ppt' in fu_jian_href or '.PDF' in fu_jian_href or '.DOC' in fu_jian_href \
or '.XLS' in fu_jian_href or '.ZIP' in fu_jian_href or '.RAR' in fu_jian_href:
category = os.path.splitext(fu_jian_href)[1]
if category not in file_name:
file_name = file_name + category
# 附件上传至文件服务器
retData = baseCore.uptoOBS(fu_jian_href, '1692', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '广西壮族自治区国资委', file_name, num,
pub_time)
id_list.append(att_id)
# 将附件链接替换
fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': content,
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1692", 'relationName': "广西壮族自治区国资委", 'labelMark': "policy"}],
'origin': '',
'organ': pub_source,
'topicClassification': '',
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
log.info(title)
num = num + 1
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
guang_xi()
\ No newline at end of file
comData/policylaw/gui_zhou.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 贵州
def gui_zhou():
"""
http://gzw.guizhou.gov.cn/zwgk/xxgkml/zcwj/ 11
http://gzw.guizhou.gov.cn/zwgk/xxgkml/qlqdhzrqd/ 1
"""
num = 0
count = 0
start_time = time.time()
for page in range(0, 11):
if page == 0:
url = 'http://gzw.guizhou.gov.cn/zwgk/xxgkml/zcwj/alist.html'
else:
url = f'http://gzw.guizhou.gov.cn/zwgk/xxgkml/zcwj/alist_{page}.html'
try:
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False).content
doc_resp = pq(resp_text)
doc_items = doc_resp('.c').items()
for doc_item in doc_items:
id_list = []
href = doc_item('a').attr('href')
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
# print(href)
# href = 'http://gzw.guizhou.gov.cn/zwgk/xxgkml/zcwj/hyzcfg/202110/t20211026_71215292.html'
title = doc_item('a').text().strip()
href_text = requests.get(url=href, headers=baseTool.headers, verify=False)
if '404 Not Found' in href_text.text:
continue
doc_href = pq(href_text.content)
# 发文机构
organ = doc_href('#NewsArticleSource').text()
pub_result = doc_href('.xxgk_xl_top').text().replace('var str = ""; var str_1 = "', '').replace(
'"; if (str == "") { document.write(str_1); } else { document.write(str); }', '')
pub_time = pub_result.split('发文日期: ')[1].split('文号:')[0].strip().replace('年', '-').replace('月',
'-').replace(
'日', ' ') + ' 00:00:00'
# origin
pub_source = pub_result.split('发布机构:')[1].split('发文日期:')[0].strip()
pub_hao = pub_result.split('文号:')[1].split('是否有效:')[0].strip()
topicClassification = pub_result.split('信息分类:')[1].split('发布机构:')[0].strip()
if pub_source == '无':
pub_source = ''
if pub_hao == '无':
pub_hao = ''
contentWithTag = doc_href('#Zoom').children()
contentWithTag = BeautifulSoup(str(contentWithTag), 'html.parser')
contentWithTag = baseTool.paserUrl(contentWithTag, href)
content = contentWithTag.text.strip()
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
fu_jian_list = contentWithTag.find_all('a')
for fu_jian in fu_jian_list:
try:
fu_jian_href = fu_jian['href']
except:
continue
file_name = fu_jian.text.strip()
if '.doc' in fu_jian_href or '.pdf' in fu_jian_href or '.xls' in fu_jian_href or '.zip' in fu_jian_href \
or '.rar' in fu_jian_href or '.ppt' in fu_jian_href or '.PDF' in fu_jian_href or '.DOC' in fu_jian_href \
or '.XLS' in fu_jian_href or '.ZIP' in fu_jian_href or '.RAR' in fu_jian_href:
category = os.path.splitext(fu_jian_href)[1]
if category not in file_name:
file_name = file_name + category
# 附件上传至文件服务器
retData = baseCore.uptoOBS(fu_jian_href, '1694', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '贵州省国资委', file_name, num, pub_time)
id_list.append(att_id)
# 将附件链接替换
fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': content,
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1694", 'relationName': "贵州省国资委", 'labelMark': "policy"}],
'origin': pub_source,
'organ': organ,
'topicClassification': topicClassification,
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
log.info(title)
count += 1
num = num + 1
except Exception as e:
pass
except Exception as e:
pass
end_time = time.time()
log.info(f'共抓取{num}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
gui_zhou()
\ No newline at end of file
comData/policylaw/gwyfile.py 0 → 100644
import json
import os
import time
from random import choice
import requests
from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 国务院文件
def get_content1():
def getPageConunt(a_list, url, headers, s):
data = {"code": "18122f54c5c", "thirdPartyCode": "thirdparty_code_107", "thirdPartyTableId": 30,
"resultFields": ["pub_url", "maintitle", "fwzh", "cwrq", "publish_time"],
"trackTotalHits": "true",
"searchFields": [{"fieldName": "maintitle", "searchWord": ""}], "isPreciseSearch": 0,
"sorts": [{"sortField": "publish_time", "sortOrder": "DESC"}], "childrenInfoIds": [[a_list[1]]],
"pageSize": 20, "pageNo": 1}
data = json.dumps(data)
ip = baseCore.get_proxy()
res = s.post(url=url, headers=headers, data=data, verify=False, proxies=ip)
# 获得结果为json格式
res_text = json.loads(res.text)
pageCount = res_text['result']['data']['pager']['pageCount']
return pageCount
def getList(a_list, url, headers, pageNo, s):
# post请求所需参数
data = {"code": "18122f54c5c", "thirdPartyCode": "thirdparty_code_107", "thirdPartyTableId": 30,
"resultFields": ["pub_url", "maintitle", "fwzh", "cwrq", "publish_time"],
"trackTotalHits": "true",
"searchFields": [{"fieldName": "maintitle", "searchWord": ""}], "isPreciseSearch": 0,
"sorts": [{"sortField": "publish_time", "sortOrder": "DESC"}], "childrenInfoIds": [[a_list[1]]],
"pageSize": 20, "pageNo": pageNo}
data = json.dumps(data)
ip = baseCore.get_proxy()
res = s.post(url=url, headers=headers, data=data, verify=False, proxies=ip)
res_text = json.loads(res.text)
page_list = res_text['result']['data']['list']
return page_list
start_time = time.time()
num = 0
# 过网站验证所需 athenaAppKey athenaAppName
athenaAppKeys = [
'ZfaiEpAY%2B%2FYj5RjJrDfj2dn%2BcS4WRoxcLidI68z5l6SH8WE8CXSVP7QkJNIhy%2Bng4mZcwCuOFKNUvj%2FH6mR7sx0sCwgkIAfq4XNfHY6Fy7fxQ0NWm%2Fx7rmB5ow5OPdW5NMdI2RzURAFCVA9aIV4a1W8TFOjbQWYOukxUFtVJibU%3D',
'XU4ULRMBYHbE0I2fNNgCBxYTAg5Dk%2FPUEN9XeKy4OOAPdcZ6DW%2BrsopeI1gUwPmq3Y%2FtZJhH3NXiXqH5RxBAoYO231FHjPaMxD6QaMlA2BUeOFxmGKnuvJnIby1k6RmrCFd6IoXSImm5RFgcVed%2FvL6Qie2o6BAkRaEUHAitK18%3D',
'IFrNB5NkaDApRpF09SoT4fVBUoi7gRF2prj4EHk8eIVSEc1yYPpAZWVDMnqc2lVmeaQcNvrvZffA2kPVdvqjUHV5lGPJccRK3epnJ5Xx3xwIfTG7iIgrjFlqK1I93E0SIP6wyZJu42ksnF3nJdZ31sLEDCBeLi3pkggFtIEIQsg%3D',
'Nkgtgnyd%2B6jfdlclssI8FB9xRTQDdWzreONdqvta2aKZMRlhWoHhdj6L%2BQRyD8InaLWJC1zCSOkIy5b%2BjjZTg80t2jPu%2F1ifcRnboIj8%2BDIYWNSxMu%2Fdxze7oPtPo6sR08%2B3tQOE3ZntyFsGT44vCpa6DgK8ee3C5S58lanYXuI%3D',
'Zcko%2F7%2F2f2EuUmKpXbWnK3JtZtVy4trUNyE2JA5jVIw2r1oxTXVNZy8KQDmnOPDfyazdOrH6VYaJWloE4MukMK4VloB%2BRy6QhEaUvm%2Fsp4Enzl7doEk%2B1sZ1Y2iUd5REIhJQ%2Bp%2BB5iJEeNTmlQuRzYU3kOjDYtXftuehRTNKiXk%3D',
'Y2guFVvdtqMPhx5s9xThqdkvbe5hPaTlV7BYhcDuK7l%2BaXUqUMUHdim3uzn9IRlbHUtOLmRk6tfPEFM%2B8vzGDvI8U48acQ8Ff6MsfOGxShrQ7kW4tr4NaoE1sBW3PNkWj1Z0K6JzSXmAS2C1zVchTUYzTlfk62ghIeDtIPsPa6s%3D',
'a1drgLsStJotfBqHp1cFQg4lTJMMbgkTjVgCv34uy4Q%2BQ86DNEdc%2Fst0dZTUWFttuyXKNIH8%2FPYSSk465lXIn4wfuG4GuZLUk6wQo5PHNCUP0%2FvIL63IUxT0DCMo7lbsPq0ncdh4aiVswJe%2F6LM9U1m9OoaNGbeIUOl%2FxIOrMnE%3D',
'SGwQuPLZq2UzfaBPSwcR8DGZa4Ckh3Amp%2Bc1tMBFsMp%2Fh7Qn%2B9nspxdI3CW9S5LlfxYQmfa%2F%2B%2BJdH%2BBnxt0ILiCA4o9TUOxx27MhN9b4CLZnD8ZJ6sOwMszdFToDAD7hE21%2FzCzxhPNzPbyMXPpeMdi6sY0O2Sd85PLDtlZv%2FYQ%3D',
'CbvDEoIrP1%2BgMOuRJFhJNUGhzHBGnwdI6lIVG1ns1ZaLTlGRXLRgMjh9nBwLGMLTZwlPskklMbygvfA4P5UGhGT%2FpqKFkZne%2FAzTK8U6oJMo5%2FNAczbHhKwG7gdepIiiI7CgeNDtP8kurkcxnVS2KA1CLo8CVzmMlLHRmMPI8ag%3D']
athenaAppKey = choice(athenaAppKeys)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
'athenaAppKey': f"{athenaAppKey}",
'athenaAppName': "%E5%9B%BD%E7%BD%91%E6%90%9C%E7%B4%A2",
'Content-Type': 'application/json;charset=UTF-8',
}
headers_ = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
}
requests.adapters.DEFAULT_RETRIES = 5
url = 'https://sousuoht.www.gov.cn/athena/forward/486B5ABFBAD0FF5743F5E82E007EF04DDD6388E7989E9EC9CC7B84917AC81A5F'
result_list = [['国令', "1108"], ['国发', "1107"], ['国函', "1106"], ['国发明电', "1105"], ['国办发', "1104"],
['国办函', "1103"],
['国办发明电', "1102"], ['其他', "1101"]]
for a_list in result_list:
s = requests.session()
s.mount('https://', HTTPAdapter(max_retries=3))
s.mount('http://', HTTPAdapter(max_retries=3))
s.keep_alive = False
pcodeJiguan = a_list[0]
try:
# pageCount = getPageConunt(a_list, url, headers, s)
# for pageNo in range(1, pageCount + 1):
pageNo = 1
try:
try:
page_list = getList(a_list, url, headers, pageNo, s)
except:
s.close()
page_list = getList(a_list, url, headers, pageNo, s)
for page in page_list:
id_list = []
# 获取所需信息
title = page['maintitle'] # 标题
pub_time1 = page['publish_time'] # 发布时间
pub_time2 = page['cwrq'] # 成文时间
pub_code = page['fwzh'] # 发文字号
href = page['pub_url'] # 网址
# 判断是否已经爬取过
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
log.info('已采集----------跳过')
time.sleep(0.5)
continue
try:
resp_href = requests.get(url=href, headers=headers_, verify=False)
resp_href.encoding = resp_href.apparent_encoding
i_html = resp_href.text
if '您访问的页面不存在或已删除' in i_html:
# log.error(f'{title}...{href}...页面不存在或已删除')
continue
i_soup = BeautifulSoup(i_html, 'html.parser')
i_soup = baseTool.paserUrl(i_soup, href)
source = str(i_soup.find_all('tbody')[0])
pub_org = source.split('<td><b>发文机关：</b></td>')[1].split('<td>')[1].split('</td>')[
0] # 发文机关
child_type = source.split('<td class="w340 zcwj_ztfl">')[1].split('</td>')[0] # 主题分类
contentWithTag = i_soup.find('div', class_='wrap mxxgkwrap mxxgkwrap_gwywj').find('table',
class_='border-table noneBorder pages_content')
# 去除扫一扫
contentWithTag.find('div', attrs={'id': 'div_div'}).decompose()
content = contentWithTag.text # 不带标签正文
fu_jian_soup = contentWithTag.find_all('a')
time.sleep(0.5)
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
log.info(f'---{href}--------{e}-------')
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xls' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1766', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '国务院文件', file_name, num, pub_time1)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
except:
log.error(f'{title}...{href}...获取内容失败')
continue
# todo:替换完成之后，将附件上传至文件服务器
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list, # 附件id
'author': '', # 作者
'content': content, # 正文不带标签
'contentWithTag': str(contentWithTag), # 正文带标签
'createDate': time_now, # 创建时间
'deleteFlag': 0, # 是否删除(0为默认，1为删除)
'id': '', #
'labels': [{'relationId': "1766", 'relationName': "国务院文件", 'labelMark': "policy"}],
# 关联标签id 关联标签名称 关联标签标识
'origin': '', # 政策发布机关
'organ': pub_org, # 政策发文机关
'topicClassification': child_type, # 政策文件分类
'issuedNumber': pub_code, # 发文字号
'publishDate': pub_time1, # 发布时间
'writtenDate': pub_time2, # 成文时间
'sid': '1697458829758697473', # 信息源id
'sourceAddress': href, # 原文链接
'summary': '', # 摘要
'title': title # 标题
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
except:
log.error(f'{pcodeJiguan}...第{pageNo}页获取列表失败')
continue
except:
log.error(f'{pcodeJiguan}...获取总数失败')
continue
end_time = time.time()
log.info(f'共抓取国务院文件{num}条数据，共耗时{end_time - start_time}')
if __name__ == "__main__":
get_content1()
\ No newline at end of file
comData/policylaw/gwyparts.py 0 → 100644
import json
import os
import time
from random import choice
import requests
from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 国务院部门文件
def get_content2():
def getTotalpage(bmfl, headers, session):
ip = baseCore.get_proxy()
pageNo = 1
time.sleep(2)
# 拼接url
url_ = f'https://sousuo.www.gov.cn/search-gov/data?t=zhengcelibrary_bm&q=&timetype=timeqb&mintime=&maxtime=&sort=score&sortType=1&searchfield=title&puborg=&pcodeYear=&pcodeNum=&filetype=&p={pageNo}&n=20&inpro=&bmfl={bmfl}&dup=&orpro=&type=gwyzcwjk'
resp = session.get(url=url_, headers=headers, verify=False, proxies=ip)
resp_text = resp.text
resp_json = json.loads(resp_text)
totalpage = resp_json['searchVO']['totalpage']
return totalpage
def getContentList(bmfl, pageNo, headers, session):
ip = baseCore.get_proxy()
url_ = f'https://sousuo.www.gov.cn/search-gov/data?t=zhengcelibrary_bm&q=&timetype=timeqb&mintime=&maxtime=&sort=score&sortType=1&searchfield=title&puborg=&pcodeYear=&pcodeNum=&filetype=&p={pageNo}&n=20&inpro=&bmfl={bmfl}&dup=&orpro=&type=gwyzcwjk'
# 请求结果为json格式
resp = session.get(url=url_, headers=headers, verify=False, proxies=ip)
resp_text = resp.text
resp_json = json.loads(resp_text)
content_list = resp_json['searchVO']['listVO']
return content_list
session = requests.session()
session.mount('https://', HTTPAdapter(max_retries=3))
session.mount('http://', HTTPAdapter(max_retries=3))
session.keep_alive = False
start_time = time.time()
num = 0
count = 0
result_list = ['外交部', '国家发展和改革委员会', '教育部', '科学技术部', '工业和信息化部', '国家民族事务委员会', '公安部', '国家安全部', '民政部', '司法部', '财政部',
'人力资源和社会保障部', '自然资源部', '生态环境部', '住房和城乡建设部', '交通运输部', '水利部', '农业农村部', '商务部', '文化和旅游部',
'国家卫生健康委员会',
'退役军人事务部',
'应急管理部', '人民银行', '审计署', '国务院国有资产监督管理委员会', '海关总署', '国家税务总局', '国家市场监督管理总局', '国家金融监督管理总局',
'国家广播电视总局',
'国家体育总局',
'国家统计局', '国家国际发展合作署', '国家医疗保障局', '国家机关事务管理局', '国家标准化管理委员会', '国家新闻出版署', '国家版权局', '国家互联网信息办公室',
'中国科学院',
'中国社会科学院', '中国工程院', '中国气象局', '中国银行保险监督管理委员会', '中国证券监督管理委员会', '国家粮食和物资储备局', '国家能源局', '国家国防科技工业局',
'国家烟草专卖局',
'国家移民管理局', '国家林业和草原局', '国家铁路局', '中国民用航空局', '国家邮政局', '国家文物局', '国家中医药管理局', '国家矿山安全监察局', '国家外汇管理局',
'国家药品监督管理局',
'国家知识产权局', '国家档案局', '国家保密局', '国家密码管理局', '国家宗教事务局', '国务院台湾事务办公室', '国家乡村振兴局', '国家电影局']
for bmfl in result_list:
# try:
# totalpage = getTotalpage(bmfl,headers,session)
# for pageNo in range(1,totalpage+1):
# for pageNo in range(1,6):
pageNo = 1
try:
try:
content_list = getContentList(bmfl, pageNo, baseTool.headers, session)
except:
session.close()
content_list = getContentList(bmfl, pageNo, baseTool.headers, session)
for content_dict in content_list:
id_list = []
href = content_dict['url'] # 详情页
title = content_dict['title'] # 标题
pub_code = content_dict['pcode'] # 发文字号
try:
pub_time = int(content_dict['pubtime'] / 1000) # 发布时间
pub_time1 = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(pub_time))
except:
pub_time1 = None
try:
p_time = int(content_dict['ptime'] / 1000) # 成文时间
pub_time2 = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(p_time))
except:
pub_time2 = None
pub_org = content_dict['puborg'] # 发文机关
try:
child_type = content_dict['childtype'] # 主题分类
except:
child_type = ''
# # 判断是否已经爬取过
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
log.info('已采集----------跳过')
time.sleep(1)
continue
try:
resp = requests.get(url=href, headers=headers, verify=False)
resp.encoding = resp.apparent_encoding
resp_text = resp.text
soup = BeautifulSoup(resp_text, 'html.parser')
soup = baseTool.paserUrl(soup, href)
time.sleep(0.5)
contentWithTag = soup.find('div', attrs={'class': 'pages_content mhide'})
content = contentWithTag.text
if content == '' or content == 'None':
log.info(f'----{href}---{title}---内容为空---')
continue
fu_jian_soup = contentWithTag.find_all('a')
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
log.info(f'---{href}--------{e}-------')
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xls' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1699', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '国务院文件', file_name, num, pub_time1)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
except:
log.error(f'{title}...{href}获取内容失败')
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list, # 附件id
'author': '', # 作者
'content': content, # 正文不带标签
'contentWithTag': str(contentWithTag), # 正文带标签
'createDate': time_now, # 创建时间
'deleteFlag': 0, # 是否删除(0为默认，1为删除)
'id': '', #
'labels': [{'relationId': "1699", 'relationName': "国务院各部委文件", 'labelMark': "policy"}],
# 关联标签id 关联标签名称 关联标签标识
'origin': '', # 政策发布机关
'organ': pub_org, # 政策发文机关
'topicClassification': child_type, # 政策文件分类
'issuedNumber': pub_code, # 发文字号
'publishDate': pub_time1, # 发布时间
'writtenDate': pub_time2, # 成文时间
'sid': '1697458829758697473', # 信息源id
'sourceAddress': href, # 原文链接
'summary': '', # 摘要
'title': title # 标题
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
count += 1
num += 1
except:
log.error(f'{bmfl}...第{pageNo}页获取信息列表失败')
continue
# except:
# log.error(f'{bmfl}...获取页数失败')
# continue
end_time = time.time()
log.info(f'共抓取国务院部门文件{count}条数据，耗时{end_time - start_time}')
if __name__ == "__main__":
get_content2()
\ No newline at end of file
comData/policylaw/gwysasac.py 0 → 100644
import os
import re
import time
from pyquery import PyQuery as pq
import requests
from bs4 import BeautifulSoup
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 国务院国有资产监督管理委员会-政策发布
def get_content3():
pathType = 'policy/gyzc/'
def getPage():
url = "http://www.sasac.gov.cn/n2588035/n2588320/n2588335/index.html"
req = requests.get(url, headers=baseTool.headers, verify=False)
req.encoding = req.apparent_encoding
soup = BeautifulSoup(req.text, 'html.parser')
# totalpage = re.findall("总页数:(.*)", soup.select('#pag_2603340')[0].text)[0]
totalpage = '17'
return int(totalpage)
def sendContent(href, headers, title, pub_time, num):
id_list = []
resp_href = requests.request("GET", href, headers=headers, verify=False)
resp_href.encoding = resp_href.apparent_encoding
soup = BeautifulSoup(resp_href.text, 'lxml')
soup = baseTool.paserUrl(soup, href)
doc_href = soup.find('div', class_='zsy_content')
try:
org_content = doc_href.select('.zsy_cotitle')[0]
org = re.findall('文章来源：(.*?)发布时间：', org_content)[0].strip()
except:
org = ''
try:
contentWithTag = doc_href.find('div', class_='zsy_comain')
except:
return
contentWithTag.select('#qr_container')[0].decompose()
contentWithTag.find('div', attrs={'id': 'div_div'}).decompose()
contentWithTag.find('div', class_='related').decompose()
contentWithTag.find('div', class_='jiathis_style_24x24').decompose()
try:
p_list = contentWithTag.findAll('p')
pub_hao = ''
for p in p_list:
p = str(p.text)
if '号' in p and '〔' in p and '〕' in p or '[' in p and ']' in p and '号' in p or '【' in p and '】' in p and '号' in p:
try:
pub_hao = p.split('日')[1].split('自')[0].strip().lstrip()
except:
pub_hao = p.strip().lstrip()
break
except:
pub_hao = ''
if len(pub_hao) > 15:
pub_hao = ''
content = contentWithTag.text
if content == '' or content == 'None':
log.info(f'----{href}----{title}----内容为空----')
return
fu_jian_soup = contentWithTag.find_all('a')
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
log.info(f'---{href}--------{e}-------')
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xls' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1642', file_name)
if retData['state']:
pass
else:
continue
try:
att_id, full_path = baseCore.tableUpdate(retData, '国务院国资委', file_name, num, pub_time)
id_list.append(att_id)
except:
continue
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list, # 附件id
'author': '', # 作者
'content': content, # 正文不带标签
'contentWithTag': str(contentWithTag), # 正文带标签
'createDate': time_now, # 创建时间
'deleteFlag': 0, # 是否删除(0为默认，1为删除)
'id': '', #
'labels': [{'relationId': "1642", 'relationName': "国务院国资委", 'labelMark': "policy"}],
# 关联标签id 关联标签名称 关联标签标识
'origin': '', # 政策发布机关
'organ': org, # 政策发文机关
'topicClassification': '', # 政策文件分类
'issuedNumber': pub_hao, # 发文字号
'publishDate': pub_time, # 发布时间
'writtenDate': None, # 成文时间
'sid': '1697458829758697473', # 信息源id
'sourceAddress': href, # 原文链接
'summary': '', # 摘要
'title': title # 标题
}
# log.info(title)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
def partTwo():
start_time = time.time()
num = 0
count = 0
totalpage = getPage()
for page in range(1, totalpage):
url = f"http://www.sasac.gov.cn/n2588035/n2588320/n2588335/index_2603340_{page}.html"
href_resp = requests.request("GET", url, headers=baseTool.headers, verify=False)
resp_text = href_resp.content.decode('UTF-8')
li_list = resp_text.split('<li>')
del (li_list[0])
for li in li_list:
id_list = []
href_ = li.split('<a href="')[1].split('" target=')[0]
title = li.split('title="')[1].split('">')[0]
href = f'http://www.sasac.gov.cn{href_.replace("../../..", "")}'
pub_time = li.split('<span>[')[1].split(']</span>')[0]
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
log.info('已采集----------跳过')
continue
sendContent(href, baseTool.headers, title, pub_time, num)
num += 1
count += 1
end_time = time.time()
log.info(f'共抓取国资委文件{count}条数据，耗时{end_time - start_time}')
def partOne():
start_time = time.time()
num = 0
count = 0
url = "http://www.sasac.gov.cn/n2588035/n2588320/n2588335/index.html"
try:
# get请求,需要取消ssl验证
href_resp = requests.request("GET", url, headers=baseTool.headers, verify=False)
resp_text = href_resp.content.decode('UTF-8')
doc_resp = pq(resp_text)
doc_items = doc_resp('.zsy_conlist li').items()
time.sleep(1)
for doc_item in doc_items:
# 获取所需数据
try:
href_ = doc_item('a').attr('href')
if href_ is None:
continue
href = f'http://www.sasac.gov.cn{href_.replace("../../..", "")}'
# 判断是否已经爬取过
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
log.info('已采集----------跳过')
continue
title = doc_item('a').attr('title')
pub_time = doc_item('span').text().replace('[', '').replace(']', '')
except:
continue
sendContent(href, baseTool.headers, title, pub_time, num)
num += 1
count += 1
except:
pass
end_time = time.time()
log.info(f'共抓取国资委文件{count}条数据，耗时{end_time - start_time}')
# partOne()
# 增量执行需要注释掉partTwo()
partTwo()
if __name__ == "__main__":
get_content3()
\ No newline at end of file
comData/policylaw/hai_nan.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 海南
def hai_nan():
pathType = 'policy/hainan/'
def hai_nan1():
# 部门文件
num = 0
count = 0
start_time = time.time()
for page in range(13):
if page == 0:
url = "http://gzw.hainan.gov.cn/zwgk_23509/zfwj/bmwj/"
else:
url = 'http://gzw.hainan.gov.cn/zwgk_23509/zfwj/bmwj/' + '/' + 'index_{}.html'.format(page)
try:
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False).content
doc_resp = pq(resp_text)
doc_items = doc_resp('.list-right_title').items()
for doc_item in doc_items:
id_list = []
pub_time = doc_item.next().text().replace('发布时间： ', '') + ' 00:00:00'
title = doc_item('a:nth-child(2)').text().strip()
href = doc_item('a:nth-child(2)').attr('href')
if '../../' in href:
href = 'http://gzw.hainan.gov.cn/zwgk_23509/' + href.replace('../../', '')
elif './' in href:
href = href.replace('./', 'http://gzw.hainan.gov.cn/zwgk_23509/zfwj/bmwj/')
try:
is_href1 = baseTool.db_storage.find_one({'网址': href.split('?')[0]})
is_href2 = baseTool.db_storage.find_one({'网址': href})
except:
is_href1 = False
is_href2 = baseTool.db_storage.find_one({'网址': href})
if is_href1 or is_href2:
num += 1
log.info('已采集=====跳过')
continue
try:
try:
# print(href)
# href = 'http://gzw.hainan.gov.cn/zwgk_23509/gkbz/zfxxgkml/202211/t20221114_3307081.html'
href_text = requests.get(url=href, headers=baseTool.headers, verify=False).content
doc_href = pq(href_text)
pub_result = doc_href('.xxgk-syxl-t1023.clear').remove('script').text().replace(' ',
'').replace(
'　　', '')
# organ
pub_source = pub_result.split('发文机关：')[1].split('成文日期：')[0].strip()
origin = ''
pub_hao = pub_result.split('文号：')[1].split('发布日期：')[0].strip()
topicClassification = pub_result.split('分类：')[1].split('发文机关：')[0].strip()
writtenDate = pub_result.split('成文日期：')[1].split('文号：')[0].strip()
# content = str(doc_href('.xxgk-syxl').children()).replace('扫一扫在手机打开当前页', '')
sp = doc_href('p[style="font-size: 16px;font-weight: bold;line-height: 30px;"]>script')
sp.remove()
content1 = doc_href('.xxgk-syxl-m-l-nr2')
soup = BeautifulSoup(str(content1), 'html.parser')
# 相对路径转化为绝对路径
contentWithTag = baseTool.paserUrl(soup, href)
# 去掉扫一扫
try:
contentWithTag.find('div', id='div_div').decompose()
except:
pass
content = contentWithTag.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
fu_jian_list = contentWithTag.find_all('a')
for fu_jian in fu_jian_list:
try:
file_name = fu_jian.text
fu_jian_href = fu_jian['href']
except:
continue
if '.doc' in fu_jian_href or '.pdf' in fu_jian_href or '.docx' in fu_jian_href or '.xlsx' in fu_jian_href or '.xls' in fu_jian_href or '.zip' in fu_jian_href \
or '.rar' in fu_jian_href or '.ppt' in fu_jian_href or '.PDF' in fu_jian_href or '.DOC' in fu_jian_href \
or '.XLS' in fu_jian_href or '.ZIP' in fu_jian_href or '.RAR' in fu_jian_href:
category = os.path.splitext(fu_jian_href)[1]
if category not in file_name:
file_name = file_name + category
# 上传至文件服务器
retData = baseCore.uptoOBS(fu_jian_href, '1677', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '海南省国资委', file_name, num,
pub_time)
id_list.append(att_id)
# 将文件服务器的链接替换
fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
except:
try:
# print(href)
resp = requests.get(url=href, headers=baseTool.headers, verify=False)
resp.encoding = resp.apparent_encoding
resp_text = resp.text
source = BeautifulSoup(resp_text, 'html.parser')
source = baseTool.paserUrl(source, href)
tbody_text = str(source.find('tbody').text)
# organ
pub_source = tbody_text.split('发文机关：')[1].split('发文日期：')[0].strip().lstrip()
origin = ''
pub_hao = tbody_text.split('文　　号：')[1].split('主 题 词：')[0].strip().lstrip()
pub_time = tbody_text.split('发文日期：')[1].split('名　　称：')[0].strip().lstrip().replace('年',
'-').replace(
'月', '-').replace('日', '')
writtenDate = None
topicClassification = tbody_text.split('分　　类：')[1].split('发文机关：')[0].strip().lstrip()
contentWithTag = source.find('div', attrs={'class': 'zx-xxxqy-nr'})
content = contentWithTag.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
fu_jian_list = source.find_all('a')
try:
for fu_jian in fu_jian_list:
try:
file_name = fu_jian.text
fu_jian_href = fu_jian['href']
except:
continue
if '.doc' in fu_jian_href or '.pdf' in fu_jian_href or '.xls' in fu_jian_href or '.zip' in fu_jian_href \
or '.rar' in fu_jian_href or '.ppt' in fu_jian_href or '.PDF' in fu_jian_href or '.DOC' in fu_jian_href \
or '.XLS' in fu_jian_href or '.ZIP' in fu_jian_href or '.RAR' in fu_jian_href:
category = os.path.splitext(fu_jian_href)[1]
if category not in file_name:
file_name = file_name + category
# print(f'----附件：{fu_jian_href}-----filename:{file_name}')
# 附件上传至文件服务器
retData = baseCore.uptoOBS(fu_jian_href, '1677', file_name)
if retData['state']:
pass
else:
continue
# 更新到数据库
att_id, full_path = baseCore.tableUpdate(retData, '海南省国资委', file_name, num,
pub_time)
id_list.append(att_id)
fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
except:
continue
except:
resp = requests.get(url=href, headers=baseTool.headers, verify=False)
resp.encoding = resp.apparent_encoding
resp_text = resp.text
source = BeautifulSoup(resp_text, 'html.parser')
# 去掉扫一扫
try:
source.find('div', id='div_div').decompose()
# 相对路径替换为绝对路径
source = baseTool.paserUrl(source, href)
except:
# 相对路径替换为绝对路径
source = baseTool.paserUrl(source, href)
pub_time = str(source.find('div', attrs={'class': 'con_div'}).text).split('来源：')[
0].lstrip().strip()
origin = \
str(source.find('div', attrs={'class': 'con_div'}).text).split('来源：')[1].split(
' 【字体：')[
0].strip().lstrip()
pub_source = ''
pub_hao = ''
writtenDate = None
topicClassification = ''
contentWithTag = source.find('div', attrs={'class': 'TRS_UEDITOR'})
content = contentWithTag.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': content,
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1677", 'relationName': "海南省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': pub_source,
'topicClassification': topicClassification,
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
log.info(title)
count += 1
num = num + 1
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
def hai_nan2():
def hai_nan_sw(page_href):
num = 0
count = 0
req = requests.get(url=page_href, headers=baseTool.headers, verify=False)
req.encoding = req.apparent_encoding
doc_resp = BeautifulSoup(req.text, 'html.parser')
doc_items = doc_resp.find_all(class_='list-right_title fon_1')
for doc_item in doc_items:
title = str(doc_item).split('target="_blank">')[1].split('</a>')[0]
href = 'https://www.hainan.gov.cn' + str(doc_item).split('href="')[1].split('" target')[0]
# print(title,href)
try:
is_href1 = baseTool.db_storage.find_one({'网址': href.split('?')[0]})
is_href2 = baseTool.db_storage.find_one({'网址': href})
except:
is_href1 = False
is_href2 = baseTool.db_storage.find_one({'网址': href})
if is_href1 or is_href2:
num += 1
log.info('已采集=====跳过')
continue
try:
# print(href)
href_text = requests.get(url=href, headers=baseTool.headers, verify=False)
doc_href = BeautifulSoup(href_text.content, 'html.parser')
# print(doc_href)
# 相对路径转化为绝对路径
doc_href = baseTool.paserUrl(doc_href, href)
# 去掉扫一扫
try:
doc_href.find('div', id='div_div').decompose()
except:
pass
pub_result = doc_href.find(class_='zwgk_comr1')
pub_result = pub_result.find_all('li')
# print(pub_result)
topicClassification = str(pub_result[0]).split('主题分类：</strong>')[1].split('</span>')[0]
# organ
pub_source = str(pub_result[1]).split('</strong>')[1].split('</span>')[0]
writtenDate = str(pub_result[1]).split('成文日期：</strong>')[1].split('</span>')[0]
pub_hao = str(pub_result[3]).split('文  号：</strong>')[1].split('</span>')[0].strip()
pub_time = str(pub_result[3]).split('发布日期：</strong>')[1].split('</span>')[0].strip()
contentWithTag = doc_href.find(class_='con_cen line mar-t2 xxgk_content_content')
content = contentWithTag.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': [],
'author': '',
'content': content,
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1677", 'relationName': "海南省国资委", 'labelMark': "policy"}],
'origin': '',
'organ': pub_source,
'topicClassification': topicClassification,
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
log.info(title)
num += 1
count += 1
href_text.close()
except:
pass
req.close()
return num
def hai_nan_szf(page_href):
num = 0
count = 0
req = requests.get(url=page_href, headers=baseTool.headers, verify=False)
req.encoding = req.apparent_encoding
doc_resp = BeautifulSoup(req.text, 'html.parser')
doc_items = doc_resp.find_all(class_='list-right_title fon_1')
for doc_item in doc_items:
id_list = []
title = str(doc_item).split('target="_blank">')[1].split('</a>')[0].replace('\n', '')
href = 'https://www.hainan.gov.cn' + str(doc_item).split('href="')[1].split('" target')[0]
# print(title,href)
try:
is_href1 = baseTool.db_storage.find_one({'网址': href.split('?')[0]})
is_href2 = baseTool.db_storage.find_one({'网址': href})
except:
is_href1 = False
is_href2 = baseTool.db_storage.find_one({'网址': href})
if is_href1 or is_href2:
num += 1
log.info('已采集=====跳过')
continue
try:
# print(href)
href_text = requests.get(url=href, headers=baseTool.headers, verify=False)
doc_href = BeautifulSoup(href_text.content, 'html.parser')
# 相对路径转化为绝对路径
doc_href = baseTool.paserUrl(doc_href, href)
# 去掉扫一扫
try:
doc_href.find('div', id='div_div').decompose()
except:
pass
# print(doc_href)
try:
pub_result = doc_href.find(class_='zwgk_comr1')
pub_result = pub_result.find_all('li')
# print(pub_result)
topicClassification = str(pub_result[0]).split('主题分类：</strong>')[1].split('</span>')[0]
# organ
origin = ''
pub_source = str(pub_result[1]).split('</strong>')[1].split('</span>')[0]
pub_hao = str(pub_result[3]).split('文  号：</strong>')[1].split('</span>')[0].strip()
writtenDate = str(pub_result[1]).split('成文日期：</strong>')[1].split('</span>')[0]
pub_time = str(pub_result[3]).split('发布日期：</strong>')[1].split('</span>')[0].strip()
contentWithTag = doc_href.find(class_='con_cen line mar-t2 xxgk_content_content')
content = contentWithTag.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
except:
# print(href)
pub_result = doc_href.find('div', attrs={'class': 'line mar-t2 con_div'})
topicClassification = ''
origin = str(pub_result.text).split('来源：')[1].split(' 【字体：')[0].lstrip().strip()
pub_source = ''
pub_time = str(pub_result.text).split('来源：')[0].lstrip().strip()
pub_hao = ''
writtenDate = None,
contentWithTag = doc_href.find('div', attrs={'class': 'xxgk_content_content'})
content = contentWithTag.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
fu_jian_list = contentWithTag.find_all('a')
for fu_jian in fu_jian_list:
try:
fu_jian_href = fu_jian['href']
except:
continue
file_name = fu_jian.text
if '.doc' in fu_jian_href or '.docx' in fu_jian_href or '.pdf' in fu_jian_href or '.xls' in fu_jian_href or '.xlsx' in fu_jian_href or '.zip' in fu_jian_href \
or '.rar' in fu_jian_href or '.ppt' in fu_jian_href or '.PDF' in fu_jian_href or '.DOC' in fu_jian_href \
or '.XLS' in fu_jian_href or '.ZIP' in fu_jian_href or '.RAR' in fu_jian_href:
category = os.path.splitext(fu_jian_href)[1]
if category not in file_name:
file_name = file_name + category
# 上传至文件服务器
retData = baseCore.uptoOBS(fu_jian_href, '1677', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '海南省国资委', file_name, num, pub_time)
id_list.append(att_id)
# todo:将返回的地址更新到soup
fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# print(f'附件：{fu_jian_href}')
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': content,
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1677", 'relationName': "海南省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': pub_source,
'topicClassification': topicClassification,
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
log.info(title)
num += 1
count += 1
href_text.close()
# save_data(result_dict)
except:
pass
req.close()
return num
def hai_nan_szfbgt(page_href):
num = 0
count = 0
req = requests.get(url=page_href, headers=baseTool.headers, verify=False)
req.encoding = req.apparent_encoding
doc_resp = BeautifulSoup(req.text, 'html.parser')
doc_items = doc_resp.find_all(class_='list-right_title fon_1')
for doc_item in doc_items:
id_list = []
title = str(doc_item).split('target="_blank">')[1].split('</a>')[0]
href = 'https://www.hainan.gov.cn' + str(doc_item).split('href="')[1].split('" target')[0]
# print(title,href)
try:
is_href1 = baseTool.db_storage.find_one({'网址': href.split('?')[0]})
is_href2 = baseTool.db_storage.find_one({'网址': href})
except:
is_href1 = False
is_href2 = baseTool.db_storage.find_one({'网址': href})
if is_href1 or is_href2:
num += 1
log.info('已采集=====跳过')
continue
try:
href_text = requests.get(url=href, headers=baseTool.headers, verify=False)
doc_href = BeautifulSoup(href_text.content, 'html.parser')
# 相对路径转化为绝对路径
doc_href = baseTool.paserUrl(doc_href, href)
# print(doc_href)
try:
# print(href)
pub_result = doc_href.find(class_='zwgk_comr1')
pub_result = pub_result.find_all('li')
# print(pub_result)
topicClassification = str(pub_result[0]).split('主题分类：</strong>')[1].split('</span>')[0]
# organ
origin = ''
pub_source = str(pub_result[1]).split('</strong>')[1].split('</span>')[0]
writtenDate = str(pub_result[1]).split('成文日期：</strong>')[1].split('</span>')[0]
try:
pub_hao = str(pub_result[3]).split('文  号：</strong>')[1].split('</span>')[0].strip()
except:
pub_hao = str(pub_result[3]).split('文       号：</strong>')[1].split('</span>')[0].strip()
pub_time = str(pub_result[3]).split('发布日期：</strong>')[1].split('</span>')[0].strip()
contentWithTag = doc_href.find(class_='con_cen line mar-t2 xxgk_content_content')
content = contentWithTag.text
except:
# print(href)
pub_result = doc_href.find('div', attrs={'class': 'line mar-t2 con_div'})
topicClassification = ''
origin = str(pub_result.text).split('来源：')[1].split(' 【字体：')[0].lstrip().strip()
pub_time = str(pub_result.text).split('来源：')[0].lstrip().strip()
pub_hao = ''
pub_source = ''
writtenDate = None,
contentWithTag = doc_href.find('div', attrs={'class': 'xxgk_content_content'})
content = contentWithTag.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
fu_jian_list = contentWithTag.find_all('a')
if fu_jian_list:
for fu_jian in fu_jian_list:
try:
fu_jian_href = fu_jian['href']
except:
continue
file_name = fu_jian.text
if '.doc' in fu_jian_href or '.docx' in fu_jian_href or '.xlsx' in fu_jian_href or '.pdf' in fu_jian_href or '.xls' in fu_jian_href or '.zip' in fu_jian_href \
or '.rar' in fu_jian_href or '.ppt' in fu_jian_href or '.PDF' in fu_jian_href or '.DOC' in fu_jian_href \
or '.XLS' in fu_jian_href or '.ZIP' in fu_jian_href or '.RAR' in fu_jian_href:
category = os.path.splitext(fu_jian_href)[1]
if category not in file_name:
file_name = file_name + category
# 上传至文件服务器
retData = baseCore.uptoOBS(fu_jian_href, '1677', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '海南省国资委', file_name, num, pub_time)
id_list.append(att_id)
fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# print(f'----附件：{fu_jian_href}')
else:
pass
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': content,
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1677", 'relationName': "海南省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': pub_source,
'topicClassification': topicClassification,
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
log.info(title)
num += 1
count += 1
href_text.close()
except:
pass
req.close()
return num
def hai_nan_zy(page_href):
num = 0
count = 0
req = requests.get(url=page_href, headers=baseTool.headers, verify=False)
req.encoding = req.apparent_encoding
doc_resp = BeautifulSoup(req.content, 'html.parser')
list_div = doc_resp.find('div', attrs={'class': 'list list_1 list_2'})
doc_items = list_div.find_all('li')
for doc_item in doc_items:
title = str(doc_item.find('a').text)
i_href = doc_item.find('a').get('href')
# https://www.gov.cn/zhengce/202307/content_6893055.htm
if 'https://www.gov.cn/zhengce/' not in i_href:
i_href = str(i_href).replace('../../', 'https://www.gov.cn/zhengce/')
try:
try:
is_href1 = baseTool.db_storage.find_one({'网址': i_href.split('?')[0]})
is_href2 = baseTool.db_storage.find_one({'网址': i_href})
except:
is_href1 = False
is_href2 = baseTool.db_storage.find_one({'网址': i_href})
if is_href1 or is_href2:
num += 1
log.info('已采集=====跳过')
continue
if i_href == 'https://www.gov.cn/jrzg/2013-11/27/content_2536600.htm':
num += 1
continue
if i_href == 'https://www.gov.cn/jrzg/2013-09/28/content_2497241.htm':
num += 1
continue
# print(f'中央----{i_href}----')
href_text = requests.get(url=i_href, headers=baseTool.headers, verify=False)
doc_href = BeautifulSoup(href_text.content, 'html.parser')
# 相对路径转化为绝对路径
doc_href = baseTool.paserUrl(doc_href, i_href)
# 去掉扫一扫
try:
doc_href.find('div', id='div_div').decompose()
except:
pass
try:
pub_result = doc_href.find('table', class_='bd1').find_all('td')
pub_time = \
str(pub_result[13]).replace('年', '-').replace('月', '-').replace('日', '').split('<td>')[
1].split('</td>')[0]
pub_source = str(pub_result[5]).split('<td>')[1].split('</td>')[0]
pub_hao = str(pub_result[11]).split('<td>')[1].split('</td>')[0]
contentWithTag = doc_href.find(class_='b12c')
content = contentWithTag.text
except:
try:
pub_result = doc_href.find(class_='pages-date')
pub_source = pub_result.find('span', class_='font').text.replace('来源：', '').strip()
pub_time = str(pub_result).split('<span')[0].split('"pages-date">')[1].split('来源')[
0].strip()
except:
pub_source = ''
pub_time = None
pub_hao = ''
contentWithTag = doc_href.find(class_='pages_content')
content = contentWithTag.text
if content == '' or content == None:
log.info(f'-----{i_href}----{title}----内容为空-----')
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': [],
'author': '',
'content': content,
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1677", 'relationName': "海南省国资委", 'labelMark': "policy"}],
'origin': '',
'organ': pub_source,
'topicClassification': '',
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': i_href,
'summary': '',
'title': title
}
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
log.info(title)
num += 1
count += 1
href_text.close()
# save_data(result_dict)
except:
pass
req.close()
return num
def start():
num = 0
count = 0
start_time = time.time()
url = "https://www.hainan.gov.cn/hainan/qzcwj/zywj.shtml"
try:
req = requests.get(url=url, headers=baseTool.headers, verify=False)
req.encoding = req.apparent_encoding
doc_resp = pq(req.text)
doc_items = doc_resp('.nzcti').items()
leibie_href_list = []
for doc_item in doc_items:
# print(doc_item)
leibie = doc_item('a').text()
leibie_href = doc_item('a').attr('href')
if '更多' in leibie:
leibie = leibie.split('更多>> ')[1]
# print(leibie)
leibie_href = 'https://www.hainan.gov.cn' + doc_item('a').attr('href')
leibie_href_list.append(leibie_href)
# 每一个类别的文件
for url in leibie_href_list:
# 翻页
if url == leibie_href_list[0]:
max_page = 23
for page in range(max_page):
if max_page == 0:
page_href = str(url) + 'home.htm'
else:
page_href = str(url) + f'home_{page}.htm'
try:
count += hai_nan_zy(page_href)
except:
pass
time.sleep(1)
elif url == leibie_href_list[1]:
# https://www.hainan.gov.cn/hainan/swygwj/list3_2.shtml
max_page = 8
for page in range(max_page):
if page == 0:
page_href = 'https://www.hainan.gov.cn/hainan/swygwj/list3.shtml'
else:
page_href = str(url).split('list3')[0] + 'list3_{}.shtml'.format(page + 1)
try:
count += hai_nan_sw(page_href)
except:
pass
elif url == leibie_href_list[2]:
max_page = 84
for page in range(max_page):
if page == 0:
page_href = 'https://www.hainan.gov.cn/hainan/szfwj/list3.shtml'
else:
page_href = str(url).split('list3')[0] + 'list3_{}.shtml'.format(page + 1)
try:
count += hai_nan_szf(page_href)
except:
pass
else:
max_page = 84
for page in range(max_page):
if page == 0:
page_href = 'https://www.hainan.gov.cn/hainan/szfbgtwj/list3.shtml'
else:
page_href = str(url).split('list3')[0] + 'list3_{}.shtml'.format(page + 1)
try:
count += hai_nan_szfbgt(page_href)
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
start()
hai_nan1()
hai_nan2()
if __name__ == "__main__":
hai_nan()
\ No newline at end of file
comData/policylaw/he_bei.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 河北
def he_bei():
num = 0
count = 0
start_time = time.time()
url = 'http://hbsa.hebei.gov.cn/Json/GFXWJ51.json'
try:
res = requests.get(url, baseTool.headers)
# print(res)
json = res.json()
# print(json)
for info in json:
title = info['title']
contentWithTag = info['content']
id = info['id']
href = 'http://hbsa.hebei.gov.cn/xxgk/GFXWJ?id=' + str(id)
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
pub_time_ = info['updated']
m = round(pub_time_ / 1000) # 四舍五入取10位时间戳（秒级）
n = time.localtime(m) # 将时间戳转换成时间元祖tuple
publishDate = time.strftime("%Y-%m-%d %H:%M:%S", n)[:10] # 格式化输出时间
origin = ''
soup = baseTool.paserUrl(str(contentWithTag), href)
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1668', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '河北省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# id_ = redefid(id_list)
contentWithTag = str(soup.prettify())
if len(contentWithTag) < 1:
if len(fu_jian_soup) < 1:
continue
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
pattern = r'(冀国.{1,}?号)|(国资.{1,}?号)'
match_list = re.findall(pattern, content)
if len(match_list) > 0:
issuedNumber = match_list[0][0]
if len(issuedNumber) > 20:
issuedNumber = ''
else:
issuedNumber = ''
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1668", 'relationName': "河北省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': "",
'topicClassification': "",
'issuedNumber': issuedNumber,
'publishDate': publishDate,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
he_bei()
\ No newline at end of file
comData/policylaw/he_nan.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 河南
def he_nan():
num = 0
count = 0
pathType = 'policy/henan/'
start_time = time.time()
for page in range(0, 7):
if page == 0:
url = 'http://gzw.henan.gov.cn/xxgk/fdzdgknr/zcfg/index.html'
else:
url = f'http://gzw.henan.gov.cn/xxgk/fdzdgknr/zcfg/index_{page}.html'
try:
resp_text = requests.get(url=url, headers=headers, verify=False)
doc_resp = pq(resp_text.content)
doc_items = doc_resp('.mt15.list-box li').items()
for doc_item in doc_items:
title = doc_item('a').text().strip()
href = doc_item('a').attr('href')
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
href_res = requests.get(url=href, headers=headers, verify=False)
href_res.encoding = href_res.apparent_encoding
href_text = href_res.text
soup = BeautifulSoup(href_text, 'html.parser')
origin = soup.select('#source')[0].text
publishDate = soup.select('#pubDate')[0].text
contentWithTag = str(soup.select('#content')[0])
# contentWithTag =doc('div[class="information-zt-show"]')
# soup=BeautifulSoup(str(contentWithTag), 'html.parser')
soup = baseTool.paserUrl(str(contentWithTag), href)
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1690', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '河南省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
contentWithTag = str(soup.prettify())
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
pattern = r'(豫国.{1,}?号)|(国.{1,}?号)'
match_list = re.findall(pattern, content)
if len(match_list) > 0:
issuedNumber = match_list[0][0]
else:
issuedNumber = ''
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1690", 'relationName': "河南省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': '',
'topicClassification': '',
'issuedNumber': issuedNumber,
'publishDate': publishDate,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
href_res.close()
resp_text.close()
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
he_nan()
\ No newline at end of file
comData/policylaw/hei_long_jiang.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 黑龙江
def hei_long_jiang():
pathType = 'policy/heilongjiang/'
num = 0
count = 0
start_time = time.time()
for page in range(1, 3):
url = f'http://gzw.hlj.gov.cn/common/search/a4e4f3e94596456db749bfb0f7937cc7?_isAgg=true&_isJson=true&_pageSize=10&_template=index&_rangeTimeGte=&_channelName=&page={page}'
try:
web = requests.get(url=url, headers=baseTool.headers, verify=False)
text = web.json()
rows = text['data']['rows']
try:
for row in range(int(rows)):
result = text['data']['results'][row]
title = result['title']
href = 'http://gzw.hlj.gov.cn' + result['url']
publishDate = result['publishedTimeStr']
list_all = text['data']['results'][row]['domainMetaList'][1]['resultList'][0]
if list_all['name'] == '文号':
pub_hao = list_all['value']
else:
pub_hao = ''
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
contentWithTag = text['data']['results'][row]['contentHtml']
href_text = requests.get(url=href, headers=baseTool.headers, verify=False)
href_text.encoding = href_text.apparent_encoding
href_text = href_text.text
doc_href = BeautifulSoup(href_text, 'html.parser')
origin = doc_href.find(class_='ly')
if origin:
origin = origin.find('b').text
else:
origin = ''
soup = baseTool.paserUrl(str(contentWithTag), href)
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1687', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '江苏省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
contentWithTag = str(soup.prettify())
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1687", 'relationName': "江苏省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': '',
'topicClassification': '',
'issuedNumber': pub_hao,
'publishDate': publishDate,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except:
pass
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
hei_long_jiang()
\ No newline at end of file
comData/policylaw/hu_bei.py 0 → 100644
import os
import time
from pyquery import PyQuery as pq
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 湖北
def hu_bei(chromr_bin=None):
num = 0
count = 0
start_time = time.time()
hrefs = []
url = 'http://gzw.hubei.gov.cn/zfxxgk/zc/gfxwj/'
chrome_driver = baseTool.driver_path
path = Service(chrome_driver)
chrome_options = webdriver.ChromeOptions()
# chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
chrome_options.add_argument("--disable-blink-features=AutomationControlled")
chrome_options.binary_location = baseTool.chromr_bin
driver = webdriver.Chrome(service=path, chrome_options=chrome_options)
driver.get(url)
time.sleep(2)
ul = driver.find_element(By.ID, 'ulList')
li_list = ul.find_elements(By.TAG_NAME, 'li')
time.sleep(1)
for li in li_list:
href = li.find_element(By.TAG_NAME, 'a').get_attribute('href')
hrefs.append(href)
for href in hrefs:
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
driver.get(href)
time.sleep(2)
dhtml = driver.page_source
if len(dhtml) < 400:
driver.get(href)
time.sleep(2)
doc = pq(dhtml)
article = doc('div[class="article"]')
adoc = pq(article)
title = adoc('h2').text()
publishDate = adoc('div[class="info"]>span:nth-child(1)').text()
origin = adoc('div[class="info"]>span:nth-child(3)').text()
organ = ''
topicClassification = adoc('td[bfdi="93"]').text()
issuedNumber = adoc('td[bfdi="101"]').text()
writtenDate = adoc('td[bfdi="98"]').text()
rmtag = adoc('p:contains("附件：")')
rmtag2 = adoc('div[class="hbgov-qrcode-content"]')
rmtag.remove()
rmtag2.remove()
contentWithTag = adoc('div[class="article-box"]')
soup = baseTool.paserUrl(str(contentWithTag), href)
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1675', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '湖北省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# id_ = redefid(id_list)
contentWithTag = str(soup.prettify())
if len(contentWithTag) < 1:
if len(fu_jian_soup) < 1:
continue
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1675", 'relationName': "湖北省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': organ,
'topicClassification': topicClassification,
'issuedNumber': issuedNumber,
'publishDate': publishDate,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except Exception as e:
pass
driver.close()
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
hu_bei()
\ No newline at end of file
comData/policylaw/hu_nan.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 湖南
def hu_nan():
num = 0
count = 0
pathType = 'policy/hunan/'
start_time = time.time()
for page in range(1, 7):
if page == 1:
# http://gzw.hunan.gov.cn/gzw/xxgk_71571/zcfg/index.html
url = 'http://gzw.hunan.gov.cn/gzw/xxgk_71571/zcfg/index.html'
else:
url = f'http://gzw.hunan.gov.cn/gzw/xxgk_71571/zcfg/index_{page}.html'
try:
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False).content
doc_resp = pq(resp_text)
doc_items = doc_resp('.table tbody tr').items()
for doc_item in doc_items:
href = 'http://gzw.hunan.gov.cn' + doc_item('a').attr('href')
publishDate = doc_item('td:nth-child(3)').text()
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
# href = 'http://gzw.hunan.gov.cn/gzw/xxgk_71571/zcfg/201109/t20110920_1942364.html'
try:
res = requests.get(url=href, headers=baseTool.headers, verify=False)
res.encoding = res.apparent_encoding
res_text = res.text
# soup = BeautifulSoup(res_text, 'html.parser')
soup = baseTool.paserUrl(res_text, href)
# pub_result = str(soup.find('div', attrs={'class': 'information-zt-list fn-clear'}).text)
# writtenDate = pub_result.split('发文日期：')[1].split('名称：')[0].strip() + ':00'
# title = pub_result.split('名称：')[1].split('主题分类：')[0].lstrip().strip()
# organ = pub_result.split('发布机构: ')[1].split('if(')[0].lstrip().strip()
doc = pq(str(soup))
organ = doc('div[class="information-zt-list fn-clear"]>ul>li:nth-child(3)').text().replace('发布机构：',
'')
if 'document.write' in organ:
organ = ''
writtenDate = doc('div[class="information-zt-list fn-clear"]>ul>li:nth-child(4)').text().replace(
'发文日期：', '')
title = doc('div[class="information-zt-list fn-clear"]>ul>li:nth-child(5)').text().replace('名称：',
'')
topicClassification = doc(
'div[class="information-zt-list fn-clear"]>ul>li:nth-child(6)').text().replace('主题分类：', '')
contentWithTag = doc('div[class="information-zt-show"]')
soup = BeautifulSoup(str(contentWithTag), 'html.parser')
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1691', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '湖南省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
contentWithTag = str(soup.prettify())
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1691", 'relationName': "湖南省国资委", 'labelMark': "policy"}],
'origin': '',
'organ': organ,
'topicClassification': topicClassification,
'issuedNumber': '',
'publishDate': publishDate,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
hu_nan()
\ No newline at end of file
comData/policylaw/jiang_su.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 江苏
def jiang_su():
num = 0
count = 0
pathType = 'policy/jiangsu/'
start_time = time.time()
pagestart = 1
pageend = 45
for page in range(1, 3):
url = f"http://jsgzw.jiangsu.gov.cn/module/web/jpage/dataproxy.jsp?startrecord={pagestart}&endrecord={pageend}&perpage=15"
pagestart = pageend + 1
pageend = pageend + 45
payload = "col=1&appid=1&webid=39&path=%2F&columnid=85683&sourceContentType=1&unitid=369983&webname=%E6%B1%9F%E8%8B%8F%E7%9C%81%E5%9B%BD%E8%B5%84%E5%A7%94&permissiontype=0"
header = {
'Connection': 'keep-alive',
'Accept': 'application/xml, text/xml, */*; q=0.01',
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Origin': 'http://jsgzw.jiangsu.gov.cn',
'Referer': 'http://jsgzw.jiangsu.gov.cn/col/col61490/index.html?uid=247686&pageNum=4',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cookie': 'JSESSIONID=ADB520E83E1FC10429D961634BAD303D; __jsluid_h=02c2c950abb71f547a79da79719246aa; _gscu_210493472=24936291qq5dvl18; _gscbrs_210493472=1; yunsuo_session_verify=60cc00825d4e2dd3dee278a301f60f1e; _gscs_210493472=24936291p77pyu18|pv:3'
}
try:
resp_text = requests.request("POST", url, headers=header, data=payload).text
li_list = re.findall('CDATA\[(.*?)\]\]></record>', str(resp_text))
for li in li_list:
a = BeautifulSoup(li, 'lxml').find('a')
href = 'https://jsgzw.jiangsu.gov.cn/' + a['href']
title = a.text
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
href_text = requests.get(url=href, headers=baseTool.headers, verify=False)
href_text.encoding = href_text.apparent_encoding
href_text = href_text.text
doc_href = BeautifulSoup(href_text, 'html.parser')
soup = baseTool.paserUrl(href_text, href)
doc = pq(str(soup))
publishDate = doc('div[class="cf tip"]>span:contains(发布日期：)').text().replace('发布日期：', '')
writtenDate = doc('table[class="xlt_table"]>tbody>tr:nth-child(1)>td:nth-child(4)').text()
organ = doc('table[class="xlt_table"]>tbody>tr:nth-child(2)>td:nth-child(2)').text()
pub_hao = doc('table[class="xlt_table"]>tbody>tr:nth-child(2)>td:nth-child(4)').text()
contentWithTag = doc('div[id="zoom"]')
if len(contentWithTag) < 1:
contentWithTag = doc('div[class="main-txt"]')
soup = baseTool.paserUrl(str(contentWithTag), href)
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1687', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '江苏省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
contentWithTag = str(soup.prettify())
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
if len(pub_hao) < 1:
pattern = r'(苏国.{1,}?号)|(国.{1,}?号)'
match_list = re.findall(pattern, content)
if len(match_list) > 0:
pub_hao = match_list[0][0]
else:
pub_hao = ''
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1687", 'relationName': "江苏省国资委", 'labelMark': "policy"}],
'origin': '',
'organ': organ,
'topicClassification': '',
'issuedNumber': pub_hao,
'publishDate': publishDate,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
jiang_su()
\ No newline at end of file
comData/policylaw/jiang_xi.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 江西
def jiang_xi():
"""
1-60
61-120
121-164
"""
num = 0
count = 0
pathType = 'policy/jiangxi/'
start_time = time.time()
startrecord = 1
endrecord = 60
for page in range(1, 3):
url = f"http://gzw.jiangxi.gov.cn/module/web/jpage/dataproxy.jsp?startrecord={startrecord}&endrecord={endrecord}&perpage=20"
startrecord = endrecord + 1
endrecord = endrecord + 60
payload = "col=1&webid=175&path=http%3A%2F%2Fgzw.jiangxi.gov.cn%2F&columnid=22977&sourceContentType=1&unitid=402016&webname=%E6%B1%9F%E8%A5%BF%E7%9C%81%E5%9B%BD%E6%9C%89%E8%B5%84%E4%BA%A7%E7%9B%91%E7%9D%A3%E7%AE%A1%E7%90%86%E5%A7%94%E5%91%98%E4%BC%9A&permissiontype=0"
header = {
'Connection': 'keep-alive',
'Accept': 'application/xml, text/xml, */*; q=0.01',
'X-Requested-With': 'XMLHttpRequest',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Origin': 'http://gzw.jiangxi.gov.cn',
'Referer': 'http://gzw.jiangxi.gov.cn/col/col22977/index.html?uid=402016&pageNum=9',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cookie': 'JSESSIONID=F601A052571881210819664F5BD38015; JSESSIONID=6E54DB27D82E844B825DD675AE19E399'
}
try:
resp_text = requests.request("POST", url, headers=header, data=payload).text
href_list = re.findall("href='(.*?)'", resp_text)
for href in href_list:
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
href_res = requests.get(url=href, headers=baseTool.headers, verify=False)
href_res.encoding = href_res.apparent_encoding
href_text = href_res.text
soup = baseTool.paserUrl(href_text, href)
doc = pq(str(soup))
try:
# origin=soup.find(text='信息来源：').text.replace('信息来源：','')
origin = doc('td:contains("信息来源：")').text().replace('信息来源：', '')
except Exception as e:
origin = ''
title = doc('tr[class="biaoti"]>td:nth-child(1)').text().replace('标题:', '')
organ = doc('div[class="xxgk-quote"]>table>tbody>tr:nth-child(1)>td:nth-child(2)').text().replace(
'发文机关:', '')
pub_hao = doc('div[class="xxgk-quote"]>table>tbody>tr:nth-child(1)>td:nth-child(3)').text().replace(
'文号:', '')
topicClassification = doc(
'div[class="xxgk-quote"]>table>tbody>tr:nth-child(2)>td:nth-child(1)').text().replace('主题分类:',
'')
writtenDate = doc(
'div[class="xxgk-quote"]>table>tbody>tr:nth-child(2)>td:nth-child(3)').text().replace('成文日期:',
'')
# pub_result = str(soup.find('div', attrs={'class': 'xxgk-quote'}).text)
# title = pub_result.split('标？？？？？？题: ')[1].split('有？？效？？性: ')[0].lstrip().strip()
# organ = pub_result.split('发文机关:')[1].split('文？？？？？？号:')[0].lstrip().strip()
# pub_hao = pub_result.split('文？？？？？？号:')[1].split('主题分类: ')[0].lstrip().strip()
# writtenDate = pub_result.split('成文日期:')[1].split('标？？？？？？题: ')[0].lstrip().strip()
contentWithTag = doc('div[id="zoom"]')
soup = baseTool.paserUrl(str(contentWithTag), href)
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1689', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '江西省国资委', file_name, num, writtenDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
contentWithTag = str(soup.prettify())
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
if len(pub_hao) < 1:
pattern = r'(赣国资.{1,}?号)|(国.{1,}?号)'
match_list = re.findall(pattern, content)
if len(match_list) > 0:
pub_hao = match_list[0][0]
else:
pub_hao = ''
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1689", 'relationName': "江西省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': organ,
'topicClassification': topicClassification,
'issuedNumber': pub_hao,
'publishDate': None,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
jiang_xi()
\ No newline at end of file
comData/policylaw/liao_ning.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 辽宁
def liao_ning():
num = 0
count = 0
start_time = time.time()
for page in range(1, 3):
url = f'https://gzw.ln.gov.cn/gzw/xxgk/zc/zcfb/aa251549-{page}.shtml'
try:
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False)
resp_text.encoding = resp_text.apparent_encoding
resp_text = resp_text.text
doc_resp = BeautifulSoup(resp_text, 'html.parser')
doc_items = doc_resp.select(
'#aa25154996104f57858a48e0b1aecca9 > div:nth-of-type(2) > div.tablist-show > div.tab-list-page')[0]
li_list = doc_items.select('li')
for li in li_list:
# print(li)
href = str(li.select('a')[0].get('href'))
if 'http' not in href:
if 'https' not in href:
href = 'https://gzw.ln.gov.cn/' + href
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
href_text = requests.get(url=href, headers=baseTool.headers, verify=False)
href_text.encoding = href_text.apparent_encoding
href_text = href_text.text
doc_href = baseTool.paserUrl(href_text, href)
doc = pq(str(doc_href))
title = doc('p[class="govxlTText"]').text().strip()
origintag = doc('p[class="govxlTText2"]').text().strip()
origin = origintag.split('文章来源：')[1].split('发布时间：')[0].strip()
publishDate = origintag.split('发布时间：')[1].strip().replace('年', '-').replace('月', '-').replace('日',
'') + ' 00:00:00'
contentWithTag = doc('div[class="TRS_Editor"]')
if len(title) < 1:
title = doc('h1[class="title"]')
issuedNumber = doc('p[class="wjh"]')
if len(contentWithTag) < 1:
contentWithTag = doc('div[class="content"]')
soup = baseTool.paserUrl(str(contentWithTag), href)
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1685', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '辽宁省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# id_ = redefid(id_list)
contentWithTag = str(soup.prettify())
if len(contentWithTag) < 1:
continue
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
pattern = r'(辽国资.{1,}?号)|(国资.{1,}?号)'
match_list = re.findall(pattern, content)
if len(match_list) > 0:
issuedNumber = match_list[0][0]
if len(issuedNumber) > 20:
issuedNumber = ''
else:
issuedNumber = ''
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1685", 'relationName': "辽宁省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': "",
'topicClassification': "",
'issuedNumber': issuedNumber,
'publishDate': publishDate,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{num}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
liao_ning()
\ No newline at end of file
comData/policylaw/nei_meng_gu.py 0 → 100644
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 内蒙古
def nei_meng_gu():
start = time.time()
num = 0
url = 'http://gzw.nmg.gov.cn/zfxxgk/zcfg/index.html'
try:
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False)
resp_text.encoding = 'utf-8'
html = resp_text.text
soup = BeautifulSoup(html, 'html.parser')
result = soup.find(class_='right_two')
li_list = result.find_all(class_='font14wr')
for a in li_list:
id_list = []
a_text = str(a)
real_href = 'https://gzw.nmg.gov.cn/zfxxgk' + a_text.split('href="..')[-1].split('" target="_blank')[0]
# # 判断是否已经爬取过
# todo:测试用 注释掉判重
is_href = baseTool.db_storage.find_one({'网址': real_href})
if is_href:
num += 1
continue
try:
# 获取所需信息
title = a_text.split('target="_blank">')[-1].split('</a>')[0]
href_text = requests.get(url=real_href, headers=baseTool.headers, verify=False)
href_text.encoding = 'utf-8'
i_html = href_text.text
i_soup = BeautifulSoup(i_html, 'html.parser')
# todo:将html中的a标签相对路径改为绝对路径
i_soup = baseTool.paserUrl(i_soup, real_href)
i_result = i_soup.find('div', id='d_laiyuan')
time_ = i_result.find_all('span')[0]
time_ = str(time_)
pub_time = time_.split('<span>')[1].split('</span>')[0].replace('发布时间：', '')
# 发布机关
origin = i_result.find_all('span')[1]
origin = str(origin)
pub_source = origin.split('<span>')[1].split('</span>')[0].replace('来源：', '')
# 发文机关
organ = origin
fwzh = i_soup.find_all('td')[7]
pub_hao_result = re.findall('〔(.*?)〕', str(fwzh))
if len(pub_hao_result) == 0:
pub_hao = ''
else:
if '内' in str(fwzh):
pub_hao = str(fwzh).split('<td>')[1].split('</td>')[0]
else:
pub_hao = ''
# 成文时间
writtenDate = i_soup.find_all('td')[9].text
topicClassification = i_soup.find_all('td')[3].text
i_content = i_soup.find(class_='d_show')
if i_content:
content = str(i_content)
else:
i_content = i_soup.find(class_='view TRS_UEDITOR trs_paper_default')
content = str(i_content)
if i_content.text == '' or i_content.text == 'None':
log.info(f'{real_href}------{title}----内容为空-----')
continue
# todo:内蒙古市的附件不在正文中，异步加载出来，替换不了标签，附件可上传att表中
fujian = i_soup.find(class_='xy_zcwjxl_downloadPC_list')
fu_jian_result = re.findall('href="(.*?)"', str(fujian))
if len(fu_jian_result) > 0:
for fu_jian_re in fu_jian_result:
if '.doc' in fu_jian_re or '.pdf' in fu_jian_re or '.xls' in fu_jian_re or '.zip' in fu_jian_re \
or '.rar' in fu_jian_re or '.ppt' in fu_jian_re or '.PDF' in fu_jian_re or '.DOC' in fu_jian_re \
or '.XLS' in fu_jian_re or '.ZIP' in fu_jian_re or '.RAR' in fu_jian_re:
fu_jian_re = str(real_href).split('/t')[0] + '/' + str(fu_jian_re).split('./')[1]
fu_jian_href = fu_jian_re
category = os.path.splitext(fu_jian_href)[1]
if category not in title:
file_name = title + category
# print(fu_jian_href)
# todo:附件上传至文件服务器
retData = baseCore.uptoOBS(fu_jian_href, '1669', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '内蒙古自治区国资委', file_name, num, pub_time)
id_list.append(att_id)
log.info(title)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': i_content.text,
'contentWithTag': content,
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1669", 'relationName': "内蒙古自治区国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': organ,
'topicClassification': topicClassification,
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': real_href,
'summary': '',
'title': title
}
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num = num + 1
except:
pass
except:
pass
end = time.time()
log.info(f'共抓取{num}条数据,共耗时{end - start}')
if __name__ == "__main__":
nei_meng_gu()
\ No newline at end of file
comData/policylaw/ning_xia.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 宁夏
def ning_xia():
num = 0
count = 0
pathType = 'policy/ningxia/'
start_time = time.time()
for page in range(0, 3):
if page == 0:
url = 'http://gzw.nx.gov.cn/zcfg/zcwj/gzwwj/index.html'
else:
url = f'http://gzw.nx.gov.cn/zcfg/zcwj/gzwwj/index_{page}.html'
try:
res = requests.get(url=url, headers=baseTool.headers, verify=False)
res.encoding = res.apparent_encoding
res_text = res.text
soup = BeautifulSoup(res_text, 'html.parser')
li_list = soup.find('div', attrs={'class': 'stdnewslist'}).find_all('li')
for li in li_list:
title = li.find('a').get('title').replace('</p>', '').replace('<p>', '')
href = url.split('index')[0] + li.find('a').get('href').replace('./', '')
publishDate = li.find('span', attrs={'class': 'stdnewslistspan'}).text
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
href_res = requests.get(url=href, headers=baseTool.headers, verify=False)
href_res.encoding = href_res.apparent_encoding
href_text = href_res.text
# soup_ = BeautifulSoup(href_text, 'html.parser')
soup_ = baseTool.paserUrl(href_text, href)
pub_result = soup_.find('table', attrs={'class': 'gk-xl-table'}).text.replace(' ', '')
writtenDate = pub_result.split('生成日期')[1].split('发文字号')[0].strip() + ' 00:00:00'
pub_hao = pub_result.split('发文字号')[1].split('公开形式')[0].strip()
organ = pub_result.split('所属机构')[1].split('有效性')[0].strip()
contentWithTag = soup_.find('div', attrs={'class': 'content'}).find('div',
attrs={'class': 'TRS_UEDITOR'})
soup = BeautifulSoup(str(contentWithTag), 'html.parser')
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1697', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '宁夏回族自治区国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# id_ = redefid(id_list)
contentWithTag = str(soup.prettify())
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
t = time.strptime(publishDate, "%Y年%m月%d日")
publishDate = time.strftime("%Y-%m-%d %H:%M:%S", t)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1697", 'relationName': "宁夏回族自治区国资委", 'labelMark': "policy"}],
'origin': '',
'organ': organ,
'topicClassification': "",
'issuedNumber': pub_hao,
'publishDate': publishDate,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
ning_xia()
\ No newline at end of file
comData/policylaw/qing_hai.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 青海
def qing_hai():
def qing_hai1():
count = 0
start_time = time.time()
url_mode = 'http://gxgz.qinghai.gov.cn/xxgk/List_wj.aspx?lmid=604'
try:
res = requests.get(url=url_mode, headers=baseTool.headers)
res.encoding = res.apparent_encoding
res_text = res.text
# page = BeautifulSoup(res_text, 'html.parser')
soup = baseTool.paserUrl(res_text, url_mode)
tr_list = soup.findAll('tr', attrs={'class': 'yhhei13'})
num = 0
for tr in tr_list:
try:
durl = tr.find('a').get('href')
is_href = baseTool.db_storage.find_one({'网址': durl})
if is_href:
num += 1
log.info('已采集----------跳过')
continue
title = tr.find('a').text
# pub_hao = tr.findAll('td')[-3].text
# writtenDate = tr.findAll('td')[-2].text
publishDate = tr.findAll('td')[-1].text
# t = time.strptime(writtenDate, "%Y年%m月%d日")
# writtenDate = time.strftime("%Y-%m-%d %H:%M:%S", t)
t = time.strptime(publishDate, "%Y年%m月%d日")
publishDate = time.strftime("%Y-%m-%d %H:%M:%S", t)
if '.html' in durl:
res_ = requests.get(url=durl, headers=baseTool.headers)
res_.encoding = res_.apparent_encoding
res_text_ = res_.text
page = BeautifulSoup(res_text_, 'html.parser')
contentWithTag = str(page.find('div', attrs={'class': 'nxgz-detail-con'}))
origin = str(page.find('div', attrs={'class': 'foot-fb'}))
soup = BeautifulSoup(contentWithTag, 'html.parser')
content = soup.text
if content == '' or content == None:
log.info(f'-----{durl}----{title}----内容为空-----')
continue
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1681', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '青海省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# id_ = redefid(id_list)
contentWithTag = str(soup.prettify())
# todo:替换完成之后，将附件上传至文件服务器
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1681", 'relationName': "青海省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': "",
'topicClassification': "",
'issuedNumber': '',
'publishDate': publishDate,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': durl,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
# print(id)
# id_list.append(id)
num += 1
count += 1
except Exception as e:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
def qing_hai2():
num = 0
count = 0
start_time = time.time()
urls = [
'http://gxgz.qinghai.gov.cn/xxgk/List_wj.aspx?lmid=627',
'http://gxgz.qinghai.gov.cn/xxgk/List_wj.aspx?lmid=559',
'http://gxgz.qinghai.gov.cn/xxgk/List_wj.aspx?lmid=64',
'http://gxgz.qinghai.gov.cn/xxgk/List_wj.aspx?lmid=65',
'http://gxgz.qinghai.gov.cn/xxgk/List_wj.aspx?lmid=558', ]
for url_mode in urls:
# print(url_mode)
try:
res = requests.get(url=url_mode, headers=baseTool.headers)
res.encoding = res.apparent_encoding
res_text = res.text
page = BeautifulSoup(res_text, 'html.parser')
page_numbers = str(page.find('span', attrs={'id': 'center_wj1_Label5'}).text)
page_numbers = int(page_numbers)
num = 0
for page_number in range(1, page_numbers + 1):
url = url_mode + f'&pages={page_number}'
# print(url + '................................')
res = requests.get(url=url, headers=baseTool.headers)
res.encoding = res.apparent_encoding
res_text = res.text
# page = BeautifulSoup(res_text, 'html.parser')
page = baseTool.paserUrl(res_text, url)
tr_list = page.findAll('tr', attrs={'class': 'yhhei13'})
for tr in tr_list:
try:
durl = tr.find('a').get('href')
is_href = baseTool.db_storage.find_one({'网址': durl})
if is_href:
num += 1
log.info('已采集----------跳过')
continue
title = tr.find('a').text
# pub_hao = tr.findAll('td')[-3].text
# writtenDate = tr.findAll('td')[-2].text
publishDate = tr.findAll('td')[-1].text
# t = time.strptime(writtenDate, "%Y年%m月%d日")
# writtenDate = time.strftime("%Y-%m-%d %H:%M:%S", t)
t = time.strptime(publishDate, "%Y年%m月%d日")
publishDate = time.strftime("%Y-%m-%d %H:%M:%S", t)
if '.html' in durl:
res_ = requests.get(url=durl, headers=baseTool.headers)
res_.encoding = res_.apparent_encoding
res_text_ = res_.text
# page = BeautifulSoup(res_text_, 'html.parser')
page = baseTool.paserUrl(res_text_, durl)
contentWithTag = str(page.find('td', attrs={'class': 'yhhei15'}))
try:
origin = str(page.find('td', attrs={'class', 'heizi12'}).text).split()[0].replace(
'来源：',
'')
except:
origin = ''
soup = BeautifulSoup(contentWithTag, 'html.parser')
content = soup.text
if content == '' or content == None:
log.info(f'-----{durl}----{title}----内容为空-----')
continue
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1681', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '青海省国资委', file_name, num,
publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# id_ = redefid(id_list)
contentWithTag = str(soup.prettify())
# todo:替换完成之后，将附件上传至文件服务器
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1681", 'relationName': "青海省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': "",
'topicClassification': "",
'issuedNumber': '',
'publishDate': publishDate,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': durl,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
# print(id)
# id_list.append(id)
num += 1
count += 1
except:
pass
res.close()
except Exception as e:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
qing_hai1()
qing_hai2()
if __name__ == "__main__":
qing_hai()
\ No newline at end of file
comData/policylaw/shan_dong.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 山东
def shan_dong():
headers = {
'Cookie': 'COLLCK=2502513302; COLLCK=2493627587',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.183'
}
start = time.time()
num = 0
count = 0
url_list = ['http://gzw.shandong.gov.cn/channels/ch06086/', 'http://gzw.shandong.gov.cn/channels/ch06088/']
for url in url_list:
try:
resp_text = requests.get(url=url, headers=headers, verify=False)
resp_text.encoding = 'utf-8'
html = resp_text.text
soup = BeautifulSoup(html, 'html.parser')
result = soup.find_all(class_='pagedContent')
for li in result:
href = li.find('a')['href']
is_href = db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
href_text = requests.get(url=href, headers=headers, verify=False)
href_text.encoding = href_text.apparent_encoding
i_html = href_text.text
i_soup = BeautifulSoup(i_html, 'html.parser')
try:
source = i_soup.find_all('tbody')[0]
title = str(source).split('标　　题：</strong>')[1].split('</td>')[0].replace('\r', '').replace('\n',
'')
pub_time = re.findall('<strong>发布日期：</strong>(.*?)</td>', str(source))
pub_time = ''.join(pub_time)
pub_hao = re.findall('<strong>发文字号：</strong>(.*?)</td>', str(source))
pub_hao = ''.join(pub_hao)
pub_source = re.findall('<strong>发文机关：</strong>(.*?)</td>', str(source))
pub_source = ''.join(pub_source)
writtenDate = re.findall('<strong>成文日期：</strong>(.*?)</td>', str(source))
writtenDate = ''.join(writtenDate)
# print(pub_time,pub_source,pub_hao)
content = i_soup.find(class_="wz_zoom scroll_cont ScrollStyle").text
contentwithtag = i_soup.find(class_="wz_zoom scroll_cont ScrollStyle")
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
if pub_hao == '无':
p_list = content.find_all('p')
for p in p_list:
p_text = p.text
if '〔' and '〕' in p_text:
pub_hao = p_text
break
else:
continue
except:
try:
title = str(i_soup.find('div', attrs={'class': 'wz_title'}).text).strip().lstrip()
except:
title = ''
source = i_soup.find('div', attrs={'id': 'nr'})
h1_list = source.find_all('h1')
for h1 in h1_list:
title = title + str(h1.text)
title.strip().lstrip()
pub_time = None
span_list = source.find_all('span')
i = 0
for span in span_list:
span_text = span.text
if '〔' and '〕' in span_text or '鲁国' in span_text or '国办发' in span_text:
pub_hao = str(span_text)
if '号' not in pub_hao:
pub_hao = pub_hao + str(span_list[i + 1].text)
break
i = i + 1
content = i_soup.find(class_="wz_zoom scroll_cont ScrollStyle").text
contentwithtag = i_soup.find(class_="wz_zoom scroll_cont ScrollStyle")
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': [],
'author': '',
'content': content,
'contentWithTag': str(contentwithtag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1674", 'relationName': "山东省国资委", 'labelMark': "policy"}],
'origin': '',
'organ': pub_source,
'topicClassification': '',
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': writtenDate,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = sendKafka(dic_news)
if flag:
save_data(dic_news)
log.info(title)
num = num + 1
count += 1
except:
pass
except:
pass
end = time.time()
log.info('共', count, '条', '...........', '共耗时', end - start, '秒')
if __name__ == "__main__":
shan_dong()
\ No newline at end of file
comData/policylaw/shan_xi.py 0 → 100644
import datetime
import os
import re
import time
import requests
from lxml import etree
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 山西
def shan_xi():
num = 0
count = 0
start_time = time.time()
for page in range(1, 7):
if page == 1:
url = 'http://gzw.shanxi.gov.cn/zxhrdgz/fzyd/zywj/'
else:
url = f'http://gzw.shanxi.gov.cn/zxhrdgz/fzyd/zywj/index_{page - 1}.shtml'
try:
res = requests.get(url, baseTool.headers)
page_text = res.text.encode("ISO-8859-1")
page_text = page_text.decode("utf-8")
tree = etree.HTML(page_text)
tr_list = tree.xpath(
'/html/body/table[3]/tbody/tr/td[2]/table/tbody/tr[3]/td/table[2]/tbody/tr[3]/td/form/table/tbody/tr')
for tr in tr_list:
href = tr.xpath('./td[1]/a/@href')
if href == []:
continue
href = href[0].replace('../../', 'http://gzw.shanxi.gov.cn/zxhrdgz/fzyd/zywj/').replace('./',
'http://gzw.shanxi.gov.cn/zxhrdgz/fzyd/zywj/')
title = tr.xpath('./td[1]/a/span//text()')[0]
publishDate_ = str(tr.xpath('./td[2]/span/text()')[0]).strip()
time_obj = datetime.datetime.strptime(publishDate_, "%Y/%m/%d")
# 将datetime对象格式化为年月日的字符串
publishDate = time_obj.strftime("%Y-%m-%d")
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
if ".pdf" in href:
content = ''
publishDate = None
origin = ''
fu_jian_soup = [href]
contentWithTag = ''
else:
res = requests.get(href, baseTool.headers)
page_text = res.text.encode("ISO-8859-1")
page_text = page_text.decode("utf-8")
page = baseTool.paserUrl(page_text, href)
doc = pq(str(page))
title = doc('title').text()
origin = ''
contentWithTag = doc('div[id="vsb_content"]')
soup = baseTool.paserUrl(str(contentWithTag), href)
if len(fu_jian_soup) < 1:
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1684', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '山西省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# id_ = redefid(id_list)
contentWithTag = str(soup.prettify())
if len(contentWithTag) < 1:
if len(fu_jian_soup) < 1:
continue
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
pattern = r'(晋国资.{1,}?号)|(国资.{1,}?号)'
match_list = re.findall(pattern, content)
if len(match_list) > 0:
issuedNumber = match_list[0][0]
if len(issuedNumber) > 20:
issuedNumber = ''
else:
issuedNumber = ''
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1684", 'relationName': "山西省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': "",
'topicClassification': "",
'issuedNumber': issuedNumber,
'publishDate': publishDate,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except Exception as e:
pass
except Exception as e:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
shan_xi()
\ No newline at end of file
comData/policylaw/shang_hai.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 上海
def shang_hai():
start = time.time()
num = 0
count = 0
for page in range(1, 7):
if page == 1:
url = 'https://www.gzw.sh.gov.cn/shgzw_flfg_zcfg_gfxwj/index.html'
else:
url = f'https://www.gzw.sh.gov.cn/shgzw_flfg_zcfg_gfxwj/index_{page}.html'
try:
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False).text
doc_resp = pq(resp_text)
doc_items = doc_resp('.gqzc_list_right ul li').items()
for doc_item in doc_items:
id_list = []
title = doc_item('a').attr('title').strip()
pub_time = doc_item('span').text() + ' 00:00:00'
href = doc_item('a').attr('href')
if 'https:/' in href:
pass
else:
href = 'https://www.gzw.sh.gov.cn' + href
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
# href = 'https://www.gzw.sh.gov.cn/shgzw_xxgk_zxgkxx/20230119/7c5e9691b2b54ff293e5d16d746d1a61.html'
href_text = requests.get(url=href, headers=baseTool.headers, verify=False).text
doc_href = pq(href_text)
doc_href_ = BeautifulSoup(href_text, 'html.parser')
# 相对路径转化为绝对路径
doc_href_ = baseTool.paserUrl(doc_href_, href)
info_list = doc_href_.find_all('span', style='text-align: center;margin-left: 42%;')
pub_source = info_list[1].find('b').text.split('信息来源：')[1]
content = doc_href_.find('div', attrs={'class': 'detail_03'})
if content == '' or content == 'None':
log.info(f'{href}-----{title}----内容为空')
continue
# 将文章中的附件字段删去
pattern = r'\d+\.'
for p in content.find_all('p')[-22:]:
p_text = p.text
if len(p_text) > 50:
continue
matches = re.findall(pattern, p_text)
for k in matches:
if k in p_text:
p.extract()
try:
pub_result = doc_href('.detail_03')
pub_result('meta')
pub_result = '沪' + str(pub_result('meta')).split('沪')[1].split('号')[0].strip() + '号'
except:
try:
pub_result = str(
'沪' + doc_href('.detail_03 ul').text().split('沪')[1].split('号')[0].strip() + '号')
except:
pub_result = str(doc_href('.detail_03 p').text().split('号')[0].strip() + '号')
if '﹝' in pub_result and '﹞' in pub_result:
pub_hao = pub_result.replace('﹝', '〔').replace('﹞', '〕')
elif '〔' in pub_result and '〕' in pub_result:
pub_hao = pub_result
elif '【' in pub_result and '】' in pub_result:
pub_hao = pub_result
elif '[' in pub_result and ']' in pub_result:
pub_hao = pub_result
else:
pub_hao = ''
if len(pub_hao) > 20:
pub_hao = ''
# todo:找到附件标签,正文内容带有附件
fu_jian_soup = content.find('ul')
if fu_jian_soup:
li_list = fu_jian_soup.find_all('a')
else:
li_list = []
for a in li_list:
fu_jian_href = a['href']
file_name = a.text
if '.doc' in fu_jian_href or '.docx' in fu_jian_href or '.pdf' in fu_jian_href or '.xls' in fu_jian_href or '.zip' in fu_jian_href \
or '.rar' in fu_jian_href or '.ppt' in fu_jian_href or '.PDF' in fu_jian_href or '.DOC' in fu_jian_href \
or '.XLS' in fu_jian_href or '.ZIP' in fu_jian_href or '.RAR' in fu_jian_href:
category = os.path.splitext(fu_jian_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(fu_jian_href, '1671', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '上海市国资委', file_name, num, pub_time)
id_list.append(att_id)
# todo:将返回的地址更新到soup
a['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
else:
continue
log.info(title)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': content.text,
'contentWithTag': str(content),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1671", 'relationName': "上海市国资委", 'labelMark': "policy"}],
'origin': pub_source,
'organ': '',
'topicClassification': '',
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num = num + 1
count += 1
except:
pass
except:
pass
end = time.time()
log.info(f'共抓取{count}条数据,共耗时{end - start}')
if __name__ == "__main__":
shang_hai()
\ No newline at end of file
comData/policylaw/shanxi.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 陕西
def shanxi():
num = 0
count = 0
start_time = time.time()
url = 'https://sxgz.shaanxi.gov.cn/newstyle/pub_newschannel.asp?chid=100127'
# url = 'https://sxgz.shaanxi.gov.cn/newstyle/pub_newschannel.asp?chid=100127'
try:
res = requests.get(url=url, headers=baseTool.headers)
res.encoding = res.apparent_encoding
res_text = res.text
soup = BeautifulSoup(res_text, 'html.parser')
# soup = paserUrl(res_text, 'https://sxgz.shaanxi.gov.cn')
# print(soup)
result = soup.find(class_='scroll_cont')
li_list = result.find_all('li')
for li in li_list:
href = li.find('a')['href']
if 'http' in str(href):
href = href
else:
href = 'https://sxgz.shaanxi.gov.cn/' + href
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
res_href = requests.get(url=href, headers=baseTool.headers)
res_href.encoding = res_href.apparent_encoding
res_text = res_href.text
# i_soup = BeautifulSoup(res_text, 'html.parser')
i_soup = baseTool.paserUrl(res_text, href)
title = i_soup.find(class_='m-gk-title').text
i_result = i_soup.find(class_='ftitle')
span_list = i_result.find_all('span')
origin = str(span_list[0]).split('<span>')[1].split('</span>')[0]
publishDate = str(span_list[2]).split('<span>')[1].split('</span>')[0]
t = time.strptime(publishDate, "%Y/%m/%d %H:%M:%S")
publishDate = time.strftime("%Y-%m-%d %H:%M:%S", t)
contentWithTag = i_soup.find(class_='scroll_cont')
soup = BeautifulSoup(str(contentWithTag), 'html.parser')
div_tag = soup.find(id='ztl')
div_tag.extract()
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1680', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '陕西省国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# id_ = redefid(id_list)
contentWithTag = str(soup.prettify())
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1680", 'relationName': "陕西省国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': "",
'topicClassification': "",
'issuedNumber': "",
'publishDate': publishDate,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
res_href.close()
except Exception as e:
pass
res.close()
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
shanxi()
\ No newline at end of file
comData/policylaw/si_chuan.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 四川
def si_chuan():
num = 0
count = 0
start_time = time.time()
for page in range(1, 3):
if page == 1:
url = 'http://gzw.sc.gov.cn/scsgzw/CU2304010701/cu_xxgk_xzgfxwj.shtml'
else:
url = 'http://gzw.sc.gov.cn/scsgzw/CU2304010701/cu_xxgk_xzgfxwj_2.shtml'
try:
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False).text
doc_resp = pq(resp_text)
doc_items_ = doc_resp('.biaobody')
doc_items = doc_items_('li').items()
for doc_item in doc_items:
id_list = []
# print(doc_item)
pub_time = doc_item('.lie4').text().strip() + ' 00:00:00'
pub_hao = doc_item('.lie3').text().strip()
title = doc_item('a').attr('title')
href = doc_item('a').attr('href')
if 'http:' not in href:
href = 'http://gzw.sc.gov.cn' + doc_item('a').attr('href')
# href = 'http://gzw.sc.gov.cn/scsgzw/CU2304010701/2018/1/22/9c5db691e09f4efdafce41763a0d7e03.shtml'
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
# print(href)
href_text = requests.get(url=href, headers=baseTool.headers, verify=False).text
doc_href = pq(href_text)
# title = str(doc_href('.xxgkzn_title').text()).replace('\n', '').replace('\r', '')
# content = str(doc_href('#scrollBox').children())
# 将doc_href转化为BeautifulSoup
doc_href = BeautifulSoup(str(doc_href), 'html.parser')
# 相对路径转化为绝对路径
doc_href = baseTool.paserUrl(doc_href, href)
contentWithTag = doc_href.find('div', id='scrollBox')
content = contentWithTag.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
fu_jian_list = doc_href.find_all('a')
for fu_jian in fu_jian_list:
try:
fu_jian_href = fu_jian['href']
except:
continue
file_name = fu_jian.text.strip()
if '.doc' in fu_jian_href or '.docx' in fu_jian_href or '.pdf' in fu_jian_href or '.xls' in fu_jian_href or '.xlsx' in fu_jian_href or '.zip' in fu_jian_href \
or '.rar' in fu_jian_href or '.ppt' in fu_jian_href or '.PDF' in fu_jian_href or '.DOC' in fu_jian_href \
or '.XLS' in fu_jian_href or '.ZIP' in fu_jian_href or '.RAR' in fu_jian_href:
category = os.path.splitext(fu_jian_href)[1]
if category not in file_name:
file_name = file_name + category
# 对附件上传至文件服务器
retData = baseCore.uptoOBS(fu_jian_href, '1678', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '四川省国资委', file_name, num, pub_time)
id_list.append(att_id)
fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# fu_jian_href_list.append(fu_jian_href)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': content,
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1678", 'relationName': "四川省国资委", 'labelMark': "policy"}],
'origin': '',
'organ': '',
'topicClassification': '',
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
log.info(title)
count += 1
num = num + 1
except Exception as e:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
si_chuan()
\ No newline at end of file
comData/policylaw/start.py 0 → 100644
import datetime
import time
from comData.policylaw.an_hui import an_hui
from comData.policylaw.bei_jing import bei_jing
from comData.policylaw.chong_qing import chong_qing
from comData.policylaw.fu_jian import fu_jian
from comData.policylaw.guang_dong import guang_dong
from comData.policylaw.guang_xi import guang_xi
from comData.policylaw.gui_zhou import gui_zhou
from comData.policylaw.gwyfile import get_content1
from comData.policylaw.gwyparts import get_content2
from comData.policylaw.gwysasac import get_content3
from comData.policylaw.hai_nan import hai_nan
from comData.policylaw.he_nan import he_nan
from comData.policylaw.hei_long_jiang import hei_long_jiang
from comData.policylaw.ji_lin import ji_lin
from comData.policylaw.jiang_su import jiang_su
from comData.policylaw.jiang_xi import jiang_xi
from comData.policylaw.liao_ning import liao_ning
from comData.policylaw.nei_meng_gu import nei_meng_gu
from comData.policylaw.shan_dong import shan_dong
from comData.policylaw.shan_xi import shan_xi
from comData.policylaw.shang_hai import shang_hai
from comData.policylaw.si_chuan import si_chuan
from comData.policylaw.tian_jin import tian_jin
from comData.policylaw.xin_jiang import xin_jiang
from comData.policylaw.yun_nan import yun_nan
from comData.policylaw.zhe_jiang import zhe_jiang
if __name__ == "__main__":
get_content1()
get_content3()
bei_jing()
nei_meng_gu()
ji_lin()
shang_hai()
zhe_jiang()
fu_jian()
shan_dong()
guang_dong()
hai_nan()
si_chuan()
guang_xi()
gui_zhou()
yun_nan()
chong_qing()
tian_jin()
xin_jiang()
shan_xi()
liao_ning()
hei_long_jiang()
jiang_su()
an_hui()
jiang_xi()
he_nan()
hu_nan()
gan_su()
ning_xia()
xi_zang()
shanxi()
qing_hai()
he_bei()
qing_hai()
get_content2()
current_time = datetime.datetime.now()
midnight_time = current_time.replace(hour=0, minute=0, second=0, microsecond=0) + datetime.timedelta(days=1)
sleep_seconds = (midnight_time - current_time).total_seconds()
time.sleep(sleep_seconds)
\ No newline at end of file
comData/policylaw/xi_zang.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 西藏
def xi_zang():
start_time = time.time()
pathType = 'policy/xizang/'
url_list = ['http://gzw.lasa.gov.cn/gzw/zccfg/common_list.shtml',
'http://gzw.lasa.gov.cn/gzw/wjzl/common_list.shtml', ]
for url in url_list:
num = 0
count = 0
try:
res = requests.get(url=url, headers=baseTool.headers)
res.encoding = res.apparent_encoding
res_text = res.text
# soup = BeautifulSoup(res_text, 'html.parser')
soup = baseTool.paserUrl(res_text, url)
result = soup.find('ul', class_='list')
li_list = result.find_all('li')
for li in li_list:
href = li.find('a')['href']
title = li.find('a').text
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
res_href = requests.get(url=href, headers=baseTool.headers)
res_href.encoding = res_href.apparent_encoding
res_href = res_href.text
# i_soup = BeautifulSoup(res_href, 'html.parser')
i_soup = baseTool.paserUrl(res_href, href)
i_result = i_soup.find(class_='inform')
div_list = i_result.find_all('div')
publishDate = str(div_list[0]).split('<div>')[1].split('</div>')[0].replace('发布时间：', '')
origin = str(div_list[1]).split('<div>')[1].split('</div>')[0].replace('来源：', '')
contentWithTag = str(i_soup.find(id='NewsContent'))
soup = BeautifulSoup(contentWithTag, 'html.parser')
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1695', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '西藏自治区国资委', file_name, num, publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# id_ = redefid(id_list)
contentWithTag = str(soup.prettify())
# todo:替换完成之后，将附件上传至文件服务器
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1695", 'relationName': "西藏自治区国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': "",
'topicClassification': "",
'issuedNumber': "",
'publishDate': publishDate,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
if __name__ == "__main__":
xi_zang()
\ No newline at end of file
comData/policylaw/xin_jiang.py 0 → 100644
import datetime
import os
import re
import time
import pandas as pd
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 新疆
def xin_jiang():
def xin_jiang1():
num = 0
count = 0
start_time = time.time()
for page in range(1, 10):
if page == 1:
url = 'http://gzw.xinjiang.gov.cn/gzw/zcwj/list_tj.shtml'
else:
url = f'http://gzw.xinjiang.gov.cn/gzw/zcwj/list_tj_{page}.shtml'
try:
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False).content
doc_resp = pq(resp_text)
doc_items = doc_resp('.list.pt20 li').items()
for doc_item in doc_items:
title = doc_item('a').attr('title').strip()
href = 'http://gzw.xinjiang.gov.cn' + doc_item('a').attr('href')
if '/gzw/zcwj/' not in href:
continue
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
# href = 'http://gzw.xinjiang.gov.cn/gzw/zcwj/201909/559cf77b5a954d028bd187d6c6e46747.shtml'
try:
href_text = requests.get(url=href, headers=baseTool.headers, verify=False)
href_text = href_text.text.encode("ISO-8859-1")
href_text = href_text.decode("utf-8")
doc_href = BeautifulSoup(href_text, 'html.parser')
publishDate = doc_href.find('span', attrs={'class', 'date'}).text.replace('日期：', '').replace(
'∶',
':') + ':00'
publishDate = publishDate.strip()
origin = doc_href.find('span', attrs={'class', 'from'}).text.replace('来源：', '').strip()
contentWithTag = str(doc_href.select('#NewsContent')[0])
soup = baseTool.paserUrl(str(contentWithTag), href)
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1682', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '新疆维吾尔自治区国资委', file_name, num,
publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# id_ = redefid(id_list)
contentWithTag = str(soup.prettify())
if len(contentWithTag) < 1:
if len(fu_jian_soup) < 1:
continue
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
pattern = r'(新国.{1,}?号)|(国资.{1,}?号)'
match_list = re.findall(pattern, content)
if len(match_list) > 0:
issuedNumber = match_list[0][0]
if len(issuedNumber) > 20:
issuedNumber = ''
else:
issuedNumber = ''
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1682", 'relationName': "新疆维吾尔自治区国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': "",
'topicClassification': "",
'issuedNumber': issuedNumber,
'publishDate': publishDate,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
except:
pass
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
def xin_jiang_jsbt():
num = 0
count = 0
start_time = time.time()
for page in range(1, 6):
if page == 1:
url = 'http://gyzc.xjbt.gov.cn/xxgk/zcfg/'
else:
url = f'http://gyzc.xjbt.gov.cn/xxgk/zcfg/index_{page}.shtml'
try:
resp_text = requests.get(url=url, headers=baseTool.headers, verify=False)
doc_resp = pq(resp_text.content)
doc_items = doc_resp('.article').items()
for doc_item in doc_items:
title = doc_item('a').text().strip()
publishDate = doc_item('.time.pull-right').text().strip() + ' 00:00:00'
href = doc_item('a').attr('href')
if 'http' not in href:
href = 'http://gyzc.xjbt.gov.cn' + href
# is_href = baseTool.db_storage.find_one({'网址': href})
# if is_href:
# num += 1
# continue
try:
href_res = requests.get(url=href, headers=baseTool.headers, verify=False)
href_res.encoding = href_res.apparent_encoding
res_text = href_res.text
soup = BeautifulSoup(res_text, 'html.parser')
pub_result = soup.find('div', attrs={'class': 'title_info'}).text
origin = pub_result.split('信息来源：')[1].split('编辑：')[0].strip()
contentWithTag = str(soup.find('div', attrs={'id': 'detail'}))
soup = baseTool.paserUrl(str(contentWithTag), href)
fu_jian_soup = soup.find_all('a')
id_list = []
for file in fu_jian_soup:
try:
file_href = file['href']
except Exception as e:
continue
if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xlsx' in file_href or '.zip' in file_href \
or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:
file_name = file.text.strip()
category = os.path.splitext(file_href)[1]
if category not in file_name:
file_name = file_name + category
retData = baseCore.uptoOBS(file_href, '1698', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '新疆维吾尔自治区国资委', file_name, num,
publishDate)
id_list.append(att_id)
# todo:将返回的地址更新到soup
file['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
# id_ = redefid(id_list)
contentWithTag = str(soup.prettify())
if len(contentWithTag) < 1:
if len(fu_jian_soup) < 1:
continue
content = soup.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
pattern = r'(新国.{1,}?号)|(国资.{1,}?号)'
match_list = re.findall(pattern, content)
if len(match_list) > 0:
issuedNumber = match_list[0][0]
if len(issuedNumber) > 20:
issuedNumber = ''
else:
issuedNumber = ''
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': str(content),
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1698", 'relationName': "新疆生产建设兵团国资委", 'labelMark': "policy"}],
'origin': origin,
'organ': "",
'topicClassification': "",
'issuedNumber': issuedNumber,
'publishDate': publishDate,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num += 1
count += 1
href_res.close()
except:
pass
resp_text.close()
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
xin_jiang1()
xin_jiang_jsbt()
if __name__ == "__main__":
xin_jiang()
\ No newline at end of file
comData/policylaw/yun_nan.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 云南
def yun_nan():
pathType = 'policy/yunnan/'
def yun_nan1():
"""
http://gzw.yn.gov.cn/yngzw/c100093/zfxxgk_gkgz.shtml 9
http://gzw.yn.gov.cn/yngzw/c100040/zfxxgk_list.shtml 1
"""
num = 0
count = 0
start_time = time.time()
for page in range(1, 6):
if page == 1:
# url = 'http://gzw.yn.gov.cn/yngzw/c100040/zfxxgk_gkgz.shtml'
url = 'http://gzw.yn.gov.cn/yngzw/c100093/zfxxgk_gkgz.shtml'
else:
url = f'http://gzw.yn.gov.cn/yngzw/c100093/zfxxgk_gkgz_{page}.shtml'
try:
resp = requests.get(url=url, headers=baseTool.headers, verify=False)
doc_resp = pq(resp.content)
doc_items = doc_resp('.gkgz_list_content li').items()
for doc_item in doc_items:
id_list = []
href = doc_item('a').attr('href')
if 'http:' not in href:
href = 'http://gzw.yn.gov.cn' + doc_item('a').attr('href')
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
fu_jian_href_list = []
# print(href)
if '.shtml' in href:
href_resp = requests.get(url=href, headers=baseTool.headers, verify=False)
href_resp.encoding = href_resp.apparent_encoding
href_text = href_resp.text
doc_href = BeautifulSoup(href_text, 'html.parser')
# 相对路径转化为绝对路径
doc_href = baseTool.paserUrl(doc_href, href)
title = doc_href.select('#gknbxq_container > div > div.zfxxgk-content.zfxxgk_content > h2')[
0].text.lstrip().strip()
pub_hao = \
str(doc_href.select('#gknbxq_container > div > div.zfxxgk-content.zfxxgk_content > p')[
0].text).split('(')[1].split(')')[0].replace('\n', '')
contentwithTag = \
doc_href.select('#gknbxq_container > div > div.zfxxgk-content.zfxxgk_content')[0]
content = contentwithTag.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
fu_jian_list = contentwithTag.find_all('a')
for fu_jian in fu_jian_list:
try:
fu_jian_href = fu_jian['href']
except:
continue
file_name = fu_jian.text
if '.doc' in fu_jian_href or '.pdf' in fu_jian_href or '.xls' in fu_jian_href or '.zip' in fu_jian_href \
or '.rar' in fu_jian_href or '.ppt' in fu_jian_href or '.PDF' in fu_jian_href or '.DOC' in fu_jian_href \
or '.XLS' in fu_jian_href or '.ZIP' in fu_jian_href or '.RAR' in fu_jian_href:
try:
category = os.path.splitext(fu_jian_href)[1]
if category not in file_name:
file_name = file_name + category
# 附件上传至文件服务器
retData = baseCore.uptoOBS(fu_jian_href, '1679', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '云南省国资委', file_name, num, '')
id_list.append(att_id)
# 将附件链接替换
fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
except:
continue
href_resp.close()
elif 'display' in href:
continue
else:
content = ''
pub_hao = ''
fu_jian_href_list.append(href)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': content,
'contentWithTag': str(contentwithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1679", 'relationName': "云南省国资委",
'labelMark': "policy"}],
'origin': '',
'organ': '',
'topicClassification': '',
'issuedNumber': pub_hao,
'publishDate': None,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
log.info(title)
num = num + 1
count += 1
except:
pass
resp.close()
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
def yun_nan2():
num = 0
count = 0
start_time = time.time()
for page in range(1, 4):
if page == 1:
# url = 'http://gzw.yn.gov.cn/yngzw/c100040/zfxxgk_gkgz.shtml'
url = 'http://gzw.yn.gov.cn/yngzw/c100095/zfxxgk_list.shtml'
else:
url = f'http://gzw.yn.gov.cn/yngzw/c100095/zfxxgk_list_{page}.shtml'
try:
res = requests.get(url=url, headers=baseTool.headers, verify=False)
page_text = res.text.encode("ISO-8859-1")
page_text = page_text.decode("utf-8")
soup = BeautifulSoup(page_text, 'html.parser')
li_list = soup.find('ul', attrs={'class': 'zfxxgk-nr-cnet'}).find_all('li')
for li in li_list:
id_list = []
title = str(li.find('a').text).lstrip().strip()
pub_time = str(li.find('span').text).replace(' ', '').replace('\n', '')
href = 'http://gzw.yn.gov.cn' + li.find('a').get('href').replace(' ', '')
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
# print(href)
if '.shtml' in href:
res_ = requests.get(href, baseTool.headers)
page_text_ = res_.text.encode("ISO-8859-1")
page_text_ = page_text_.decode("utf-8")
page = BeautifulSoup(page_text_, 'html.parser')
# 相对路径转化为绝对路径
page = baseTool.paserUrl(page, href)
pub_hao = ''
try:
pub_hao_list = page.find('p', attrs={'class': 'MsoNormal'}).findAll('span')
for a in pub_hao_list:
pub_hao = pub_hao + str(a.text)
if '﹝' not in pub_hao and '﹞' not in pub_hao:
pub_hao = ''
except:
pub_hao = ''
contentwithTag = page.find('div', attrs={'class': 'zfxxgk-right'})
content = contentwithTag.text
if content == '' or content == None:
log.info(f'-----{href}----{title}----内容为空-----')
continue
fu_jian_list = contentwithTag.find_all('a')
for fu_jian in fu_jian_list:
try:
fu_jian_href = fu_jian['href']
except:
continue
file_name = fu_jian.text
if '.doc' in fu_jian_href or '.pdf' in fu_jian_href or '.xls' in fu_jian_href or '.zip' in fu_jian_href \
or '.rar' in fu_jian_href or '.ppt' in fu_jian_href or '.PDF' in fu_jian_href or '.DOC' in fu_jian_href \
or '.XLS' in fu_jian_href or '.ZIP' in fu_jian_href or '.RAR' in fu_jian_href:
# print(fu_jian_href)
try:
category = os.path.splitext(fu_jian_href)[1]
if category not in file_name:
file_name = file_name + category
# 附件上传至文件服务器
retData = baseCore.uptoOBS(fu_jian_href, '1679', file_name)
if retData['state']:
pass
else:
continue
att_id, full_path = baseCore.tableUpdate(retData, '云南省国资委', file_name, num,
pub_time)
id_list.append(att_id)
# 将附件链接替换
fu_jian['href'] = 'http:zzsn.luyuen.com/' + str(full_path)
except:
continue
res_.close()
elif 'display' in href:
continue
else:
continue
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
dic_news = {
'attachmentIds': id_list,
'author': '',
'content': content,
'contentWithTag': str(contentwithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1679", 'relationName': "云南省国资委",
'labelMark': "policy"}],
'origin': '',
'organ': '',
'topicClassification': '',
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
log.info(title)
count += 1
num = num + 1
except:
pass
res.close()
except:
pass
end_time = time.time()
log.info(f'共抓取{count}条数据,共耗时{end_time - start_time}')
yun_nan1()
yun_nan2()
if __name__ == "__main__":
yun_nan()
\ No newline at end of file
comData/policylaw/zhe_jiang.py 0 → 100644
import datetime
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from pyquery import PyQuery as pq
from ClassTool import ClassTool
baseTool = ClassTool()
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# 浙江
def zhe_jiang():
start = time.time()
num = 0
count = 0
url = 'http://gzw.zj.gov.cn/col/col1229430928/index.html'
try:
res = requests.get(url, baseTool.headers).content
soup = BeautifulSoup(res, 'html.parser')
# print(soup)
# recordset = soup.find('recordset')
list_li = re.findall('CDATA\[\\n(.*?)\]\]></record>', str(soup))
# print(list_li)
for li in list_li:
fj_href_list = []
li = BeautifulSoup(li, 'lxml')
href = li.find('a')['href']
pub_time = li.find('a').find('span').text
title = li.find('a').text.replace(pub_time, '').strip()
# log.info(title)
if 'http' in href:
href = href
else:
href = 'http://gzw.zj.gov.cn/' + href
is_href = baseTool.db_storage.find_one({'网址': href})
if is_href:
num += 1
continue
try:
href_text = requests.get(url=href, headers=baseTool.headers, verify=False)
href_text.encoding = href_text.apparent_encoding
i_html = href_text.text
i_soup = BeautifulSoup(i_html, 'html.parser')
# 将相对路径转化为绝对路径
i_soup = baseTool.paserUrl(i_soup, href)
# g_xxgk_table cf
i_info = i_soup.find_all(class_='g_xxgk_td')
if len(i_info) != 0:
try:
pub_source = str(i_info[4]).split('"g_xxgk_td">')[1].split('</div>')[0]
# pub_time = str(i_info[5]).split('"g_xxgk_td">')[1].split('</div>')[0]
pub_hao = str(i_info[2]).split('"g_xxgk_td">')[1].split('</div>')[0]
content = i_soup.find(class_='g_content').text
contentWithTag = str(i_soup.find(class_='g_content'))
except:
# pub_source = str(i_info[3])
# print(pub_source)
pub_source = str(i_info[2]).split('"g_xxgk_td">')[1].split('</div>')[0]
# pub_time = str(i_info[3]).split('"g_xxgk_td">')[1].split('</div>')[0]
pub_hao = ''
content = i_soup.find(class_='g_content').text
contentWithTag = str(i_soup.find(class_='g_content'))
else:
try:
source = i_soup.find('span', class_='rich_media_meta rich_media_meta_nickname')
pub_source = source.find('a').text
time_ = i_soup.find('em', id='publish_time')
pub_time = time_.text
pub_hao = ''
content = i_soup.find(
class_='zh_CN wx_wap_page wx_wap_desktop_fontsize_2 mm_appmsg comment_feature discuss_tab appmsg_skin_default appmsg_style_default pages_skin_pc not_in_mm').text
contentWithTag = str(i_soup.find(
class_='zh_CN wx_wap_page wx_wap_desktop_fontsize_2 mm_appmsg comment_feature discuss_tab appmsg_skin_default appmsg_style_default pages_skin_pc not_in_mm'))
except:
try:
source = i_soup.find_all(class_='ant-space-item')
# pub_time = str(source[1]).split('<span>')[1].split('</span>')[0]
pub_source = str(source[0]).split('<span>')[1].split('</span>')[0].replace('来源：', '')
pub_hao = ''
content = i_soup.find(class_='index_wrapper__L_zqV').text
contentWithTag = str(i_soup.find(class_='index_wrapper__L_zqV'))
except:
source = i_soup.find('div', class_='zsy_cotitle').find('p').text
pub_source = source.split('文章来源：')[1].split('发布时间：')[0]
pub_hao = ''
content = i_soup.find('div', class_='zsy_comain').replace('扫一扫在手机打开当前页', '').strip().text
contentWithTag = str(i_soup.find('div', class_='zsy_comain')).replace('扫一扫在手机打开当前页',
'').strip()
# fujian_list = i_soup.find(class_='related').find_all('li')
# for fujian in fujian_list:
# # print(fujian)
# fujian_href = 'http://www.sasac.gov.cn/' + str(fujian.find('a')['href']).replace('../', '')
# fj_href_list.append(fujian_href)
# print(fj_href_list)
log.info(title)
time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# todo:传kafka字段
if content == '' or content == 'None':
log.info(f'{href}-----{title}----内容为空')
continue
dic_news = {
'attachmentIds': [],
'author': '',
'content': content,
'contentWithTag': str(contentWithTag),
'createDate': time_now,
'deleteFlag': 0,
'id': '',
'labels': [{'relationId': "1672", 'relationName': "浙江省国资委", 'labelMark': "policy"}],
'origin': pub_source,
'organ': pub_source,
'topicClassification': '',
'issuedNumber': pub_hao,
'publishDate': pub_time,
'writtenDate': None,
'sid': '1697458829758697473',
'sourceAddress': href,
'summary': '',
'title': title
}
# print(dic_news)
flag = baseTool.sendKafka(dic_news)
if flag:
baseTool.save_data(dic_news)
num = num + 1
count += 1
except:
pass
except:
pass
end = time.time()
log.info(f'共抓取{num}条数据,共耗时{end - start}')
if __name__ == "__main__":
zhe_jiang()
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论