Skip to content

Z
zzsn_spider
提交 e943c06c 作者: XveLingKun
操作

20241021

上级 6fc56b71
正在显示 包含 739 行增加 和 206 行删除
1018test.py 0 → 100644
# -*- coding: utf-8 -*-
import nltk
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
from nltk.probability import FreqDist
from nltk.stem import PorterStemmer
from nltk import pos_tag, ne_chunk
from collections import defaultdict
from heapq import nlargest
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
nltk.download('maxent_ne_chunker')
nltk.download('words')
def generate_summary(text, num_sentences=3):
# 分句
sentences = sent_tokenize(text)
# 分词
words = word_tokenize(text.lower())
# 去除停用词
stop_words = set(stopwords.words("english"))
filtered_words = [word for word in words if word not in stop_words]
# 词频统计
freq_dist = FreqDist(filtered_words)
# 词干化
stemmer = PorterStemmer()
stemmed_words = [stemmer.stem(word) for word in filtered_words]
# 句子得分
sentence_scores = defaultdict(int)
for i, sentence in enumerate(sentences):
for word in word_tokenize(sentence.lower()):
if word in freq_dist:
sentence_scores[i] += freq_dist[word]
# 选择最高得分的句子
summary_sentences = nlargest(num_sentences, sentence_scores, key=sentence_scores.get)
# 生成摘要
summary = ' '.join([sentences[i] for i in sorted(summary_sentences)])
return summary
text = """中国共产党第二十次全国代表大会审查、批准十九届中央纪律检查委员会工作报告。大会充分肯定十九届中央纪律检查委员会的工作。\n\n\n\n大会认为，党的十九大以来，在以习近平同志为核心的党中央坚强领导下，各级纪律检查委员会深入学习贯彻习近平新时代中国特色社会主义思想，认真落实新时代党的建设总要求，坚定贯彻党的自我革命战略部署，深入落实全面从严治党战略方针，忠实履行党章赋予的职责，坚定不移推进党风廉政建设和反腐败斗争，推动新时代纪检监察工作高质量发展，紧紧围绕党和国家工作大局发挥监督保障执行、促进完善发展作用，为全面建成小康社会、实现第一个百年奋斗目标，迈上全面建设社会主义现代化国家新征程、向第二个百年奋斗目标进军提供坚强保障，向党和人民交上了优异答卷。\n\n\n\n大会要求，高举中国特色社会主义伟大旗帜，坚持马克思列宁主义、毛泽东思想、邓小平理论、“三个代表”重要思想、科学发展观，全面贯彻习近平新时代中国特色社会主义思想，贯彻落实党的二十大作出的战略部署，深刻领悟“两个确立”的决定性意义，增强“四个意识”、坚定“四个自信”、做到“两个维护”，弘扬伟大建党精神，坚定不移全面从严治党，深入推进新时代党的建设新的伟大工程，推动完善党的自我革命制度规范体系，坚持以严的基调强化正风肃纪，一体推进不敢腐、不能腐、不想腐，坚决打赢反腐败斗争攻坚战持久战，锻造忠诚干净担当的纪检监察队伍，为全面建设社会主义现代化国家、全面推进中华民族伟大复兴而团结奋斗。"""
# 生成摘要
summary = generate_summary(text)
print(summary)
612test.py
......@@ -1512,6 +1512,33 @@ if __name__ == "__main__":
# print(aaa)
# aaa = int("07")
# print(aaa)
keywords = "(党建工作)+(生产经营)+(深度融合)"
kwList = getkeywords(keywords)
print(kwList)
\ No newline at end of file
title = "党建论文│工控科产党委“1+2+V”大党建工作格局推动党建工作与生产经营深度融合"
content = "党建工作和深度融合"
keywords = "(浙江|北京)+(尼日利亚|科特迪瓦)+(活动|访问)"
keywords_split = getkeywords(keywords)
print(keywords_split)
tf_title = 0 # 统计当前规则中的关键词在标题中出现的次数
tf_content = 0 # 统计当前规则中的关键词在内容中出现的次数
for kw in keywords_split:
if "+" in kw:
# todo:2024-10-15 关键词需要同时出现 若没有同时出现则分数为0
kws = kw.split("+")
for k in kws:
c_t = str(title).lower().count(k)
c_c = str(content).lower().count(k)
if c_c:
# 如果文章中出现
tf_content += c_c
else:
tf_content = 0
break
if c_t:
tf_title += c_t
else:
tf_title = 0
break
else:
tf_title += str(title).lower().count(kw)
tf_content += str(content).lower().count(kw)
print(tf_title)
print(tf_content)
\ No newline at end of file
comData/Tyc/更新法人数据.py 0 → 100644
# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-
import json
import re
import time
import datetime
import pymongo
import requests
from bs4 import BeautifulSoup
from dateutil.relativedelta import relativedelta
from kafka import KafkaProducer
import urllib3
from retry import retry
from selenium.webdriver.support.wait import WebDriverWait
db_storage = pymongo.MongoClient('mongodb://1.95.69.135:27017/', username='admin', password='ZZsn@9988').ZZSN[
'天眼查登录信息']
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
import sys
# sys.path.append('D:\\KK\\zzsn_spider\\base')
sys.path.append('D:\\PycharmProjects\\zzsn\\base')
import BaseCore
baseCore = BaseCore.BaseCore()
cnx_ = baseCore.cnx
cursor_ = baseCore.cursor
cnx = baseCore.cnx_
cursor = baseCore.cursor_
log = baseCore.getLogger()
def sql_11():
sql = "SELECT * FROM sys_base_enterprise WHERE legal_person like '%关联企业%'"
cursor.execute(sql)
data_list = cursor.fetchall()
for data in data_list:
social_code = data[1]
legal_person = data[4]
print(social_code, legal_person)
legal_person_update = legal_person.split('关联企业')[0]
update_sql = "UPDATE sys_base_enterprise SET legal_person = '%s' WHERE social_credit_code = '%s'" %(legal_person_update, social_code)
cursor.execute(update_sql)
cnx.commit()
if __name__ == "__main__":
sql_11()
comData/dingzhi/dfsm_sasac.py
......@@ -114,7 +114,7 @@ def two_dfsm_mtgc():
info_code = 'IN-20240129-0019'
result_dict = {
'id': '',
'sid': '1751849444877144065',
'sid': '1846847035718762497',
'title': title,
'organ': pub_source,
'origin': '国务院国有资产监督管理委员会',
......
comData/dingzhi/gzyw_sasac.py
......@@ -125,7 +125,7 @@ def gzyw():
info_code = 'IN-20240129-0002'
result_dict = {
'id':'',
'sid':'1751810519211053058',
'sid':'1846847035718762497',
'title': title,
'organ': pub_source,
'origin': '国务院国有资产监督管理委员会',
......
comData/important_meeting/中国共产党历次全国代表.py
url = "http://cpc.people.com.cn/GB/64162/64168/448520/index.html"
url = "http://cpc.people.com.cn/GB/64162/64168/448520/index.html"
......@@ -17,11 +17,12 @@ r = redis.Redis(host='114.116.90.53', port=6380, password='RPHZgkDQ4zGJ', db=5)
def getRequest(url):
req = requests.get(url)
print(req)
# print(req)
req.encoding = req.apparent_encoding
soup = BeautifulSoup(req.text, 'html.parser')
# print(soup)
req.close()
return soup
......@@ -58,23 +59,41 @@ if __name__ == "__main__":
# print(list_href)
for a in list_href[1:]:
href = a.get('href')
if "http" in href:
if href:
pass
else:
continue
if "http" in href:
continue
else:
href = "http://cpc.people.com.cn" + href
href = "http://cpc.people.com.cn/20th/n1/2023/0228/c64094-32633204.html"
# href = "http://cpc.people.com.cn/20th/n1/2022/1017/c448334-32546342.html"
if is_member_containing_string(info_code, href):
continue
soup_detail = getRequest(href)
div_part = soup_detail.find('div', class_="text_c")
try:
title = div_part.find('h1').text.strip()
publishDate = div_part.find('p', class_="sou").text.split('来源')[0]
publishDateList = div_part.find_all('p', class_="sou")
publishDate, origin = "", ""
for k in publishDateList:
if len(k.text) > 4:
publishDate = k.text.split('来源')[0]
origin = k.text.split('来源')[1]
break
else:
continue
if publishDate:
pass
else:
continue
year = int(publishDate[:4])
if year < 2023:
continue
print(href)
origin = div_part.find('p', class_="sou").text.split('来源')[1]
# origin = div_part.find('p', class_="sou").text.split('来源')[1]
contentWithTag = div_part.find('div', class_="show_text")
content = contentWithTag.text.strip()
time_now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
......@@ -94,6 +113,10 @@ if __name__ == "__main__":
'createDate': time_now,
}
log.info(dic_info)
if sendKafka(dic_info):
r.sadd(info_code, href)
except Exception as e:
log.info(f"出错 {e} ----- {href}")
time.sleep(3)
google_comm/config.ini
......@@ -13,7 +13,7 @@ url=jdbc:mysql://114.115.159.144:3306/caiji?useUnicode=true&characterEncoding=ut
[kafka]
bootstrap_servers=1.95.78.131:9092
topic=keyWordsInfo
groupId=python_google
groupId=python_google1
[selenium]
;chrome_driver=C:\Users\WIN10\DataspellProjects\crawlerProjectDemo\tmpcrawler\cmd100\chromedriver.exe
......
google_comm/googleSpider.py
......@@ -34,7 +34,7 @@ from bs4 import BeautifulSoup
class GoogleSpider(object):
def __init__(self, threadId, searchkw, wordsCode, sid, item, bangdan_name):
def __init__(self, threadId, searchkw, wordsCode, sid):
# 创建ConfigParser对象
self.config = configparser.ConfigParser()
# 读取配置文件
......@@ -65,8 +65,8 @@ class GoogleSpider(object):
self.wordsCode = wordsCode
self.sid = sid
self.threadId = threadId
self.item = item
self.bangdan_name = bangdan_name
# self.item = item
# self.bangdan_name = bangdan_name
def createDriver(self):
chrome_driver = self.config.get('selenium', 'chrome_driver')
......@@ -202,15 +202,18 @@ class GoogleSpider(object):
lang = ''
try:
lang = self.detect_language(title)
raw_html = self.webDriver(url)
sm = SmartExtractor(lang)
article = sm.extract_by_url(url=url)
article = sm.extract_by_html(raw_html, title)
# article = sm.extract_by_html(raw_html)
content = article.cleaned_text
contentWithTag = article.text
except Exception as e:
try:
lang = self.detect_language(title)
raw_html = self.webDriver(url)
sm = SmartExtractor(lang)
article = sm.extract_by_html(raw_html)
article = sm.extract_by_html(raw_html, title)
content = article.cleaned_text
contentWithTag = article.text
except Exception as e:
......
google_comm/googletaskJob.py
......@@ -18,6 +18,7 @@ from baseCore import BaseCore
from queue import Queue
import configparser
class GoogleTaskJob(object):
def __init__(self):
# 创建ConfigParser对象
......@@ -33,7 +34,7 @@ class GoogleTaskJob(object):
bootstrap_servers = self.config.get('kafka', 'bootstrap_servers')
# 要订阅的主题
topic = self.config.get('kafka', 'topic')
groupId=self.config.get('kafka', 'groupId')
groupId = self.config.get('kafka', 'groupId')
consumer = KafkaConsumer(topic, group_id=groupId,
bootstrap_servers=[bootstrap_servers],
value_deserializer=lambda m: json.loads(m.decode('utf-8')))
......@@ -41,119 +42,111 @@ class GoogleTaskJob(object):
for record in consumer:
try:
logger.info(f"value:{record.value}")
keymsg=record.value
keymsg = record.value
if keymsg:
break
else:
continue
#print("%s:%d:%d: key=%s value=%s" % (msg.topic, msg.partition, msg.offset, msg.key, msg.value))
# print("%s:%d:%d: key=%s value=%s" % (msg.topic, msg.partition, msg.offset, msg.key, msg.value))
except Exception as e:
logger.info("msg.value error:",e)
logger.info("msg.value error:", e)
except KeyboardInterrupt as e:
keymsg={}
keymsg = {}
finally:
consumer.close()
return keymsg
def getkeyFromredis(self, codeid):
def getkeyFromredis(self,codeid):
kvalue=self.r.get('KEY_WORDS_TO_REDIS::'+codeid)
kvalue=kvalue.decode('utf-8')
kvalue=json.loads(kvalue)
kvalue = self.r.get('KEY_WORDS_TO_REDIS::' + codeid)
kvalue = kvalue.decode('utf-8')
kvalue = json.loads(kvalue)
return kvalue
def getkeywords(self,keywords):
kwList=[]
def getkeywords(self, keywords):
keywords = keywords.replace('（', '(').replace('）', ')')
kwList = []
if ')+(' in keywords:
k1List=keywords.split('+')
kk2=[]
k1List = keywords.split('+')
kk2 = []
for k2 in k1List:
k2=k2.strip("()")
k2List=k2.split('|')
k2 = k2.strip("()")
k2List = k2.split('|')
kk2.append(k2List)
if len(kk2)==2:
if len(kk2) == 2:
result = list(itertools.product(kk2[0], kk2[1]))
elif len(kk2)==3:
result = list(itertools.product(kk2[0], kk2[1],kk2[2]))
elif len(kk2)==4:
result = list(itertools.product(kk2[0], kk2[1],kk2[2],kk2[3]))
elif len(kk2) == 3:
result = list(itertools.product(kk2[0], kk2[1], kk2[2]))
elif len(kk2) == 4:
result = list(itertools.product(kk2[0], kk2[1], kk2[2], kk2[3]))
for res in result:
kwstr=''
kwstr = ''
for kw in res:
kwstr+=kw+"+"
kwstr += kw + "+"
kwList.append(kwstr.strip('+'))
elif '+(' in keywords:
k1List=keywords.split('+')
kk2=[]
k1List = keywords.split('+')
kk2 = []
for k2 in k1List:
k2=k2.strip("()")
k2List=k2.split('|')
k2 = k2.strip("()")
k2List = k2.split('|')
kk2.append(k2List)
if len(kk2)==2:
if len(kk2) == 2:
result = list(itertools.product(kk2[0], kk2[1]))
for res in result:
kwstr=''
kwstr = ''
for kw in res:
kwstr+=kw+"+"
kwstr += kw + "+"
kwList.append(kwstr.strip('+'))
else:
k3=keywords.split("|")
kwList=k3
k3 = keywords.split("|")
kwList = k3
return kwList
def paserKeyMsg(self,keymsg):
def paserKeyMsg(self, keymsg):
num = 1
logger.info('----------')
wordsCode=keymsg['wordsCode']
id=keymsg['id']
wordsCode = keymsg['wordsCode']
id = keymsg['id']
try:
searchEngines=keymsg['searchEngines']
searchEngines = keymsg['searchEngines']
if 'java.util.ArrayList' in searchEngines:
searchEngines = searchEngines[1]
except Exception as e:
searchEngines=[]
kwList=[]
searchEngines = []
kwList = []
if searchEngines:
if '3' in searchEngines:
keyword=keymsg['keyWord']
keymsglist=self.getkeywords(keyword)
if '4' in searchEngines:
keyword = keymsg['keyWord']
keymsglist = self.getkeywords(keyword)
for kw in keymsglist:
kwmsg={
'kw':kw,
'wordsCode':wordsCode,
'sid':id
kwmsg = {
'kw': kw,
'wordsCode': wordsCode,
'sid': id
}
kwList.append(kwmsg)
else:
logger.info('+++++')
keyword=keymsg['keyWord']
keymsglist=self.getkeywords(keyword)
for kw in keymsglist:
kwmsg={
'kw':kw,
'wordsCode':wordsCode,
'sid':id
}
kwList.append(kwmsg)
kwList.append((num, kwmsg))
num += 1
return kwList
def runSpider(self,kwmsg):
searchkw=kwmsg['kw']
wordsCode=kwmsg['wordsCode']
sid=kwmsg['sid']
googleSpider=GoogleSpider(searchkw,wordsCode,sid)
def runSpider(self, threadId, kwmsg):
searchkw = kwmsg['kw']
wordsCode = kwmsg['wordsCode']
sid = kwmsg['sid']
googleSpider = GoogleSpider(threadId, searchkw, wordsCode, sid)
try:
googleSpider.get_page_html()
googleSpider.get_detail_html()
except Exception as e:
logger.info('百度搜索异常'+searchkw)
logger.info('百度搜索异常' + searchkw)
finally:
googleSpider.driver.quit()
logger.info("关键词采集结束！"+searchkw)
logger.info("关键词采集结束！" + searchkw)
if __name__ == '__main__':
# ss='道地西洋参+(销售市场|交易市场|直播带货|借助大会平台|网店|微商|电商|农民博主|推介宣传|高品质定位|西洋参产品经营者加盟|引进龙头企业|西洋参冷风库|建设农旅中心|农产品展销中心|精品民宿|温泉)'
......@@ -161,15 +154,16 @@ if __name__ == '__main__':
# print(keymsglist)
# 创建Redis连接
googleTaskJob=GoogleTaskJob()
baseCore=BaseCore()
logger=baseCore.getLogger()
googleTaskJob = GoogleTaskJob()
baseCore = BaseCore()
logger = baseCore.getLogger()
print('---------------')
while True:
# while True:
for i in range(50):
try:
try:
keymsg=googleTaskJob.getkafka()
kwList=googleTaskJob.paserKeyMsg(keymsg)
keymsg = googleTaskJob.getkafka()
kwList = googleTaskJob.paserKeyMsg(keymsg)
except Exception as e:
logger.info("从kafka拿取信息失败！")
time.sleep(5)
......@@ -178,7 +172,7 @@ if __name__ == '__main__':
# 创建一个线程池，指定线程数量为4
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
# 提交任务给线程池，每个任务处理一个数据
results = [executor.submit(googleTaskJob.runSpider, data) for data in kwList]
results = [executor.submit(googleTaskJob.runSpider, num, data) for num, data in kwList]
# 获取任务的执行结果
for future in concurrent.futures.as_completed(results):
try:
......@@ -190,4 +184,3 @@ if __name__ == '__main__':
logger.info(f"任务执行exception: {e}")
except Exception as e:
logger.info('采集异常')
google_comm/googletaskJob_loc.py
......@@ -18,6 +18,7 @@ from baseCore import BaseCore
from queue import Queue
import configparser
class GoogleTaskJob(object):
def __init__(self):
# 创建ConfigParser对象
......@@ -27,155 +28,132 @@ class GoogleTaskJob(object):
self.r = redis.Redis(host=self.config.get('redis', 'host'),
port=self.config.get('redis', 'port'),
password=self.config.get('redis', 'pass'), db=0)
self.r_6 = redis.Redis(host=self.config.get('redis', 'host'),
port=self.config.get('redis', 'port'),
password=self.config.get('redis', 'pass'), db=6)
def getkafka(self):
# Kafka集群的地址
bootstrap_servers = self.config.get('kafka', 'bootstrap_servers')
# 要订阅的主题
topic = self.config.get('kafka', 'topic')
groupId=self.config.get('kafka', 'groupId')
groupId = self.config.get('kafka', 'groupId')
consumer = KafkaConsumer(topic, group_id=groupId,
bootstrap_servers=[bootstrap_servers],
value_deserializer=lambda m: json.loads(m.decode('utf-8')))
try:
for record in consumer:
try:
logger.info("value:",record.value)
keymsg=record.value
logger.info("value:", record.value)
keymsg = record.value
if keymsg:
break
else:
continue
#print("%s:%d:%d: key=%s value=%s" % (msg.topic, msg.partition, msg.offset, msg.key, msg.value))
# print("%s:%d:%d: key=%s value=%s" % (msg.topic, msg.partition, msg.offset, msg.key, msg.value))
except Exception as e:
logger.info("msg.value error:",e)
logger.info("msg.value error:", e)
except KeyboardInterrupt as e:
keymsg={}
keymsg = {}
finally:
consumer.close()
return keymsg
def getkeyFromredis(self, codeid):
def getkeyFromredis(self,codeid):
kvalue=self.r.get('KEY_WORDS_TO_REDIS::'+codeid)
kvalue=kvalue.decode('utf-8')
kvalue=json.loads(kvalue)
kvalue = self.r.get('KEY_WORDS_TO_REDIS::' + codeid)
kvalue = kvalue.decode('utf-8')
kvalue = json.loads(kvalue)
return kvalue
def getkeywords(self,keywords):
kwList=[]
def getkeywords(self, keywords):
kwList = []
if ')+(' in keywords:
k1List=keywords.split('+')
kk2=[]
k1List = keywords.split('+')
kk2 = []
for k2 in k1List:
k2=k2.strip("()")
k2List=k2.split('|')
k2 = k2.strip("()")
k2List = k2.split('|')
kk2.append(k2List)
if len(kk2)==2:
if len(kk2) == 2:
result = list(itertools.product(kk2[0], kk2[1]))
elif len(kk2)==3:
result = list(itertools.product(kk2[0], kk2[1],kk2[2]))
elif len(kk2)==4:
result = list(itertools.product(kk2[0], kk2[1],kk2[2],kk2[3]))
elif len(kk2) == 3:
result = list(itertools.product(kk2[0], kk2[1], kk2[2]))
elif len(kk2) == 4:
result = list(itertools.product(kk2[0], kk2[1], kk2[2], kk2[3]))
for res in result:
kwstr=''
kwstr = ''
for kw in res:
kwstr+=kw+"+"
kwstr += kw + "+"
kwList.append(kwstr.strip('+'))
elif '+(' in keywords:
k1List=keywords.split('+')
kk2=[]
k1List = keywords.split('+')
kk2 = []
for k2 in k1List:
k2=k2.strip("()")
k2List=k2.split('|')
k2 = k2.strip("()")
k2List = k2.split('|')
kk2.append(k2List)
if len(kk2)==2:
if len(kk2) == 2:
result = list(itertools.product(kk2[0], kk2[1]))
for res in result:
kwstr=''
kwstr = ''
for kw in res:
kwstr+=kw+"+"
kwstr += kw + "+"
kwList.append(kwstr.strip('+'))
else:
k3=keywords.split("|")
kwList=k3
k3 = keywords.split("|")
kwList = k3
return kwList
def paserKeyMsg(self,keymsg):
def paserKeyMsg(self, keymsg):
num = 1
logger.info('----------')
wordsCode=keymsg['wordsCode']
id=keymsg['id']
keyword=keymsg['keyWord']
wordsCode = keymsg['wordsCode']
id = keymsg['id']
try:
searchEngines = keymsg['searchEngines']
if 'java.util.ArrayList' in searchEngines:
searchEngines = searchEngines[1]
except Exception as e:
searchEngines = []
kwList = []
keymsglist=self.getkeywords(keyword)
if searchEngines:
if '3' in searchEngines:
keyword = keymsg['keyWord']
keymsglist = self.getkeywords(keyword)
for kw in keymsglist:
kwmsg={
'kw':kw,
'wordsCode':wordsCode,
'sid':id
kwmsg = {
'kw': kw,
'wordsCode': wordsCode,
'sid': id
}
kwList.append((num,kwmsg))
kwList.append((num, kwmsg))
num += 1
return kwList
def runSpider(self,threadId,kwmsg, item, bangdan_name):
def runSpider(self, threadId, kwmsg):
if 'lay' in kwmsg['kw']:
com_name = item.split('|')[2]
else:
com_name = item.split('|')[1]
searchkw = com_name + ' ' + kwmsg['kw']
print(f'======拼接的关键词是{searchkw}=={com_name}====')
searchkw = kwmsg['kw']
wordsCode = kwmsg['wordsCode']
sid = kwmsg['sid']
googleSpider = GoogleSpider(threadId, searchkw, wordsCode, sid, item, bangdan_name)
googleSpider = GoogleSpider(threadId, searchkw, wordsCode, sid)
try:
googleSpider.get_page_html()
except Exception as e:
logger.info('google搜索异常'+searchkw)
logger.info('google搜索异常' + searchkw)
finally:
googleSpider.driver.quit()
try:
googleSpider.get_detail_html()
except Exception as e:
logger.info('google搜索异常'+searchkw)
logger.info('google搜索异常' + searchkw)
finally:
googleSpider.driver.quit()
logger.info("关键词采集结束！"+searchkw)
logger.info("关键词采集结束！" + searchkw)
def get_comname(self):
# todo:读取redis里的企业名称添加到关键词上
# ZZSN22080900000001|沃尔玛|WMT|1
item = baseCore.redicPullData('GOOGLE_KEYWORDS:COMPANY_NAME:2023_500')
# item = 'ZZSN22080900000001|沃尔玛|WMT|1'
if item:
return item
else:
logger.info('====已无企业===')
return None
# 从Redis的List中获取并移除一个元素
def redicPullData(key, r):
try:
r.ping()
except:
r = redis.Redis(host="114.116.90.53", port=6380, password='RPHZgkDQ4zGJ', db=6)
item = r.lpop(key)
return item.decode() if item else None
import random
if __name__ == '__main__':
# ss='道地西洋参+(销售市场|交易市场|直播带货|借助大会平台|网店|微商|电商|农民博主|推介宣传|高品质定位|西洋参产品经营者加盟|引进龙头企业|西洋参冷风库|建设农旅中心|农产品展销中心|精品民宿|温泉)'
......@@ -183,32 +161,16 @@ if __name__ == '__main__':
# print(keymsglist)
# 创建Redis连接
googleTaskJob=GoogleTaskJob()
baseCore=BaseCore()
logger=baseCore.getLogger()
googleTaskJob = GoogleTaskJob()
baseCore = BaseCore()
logger = baseCore.getLogger()
print('---------------')
while True:
try:
# try:
# googleTaskJob.r.ping()
# except:
# googleTaskJob.r = redis.Redis(host="114.116.90.53", port=6380, password='RPHZgkDQ4zGJ', db=6)
# all_keys = 'GOOGLE_KEYWORDS:COMPANY_NAME*'
# keys = googleTaskJob.r.scan_iter(f"{key}*")
# for key in keys:
item = googleTaskJob.get_comname()
bangdan_name = '2023年世界500强'
if item:
pass
else:
break
codeList = [
'KW-20240516-0002'
]
for codeid in codeList:
codeids = ['KW-20220113-0004']
for codeid in codeids:
try:
#keymsg=baiduTaskJob.getkafka()
# keymsg=baiduTaskJob.getkafka()
keymsg = googleTaskJob.getkeyFromredis(codeid)
kwList = googleTaskJob.paserKeyMsg(keymsg)
# kwList=reversed(kwList)
......@@ -221,9 +183,9 @@ if __name__ == '__main__':
continue
if kwList:
# 创建一个线程池，指定线程数量为4
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
# 提交任务给线程池，每个任务处理一个数据
results = [executor.submit(googleTaskJob.runSpider, num, data, item, bangdan_name) for num, data in kwList]
results = [executor.submit(googleTaskJob.runSpider, num, data) for num, data in kwList]
# 获取任务的执行结果
for future in concurrent.futures.as_completed(results):
try:
......@@ -234,5 +196,4 @@ if __name__ == '__main__':
# 处理任务执行过程中的异常
logger.info(f"任务执行exception: {e}")
except Exception as e:
logger.info(f'采集异常{e}')
logger.info('采集异常')
google_comm/smart_extractor.py
......@@ -129,8 +129,31 @@ class SmartExtractor:
"""
按HTML采集内容
"""
# todo 1018去除head标签 将textarea标签替换成div标签
from bs4 import BeautifulSoup
html_ = BeautifulSoup(html, 'html.parser')
head_tag = html_.find('head')
if head_tag:
head_tag.decompose()
aside_tag = html_.find('aside')
if aside_tag:
aside_tag.decompose()
textarea_tags = html_.find_all('textarea')
if textarea_tags:
for textarea in textarea_tags:
try:
# 创建一个新的 <div> 标签
div_tag = html_.new_tag('div')
# 将 <textarea> 的内容移动到新的 <div> 标签中
div_tag.string = textarea.string
# 替换 <textarea> 标签
textarea.replace_with(div_tag)
except Exception as e:
continue
# 采集正文：传入html
article = self.goose.extract(raw_html=html)
article = self.goose.extract(raw_html=str(html_))
return self.get_extraction_result(article, link_text)
......
gwzk/BaseCore.py
......@@ -826,10 +826,12 @@ class BaseCore:
'create_time': '', 'page_size': '', 'content': ''}
for i in range(0, 3):
try:
response = requests.get(pdf_url, headers=headers, verify=False, timeout=20)
ip_dic = {'https': 'http://127.0.0.1:1080', 'http': 'http://127.0.0.1:1080'}
response = requests.get(pdf_url, headers=headers, proxies=ip_dic, verify=False, timeout=20)
# response = requests.get(pdf_url, verify=False, timeout=20)
file_size = int(response.headers.get('Content-Length'))
break
except:
except Exception as e:
time.sleep(3)
continue
page_size = 0
......
gwzk/europa_1019.py 0 → 100644
"""
国外智库-欧盟 经合组织
"""
import json
import time
import pymongo
from bs4 import BeautifulSoup
import requests
from datetime import datetime
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from kafka import KafkaProducer
from retry import retry
import chardet
import BaseCore
baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()
db_storage = pymongo.MongoClient('mongodb://1.95.69.135:27017', username='admin', password='ZZsn@9988').ZZSN[
'国外智库']
@retry(tries=2, delay=5)
def sendKafka(dic):
producer = KafkaProducer(bootstrap_servers=['1.95.78.131:9092'], max_request_size=1024 * 1024 * 20)
kafka_result = producer.send("research_center_fourth",
json.dumps(dic, ensure_ascii=False).encode('utf8'))
log.info(f'{dic["sourceAddress"]}传输成功')
def secrchATT(item_id, retData, type_id, order_by):
sel_sql = '''select id from clb_sys_attachment where item_id = %s and path = %s and type_id=%s and order_by=%s '''
baseCore.cursor_.execute(sel_sql, (item_id, retData['path'], type_id, order_by))
selects = baseCore.cursor_.fetchone()
return selects
# 插入到att表 返回附件id
def tableUpdate(retData, file_name, num, publishDate,origin):
item_id = retData['item_id']
type_id = retData['type_id']
group_name = retData['group_name']
path = retData['path']
full_path = retData['full_path']
category = retData['category']
file_size = retData['file_size']
status = retData['status']
create_by = retData['create_by']
page_size = retData['page_size']
create_time = retData['create_time']
order_by = num
object_key = full_path.split('https://zzsn.obs.cn-north-1.myhuaweicloud.com/')[1]
Upsql = '''insert into clb_sys_attachment(name,type_id,item_id,group_name,path,full_path,category,file_size,order_by,status,create_by,create_time,object_key,bucket_name,publish_time,source) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''
values = (
file_name+'.pdf', type_id, item_id, group_name, path, full_path, category, file_size, order_by,
status, create_by,
create_time, object_key, 'zzsn', publishDate,origin)
baseCore.cursor_.execute(Upsql, values) # 插入
baseCore.cnx_.commit() # 提交
baseCore.getLogger().info("更新完成:{}".format(Upsql))
selects = secrchATT(item_id, retData, type_id, order_by)
id = selects[0]
return id
def save_data(dic_news):
aaa_dic = {
'附件id': dic_news['attachmentIds'],
'网址': dic_news['sourceAddress'],
'tid': '',
'来源': f"经济合作与发展组织",
'创建时间': dic_news['createDate'],
'带标签内容': dic_news['contentWithTag'][:100],
'发布时间': dic_news['publishDate'],
'标题': dic_news['title']
}
db_storage.insert_one(aaa_dic)
@retry(tries=2, delay=5)
def translate(title, contentWithTag):
headers = {
'Content-Type': 'application/json',
}
dic_info = {
'title': title,
'summary': '',
'content': '',
'contentWithTag': str(contentWithTag)
}
dic_info = json.dumps(dic_info)
# req = requests.post('http://114.116.45.224:5010/translate', data=dic_info, headers=headers)
req = requests.post('http://116.63.179.212:7865/translate', data=dic_info, headers=headers)
dataJson = req.json()
if dataJson['status'] == 'failed':
raise
titleRaw = dataJson['title']
contentWithTagRaw = dataJson['contentWithTag']
titleRaw = BeautifulSoup(titleRaw,'html.parser')
titleRaw = titleRaw.text
contentWithTagRaw = BeautifulSoup(contentWithTagRaw,'html.parser')
return titleRaw, contentWithTagRaw
def doJob():
num = 1
url = 'https://www.oecd-ilibrary.org/economics/oecd-policy-responses-on-the-impacts-of-the-war-in-ukraine_dc825602-en?page=1'
url = 'https://www.oecd-ilibrary.org/economics/oecd-policy-responses-on-the-impacts-of-the-war-in-ukraine_dc825602-en?page=1'
headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cache-Control': 'max-age=0',
# 'Cookie': 'JSESSIONID=BHezogPwi8NJVECsKXCXqijdQ00-yMJHw_gR8wiC.ip-10-240-5-121; __cf_bm=c2byUypnSjXPS_UFDM7BMRGDxN6AQEkNVUjzw9HuSq8-1707054653-1-AbbI7JWWkfWKVGi8SKI06f0jGEjPdk5kvHAIRRpBHSSSnmxj1IcvGUT8+/O6R0U2RLZJECZdUzZIXAwFuEz5lPo=; _gcl_au=1.1.201344533.1707054655; _gid=GA1.2.557164000.1707054655; cb-enabled=enabled; cf_clearance=6tK6.WKHJbXXoV4NTgbyHRhetRxMdWPZofwlv01F65Y-1707054656-1-AfrYlWnLLZFC1sKxeFVQintPrZnjvjoJSZwRRhAYwqRHGdWbU5IFZQDJZJM21l20Tj6gk4JxNobWT0wGzp1Dgjw=; _ce.irv=new; cebs=1; _ce.clock_event=1; _ce.clock_data=72%2C123.149.3.159%2C1%2C9c1ce27f08b16479d2e17743062b28ed; custom_cookie_AB=1; AWSALB=I/eGQ0glcxuROskD1JKEl/dqsqElpmo/MnwLboJZJB2QthQFFWnLA3gzuJTskEaZxJD7VuWEEsqjhLVvhq4q2Wt0RebuRhukeHpKvgmGMelxpn/RiDmehyvxTOiS; AWSALBCORS=I/eGQ0glcxuROskD1JKEl/dqsqElpmo/MnwLboJZJB2QthQFFWnLA3gzuJTskEaZxJD7VuWEEsqjhLVvhq4q2Wt0RebuRhukeHpKvgmGMelxpn/RiDmehyvxTOiS; _gat_UA-1887794-2=1; _dc_gtm_UA-136634323-1=1; _ga_F5XZ540Q4V=GS1.1.1707054655.1.1.1707055119.7.0.0; _ga=GA1.1.1014316406.1707054655; _ga_F7KSNTXTRX=GS1.1.1707054655.1.1.1707055119.0.0.0; cebsp_=5; _ce.s=v~212f033193b9432855ae8335d6d3969cc1f8b751~lcw~1707055134688~lva~1707054658247~vpv~0~v11.fhb~1707054659602~v11.lhb~1707055126493~v11.cs~325107~v11.s~6d7ba630-c364-11ee-aba8-136dbbf9a447~v11.sla~1707055134688~v11.send~1707055135439~lcw~1707055135439',
'Cookie':'_gcl_au=1.1.520819919.1729044001; cb-enabled=enabled; _gid=GA1.2.373936706.1729044001; JSESSIONID=BF8QdIzLc1AH1oHxfbMWFJiH-sPk_AlC6bTARQNt.ip-10-240-5-88; cf_chl_rc_m=1; __cf_bm=K.xgQs4g2IfOkKpkgMwvUZcCBKtUImufNP5MBWTepPA-1729046354-1.0.1.1-cVxehK2ompgx38nEFnmth4KJgF1z2gq8qFmuglu3zVPd6wHIaf2M0_mKIWrbQReJcCj5teEcMQi10y3zcV3DYg; _ga_F5XZ540Q4V=GS1.1.1729044000.1.1.1729046354.59.0.0; _ga=GA1.1.1064961592.1729044001; custom_cookie_AB=2; _ga_F7KSNTXTRX=GS1.1.1729044000.1.1.1729046354.0.0.0; cf_clearance=Nb8YewEVMb7xGIha8yYsBooN5sd6T0BQdlVnsWL0KpA-1729046354-1.2.1.1-STcJJ8GoM7birxfchMeS3hnoBgVLTIMb_13nMjNNGNZ0jw.Ja9VM.Z0Ilo6b5Txz.a2RUhqDJ6kBsPh6clCc5io2.dWAL_IZPhvxHGCcOgA0CZgD0FFTWWDEBSuSbgCFvW9.jZ9qTtaXEUUAmth2b_bhHBhKIJMlq3ZaKIABY2UUkagw4n2rEHCKFp2uol96aU7NaOTxdcXywnJwm7gKAGUdojVyPZLqBr0RsJRmw.sjtgM9M0V8wkYP0Dn4kJliUuefFEFQsD42xWwZ2RcMA9myPonLZ2WFEc_DJEhpN_WiCDEeuE.VWq1KjCSmjND81N97woKrqZHebtNXNGRUIXfQsY_4RAGqwcaRaXjTrkdkdL0ixrLGMyn.pHAw4w2slGPoKSx7QqK6cEi5HHoNDBtnulpTcjo.ng1UE5T.KC7QaAcq65RU.2fPgFGLWAYl; AWSALB=eC10QptE8CcUMLJ+tGsk1ioPWWu6X2O2keMzHHthze0Rm9x05Nu29Byk5bpDJEQktI67lEJ4zhczRPiS5CAZcYKSGqmOow9HohPAtTJ5OguiZoL1EdbS1uy1TTgo; AWSALBCORS=eC10QptE8CcUMLJ+tGsk1ioPWWu6X2O2keMzHHthze0Rm9x05Nu29Byk5bpDJEQktI67lEJ4zhczRPiS5CAZcYKSGqmOow9HohPAtTJ5OguiZoL1EdbS1uy1TTgo',
#'Referer': 'https://www.oecd-ilibrary.org/economics/oecd-policy-responses-on-the-impacts-of-the-war-in-ukraine_dc825602-en?page=2',
'Sec-Ch-Ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"Windows"',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-User': '?1',
'content-type': 'application/javascript; charset=UTF-8',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
}
chrome_driver = 'D:\cmd100\chromedriver.exe'
path = Service(chrome_driver)
chrome_options = webdriver.ChromeOptions()
chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
chrome_options.add_experimental_option('useAutomationExtension', False)
chrome_options.binary_location = 'D:\Google\Chrome\Application\chrome.exe'
driver = webdriver.Chrome(service=path, options=chrome_options)
# 访问页面
driver.get(url)
# 可选：等待页面加载完成
wait = WebDriverWait(driver, 10) # 最长等待时间为10秒
# 获取页面内容
html_content = driver.page_source
# req = requests.get(url=url, headers=headers)
# detected_encoding = chardet.detect(req.content)['encoding']
# print(f"Detected Encoding: {detected_encoding}")
#
# # 使用检测到的编码方式解码响应内容
# content = req.content.decode(detected_encoding, errors='replace')
# 使用BeautifulSoup解析
soup = BeautifulSoup(html_content, 'html.parser')
# soup = BeautifulSoup(req.text, 'lxml')
# log.info(soup)
# print(soup)
div_part = soup.find_all('div', class_='col-xs-12 body-section')[1]
div_list = div_part.find_all('div', class_='row panel')
for div in div_list:
start_time = time.time()
title = div.find('div', class_='col-lg-7 col-xs-12 resume-item').find('p', class_='intro-item').find('strong', class_='book-title').text
href = 'https://www.oecd-ilibrary.org' + div.find('div', class_='col-lg-7 col-xs-12 resume-item').find('p', class_='intro-item').find('a')['href']
print(href)
is_href = db_storage.find_one({'网址': href})
if is_href:
log.info(f'{href}===已采集')
continue
pubtime_ = div.find('div', class_='col-lg-7 col-xs-12 resume-item').find('p', class_='intro-item').find('strong', class_='book-title gray').text
# 定义原始时间的格式
time_format = "%d %b %Y"
# 转换为标准时间
standard_time = datetime.strptime(pubtime_, time_format).strftime("%Y-%m-%d")
if standard_time > '2023-01-30':
pass
else:
break
year = standard_time[:4]
pdf_part = div.find('div', class_='col-lg-5 col-xs-12 actions-item').find('ul', class_='actions').find_all('li')[1].find('a').get('href')
pdf_url = 'https://www.oecd-ilibrary.org' + pdf_part
driver.get(href)
# 可选：等待页面加载完成
wait = WebDriverWait(driver, 10) # 最长等待时间为10秒
# 获取页面内容
xiangqing_content = driver.page_source
soup_news = BeautifulSoup(xiangqing_content, 'html.parser')
# print(title, standard_time, pdf_url, href)
contentWithTag = soup_news.find('div', class_='description js-desc-fade show-all')
content = contentWithTag.get_text()
print(title,contentWithTag)
# # todo:翻译
# try:
# titleRaw, contentWithTagRaw = translate(str(title), str(contentWithTag))
# log.info(f'{href}===翻译成功')
# except Exception as e:
# log.error(f'{href}===翻译失败==={e}')
# continue
header = {'Content-Type': 'application/json'}
retData = baseCore.uptoOBS(pdf_url, title, 15, '', pathType, taskType, start_time, create_by, header)
num += 1
id_list = []
if retData['state']:
att_id = tableUpdate(retData, title, num, standard_time, '经济合作与发展组织')
if att_id:
id_list.append(att_id)
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
lang = baseCore.detect_language(content)
# contentRaw = contentWithTagRaw.text
# contentWithTagRaw = str(contentWithTagRaw)
dic = {
'id': f'1620244462491893761{int(time.time())}',
'subjectId': '1620244462491893761',
'checkStatus': 1,
'deleteFlag': 0,
'topNum': 0,
'content': content,
# 'contentRaw': contentRaw, # 未翻译成功
'contentRaw': "",
'contentWithTag': str(contentWithTag),
# 'contentWithTagRaw': contentWithTagRaw, # 未翻译成功
'contentWithTagRaw': "",
'createDate': now,
'labels': [
{'labelMark': 'organization', 'relationId': '1619903523269271554', 'relationName': '经济合作与发展组织'}],
'lang': lang,
'origin': '经济合作与发展组织',
'publishDate': standard_time,
'sourceAddress': href,
'title': title,
# 'titleRaw': titleRaw, # 未翻译成功
'titleRaw': "",
'updateDate': now,
'attachmentIds':id_list
}
sendKafka(dic)
try:
save_data(dic)
except:
log.error(f'{href}===数据库保存失败')
# break
if __name__ == "__main__":
pathType = 'PolicyDocuments/'
taskType = '国外智库-经合组织'
create_by = 'XueLingKun'
doJob()
习近平讲话/1.py
# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-
......
百度采集/baidu_comm/test.py
from baiduSpider import BaiduSpider
from baiduSpider import BaiduSpider
......@@ -2,13 +2,15 @@ from baiduSpider import BaiduSpider
searchkw, wordsCode, sid = '', '', ''
baidu = BaiduSpider(searchkw, wordsCode, sid)
import requests
# url = 'https://baijiahao.baidu.com/s?id=1784907851792547880&wfr=spider&for=pc'
url = 'https://www.163.com/dy/article/IKD3M2P20514IPKH.html'
title = '“一带一路”商学院联盟副秘书长解奕炯：临沂在国际化物流建设中一定能“先行一步”'
# url = 'https://www.thepaper.cn/newsDetail_forward_26661172'
url = 'https://finance.huanqiu.com/article/9CaKrnK5O7o' # 澎湃新闻 虎嗅APP 经济观察网
title = '中国建材集团董事长宋志平:激发和保护企业家精神'
try:
detailurl = url
title = title
content, contentWithTag = baidu.extractorMsg(detailurl, title)
content, contentWithTag,title = baidu.extractorMsg(detailurl, title)
contentWithTag = baidu.rmTagattr(contentWithTag, detailurl)
except Exception as e:
content = ''
......
百度采集/baidu_comm/test_1.py 0 → 100644
from lxml import etree
from lxml import etree
from bs4 import BeautifulSoup
from goose3 import Goose
from goose3.text import StopWordsChinese, StopWordsKorean, StopWordsArabic
# 创建 Goose 实例
g = Goose()
goose = Goose({'stopwords_class': StopWordsChinese})
with open(r"C:\Users\EDY\Desktop\2.html", "r", encoding='utf-8') as f:
raw_xml = f.read()
print(type(raw_xml))
article = goose.extract(raw_html=raw_xml)
# 解析 XML
# root = etree.fromstring(raw_xml)
# root = BeautifulSoup(raw_xml, 'html.parser')
# 假设 article 是一个对象，它的 raw_doc 属性是我们刚刚解析得到的 root
# article = type('Article', (object,), {'raw_doc': root})()
# title_element 是我们要查找的元素名称
title_element_list = [
'h1',
'h2',
'h3',
'div',
'span',
'td',
'p',
'title'
]
for title_element in title_element_list:
# 使用 XPath 查询提取所有 title 元素
element_list = article.raw_doc.xpath(f'//{title_element}')
# element_list = article.raw_doc.getroottree().xpath(f'//{title_element}')
# 输出结果
print(element_list)
for element in element_list:
# 取纯文本内容，包括子元素
text = etree.tounicode(element, method='text').strip()
text_no_space = text.replace(" ", "")
print(text_no_space)
百度采集/baidu_comm/test_2.py 0 → 100644
from goose3 import Goose
from goose3 import Goose
# 创建 Goose 实例
g = Goose()
# 示例 HTML 字符串
html = """
<html>
<head>
<title>示例文章标题</title>
</head>
<body>
<h1>示例文章标题</h1>
<p>这是文章的第一段。</p>
<p>这是文章的第二段。</p>
</body>
</html>
"""
# 使用 Goose 提取文章内容
article = g.extract(raw_html=html)
# 输出提取的信息
print(article.title) # 输出文章标题
print(article.cleaned_text) # 输出清理后的正文文本
print(article.meta_description) # 输出元描述信息
print(article.publish_date) # 输出发布日期
print(article.top_image) # 输出顶部图片信息
print(article.authors) # 输出作者信息
\ No newline at end of file
百度采集/baidu_comm/测试智能解析.py 0 → 100644
# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-
import pandas as pd
import pymongo
from baiduSpider import BaiduSpider
searchkw, wordsCode, sid = '', '', ''
baidu = BaiduSpider(searchkw, wordsCode, sid)
import urllib3
db_storage = pymongo.MongoClient('mongodb://1.95.69.135:27017/', username='admin', password='ZZsn@9988').ZZSN[
'天眼查登录信息']
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
import sys
sys.path.append('D:\\PycharmProjects\\zzsn\\base')
import BaseCore
baseCore = BaseCore.BaseCore()
cnx_ = baseCore.cnx
cursor_ = baseCore.cursor
cnx = baseCore.cnx_
cursor = baseCore.cursor_
log = baseCore.getLogger()
def selectSql():
sql = """SELECT
id,
title,
publishDate,
origin,
detailurl,
state,
keyword,
CAST(content AS CHAR),
create_time,
sid,
wordsCode
FROM
baidu_search_result_
WHERE
wordsCode='EVENT_KW-20241017-linshi-0001'"""
cursor_.execute(sql)
return cursor_.fetchall()
if __name__ == "__main__":
resultList = selectSql()
df_list = []
for result in resultList:
id_ = result[0]
title = result[1]
publishDate = result[2]
origin = result[3]
url = result[4]
try:
detailurl = url
content, contentWithTag, title = baidu.extractorMsg(detailurl, title)
contentWithTag = baidu.rmTagattr(contentWithTag, detailurl)
except Exception as e:
content = ''
contentWithTag = ''
detailmsg = {
'title': title,
'detailurl': url,
'content': content,
'contentHtml': contentWithTag,
'origin': origin
}
df_list.append(detailmsg)
df = pd.DataFrame(df_list)
df.to_excel("./测试结果.xlsx", index=False)
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论