Skip to content

Z
zzsn_spider
提交 fe5f0932 作者: 刘伟刚
操作

python相关脚本提交

上级 a5dbff9a
正在显示 包含 1090 行增加 和 96 行删除
comData/caiwushuju/RedisPPData.py
......@@ -15,6 +15,23 @@ def conn11():
return conn,cursor
#企业公告
def shizhiCodeFromSql():
conn,cursor=conn11()
try:
gn_query = "select securities_code from sys_base_enterprise_ipo where category in ('4','5','6') "
cursor.execute(gn_query)
gn_result = cursor.fetchall()
gn_social_list = [item[0] for item in gn_result]
print('shizhiCodeFromSql开始将股票代码放入redis=======')
for item in gn_social_list:
r.rpush('NoticeEnterprise:shizhi_code', item)
print('shizhiCodeFromSql将股票代码放入redis结束')
except Exception as e:
log.info("数据查询异常")
finally:
cursor.close()
conn.close()
#企业公告
def yahooCodeFromSql():
conn,cursor=conn11()
try:
......@@ -22,10 +39,10 @@ def yahooCodeFromSql():
cursor.execute(gn_query)
gn_result = cursor.fetchall()
gn_social_list = [item[0] for item in gn_result]
print('=======')
print('yahooCodeFromSql开始将股票代码放入redis=======')
for item in gn_social_list:
r.rpush('NoticeEnterprise:securities_code', item)
print('将股票代码放入redis结束')
print('yahooCodeFromSql将股票代码放入redis结束')
except Exception as e:
log.info("数据查询异常")
finally:
......@@ -38,28 +55,23 @@ def yahooCode_task():
# 每天执行一次
# scheduler.add_job(yahooCodeFromSql, 'cron', hour=0,minute=0)
#3天执行一次
scheduler.add_job(yahooCodeFromSql, 'interval', days=3)
scheduler.add_job(yahooCodeFromSql, 'cron', day='*/3', hour=0, minute=0)
# 每天执行一次
scheduler.add_job(shizhiCodeFromSql, 'cron', hour=10,minute=0)
try:
yahooCodeFromSql() # 定时开始前执行一次
shizhiCodeFromSql() # 定时开始前执行一次
scheduler.start()
except Exception as e:
print('定时采集异常', e)
pass
if __name__ == "__main__":
start = time.time()
# NoticeEnterprise()
# AnnualEnterpriseIPO()
# AnnualEnterprise()
# BaseInfoEnterpriseAbroad()
# NewsEnterprise_task()
# NewsEnterprise()
# BaseInfoEnterprise()
# FBS()
# NoticeEnterprise_task()
# AnnualEnterprise_task()
# NoticeEnterprise()
yahooCode_task()
# yahooShizhiCode_task()
log.info(f'====={basecore.getNowTime(1)}=====添加数据成功======耗时：{basecore.getTimeCost(start,time.time())}===')
# cnx.close()
# cursor.close()
......
comData/caiwushuju/huilv.py
......@@ -115,69 +115,155 @@ class CurrencyRate(object):
if __name__ == '__main__':
result_list1 = [
[
'人民币',
'CNY'],
[
'美元',
'USD'],
[
'欧元',
'EUR'],
[
'瑞士法郎',
'CHF'],
[
'加元',
'CAD'],
[
'波兰兹罗提',
'PLN'],
[
'英镑',
'GBP'],
[
'澳元',
'AUD'],
[
'泰铢',
'THB'],
[
'沙特里亚尔',
'SAR'],
[
'巴西里亚伊',
'BRL'],
[
'新土耳其新里拉',
'TRY'],
[
'新台币',
'TWD'],
[
'印度卢比',
'INR'],
[
'墨西哥比索',
'MXN'],
[
'日元',
'JPY'],
[
'瑞典克朗',
'SEK'],
[
'韩元',
'KRW'],
[
'俄罗斯卢布',
'RUB'],
[
'新加坡元',
'SGD'],
[
'港币',
'HKD']]
['人民币','CNY'],
['港元','HKD'],
['台币','TWD'],
['欧元','EUR'],
['美元','USD'],
['英镑','GBP'],
['澳元','AUD'],
['韩元','KRW'],
['日元','JPY'],
['澳元','AUD'],
['阿尔巴尼亚列克','ALL'],
['阿尔及利亚第纳尔','DZD'],
['阿根廷比索','ARS'],
['阿鲁巴岛弗罗林','AWG'],
['澳元','AUD'],
['埃及镑','EGP'],
['埃塞俄比亚比尔','ETB'],
['澳门元','MOP'],
['阿曼里亚尔','OMR'],
['阿联酋迪拉姆','AED'],
['巴哈马元','BSD'],
['巴林第纳尔','BHD'],
['巴巴多斯元','BBD'],
['白俄罗斯卢布','BYR'],
['伯利兹元','BZD'],
['百慕大元','BMD'],
['不丹卢比','BTN'],
['玻利维亚诺','BOB'],
['博茨瓦纳普拉','BWP'],
['巴西里亚伊','BRL'],
['保加利亚列瓦','BGN'],
['布隆迪法郎','BIF'],
['冰岛克朗','ISK'],
['巴基斯坦卢比','PKR'],
['巴拿马巴尔博亚','PAB'],
['巴布亚新几内亚基那','PGK'],
['巴拉圭瓜拉尼','PYG'],
['波兰兹罗提','PLN'],
['朝鲜圆','KPW'],
['多哥非洲共同体法郎','XOF'],
['丹麦克朗','DKK'],
['多米尼加比索','DOP'],
['俄罗斯卢布','RUB'],
['佛得角埃斯库多','CVE'],
['福克兰群岛镑','FKP'],
['斐济元','FJD'],
['菲律宾比索','PHP'],
['港元','HKD'],
['刚果中非共同体法郎','XAF'],
['哥伦比亚比索','COP'],
['哥斯达黎加科朗','CRC'],
['古巴比索','CUP'],
['格林纳达东加勒比元','XCD'],
['冈比亚达拉西','GMD'],
['圭亚那元','GYD'],
['韩元','KRW'],
['海地古德','HTG'],
['洪都拉斯伦皮拉','HNL'],
['哈萨克斯坦腾格','KZT'],
['柬埔寨利尔斯','KHR'],
['加拿大元','CAD'],
['捷克克朗','CZK'],
['吉布提法郎','DJF'],
['几内亚法郎','GNF'],
['科摩罗法郎','KMF'],
['克罗地亚库纳','HRK'],
['肯尼亚先令','KES'],
['科威特第纳尔','KWD'],
['卡塔尔利尔','QAR'],
['老挝基普','LAK'],
['拉脱维亚拉图','LVL'],
['黎巴嫩镑','LBP'],
['莱索托洛提','LSL'],
['利比里亚元','LRD'],
['利比亚第纳尔','LYD'],
['立陶宛里塔斯','LTL'],
['列斯荷兰盾','ANG'],
['罗马尼亚新列伊','RON'],
['卢旺达法郎','RWF'],
['美元','USD'],
['孟加拉塔卡','BDT'],
['马其顿第纳尔','MKD'],
['马拉维克瓦查','MWK'],
['马来西亚林吉特','MYR'],
['马尔代夫卢非亚','MVR'],
['毛里塔尼亚乌吉亚','MRO'],
['毛里求斯卢比','MUR'],
['墨西哥比索','MXN'],
['摩尔多瓦列伊','MDL'],
['蒙古图格里克','MNT'],
['摩洛哥道拉姆','MAD'],
['缅甸元','MMK'],
['秘鲁索尔','PEN'],
['纳米比亚元','NAD'],
['尼泊尔卢比','NPR'],
['尼加拉瓜科多巴','NIO'],
['尼日利亚奈拉','NGN'],
['挪威克朗','NOK'],
['南非兰特','ZAR'],
['欧元','EUR'],
['日元','JPY'],
['瑞典克朗','SEK'],
['瑞士法郎','CHF'],
['萨尔瓦多科朗','SVC'],
['萨摩亚塔拉','WST'],
['圣多美多布拉','STD'],
['沙特阿拉伯里亚尔','SAR'],
['塞舌尔法郎','SCR'],
['塞拉利昂利昂','SLL'],
['所罗门群岛元','SBD'],
['索马里先令','SOS'],
['斯里兰卡卢比','LKR'],
['圣赫勒拿群岛磅','SHP'],
['斯威士兰里兰吉尼','SZL'],
['台币','TWD'],
['土耳其新里拉','TRY'],
['太平洋法郎','XPF'],
['坦桑尼亚先令','TZS'],
['泰国铢','THB'],
['汤加潘加','TOP'],
['特立尼达和多巴哥元','TTD'],
['突尼斯第纳尔','TND'],
['文莱元','BND'],
['危地马拉格查尔','GTQ'],
['乌克兰赫夫米','UAH'],
['乌拉圭新比索','UYU'],
['瓦努阿图瓦图','VUV'],
['越南盾','VND'],
['匈牙利福林','HUF'],
['新西兰元','NZD'],
['新加坡元','SGD'],
['叙利亚镑','SYP'],
['英镑','GBP'],
['印度卢比','INR'],
['印度尼西亚卢比(盾)','IDR'],
['伊朗里亚尔','IRR'],
['伊拉克第纳尔','IQD'],
['以色列镑','ILS'],
['牙买加元','JMD'],
['约旦第纳尔','JOD'],
['也门里亚尔','YER'],
['智利比索','CLP'],
['直布罗陀镑','GIP'],
['铜价盎司','XCP'],
['金价盎司','XAU'],
['钯价盎司','XPD'],
['铂价盎司','XPT'],
['银价盎司','XAG']
]
result_list2 = [
'USD',
'CNY']
......
comData/caiwushuju/上市企业市值.py
# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-
......@@ -50,11 +50,15 @@ class Shizhi(object):
charset='utf8')
cursor = conn.cursor()
return conn,cursor
def getCodeFromRedis(self):
securitiescode=self.r.lpop('NoticeEnterprise:shizhi_code')
securitiescode = securitiescode.decode('utf-8')
return securitiescode
def getmarketCap(self):
def getmarketCap(self,securitiescode):
conn,cursor=self.conn11()
try:
sql1 = """select social_credit_code,securities_code,securities_short_name from sys_base_enterprise_ipo where category in ('4','5','6') """ # and stock_code = "SYNH"
sql1 = f"select social_credit_code,securities_code,securities_short_name from sys_base_enterprise_ipo where securities_code='{securitiescode}' " # and stock_code = "SYNH"
cursor.execute(sql1)
result_data = cursor.fetchall()
except Exception as e:
......@@ -75,21 +79,24 @@ class Shizhi(object):
url = f'https://finance.yahoo.com/quote/{stock2}?p={stock2}'
try:
self.logger.info(f'正在采集：{url}')
# 设置页面加载超时时间为10秒
self.driver.set_page_load_timeout(60)
self.driver.get(url)
# 等待页面加载完成
wait = WebDriverWait(self.driver, 300)
wait = WebDriverWait(self.driver, 60)
wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
time.sleep(5)
doc_resp = pq(self.driver.page_source)
unit=doc_resp('div[id="quote-header-info"]>div:nth-child(2)>div:nth-child(1)>div:nth-child(2)>span')
currency = unit.text().split("Currency in ")[1]
currency =currency.split("(")[0]
market_cap=doc_resp('td[data-test="MARKET_CAP-value"]')
marketcap=market_cap.text()
if marketcap and marketcap!='N/A':
# 获取当前时间
current_time = datetime.datetime.now()
currentdate = current_time.strftime("%Y-%m-%d")
print(f'信用代码：{social_credit_code} 股票代码：{stock} 币种：{currency} 市值：{marketcap} 日期：{currentdate}')
self.logger.info(f'信用代码：{social_credit_code} 股票代码：{stock} 币种：{currency} 市值：{marketcap} 日期：{currentdate}')
# market_url = f'http://192.168.1.39:8088/sync/marketValue'
market_url = f'http://114.115.236.206:8088/sync/marketValue'
param= {
......@@ -103,26 +110,33 @@ class Shizhi(object):
resp = requests.post(market_url,json=param)
# 检查响应状态码
if resp.status_code == 200:
print("请求成功")
self.logger.info("调用接口请求成功")
# 打印响应内容
print(resp.content)
self.logger.info(resp.content)
else:
print("请求失败")
self.logger.info("调用接口请求失败")
except:
with open('雅虎财经-财务数据_发送错误ID.txt', 'a', encoding='utf8')as f:
f.write(stock + '\n')
self.logger.info("调用接口请求失败")
except Exception as e:
self.driver.close()
self.logger.info('请求异常！重新打开浏览器')
self.driver.quit()
self.driver=self.get_webdriver()
print(e)
except Exception as e:
print(e)
self.driver.close()
self.driver.quit()
self.driver=self.get_webdriver()
self.logger.info(f'{securitiescode}股票的市值采集结束')
if __name__ == '__main__':
shizhi=Shizhi()
shizhi.getmarketCap()
\ No newline at end of file
# shizhi.getmarketCap()
while True:
securitiescode=''
try:
securitiescode=shizhi.getCodeFromRedis()
shizhi.getmarketCap(securitiescode)
except Exception as e:
shizhi.logger.info("redis为空等待5分钟")
if securitiescode:
shizhi.r.rpush('NoticeEnterprise:shizhi_code',securitiescode)
else:
time.sleep(300)
\ No newline at end of file
comData/dingzhi/bmfw.py 0 → 100644
import datetime
import time
import redis
import requests
import urllib3
from pyquery import PyQuery as pq
import json
from kafka import KafkaProducer
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
#务院政策问答平台最新发布信息采集
def reqHtml(url,data,header):
try:
proxy = {'https': 'http://127.0.0.1:1080', 'http': 'http://127.0.0.1:1080'}
json_data=json.dumps(data)
response = requests.post(url,data=json_data,headers=header,verify=False,timeout=10)
print(response.status_code)
html=response.text
except Exception as e:
html=''
return html
def page_list():
header = {
'Host':'xcx.www.gov.cn',
'Connection':'keep-alive',
'Content-Length':'72',
'x-tif-openid':'ojyj-41lGcemgsREMHBh1ac7iZUw',
'x-tif-did':'pb5XUGL1Zm',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/7.0.20.1781(0x6700143B) NetType/WIFI MiniProgramEnv/Windows WindowsWechat/WMPF XWEB/8379',
'x-tif-sid':'e1436792814f1c6845af4d84cbc4ad9957',
'Content-Type':'application/json',
'xweb_xhr':'1',
'dgd-pre-release':'0',
'x-yss-page':'publicService/pages/policyQALibrary/index/index',
'x-yss-city-code':'4400',
'Accept':'*/*',
'Sec-Fetch-Site':'cross-site',
'Sec-Fetch-Mode':'cors',
'Sec-Fetch-Dest':'empty',
'Referer':'https://servicewechat.com/wxbebb3cdd9b331046/713/page-frame.html',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'zh-CN,zh'
}
url='https://xcx.www.gov.cn/ebus/gwymp/api/r/faqlib/GetPolicyList'
for i in range(1,445):
print(f'采集第{i}页数据')
k=i
da='{"filterType":"","departmentid":"","keyword":"","page_size":15,"page":[k]}'
data=da.replace('[k]',str(k))
try:
data=json.loads(data)
lhtml=reqHtml(url,data,header)
hjson=json.loads(lhtml)
data=hjson['data']['list']
except Exception as e:
print(e)
time.sleep(60)
continue
for ss in data:
id=ss['id']
durl=f'https://xcx.www.gov.cn/ebus/gwymp/api/r/faqlib/GetPolicy'
sourceAddress=f'https://bmfw.www.gov.cn/zcdwpt/index.html#/detail?id={id}'
try:
flag=r.sismember('IN-20230829-0146',sourceAddress)
if flag:
print('信息已采集入库过')
continue
except Exception as e:
continue
ss['url']=durl
ss['sourceAddress']=sourceAddress
detailpaser(ss)
# time.sleep(5)
def detailpaser(dmsg):
hh={
'Host':'xcx.www.gov.cn',
'Connection':'keep-alive',
'Content-Length':'14',
'x-tif-openid':'ojyj-41lGcemgsREMHBh1ac7iZUw',
'x-tif-did':'pb5XUGL1Zm',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/7.0.20.1781(0x6700143B) NetType/WIFI MiniProgramEnv/Windows WindowsWechat/WMPF XWEB/8379',
'x-tif-sid':'e1436792814f1c6845af4d84cbc4ad9957',
'Content-Type':'application/json',
'xweb_xhr':'1',
'dgd-pre-release':'0',
'x-yss-page':'publicService/pages/policyQALibrary/detail/detail',
'x-yss-city-code':'4400',
'Accept':'*/*',
'Sec-Fetch-Site':'cross-site',
'Sec-Fetch-Mode':'cors',
'Sec-Fetch-Dest':'empty',
'Referer':'https://servicewechat.com/wxbebb3cdd9b331046/713/page-frame.html',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'zh-CN,zh'
}
try:
durl=dmsg['url']
id=str(dmsg['id'])
data={"id":id}
json_data=json.dumps(data)
response = requests.post(durl,data=json_data,headers=hh,verify=False,timeout=10)
dhtml=response.text
dd=json.loads(dhtml)
sendTokafka(dd)
except Exception as e:
print(e)
print(dhtml)
def sendTokafka(ddata):
dd=ddata['data']
title=dd['title']
id=dd['id']
content=dd['content']
contentWithTag=dd['content']
publishTime=dd['publishTime']
time_format='%Y年%m月%d日'
publishDate=str(datetime.datetime.strptime(publishTime, time_format))
origin=dd['departmentName']
sourceAddress=f'https://bmfw.www.gov.cn/zcdwpt/index.html#/detail?id={id}'
sid='1696404919115825153'
info_code='IN-20230829-0146'
aa_dict = {
'content': content,
'contentWithTag': contentWithTag,
'id': '',
'sid': sid,
'origin': origin,
'publishDate': publishDate,
'sourceAddress': sourceAddress,
'title': title,
'source': 'python定制采集',
'type': ''
}
producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
try:
kafka_result = producer.send("crawlerInfo", json.dumps(aa_dict, ensure_ascii=False).encode('utf8'))
r.sadd(info_code,sourceAddress)
print('发送kafka成功！')
except Exception as e:
print(e)
finally:
producer.close()
# r.close()
if __name__ == '__main__':
r = redis.Redis(host='114.115.236.206', port=6379,password='clbzzsn', db=5)
page_list()
comData/dingzhi/miit.py 0 → 100644
from datetime import datetime
from urllib.parse import urljoin
import redis
import requests
import urllib3
from bs4 import BeautifulSoup
from kafka import KafkaProducer
from pyquery import PyQuery as pq
import json
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def reqHtml(url):
try:
proxy = {'https': 'http://127.0.0.1:1080', 'http': 'http://127.0.0.1:1080'}
header={
'Accept':'*/*',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'zh-CN,zh;q=0.9',
'Cache-Control':'no-cache',
'Connection':'keep-alive',
'Cookie':'__jsluid_s=d344baee4a1e027b745a48855ff6539d',
'Host':'www.miit.gov.cn',
'Pragma':'no-cache',
'Referer':'https://www.miit.gov.cn/zwgk/zcjd/index.html',
'Sec-Fetch-Dest':'empty',
'Sec-Fetch-Mode':'cors',
'Sec-Fetch-Site':'same-origin',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
'X-Requested-With':'XMLHttpRequest',
'sec-ch-ua':'"Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"',
'sec-ch-ua-mobile':'?0',
'sec-ch-ua-platform':'"Windows"'
}
response = requests.get(url,headers=header,verify=False,timeout=10)
code=response.status_code
print(f'url:{url} 信息的采集状态码{code}')
html=response.text
except Exception as e:
html=''
return html
# 将html中的相对地址转换成绝对地址
def paserUrl(html,listurl):
soup = BeautifulSoup(html, 'html.parser')
# 获取所有的<a>标签和<img>标签
links = soup.find_all(['a', 'img'])
# 遍历标签，将相对地址转换为绝对地址
for link in links:
if 'href' in link.attrs:
link['href'] = urljoin(listurl, link['href'])
elif 'src' in link.attrs:
link['src'] = urljoin(listurl, link['src'])
return soup
def page_list():
for i in range(1,27):
print(f"采集到第{i}页！！")
aurl='https://www.miit.gov.cn/api-gateway/jpaas-publish-server/front/page/build/unit?webId=8d828e408d90447786ddbe128d495e9e&pageId=1b56e5adc362428299dfc3eb444fe23a&parseType=buildstatic&pageType=column&tagId=右侧内容&tplSetId=209741b2109044b5b7695700b2bec37e&paramJson={"pageNo":[i],"pageSize":"24"}'
url=aurl.replace('[i]',str(i))
html=reqHtml(url)
text=json.loads(html)
html=text['data']['html']
soup=paserUrl(html,'https://www.miit.gov.cn/zwgk/zcjd/index.html')
html=str(soup.prettify())
doc=pq(html)
ll=doc('li[class="cf"]')
for list in ll:
ldoc=pq(list)
title=ldoc('a').text()
url=ldoc('a').attr('href')
# url='https://www.miit.gov.cn'+url
try:
flag=r.sismember('IN-20230829-0199',url)
if flag:
print(f'信息已采集入库{title}')
continue
except Exception as e:
continue
publishdate=ldoc('span').text()
dmsg={
'title':title,
'url':url,
'publishdate':publishdate
}
print(f'列表信息： title:{title} url:{url} time:{publishdate}')
detail(dmsg)
def detail(dmsg):
try:
durl=dmsg['url']
title=dmsg['title']
publishTime=dmsg['publishdate']
html=reqHtml(durl)
soup=paserUrl(html,durl)
con=soup.select('div[id="con_con"]')[0]
contentWithTag=con.prettify()
content=con.text
if content:
pass
else:
content=contentWithTag
ddata={
'title':title,
'publishTime':publishTime,
'sourceAddress':durl,
'content':content,
'contentWithTag':contentWithTag,
'origin':'中华人民共和国工业和信息化部-政务公开-政策解读',
}
sendTokafka(ddata)
except Exception as e:
print(e)
def sendTokafka(ddata):
title=ddata['title']
content=ddata['content']
contentWithTag=ddata['contentWithTag']
publishTime=ddata['publishTime']
sourceAddress=ddata['sourceAddress']
origin=ddata['origin']
time_format='%Y-%m-%d'
publishDate=str(datetime.strptime(publishTime, time_format))
sid='1696452056436424706'
info_code='IN-20230829-0199'
aa_dict = {
'content': content,
'contentWithTag': contentWithTag,
'id': '',
'sid': sid,
'origin': origin,
'publishDate': publishDate,
'sourceAddress': sourceAddress,
'title': title,
'source': 'python定制采集',
'type': ''
}
producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
try:
kafka_result = producer.send("crawlerInfo", json.dumps(aa_dict, ensure_ascii=False).encode('utf8'))
r.sadd(info_code,sourceAddress)
print('发送kafka结束')
except Exception as e:
print(e)
print('发送kafka异常！')
finally:
producer.close()
# r.close()
if __name__ == '__main__':
r = redis.Redis(host='114.115.236.206', port=6379,password='clbzzsn', db=5)
page_list()
print('采集结束===')
\ No newline at end of file
comData/dingzhi/采集说明.md 0 → 100644
主要采集了两个网站
1、中国政府网-最新发布 数据是从微信小程序中获取的跟网站的数据是一致的 http://bmfw.www.gov.cn/zcdwpt/index.html#/
2、中华人民共和国工业和信息化部-政务公开-政策解读 http://bmfw.www.gov.cn/zcdwpt/index.html#/
主要是获取对应的资讯信息
comData/wanfangdataCrawler/yejinQiye.py 0 → 100644
# -*- coding: utf-8 -*-
import datetime
import time
import pymysql
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from pyquery import PyQuery as pq
from openpyxl import Workbook
import pandas as pd
class WanfangSpider(object):
def __init__(self):
pass
def req(self,url):
header={
"accept":"*/*",
"connection":"Keep-Alive",
"user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"
}
res = requests.get(url,headers=header)
if res.status_code==200:
text=res.text
print('请求成功！')
else:
text=''
print('请求失败！')
return text
# 将html中的相对地址转换成绝对地址
def paserUrl(self,html,listurl):
soup = BeautifulSoup(html, 'html.parser')
# 获取所有的<a>标签和<img>标签
links = soup.find_all(['a', 'img'])
# 遍历标签，将相对地址转换为绝对地址
for link in links:
if 'href' in link.attrs:
link['href'] = urljoin(listurl, link['href'])
elif 'src' in link.attrs:
link['src'] = urljoin(listurl, link['src'])
return soup
def pageList(self,start,end):
listmsg=[]
# for num in range(1,1321):
for num in range(start,end):
url=f'https://kms.wanfangdata.com.cn/IndustryYJ/Search/Cecdb?q=%E5%86%B6%E9%87%91%2B%E5%86%B6%E7%82%BC&PageNumber={num}'
html=self.req(url)
soup=self.paserUrl(html,url)
text=str(soup.prettify())
doc=pq(text)
liTag=doc('li[class="rt-wrap"]')
# print(liTag)
for li in liTag:
lidoc=pq(li)
title=lidoc('a[class="title"]').text()
turl=lidoc('a[class="title"]').attr('href')
msg={
'title':title,
'turl':turl
}
print(f'title:{title} url:{url}')
listmsg.append(msg)
return listmsg
def detailMsg(self,msg):
detailList=[]
turl = msg['turl']
title = msg['title']
html=self.req(turl)
soup=self.paserUrl(html,turl)
dtext=str(soup.prettify())
ddoc=pq(dtext)
a1=ddoc('table[class="detail-md"]>tr:nth-child(2)>td:nth-child(1)').text().replace(":","")
institutionType=ddoc('table[class="detail-md"]>tr:nth-child(2)>td:nth-child(2)').text()
a2=ddoc('table[class="detail-md"]>tr:nth-child(3)>td:nth-child(1)').text().replace(":","")
formerName=ddoc('table[class="detail-md"]>tr:nth-child(3)>td:nth-child(2)').text()
a3=ddoc('table[class="detail-md"]>tr:nth-child(4)>td:nth-child(1)').text().replace(":","")
leader=ddoc('table[class="detail-md"]>tr:nth-child(4)>td:nth-child(2)').text()
a4=ddoc('table[class="detail-md"]>tr:nth-child(5)>td:nth-child(1)').text().replace(":","")
establishmentDate=ddoc('table[class="detail-md"]>tr:nth-child(5)>td:nth-child(2)').text()
a5=ddoc('table[class="detail-md"]>tr:nth-child(6)>td:nth-child(1)').text().replace(":","")
introduction=ddoc('table[class="detail-md"]>tr:nth-child(6)>td:nth-child(2)').text()
a6=ddoc('table[class="detail-md"]>tr:nth-child(7)>td:nth-child(1)').text().replace(":","")
classification=ddoc('table[class="detail-md"]>tr:nth-child(7)>td:nth-child(2)').text()
a7=ddoc('table[class="detail-md"]>tr:nth-child(8)>td:nth-child(1)').text().replace(":","")
keywords=ddoc('table[class="detail-md"]>tr:nth-child(8)>td:nth-child(2)').text()
a8=ddoc('table[class="detail-md"]>tr:nth-child(9)>td:nth-child(1)').text().replace(":","")
researchEquipment=ddoc('table[class="detail-md"]>tr:nth-child(9)>td:nth-child(2)').text()
a9=ddoc('table[class="detail-md"]>tr:nth-child(10)>td:nth-child(1)').text().replace(":","")
researchAreas=ddoc('table[class="detail-md"]>tr:nth-child(10)>td:nth-child(2)').text()
a10=ddoc('table[class="detail-md"]>tr:nth-child(11)>td:nth-child(1)').text().replace(":","")
awards=ddoc('table[class="detail-md"]>tr:nth-child(11)>td:nth-child(2)').text()
a11=ddoc('table[class="detail-md"]>tr:nth-child(12)>td:nth-child(1)').text().replace(":","")
internalDepartments=ddoc('table[class="detail-md"]>tr:nth-child(12)>td:nth-child(2)').text()
a12=ddoc('table[class="detail-md"]>tr:nth-child(13)>td:nth-child(1)').text().replace(":","")
subsidiaryInstitutions=ddoc('table[class="detail-md"]>tr:nth-child(13)>td:nth-child(2)').text()
a13=ddoc('table[class="detail-md"]>tr:nth-child(14)>td:nth-child(1)').text().replace(":","")
productInformation=ddoc('table[class="detail-md"]>tr:nth-child(14)>td:nth-child(2)').text()
a14=ddoc('table[class="detail-md"]>tr:nth-child(15)>td:nth-child(1)').text().replace(":","")
publicationJournals=ddoc('table[class="detail-md"]>tr:nth-child(15)>td:nth-child(2)').text()
a15=ddoc('table[class="detail-md"]>tr:nth-child(16)>td:nth-child(1)').text().replace(":","")
mailingAddress=ddoc('table[class="detail-md"]>tr:nth-child(16)>td:nth-child(2)').text()
a16=ddoc('table[class="detail-md"]>tr:nth-child(17)>td:nth-child(1)').text().replace(":","")
tel=ddoc('table[class="detail-md"]>tr:nth-child(17)>td:nth-child(2)').text()
a17=ddoc('table[class="detail-md"]>tr:nth-child(18)>td:nth-child(1)').text().replace(":","")
faxNumber=ddoc('table[class="detail-md"]>tr:nth-child(18)>td:nth-child(2)').text()
a18=ddoc('table[class="detail-md"]>tr:nth-child(19)>td:nth-child(1)').text().replace(":","")
email=ddoc('table[class="detail-md"]>tr:nth-child(19)>td:nth-child(2)').text()
a19=ddoc('table[class="detail-md"]>tr:nth-child(20)>td:nth-child(1)').text().replace(":","")
website=ddoc('table[class="detail-md"]>tr:nth-child(20)>td:nth-child(2)').text()
a20=ddoc('table[class="detail-md"]>tr:nth-child(21)>td:nth-child(1)').text().replace(":","")
web=ddoc('table[class="detail-md"]>tr:nth-child(21)>td:nth-child(2)').text()
detailmsg={
'title':title,
'turl':turl,
a1:institutionType,
a2:formerName,
a3:leader,
a4:establishmentDate,
a5:introduction,
a6:classification,
a7:keywords,
a8:researchEquipment,
a9:researchAreas,
a10:awards,
a11:internalDepartments,
a12:subsidiaryInstitutions,
a13:productInformation,
a14:publicationJournals,
a15:mailingAddress,
a16:tel,
a17:faxNumber,
a18:email,
a19:website,
a20:web
}
detailList.append(detailmsg)
self.writerToExcel(detailList)
def conn144(self):
conn = pymysql.Connect(host='114.115.159.144', port=3306, user='caiji', passwd='zzsn9988', db='caiji',
charset='utf8')
cursor = conn.cursor()
return conn,cursor
def dataToSql(self,detailmsg):
conn,cursor=self.conn144()
try:
# 检查记录是否存在
# 获取当前时间
current_time = datetime.datetime.now()
# 将时间转换为字符串
currentdate = current_time.strftime("%Y-%m-%d %H:%M:%S")
except Exception as e:
print('+++++')
finally:
cursor.close()
conn.close()
# 将数据追加到excel
def writerToExcel(self,detailList):
# filename='baidu搜索.xlsx'
# 读取已存在的xlsx文件
existing_data = pd.read_excel(filename)
# 创建新的数据
new_data = pd.DataFrame(data=detailList)
# 将新数据添加到现有数据的末尾
combined_data = existing_data.append(new_data, ignore_index=True)
# 将结果写入到xlsx文件
combined_data.to_excel(filename, index=False)
print('保存成功！！')
if __name__ == '__main__':
wanfang=WanfangSpider()
for num in range(801,1321,100):
filename=f'企业_{num}.xlsx'
# # 创建一个工作簿
workbook = Workbook()
workbook.save(filename)
start=num
end=num+100
lsitmsg=wanfang.pageList(start,end)
for msg in lsitmsg:
wanfang.detailMsg(msg)
\ No newline at end of file
comData/wanfangdataCrawler/采集说明.md 0 → 100644
采集说明：
机构和企业的数据采集是从万方网站上获取冶金相关的机构和企业的基本信息
万方的地址：https://kms.wanfangdata.com.cn/
<img alt="img.png" height="200" src="img.png" width="200"/>
入口
地址
机构：https://kms.wanfangdata.com.cn/IndustryYJ/Search/Cecdb?q=%E5%86%B6%E9%87%91%2B%E5%86%B6%E7%82%BC%20%E6%9C%BA%E6%9E%84%3Acsi&f=Inst.Type
企业：https://kms.wanfangdata.com.cn/IndustryYJ/Search/Cecdb?q=%E5%86%B6%E9%87%91%2B%E5%86%B6%E7%82%BC%20%E6%9C%BA%E6%9E%84%3Acecdb&f=Inst.Type
<img alt="img_1.png" height="200" src="img_1.png" width="200"/>
主要是获取基本的信息
test/pipei2.py 0 → 100644
# -*- coding: utf-8 -*-
import pymysql
import pandas as pd
from tqdm import tqdm
import xlsxwriter
import openpyxl
from urllib.parse import urlparse
def pipeiName(qiyedatas):
sql1 = """select id, info_source_code, web_site_name, site_name , site_uri from info_source WHERE web_site_name like '%[name]%' """
cont=1;
qynot=[]
qyin=[]
for qy in qiyedatas:
name=qy['name']
if name is None:
uqynot,uqyin=pipeiURL(qy)
if uqynot:
qynot.append(uqynot[0])
if uqyin:
qyin.append(uqyin[0])
continue
if ''==name:
uqynot,uqyin=pipeiURL(qy)
if uqynot:
qynot.append(uqynot[0])
if uqyin:
qyin.append(uqyin[0])
continue
try:
sql2=sql1.replace("[name]",name)
cursor.execute(sql2)
except Exception as e:
uqynot,uqyin=pipeiURL(qy)
if uqynot:
qynot.append(uqynot[0])
if uqyin:
qyin.append(uqyin[0])
continue
result_data = cursor.fetchall()
if(len(result_data)<1):
uqynot,uqyin=pipeiURL(qy)
if uqynot:
qynot.append(uqynot[0])
if uqyin:
qyin.append(uqyin[0])
else:
cont+=1
print(cont)
qyin.append(qy)
# for row2 in tqdm(result_data):
# try:
# rd = {'id': row2[0],
# '编码': row2[1],
# '网站名称': row2[2],
# '栏目名称': row2[3],
# '栏目地址': row2[4],
# '企业名称': qy['name']
# }
# qyin.append(rd)
# except Exception as e:
# print(e)
# print("查询失败！！"+sql2)
print(qyin)
df_in = pd.DataFrame(data=qyin)
df_in.to_excel('n企业情况在平台中有数据2.xlsx', engine='xlsxwriter', index=False)
print(qynot)
df_out = pd.DataFrame(data=qynot)
df_out.to_excel('n企业情况在平台中没有数据2.xlsx', engine='xlsxwriter', index=False)
def pipeiURL(qy):
uqynot=[]
uqyin=[]
url=qy['url']
sql1 = """select id, info_source_code, web_site_name, site_name , site_uri from info_source WHERE site_uri like '%[url]%' """
if url is None:
uqynot.append(qy)
return uqynot,uqyin
try:
parsed_url = urlparse(url)
domain = parsed_url.netloc
if ''==domain:
uqynot.append(qy)
return uqynot,uqyin
except Exception as e:
uqynot.append(qy)
return uqynot,uqyin
sql2=sql1.replace("[url]",domain)
cursor.execute(sql2)
result_data = cursor.fetchall()
# if(len(result_data)>1):
if(len(result_data)<1):
uqynot.append(qy)
else:
uqyin.append(qy)
# for row2 in tqdm(result_data):
# try:
# rd = {'id': row2[0],
# '编码': row2[1],
# '网站名称': row2[2],
# '栏目名称': row2[3],
# '栏目地址': row2[4],
# '企业名称': qy['name']
# }
# uqyin.append(rd)
# except Exception as e:
# print(e)
# print("查询失败！！"+sql2)
return uqynot,uqyin
if __name__ == '__main__':
# 打开Excel文件
workbook = openpyxl.load_workbook('name.xlsx')
# 获取工作表对象
worksheet = workbook.active
qiyedatas=[]
# 遍历工作表的行
for row in worksheet.iter_rows(values_only=True):
qiyemsg={
'yname':row[0],
'name':row[1],
'url':row[2]
}
qiyedatas.append(qiyemsg)
# 打印每行的数据
# print(row)
conn = pymysql.Connect(host='114.116.44.11', port=3306, user='root', passwd='f7s0&7qqtK', db='clb_project',
charset='utf8')
cursor = conn.cursor()
pipeiName(qiyedatas)
tmp/usVsRussia/cis.py 0 → 100644
# -*- coding: utf-8 -*-
import time
import urllib
import requests
from pyquery import PyQuery as pq
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import wget
from openpyxl import Workbook
import pandas as pd
def createDriver():
chrome_driver =r'C:\Users\WIN10\DataspellProjects\crawlerProjectDemo\tmpcrawler\cmd100\chromedriver.exe'
path = Service(chrome_driver)
chrome_options = webdriver.ChromeOptions()
chrome_options.binary_location =r'D:\crawler\baidu_crawler\tool\Google\Chrome\Application\chrome.exe'
# 设置代理
# proxy = "127.0.0.1:8080" # 代理地址和端口
# chrome_options.add_argument('--proxy-server=http://' + proxy)
driver = webdriver.Chrome(service=path,chrome_options=chrome_options)
return driver
def listPage():
driver=createDriver()
for i in range(0,6):
size=i*20
url = f'https://www.bis.doc.gov/index.php/smart-search?searchword=Russia&searchphrase=all&start={size}'
driver.get(url)
html=driver.page_source
soup=paserUrl(html,url)
text=str(soup.prettify())
doc=pq(text)
titles=doc('dl[class="search-results"]>dt')
dates=doc('dl[class="search-results"]>dd[class="result-created"]')
for i in range(0,len(titles)):
detailList=[]
tt=titles[i]
dd=dates[i]
dddoc=pq(dd)
ttdoc=pq(tt)
title=ttdoc('a').text()
date=dddoc('dd[class="result-created"]').text()
url=ttdoc('a').attr('href')
pdfurl,content=detail(driver,url)
if pdfurl:
pdfpath="D:/cis/"+title+".pdf"
download_file(pdfurl,pdfpath)
else:
pdfpath=''
detailmsg={
"title":title,
"date":date,
"url":url,
"content":content,
"pdfurl":pdfurl,
"pdfpath":pdfpath,
}
detailList.append(detailmsg)
writerToExcel(detailList)
# print(f'title:{title} date:{date} url:{url}')
def detail(driver,url):
k=0
html=''
while k<5:
k+=1
try:
# proxy = {'https': 'http://127.0.0.1:1080', 'http': 'http://127.0.0.1:1080'}
# response = requests.get(url, proxies=proxy, verify=False,timeout=10)
# html=response.text
driver.get(url)
time.sleep(3)
html=driver.page_source
soup=paserUrl(html,url)
html=str(soup.prettify())
except Exception as e:
html=''
if html:
break
text=paserUrl(html,url)
docc=pq(text.encode('utf-8'))
try:
pdfurl= docc('div[class="docman_download"]>a').attr('href')
except Exception as e:
pdfurl=''
try:
content=docc('div[class="item-page"]').text()
except Exception as e:
content=''
# print(url)
return pdfurl,content
# 将数据追加到excel
def writerToExcel(detailList):
# filename='baidu搜索.xlsx'
# 读取已存在的xlsx文件
existing_data = pd.read_excel(filename)
# 创建新的数据
new_data = pd.DataFrame(data=detailList)
# 将新数据添加到现有数据的末尾
combined_data = existing_data.append(new_data, ignore_index=True)
# 将结果写入到xlsx文件
combined_data.to_excel(filename, index=False)
def download_file(url, save_path):
k=1
while True:
if k>5:
print(url)
break
k+=1
try:
header = {
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'zh-CN,zh;q=0.9',
'Cache-Control':'no-cache',
'Connection':'keep-alive',
'Cookie':'b099bcecf0be876536bb9d4826b25ba8=e2horegllcbddejsiveijp7of0; cookiesession1=678A3E12247313FBD6F74569925F4EFD; _ga=GA1.1.840765784.1693040959; __cf_bm=4DTZeDEU67Xjr5nt9OsbE1g1UTdVuOGdQlhj4KD5U2I-1693190695-0-AW81rfvAFUnclDkFVJYqD8+RWrC8FngMzW0dJ+bVHA+JwmPUVpc9/ogA0jhXrKLFYWun2BoK0R/hqWgGZAw/I1Y=; referrer_site=https%3A%2F%2Fwww.bis.doc.gov%2Findex.php%2Fsmart-search%3Fsearchword%3DRussia%26searchphrase%3Dall; csrf_token=a0d03e256a36d037708a809220564f407dee78bc; _ga_TPRT7QB30Y=GS1.1.1693190696.4.1.1693190720.0.0.0',
'Host':'www.bis.doc.gov',
'Pragma':'no-cache',
'Referer':'https://www.bis.doc.gov/index.php/documents/product-guidance/3300-russia-medical-related-license-application-guidance-fpd-final-incorp-occ-and-3f-cmts-clean-071323',
'Sec-Fetch-Dest':'document',
'Sec-Fetch-Mode':'navigate',
'Sec-Fetch-Site':'same-origin',
'Sec-Fetch-User':'?1',
'Upgrade-Insecure-Requests':'1',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
'sec-ch-ua':'"Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"',
'sec-ch-ua-mobile':'?0',
'sec-ch-ua-platform':'"Windows"'
}
proxy = {'https': 'http://127.0.0.1:1080', 'http': 'http://127.0.0.1:1080'}
response = requests.get(url, proxies=proxy, headers=header, verify=False,timeout=10)
# response = requests.get(url,verify=False)
with open(save_path, 'wb') as file:
file.write(response.content)
break
except Exception as e:
time.sleep(5)
print(e)
return save_path
def download_file3(url, save_path):
header = {
'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'zh-CN,zh;q=0.9',
'Cache-Control':'no-cache',
'Connection':'keep-alive',
'Cookie':'b099bcecf0be876536bb9d4826b25ba8=e2horegllcbddejsiveijp7of0; cookiesession1=678A3E12247313FBD6F74569925F4EFD; _ga=GA1.1.840765784.1693040959; __cf_bm=4DTZeDEU67Xjr5nt9OsbE1g1UTdVuOGdQlhj4KD5U2I-1693190695-0-AW81rfvAFUnclDkFVJYqD8+RWrC8FngMzW0dJ+bVHA+JwmPUVpc9/ogA0jhXrKLFYWun2BoK0R/hqWgGZAw/I1Y=; referrer_site=https%3A%2F%2Fwww.bis.doc.gov%2Findex.php%2Fsmart-search%3Fsearchword%3DRussia%26searchphrase%3Dall; csrf_token=a0d03e256a36d037708a809220564f407dee78bc; _ga_TPRT7QB30Y=GS1.1.1693190696.4.1.1693190720.0.0.0',
'Host':'www.bis.doc.gov',
'Pragma':'no-cache',
'Referer':'https://www.bis.doc.gov/index.php/documents/product-guidance/3300-russia-medical-related-license-application-guidance-fpd-final-incorp-occ-and-3f-cmts-clean-071323',
'Sec-Fetch-Dest':'document',
'Sec-Fetch-Mode':'navigate',
'Sec-Fetch-Site':'same-origin',
'Sec-Fetch-User':'?1',
'Upgrade-Insecure-Requests':'1',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
'sec-ch-ua':'"Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"',
'sec-ch-ua-mobile':'?0',
'sec-ch-ua-platform':'"Windows"'
}
req = urllib.request.Request(url, headers=header)
wget.download(url,save_path)
# 将html中的相对地址转换成绝对地址
def paserUrl(html,listurl):
soup = BeautifulSoup(html, 'html.parser')
# 获取所有的<a>标签和<img>标签
links = soup.find_all(['a', 'img'])
# 遍历标签，将相对地址转换为绝对地址
for link in links:
if 'href' in link.attrs:
link['href'] = urljoin(listurl, link['href'])
elif 'src' in link.attrs:
link['src'] = urljoin(listurl, link['src'])
return soup
if __name__ == '__main__':
# # 创建一个工作簿
filename='cis.xlsx'
workbook = Workbook()
workbook.save(filename)
listPage()
# driver=createDriver()
# url='https://www.bis.doc.gov/index.php/policy-guidance/deemed-exports/deemed-exports-faqs/faq/116-what-areas-are-considered-russia-for-purposes-of-these-sanctions'
# detail(driver,url)
tmp/usVsRussia/pravo.py
......@@ -30,7 +30,7 @@ cursor = baseCore.cursor
def job_2():
log.info('----开始采集---俄罗斯国家杂志----')
path = 'D:chrome/chromedriver.exe'
path = r'C:\Users\WIN10\DataspellProjects\crawlerProjectDemo\tmpcrawler\cmd100\chromedriver.exe'
driverContent = baseCore.buildDriver(path, headless=False)
url = 'http://publication.pravo.gov.ru/documents/block/president'
req = requests.get(url,headers)
......@@ -45,4 +45,3 @@ def job_2():
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论