Skip to content
项目
群组
代码片段
帮助
当前项目
正在载入...
登录 / 注册
切换导航面板
Z
zzsn_spider
概览
概览
详情
活动
周期分析
版本库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
统计图
问题
0
议题
0
列表
看板
标记
里程碑
合并请求
1
合并请求
1
CI / CD
CI / CD
流水线
作业
日程表
图表
维基
Wiki
代码片段
代码片段
成员
成员
折叠边栏
关闭边栏
活动
图像
聊天
创建新问题
作业
提交
问题看板
Open sidebar
丁双波
zzsn_spider
Commits
017e1b47
提交
017e1b47
authored
8月 15, 2023
作者:
薛凌堃
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'origin/master'
上级
75a87d7f
734b4ae2
隐藏空白字符变更
内嵌
并排
正在显示
4 个修改的文件
包含
58 行增加
和
40 行删除
+58
-40
BaseCore.py
base/BaseCore.py
+9
-9
雅虎财经_企业动态.py
comData/yhcj/雅虎财经_企业动态.py
+23
-19
stealth.min.js
test/stealth.min.js
+0
-0
test.py
test/test.py
+26
-12
没有找到文件。
base/BaseCore.py
浏览文件 @
017e1b47
...
@@ -421,15 +421,15 @@ class BaseCore:
...
@@ -421,15 +421,15 @@ class BaseCore:
chrome_options
.
add_experimental_option
(
'useAutomationExtension'
,
False
)
chrome_options
.
add_experimental_option
(
'useAutomationExtension'
,
False
)
chrome_options
.
add_argument
(
'lang=zh-CN,zh,zh-TW,en-US,en'
)
chrome_options
.
add_argument
(
'lang=zh-CN,zh,zh-TW,en-US,en'
)
chrome_options
.
add_argument
(
self
.
getRandomUserAgent
())
chrome_options
.
add_argument
(
'user-agent='
+
self
.
getRandomUserAgent
())
# 'user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36')
# 'user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36')
driver
=
webdriver
.
Chrome
(
chrome_options
=
chrome_options
,
service
=
service
)
driver
=
webdriver
.
Chrome
(
chrome_options
=
chrome_options
,
service
=
service
)
with
open
(
'../../base/
stealth.min.js'
)
as
f
:
# with open(r'F:\zzsn\zzsn_spider\base\
stealth.min.js') as f:
js
=
f
.
read
()
#
js = f.read()
#
driver
.
execute_cdp_cmd
(
"Page.addScriptToEvaluateOnNewDocument"
,
{
#
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
"source"
:
js
#
"source": js
})
#
})
return
driver
return
driver
# 根据社会信用代码获取企业信息
# 根据社会信用代码获取企业信息
...
@@ -481,7 +481,7 @@ class BaseCore:
...
@@ -481,7 +481,7 @@ class BaseCore:
def
writerToExcel
(
self
,
detailList
,
filename
):
def
writerToExcel
(
self
,
detailList
,
filename
):
# filename='baidu搜索.xlsx'
# filename='baidu搜索.xlsx'
# 读取已存在的xlsx文件
# 读取已存在的xlsx文件
existing_data
=
pd
.
read_excel
(
filename
,
engine
=
'openpyxl'
,
dtype
=
str
)
existing_data
=
pd
.
read_excel
(
filename
,
engine
=
'openpyxl'
)
# 创建新的数据
# 创建新的数据
new_data
=
pd
.
DataFrame
(
data
=
detailList
)
new_data
=
pd
.
DataFrame
(
data
=
detailList
)
# 将新数据添加到现有数据的末尾
# 将新数据添加到现有数据的末尾
...
...
comData/yhcj/雅虎财经_企业动态.py
浏览文件 @
017e1b47
# 雅虎财
经企业动态获取
# 雅虎财
经企业动态获取
...
@@ -8,9 +8,9 @@ import sys
...
@@ -8,9 +8,9 @@ import sys
from
selenium.webdriver.support
import
expected_conditions
as
EC
from
selenium.webdriver.support
import
expected_conditions
as
EC
from
selenium.webdriver.support.wait
import
WebDriverWait
from
selenium.webdriver.support.wait
import
WebDriverWait
sys
.
path
.
append
(
'D:/zzsn_spider/base'
)
from
base
import
BaseCore
import
BaseCore
from
base.smart
import
smart_extractor
from
smart
import
smart_extractor
import
urllib3
import
urllib3
urllib3
.
disable_warnings
(
urllib3
.
exceptions
.
InsecureRequestWarning
)
urllib3
.
disable_warnings
(
urllib3
.
exceptions
.
InsecureRequestWarning
)
baseCore
=
BaseCore
.
BaseCore
()
baseCore
=
BaseCore
.
BaseCore
()
...
@@ -49,6 +49,8 @@ def getZx(xydm, url, title, cnx, path):
...
@@ -49,6 +49,8 @@ def getZx(xydm, url, title, cnx, path):
content
=
contentElement
.
replace
(
"'"
,
"''"
)
content
=
contentElement
.
replace
(
"'"
,
"''"
)
driverContent
.
close
()
driverContent
.
close
()
# driverContent.quit()
# 动态信息列表
# 动态信息列表
list_info
=
[
list_info
=
[
xydm
,
xydm
,
...
@@ -159,28 +161,24 @@ def getLastUrl():
...
@@ -159,28 +161,24 @@ def getLastUrl():
def
scroll
(
xydm
,
name
,
gpdm
):
def
scroll
(
xydm
,
name
,
gpdm
):
last_url_
=
''
last_url_
=
''
try
:
last_url
=
getLastUrl
()
except
:
log
.
error
(
f
"{name}--{gpdm}--获取不到最后一条链接"
)
while
True
:
while
True
:
js
=
"var q=document.documentElement.scrollTop=100000"
js
=
"var q=document.documentElement.scrollTop=100000"
driver
.
execute_script
(
js
)
driver
.
execute_script
(
js
)
time
.
sleep
(
1
)
time
.
sleep
(
1
)
try
:
try
:
last_url
_
=
getLastUrl
()
last_url
=
getLastUrl
()
except
Exception
as
e
:
except
Exception
as
e
:
log
.
error
(
f
"{name}--{gpdm}--获取不到最后一条链接"
)
log
.
error
(
f
"{name}--{gpdm}--获取不到最后一条链接"
)
break
break
try
:
#
try:
selects
=
selectUrl
(
last_url_
,
xydm
)
#
selects = selectUrl(last_url_,xydm)
except
:
#
except:
break
#
break
if
selects
:
#
if selects:
break
#
break
if
last_url_
==
last_url
:
if
last_url_
==
last_url
:
break
break
last_url
=
last_url_
last_url
_
=
last_url
#采集失败的公众号 重新放入redis
#采集失败的公众号 重新放入redis
def
rePutIntoR
(
item
):
def
rePutIntoR
(
item
):
...
@@ -188,7 +186,7 @@ def rePutIntoR(item):
...
@@ -188,7 +186,7 @@ def rePutIntoR(item):
if
__name__
==
"__main__"
:
if
__name__
==
"__main__"
:
path
=
r'
D:\zzsn_spider\comData\cmd6
\chromedriver.exe'
path
=
r'
F:\spider\115
\chromedriver.exe'
driver
=
baseCore
.
buildDriver
(
path
)
driver
=
baseCore
.
buildDriver
(
path
)
cnx
=
pymysql
.
connect
(
host
=
'114.116.44.11'
,
user
=
'root'
,
password
=
'f7s0&7qqtK'
,
db
=
'dbScore'
,
charset
=
'utf8mb4'
)
cnx
=
pymysql
.
connect
(
host
=
'114.116.44.11'
,
user
=
'root'
,
password
=
'f7s0&7qqtK'
,
db
=
'dbScore'
,
charset
=
'utf8mb4'
)
cursor
=
cnx
.
cursor
()
cursor
=
cnx
.
cursor
()
...
@@ -198,6 +196,9 @@ if __name__ == "__main__":
...
@@ -198,6 +196,9 @@ if __name__ == "__main__":
social_code
=
baseCore
.
redicPullData
(
'NewsEnterprise:gwqy_socialCode'
)
social_code
=
baseCore
.
redicPullData
(
'NewsEnterprise:gwqy_socialCode'
)
# 判断 如果Redis中已经没有数据,则等待
# 判断 如果Redis中已经没有数据,则等待
if
not
social_code
:
time
.
sleep
(
20
)
continue
if
social_code
==
'None'
:
if
social_code
==
'None'
:
time
.
sleep
(
20
)
time
.
sleep
(
20
)
continue
continue
...
@@ -283,7 +284,10 @@ if __name__ == "__main__":
...
@@ -283,7 +284,10 @@ if __name__ == "__main__":
state
=
0
state
=
0
takeTime
=
baseCore
.
getTimeCost
(
start_time
,
time
.
time
())
takeTime
=
baseCore
.
getTimeCost
(
start_time
,
time
.
time
())
baseCore
.
recordLog
(
xydm
,
taskType
,
state
,
takeTime
,
news_url
,
exception
)
baseCore
.
recordLog
(
xydm
,
taskType
,
state
,
takeTime
,
news_url
,
exception
)
break
# 增量使用
# break
# 全量使用
continue
title
=
a_ele
.
text
.
lstrip
()
.
strip
()
.
replace
(
"'"
,
"''"
)
title
=
a_ele
.
text
.
lstrip
()
.
strip
()
.
replace
(
"'"
,
"''"
)
exception
=
getZx
(
xydm
,
news_url
,
title
,
cnx
,
path
)
exception
=
getZx
(
xydm
,
news_url
,
title
,
cnx
,
path
)
if
exception
==
''
:
if
exception
==
''
:
...
@@ -311,7 +315,7 @@ if __name__ == "__main__":
...
@@ -311,7 +315,7 @@ if __name__ == "__main__":
log
.
info
(
'===========连接已被关闭========等待重新连接==========='
)
log
.
info
(
'===========连接已被关闭========等待重新连接==========='
)
driver
.
quit
()
driver
.
quit
()
driver
=
baseCore
.
buildDriver
(
path
)
driver
=
baseCore
.
buildDriver
(
path
)
time
.
sleep
(
1200
)
time
.
sleep
(
5
)
continue
continue
cursor
.
close
()
cursor
.
close
()
...
...
test/stealth.min.js
0 → 100644
浏览文件 @
017e1b47
This source diff could not be displayed because it is too large. You can
view the blob
instead.
test/test.py
浏览文件 @
017e1b47
import
time
from
selenium
import
webdriver
from
selenium.webdriver.chrome.service
import
Service
from
base.BaseCore
import
BaseCore
from
base.BaseCore
import
BaseCore
baseCore
=
BaseCore
()
baseCore
=
BaseCore
()
...
@@ -6,13 +8,25 @@ log =baseCore.getLogger()
...
@@ -6,13 +8,25 @@ log =baseCore.getLogger()
if
__name__
==
'__main__'
:
if
__name__
==
'__main__'
:
log
.
info
(
"ok"
)
path
=
r'F:\spider\115\chromedriver.exe'
#获取流水号
driver
=
baseCore
.
buildDriver
(
path
,
headless
=
False
)
print
(
baseCore
.
getNextSeq
())
# service = Service(r'F:\spider\115\chromedriver.exe')
print
(
baseCore
.
getNextSeq
())
# chrome_options = webdriver.ChromeOptions()
# 获取随机agent
# # chrome_options.add_argument('--headless')
print
(
baseCore
.
getRandomUserAgent
())
# # chrome_options.add_argument('--disable-gpu')
# 获取代理池
# chrome_options.add_experimental_option(
print
(
baseCore
.
get_proxy
())
# "excludeSwitches", ["enable-automation"])
# 释放相关资源
# chrome_options.add_experimental_option('useAutomationExtension', False)
baseCore
.
close
()
# chrome_options.add_argument('lang=zh-CN,zh,zh-TW,en-US,en')
\ No newline at end of file
# chrome_options.add_argument('user-agent='+baseCore.getRandomUserAgent())
#
# bro = webdriver.Chrome(chrome_options=chrome_options, service=service)
# with open('stealth.min.js') as f:
# js = f.read()
#
# bro.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
# "source": js
# })
gpdm
=
'9021.T'
url
=
f
"https://finance.yahoo.com/quote/{gpdm}/press-releases?p={gpdm}"
driver
.
get
(
url
)
\ No newline at end of file
编写
预览
Markdown
格式
0%
重试
或
添加新文件
添加附件
取消
您添加了
0
人
到此讨论。请谨慎行事。
请先完成此评论的编辑!
取消
请
注册
或者
登录
后发表评论