Skip to content
项目
群组
代码片段
帮助
当前项目
正在载入...
登录 / 注册
切换导航面板
Z
zzsn_spider
概览
概览
详情
活动
周期分析
版本库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
统计图
问题
0
议题
0
列表
看板
标记
里程碑
合并请求
1
合并请求
1
CI / CD
CI / CD
流水线
作业
日程表
图表
维基
Wiki
代码片段
代码片段
成员
成员
折叠边栏
关闭边栏
活动
图像
聊天
创建新问题
作业
提交
问题看板
Open sidebar
丁双波
zzsn_spider
Commits
1c479868
提交
1c479868
authored
2月 20, 2024
作者:
薛凌堃
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
天眼查基本信息更新
上级
c442ebb4
显示空白字符变更
内嵌
并排
正在显示
1 个修改的文件
包含
780 行增加
和
0 行删除
+780
-0
baseinfotyc_update.py
comData/Tyc/baseinfotyc_update.py
+780
-0
没有找到文件。
comData/Tyc/baseinfotyc_update.py
0 → 100644
浏览文件 @
1c479868
# -*- coding: utf-8 -*-
import
json
import
re
import
time
import
datetime
import
pymongo
import
requests
from
bs4
import
BeautifulSoup
from
kafka
import
KafkaProducer
import
urllib3
from
selenium.webdriver.support.wait
import
WebDriverWait
db_storage
=
pymongo
.
MongoClient
(
'mongodb://114.115.221.202:27017/'
,
username
=
'admin'
,
password
=
'ZZsn@9988'
)
.
ZZSN
[
'天眼查登录信息'
]
urllib3
.
disable_warnings
(
urllib3
.
exceptions
.
InsecureRequestWarning
)
import
sys
# sys.path.append('D:\\KK\\zzsn_spider\\base')
sys
.
path
.
append
(
'D:
\\
kkwork
\\
zzsn_spider
\\
base'
)
import
BaseCore
baseCore
=
BaseCore
.
BaseCore
()
cnx_
=
baseCore
.
cnx
cursor_
=
baseCore
.
cursor
cnx
=
baseCore
.
cnx_
cursor
=
baseCore
.
cursor_
log
=
baseCore
.
getLogger
()
from
classtool
import
Token
,
File
,
Tag
token
=
Token
()
file
=
File
()
tag
=
Tag
()
from
selenium
import
webdriver
from
selenium.webdriver.common.by
import
By
def
create_driver
():
path
=
r'D:\soft\msedgedriver.exe'
# options = webdriver.EdgeOptions()
options
=
{
"browserName"
:
"MicrosoftEdge"
,
"ms:edgeOptions"
:
{
"extensions"
:
[],
"args"
:
[
"--start-maximized"
]
# 添加最大化窗口运作参数
}
}
session
=
webdriver
.
Edge
(
executable_path
=
path
,
capabilities
=
options
)
return
session
# 发送数据
def
sendkafka
(
post_data
):
try
:
producer
=
KafkaProducer
(
bootstrap_servers
=
[
'114.115.159.144:9092'
],
api_version
=
(
2
,
0
,
2
))
kafka_result
=
producer
.
send
(
"enterpriseInfo"
,
json
.
dumps
(
post_data
,
ensure_ascii
=
False
)
.
encode
(
'utf8'
))
print
(
kafka_result
.
get
(
timeout
=
10
))
except
:
exception
=
'kafka传输失败'
state
=
0
takeTime
=
baseCore
.
getTimeCost
(
start_time
,
time
.
time
())
baseCore
.
recordLog
(
social_code
,
taskType
,
state
,
takeTime
,
''
,
exception
)
log
.
info
(
f
"{com_name}--{social_code}--kafka传输失败"
)
# 检查登陆状态
def
checklogin
(
key
):
t
=
int
(
time
.
time
())
# url = 'https://www.tianyancha.com/search?key=%E4%B8%AD%E5%9B%BD%E7%9F%B3%E6%B2%B9%E5%8C%96%E5%B7%A5%E9%9B%86%E5%9B%A2%E6%9C%89%E9%99%90%E5%85%AC%E5%8F%B8&sessionNo=1706594186.22975563'
url
=
f
'https://www.tianyancha.com/search?key={key}&sessionNo={t}'
driver
.
get
(
url
)
time
.
sleep
(
2
)
page_source
=
driver
.
page_source
soup
=
BeautifulSoup
(
page_source
,
'html.parser'
)
# todo:检查未登录状态
# if soup.find('title').text == '会员登录 - 企查查':
# log.info('状态---未登录')
# soup = ''
# return soup
return
soup
# 合并基本信息和工商信息字段
def
getinfo
(
dict1
,
dict2
):
# 取出两个字典的key值集合
keys1
=
set
(
dict1
.
keys
())
keys2
=
set
(
dict2
.
keys
())
# 取出并集
union_keys
=
keys1
|
keys2
# 根据并集的key值,从两个字典中取出value值,组成新的字典
result_dict
=
{
key
:
dict1
.
get
(
key
,
None
)
or
dict2
.
get
(
key
,
None
)
for
key
in
union_keys
}
return
result_dict
def
dic_handle
(
result_dic
):
zxss
=
[
'北京市'
,
'天津市'
,
'上海市'
,
'重庆市'
]
try
:
company_name
=
result_dic
[
'企业名称'
]
except
:
company_name
=
None
try
:
CreditCode
=
result_dic
[
'统一社会信用代码'
]
except
:
CreditCode
=
None
try
:
OperName
=
result_dic
[
'法定代表人'
]
except
:
OperName
=
None
try
:
PhoneNumber
=
result_dic
[
'电话'
]
except
:
PhoneNumber
=
None
try
:
WebSite
=
result_dic
[
'网址'
]
except
:
WebSite
=
None
try
:
Email
=
result_dic
[
'邮箱'
]
except
:
Email
=
None
try
:
Desc
=
result_dic
[
'简介'
]
except
:
Desc
=
None
try
:
Status
=
result_dic
[
'经营状态'
]
except
:
try
:
Status
=
result_dic
[
'公司现状'
]
except
:
Status
=
None
try
:
StartDate
=
result_dic
[
'成立日期'
]
except
:
StartDate
=
None
try
:
RecCap
=
result_dic
[
'实缴资本'
]
except
:
RecCap
=
None
try
:
RegistCapi
=
result_dic
[
'注册资本'
]
except
:
RegistCapi
=
None
try
:
CheckDate
=
result_dic
[
'核准日期'
]
except
:
CheckDate
=
None
try
:
OrgNo
=
result_dic
[
'组织机构代码'
]
except
:
OrgNo
=
None
try
:
No
=
result_dic
[
'工商注册号'
]
except
:
No
=
None
try
:
taxpayerNo
=
result_dic
[
'纳税人识别号'
]
except
:
taxpayerNo
=
None
try
:
EconKind
=
result_dic
[
'企业类型'
]
except
:
EconKind
=
None
try
:
TermStart
=
result_dic
[
'营业期限'
]
.
split
(
'至'
)[
0
]
except
:
TermStart
=
None
try
:
TeamEnd
=
result_dic
[
'营业期限'
]
.
split
(
'至'
)[
1
]
except
:
TeamEnd
=
None
try
:
TaxpayerType
=
result_dic
[
'纳税人资质'
]
except
:
TaxpayerType
=
None
try
:
SubIndustry
=
result_dic
[
'国标行业'
]
except
:
SubIndustry
=
None
# try:
# region = result_dic['所属地区']
# except:
# region = None
# try:
# pattern = r'^(.*?省|.*?自治区)?(.*?市|.*?自治州)?(.*?区|.*?县|.*?自治县|.*?市辖区)?(.*?区|.*?县|.*?自治县|.*?市辖区)?$'
# matches = re.match(pattern, region)
# Province = matches.group(1)
# City = matches.group(2)
# County = matches.group(3)
# if Province is None:
# for zxs in zxss:
# if zxs in region:
# Province = zxs
# break
# except:
# Province = None
# City = None
# County = None
try
:
BelongOrg
=
result_dic
[
'登记机关'
]
except
:
BelongOrg
=
None
try
:
Info
=
result_dic
[
'人员规模'
]
except
:
Info
=
None
try
:
can_bao
=
result_dic
[
'参保人数'
]
except
:
can_bao
=
None
try
:
OriginalName
=
result_dic
[
'曾用名'
]
except
:
OriginalName
=
None
try
:
EnglishName
=
result_dic
[
'英文名称'
]
except
:
EnglishName
=
None
try
:
IxCode
=
result_dic
[
'进出口企业代码'
]
except
:
IxCode
=
None
try
:
Address
=
result_dic
[
'地址'
]
except
:
Address
=
None
try
:
Scope
=
result_dic
[
'经营范围'
]
except
:
Scope
=
None
aa_dict
=
{
'name'
:
company_name
,
# 企业名称
'shortName'
:
None
,
# 企业简称
'socialCreditCode'
:
CreditCode
,
# 统一社会信用代码
'legalPerson'
:
OperName
,
# 法定代表人
'officialPhone'
:
PhoneNumber
,
# 电话
'officialUrl'
:
WebSite
,
# 官网
'officialEmail'
:
Email
,
# 邮箱
'briefInfo'
:
Desc
,
# 简介
'registerStatus'
:
Status
,
# 登记状态
'incorporationDate'
:
StartDate
,
# 成立日期
'capital'
:
RegistCapi
,
# 注册资本
'paidCapital'
:
RecCap
,
# 实缴资本
'approvalDate'
:
CheckDate
,
# 核准日期
'organizationCode'
:
OrgNo
,
# 组织机构代码
'registerNo'
:
No
,
# 工商注册号
'taxpayerNo'
:
taxpayerNo
,
# 纳税人识别号
'type'
:
EconKind
,
# 企业类型
'businessStartDate'
:
TermStart
,
# 营业期限自
'businessEndDate'
:
TeamEnd
,
# 营业期限至
'taxpayerQualification'
:
TaxpayerType
,
# 纳税人资质
'industry'
:
SubIndustry
,
# 所属行业
'region'
:
None
,
'province'
:
None
,
# 所属省
'city'
:
None
,
# 所属市
'county'
:
None
,
# 所属县
'registerDepartment'
:
BelongOrg
,
# 登记机关
'scale'
:
Info
,
# 人员规模
'insured'
:
can_bao
,
# 参保人数
'beforeName'
:
OriginalName
,
# 曾用名
'englishName'
:
EnglishName
,
# 英文名
'importExportEnterpriseCode'
:
IxCode
,
# 进出口企业代码
'address'
:
Address
,
# 地址
'businessRange'
:
Scope
,
# 经营范围
'status'
:
0
,
# 状态
}
return
aa_dict
# 获取基本信息
def
baseinfo
(
com_soup
):
baseinfo
=
com_soup
.
find
(
'div'
,
class_
=
'index_detail__JSmQM'
)
cominfo_list
=
baseinfo
.
find_all
(
'div'
,
class_
=
'index_detail-info-item__oAOqL'
)
#name
data
=
{}
for
cominfo
in
cominfo_list
:
name
=
cominfo
.
find
(
'span'
,
class_
=
'index_detail-label__oRf2J'
)
.
text
.
replace
(
':'
,
''
)
.
replace
(
' '
,
''
)
# print(name)
tag
.
deletep
(
cominfo
,
'span'
,
'class'
,
'index_detail-label__oRf2J'
)
tag
.
deletep
(
cominfo
,
'i'
,
'class'
,
'index_detail-text-desc__myXYK'
)
# print(info)
value
=
cominfo
.
text
.
replace
(
''
,
''
)
.
replace
(
'
\ue657
'
,
''
)
.
replace
(
'
\ue655
'
,
''
)
if
name
==
'法定代表人'
:
try
:
value
=
cominfo
.
find
(
'a'
)
.
text
except
:
value
=
None
if
name
==
'电话'
:
try
:
value
=
cominfo
.
find
(
'span'
)
.
text
except
:
value
=
None
if
name
==
'邮箱'
:
try
:
value
=
cominfo
.
find
(
'a'
)
.
text
except
:
value
=
None
if
name
==
'网址'
:
try
:
value
=
cominfo
.
find
(
'a'
)
.
text
except
:
value
=
None
if
name
==
'地址'
:
try
:
value
=
cominfo
.
find
(
'span'
)
.
text
except
:
value
=
None
data
[
name
]
=
value
# print("==================")
briefTag
=
baseinfo
.
find
(
'div'
,
class_
=
'index_detail-linewrap__AKtCa index_-intro__ma3Qd'
)
span_list
=
briefTag
.
find_all
(
'span'
)
for
span
in
span_list
:
if
len
(
span
.
attrs
)
==
0
:
data
[
'简介'
]
=
span
.
text
.
split
(
'通过天眼查大数据分析'
)[
0
]
break
return
data
# 采集准备
def
redaytowork
(
com_name
,
social_code
,
file_name
):
log
.
info
(
f
'----当前企业{social_code}-{com_name}--开始处理---'
)
count
=
0
# 如果没有信用代码 就通过名字搜索 如果有信用代码 就通过信用代码
if
social_code
:
soup
=
checklogin
(
social_code
)
else
:
soup
=
checklogin
(
com_name
)
if
not
soup
:
log
.
info
(
"登录失效===重新放入redis"
)
baseCore
.
r
.
lpush
(
'UpdateBasdeInfo:SocialCode_CompanyName'
,
company_field
)
# token.updateTokeen(id_cookie,2)
# log.info('=====已重新放入redis,失效cookies已删除======')
time
.
sleep
(
20
)
return
count
else
:
try
:
searchinfo
=
soup
.
find
(
'div'
,
class_
=
'index_content-tool-title__K1Z6C'
)
.
find
(
'span'
,
class_
=
'index_title-count__lDSjB'
)
.
text
except
:
try
:
# todo:可能是搜不到该企业
errormessage
=
soup
.
find
(
'div'
,
class_
=
'index_no-data-reason-title__V3gFY'
)
.
text
if
'抱歉'
in
errormessage
:
log
.
info
(
'=====搜索不到该企业===='
)
data
=
[
com_name
,
social_code
]
# todo:搜不到的企业需要返回到一个表格中
file
.
appenddata
(
file_name
,
'需处理企业'
,
data
)
return
count
except
:
log
.
info
(
"登录失效===重新放入redis"
)
baseCore
.
r
.
lpush
(
'UpdateBasdeInfo:SocialCode_CompanyName'
,
company_field
)
# token.updateTokeen(id_cookie,2)
log
.
info
(
'=====已重新放入redis,cookies已封号======'
)
time
.
sleep
(
20
)
return
count
# 开始采集
try
:
if
spiderwork
(
soup
,
com_name
,
file_name
):
count
+=
1
log
.
info
(
f
'采集{com_name}成功=======耗时{baseCore.getTimeCost(start_time, time.time())}'
)
# token.updateTokeen(id_cookie,3)
return
count
else
:
return
count
except
Exception
as
e
:
log
.
info
(
f
'====={social_code}=====获取基本信息失败,重新放入redis====='
)
baseCore
.
r
.
lpush
(
'UpdateBasdeInfo:SocialCode_CompanyName'
,
company_field
)
# token.updateTokeen(id_cookie,2)
log
.
info
(
'=====已重新放入redis,cookies已封号======'
)
return
count
def
ifbeforename
(
company_url
):
driver
.
get
(
company_url
)
time
.
sleep
(
2
)
com_soup
=
BeautifulSoup
(
driver
.
page_source
,
'html.parser'
)
try
:
businessinfo
=
com_soup
.
find
(
'table'
,
{
'class'
:
'index_tableBox__ZadJW'
})
except
:
businessinfo
=
''
if
businessinfo
:
try
:
name
=
businessinfo
.
find
(
'span'
,
class_
=
'index_history-gray-tags__o8mkl'
)
.
text
value
=
businessinfo
.
find
(
'span'
,
class_
=
'index_copy-text__ri7W6'
)
.
text
.
replace
(
'展开'
,
''
)
.
replace
(
' '
,
''
)
.
replace
(
'…'
,
''
)
.
replace
(
'
\n
'
,
''
)
.
replace
(
'复制'
,
''
)
.
split
(
'('
)[
0
]
except
:
name
=
'曾用名'
value
=
''
return
value
else
:
return
''
#解析时间
def
paserTime
(
publishtime
):
timeType
=
[
'年前'
,
'月前'
,
'周前'
,
'前天'
,
'昨天'
,
'天前'
,
'今天'
,
'小时前'
,
'分钟前'
]
current_datetime
=
datetime
.
datetime
.
now
()
publishtime
=
publishtime
.
strip
()
print
(
publishtime
)
try
:
if
'年前'
in
publishtime
:
numbers
=
re
.
findall
(
r'\d+'
,
publishtime
)
day
=
int
(
numbers
[
0
])
delta
=
datetime
.
timedelta
(
days
=
365
*
day
)
publishtime
=
current_datetime
-
delta
elif
'月前'
in
publishtime
:
numbers
=
re
.
findall
(
r'\d+'
,
publishtime
)
day
=
int
(
numbers
[
0
])
delta
=
datetime
.
timedelta
(
months
=
day
)
publishtime
=
current_datetime
-
delta
elif
'周前'
in
publishtime
:
numbers
=
re
.
findall
(
r'\d+'
,
publishtime
)
day
=
int
(
numbers
[
0
])
delta
=
datetime
.
timedelta
(
weeks
=
day
)
publishtime
=
current_datetime
-
delta
elif
'天前'
in
publishtime
:
numbers
=
re
.
findall
(
r'\d+'
,
publishtime
)
day
=
int
(
numbers
[
0
])
delta
=
datetime
.
timedelta
(
days
=
day
)
publishtime
=
current_datetime
-
delta
elif
'前天'
in
publishtime
:
delta
=
datetime
.
timedelta
(
days
=
2
)
publishtime
=
current_datetime
-
delta
elif
'昨天'
in
publishtime
:
current_datetime
=
datetime
.
datetime
.
now
()
delta
=
datetime
.
timedelta
(
days
=
1
)
publishtime
=
current_datetime
-
delta
elif
'今天'
in
publishtime
or
'小时前'
in
publishtime
or
'分钟前'
in
publishtime
:
if
'小时'
in
publishtime
:
hour
=
publishtime
.
split
(
"小时"
)[
0
]
else
:
hour
=
0
if
hour
!=
0
:
min
=
publishtime
.
split
(
"小时"
)[
1
]
.
split
(
"分钟"
)[
0
]
else
:
min
=
publishtime
.
split
(
"分钟"
)[
0
]
delta
=
datetime
.
timedelta
(
hours
=
int
(
hour
),
minutes
=
int
(
min
))
publishtime
=
current_datetime
-
delta
elif
'年'
in
publishtime
and
'月'
in
publishtime
:
time_format
=
'
%
Y年
%
m月
%
d日'
publishtime
=
datetime
.
datetime
.
strptime
(
publishtime
,
time_format
)
elif
'月'
in
publishtime
and
'日'
in
publishtime
:
current_year
=
current_datetime
.
year
time_format
=
'
%
Y年
%
m月
%
d日'
publishtime
=
str
(
current_year
)
+
'年'
+
publishtime
publishtime
=
datetime
.
datetime
.
strptime
(
publishtime
,
time_format
)
except
Exception
as
e
:
print
(
'时间解析异常!!'
)
return
publishtime
# 采集基本信息和工商信息
def
spiderinfo
(
company_url
,
receptname
,
file_name
):
qccid
=
company_url
.
split
(
'company/'
)[
1
]
log
.
info
(
f
'====={qccid}====='
)
driver
.
get
(
company_url
)
# req_ = s.get(headers=headers, url=company_url)
page_source_detail
=
driver
.
page_source
com_soup
=
BeautifulSoup
(
page_source_detail
,
'html.parser'
)
#todo:天眼查更新时间 正常请求不到 需要使用模拟浏览器
try
:
sourceUpdateTime_
=
com_soup
.
find
(
'div'
,
class_
=
'index_detail-refresh__6W7U4'
)
.
find
(
'span'
)
.
text
pattern
=
r'\d{4}-\d{2}-\d{2}'
matched
=
re
.
findall
(
pattern
,
sourceUpdateTime_
)
if
matched
:
sourceUpdateTime
=
sourceUpdateTime_
else
:
sourceUpdateTime
=
paserTime
(
sourceUpdateTime_
)
.
strftime
(
"
%
Y-
%
m-
%
d
%
H:
%
M:
%
S"
)
except
:
log
.
info
(
f
'天眼查无该企业{social_code}'
)
return
try
:
businessinfo
=
com_soup
.
find
(
'table'
,
{
'class'
:
'index_tableBox__ZadJW'
})
except
:
businessinfo
=
''
if
businessinfo
:
data_baseinfo
=
baseinfo
(
com_soup
)
# print(data_baseinfo)
tr_list
=
businessinfo
.
find_all
(
'tr'
)
dic_buseniss
=
{}
for
tr
in
tr_list
:
# td_count = len(tr.find_all('td'))
# print(td_count)
td_list
=
tr
.
find_all
(
'td'
)
td_count
=
len
(
td_list
)
name_list
=
[
td_list
[
i
]
.
text
for
i
in
range
(
td_count
)
if
i
%
2
==
0
]
# print(name_list)
# value_list = [td_list[i].text for i in range(td_count) if i % 2 != 0]
value_list
=
[]
for
i
in
range
(
td_count
):
if
i
%
2
!=
0
:
value_tag
=
td_list
[
i
]
# print(value_tag)
# print("==============")
tag
.
deletep
(
value_tag
,
'span'
,
'class'
,
'index_history-operate__t3kjv'
)
tag
.
deletep
(
value_tag
,
'div'
,
'class'
,
'_efcb8'
)
tag
.
deletep
(
value_tag
,
'span'
,
'class'
,
'index_legal-bottom-info__bYvYZ'
)
tag
.
deletep
(
value_tag
,
'a'
,
'class'
,
'ml8 link-click'
)
tag
.
deletep
(
value_tag
,
'span'
,
'class'
,
'index_report-jump__z__UW'
)
tag
.
deletep
(
value_tag
,
'span'
,
'class'
,
'index_branch-report__Nyf_Y'
)
# for value_tag in value_tag_list:
value_list
.
append
(
value_tag
.
text
.
replace
(
'
\xa0
'
,
''
))
# print(value_list)
if
len
(
name_list
)
==
len
(
value_list
):
for
i
in
range
(
len
(
name_list
)):
dic_buseniss
[
name_list
[
i
]]
=
value_list
[
i
]
if
'曾用名'
in
value_list
[
i
]:
dic_buseniss
[
'曾用名'
]
=
value_list
[
i
]
.
split
(
'曾用名'
)[
1
]
.
split
(
'更多'
)[
0
]
dic_buseniss
[
name_list
[
i
]]
=
value_list
[
i
]
.
split
(
'曾用名'
)[
0
]
if
name_list
[
i
]
==
'法定代表人'
:
value_list
[
i
]
=
value_list
[
i
]
.
split
(
'任职'
)[
0
]
dic_buseniss
[
name_list
[
i
]]
=
value_list
[
i
]
try
:
del
dic_buseniss
[
'天眼评分'
]
except
:
pass
# print(dic_buseniss)
result_dict
=
getinfo
(
dic_buseniss
,
data_baseinfo
)
# 主要针对香港台湾企业,社会信用代码传为给定的
try
:
result_dict
[
'统一社会信用代码'
]
except
:
# log.info('未获取到统一社会信用代码')
if
social_code
:
result_dict
[
'统一社会信用代码'
]
=
social_code
else
:
# 如果未给定社会信用代码,则返回
return
False
if
result_dict
[
'企业名称'
]
.
startswith
(
'('
)
and
result_dict
[
'企业名称'
]
.
endswith
(
')'
):
result_dict
[
'企业名称'
]
=
result_dict
[
'企业名称'
][
1
:
-
1
]
if
result_dict
[
'企业名称'
]
==
'-'
and
com_name
:
result_dict
[
'企业名称'
]
=
com_name
elif
not
com_name
:
return
False
else
:
pass
# print(result_dict)
# 采集成功的企业
data
=
[
com_name
,
result_dict
[
'企业名称'
],
social_code
,
result_dict
[
'统一社会信用代码'
]]
file
.
appenddata
(
file_name
,
'获取基本信息成功企业'
,
data
)
# 将字段转化成英文驼峰
aa_dic
=
dic_handle
(
result_dict
)
aa_dic
[
'sourceUpdateTime'
]
=
sourceUpdateTime
aa_dic
[
'qccId'
]
=
qccid
log
.
info
(
aa_dic
)
# sendkafka(aa_dic)
# print(aa_dic)
# post_url = 'http://192.168.1.41:8088/enterprise/check/judge'
# dic_info = json.dumps(aa_dic)
# req = requests.post(post_url, data=dic_info)
else
:
data_baseinfo
=
baseinfo
(
com_soup
)
# 主要针对香港台湾企业,社会信用代码传为给定的
try
:
data_baseinfo
[
'统一社会信用代码'
]
except
:
log
.
info
(
'未获取到统一社会信用代码'
)
if
social_code
:
data_baseinfo
[
'统一社会信用代码'
]
=
social_code
else
:
# 如果未给定社会信用代码,则返回
return
False
if
data_baseinfo
[
'企业名称'
]
.
startswith
(
'('
)
and
data_baseinfo
[
'企业名称'
]
.
endswith
(
')'
):
data_baseinfo
[
'企业名称'
]
=
data_baseinfo
[
'企业名称'
][
1
:
-
1
]
if
data_baseinfo
[
'企业名称'
]
==
'-'
and
com_name
:
data_baseinfo
[
'企业名称'
]
=
com_name
elif
not
com_name
:
return
False
else
:
pass
# 采集成功的企业
data
=
[
com_name
,
data_baseinfo
[
'企业名称'
],
social_code
,
data_baseinfo
[
'统一社会信用代码'
]]
file
.
appenddata
(
file_name
,
'获取基本信息成功企业'
,
data
)
# 将字段转化成英文驼峰
aa_dic
=
dic_handle
(
data_baseinfo
)
aa_dic
[
'sourceUpdateTime'
]
=
sourceUpdateTime
aa_dic
[
'qccId'
]
=
qccid
# sendkafka(aa_dic)
log
.
info
(
aa_dic
)
# post_url = 'http://192.168.1.41:8088/enterprise/check/judge'
# dic_info = json.dumps(aa_dic)
# req = requests.post(post_url, data=dic_info)
def
remove_parentheses
(
text
):
# 清除中文小括号
text
=
re
.
sub
(
r'(|)'
,
''
,
text
)
# 清除英文小括号
text
=
re
.
sub
(
r'\(|\)'
,
''
,
text
)
return
text
.
replace
(
' '
,
''
)
# 判断名称是否统一
def
spiderwork
(
soup
,
receptname
,
file_name
):
company_url
=
''
try
:
company_list
=
soup
.
find_all
(
'div'
,
class_
=
'index_search-box__7YVh6'
)
except
:
log
.
info
(
f
'====={social_code}=====获取基本信息失败,重新放入redis====='
)
baseCore
.
r
.
lpush
(
'UpdateBasdeInfo:SocialCode_CompanyName'
,
company_field
)
# token.updateTokeen(id_cookie,2)
log
.
info
(
'=====已重新放入redis,cookies已封号======'
)
return
False
# receptname = '小米通讯技术有限公司'
for
compamy
in
company_list
:
info_t
=
compamy
.
find
(
'div'
,
class_
=
'index_name__qEdWi'
)
getname
=
info_t
.
find
(
'span'
)
.
text
log
.
info
(
f
'接收到的企业名称--{receptname}---采到的企业名称--{getname}'
)
if
receptname
and
getname
==
receptname
:
company_url
=
info_t
.
find
(
'a'
)[
'href'
]
break
elif
not
receptname
:
company_url
=
info_t
.
find
(
'a'
)[
'href'
]
break
else
:
jian_name
=
remove_parentheses
(
baseCore
.
hant_2_hans
(
getname
))
if
remove_parentheses
(
receptname
)
==
jian_name
:
log
.
info
(
f
'接收到的企业名称--{receptname}---转化成简体字的企业名称--{jian_name}'
)
company_url
=
info_t
.
find
(
'a'
)[
'href'
]
break
else
:
continue
if
company_url
:
# 采集基本信息和工商信息
spiderinfo
(
company_url
,
receptname
,
file_name
)
else
:
# 判断是否是曾用名
getname
=
''
for
child
in
company_list
[
0
]
.
find_all
():
if
child
.
has_attr
(
'class'
):
# print(child['class'])
if
'index_name'
in
child
[
'class'
][
0
]:
getname
=
child
.
text
company_url
=
child
.
find
(
'a'
)[
'href'
]
break
# tr = company_list[:1][0]
# info_t = tr.find('div', class_='index_name__qEdWi')
# getname = info_t.find('span').text
if
getname
:
log
.
info
(
f
'------可能是曾用名------接收到的企业名称--{receptname}---采到的企业名称--{getname}'
)
beforename
=
ifbeforename
(
company_url
)
if
beforename
==
receptname
:
spiderinfo
(
company_url
,
receptname
,
file_name
)
else
:
# 没有搜到相同的企业名称
data
=
[
com_name
,
social_code
]
file
.
appenddata
(
file_name
,
'需处理企业'
,
data
)
time
.
sleep
(
2
)
return
False
else
:
# 没有搜到相同的企业名称
data
=
[
com_name
,
social_code
]
file
.
appenddata
(
file_name
,
'需处理企业'
,
data
)
time
.
sleep
(
2
)
return
False
return
True
def
login
():
driver
=
create_driver
()
url
=
'https://www.tianyancha.com/'
driver
.
get
(
url
)
driver
.
maximize_window
()
# time.sleep(10)
cookies_list
,
id_cookie
=
token
.
get_cookies
()
for
cookie
in
cookies_list
:
driver
.
add_cookie
(
cookie
)
time
.
sleep
(
5
)
driver
.
refresh
()
# url_test = 'https://www.qcc.com/firm/a5f5bb3776867b3e273cd034d6fb4baa.html'
# driver.get(url_test)
# # driver.get('https://www.qcc.com/')
time
.
sleep
(
60
)
return
driver
,
id_cookie
if
__name__
==
'__main__'
:
taskType
=
'基本信息/天眼查'
# driver = create_driver()
# #手动登录
# driver.get('https://www.tianyancha.com/')
#todo:绕过验证使用cookies登录
driver
,
id_cookie
=
login
()
while
True
:
nowtime
=
baseCore
.
getNowTime
(
1
)
.
replace
(
'-'
,
''
)[:
8
]
file_name
=
f
'./data/国内企业基本信息更新.xlsx'
file
.
createFile
(
file_name
)
start_time
=
time
.
time
()
# 获取企业信息
company_field
=
baseCore
.
redicPullData
(
'UpdateBasdeInfo:SocialCode_CompanyName'
)
# company_field = '913100006073602992|光明乳业股份有限公司'
if
company_field
==
'end'
:
# 本轮处理完毕,需要发送邮件,并且进入下一轮
baseCore
.
sendEmail
(
file_name
)
time
.
sleep
(
20
)
file
.
deleteFile
(
file_name
)
continue
if
company_field
==
''
or
company_field
is
None
:
# 本轮结束后没有新增的企业要采集
file
.
deleteFile
(
file_name
)
flag
=
True
while
flag
:
log
.
info
(
'--------已没有数据---------'
)
time
.
sleep
(
30
)
if
not
baseCore
.
check_mysql_conn
(
cnx_
):
# 144数据库
cnx_
=
baseCore
.
cnx
cursor_
=
cnx_
.
cursor
()
log
.
info
(
'===11数据库重新连接成功==='
)
company_field
=
baseCore
.
redicPullData
(
'UpdateBasdeInfo:SocialCode_CompanyName'
)
if
company_field
:
flag
=
False
log
.
info
(
"-----已添加数据------"
)
baseCore
.
r
.
lpush
(
'UpdateBasdeInfo:SocialCode_CompanyName'
,
company_field
)
continue
continue
# company_field_ = f'|{company_field}'
social_code
=
company_field
.
split
(
'|'
)[
0
]
com_name
=
company_field
.
split
(
'|'
)[
1
]
.
replace
(
' '
,
''
)
if
'ZZSN'
in
social_code
and
'ZD'
in
social_code
:
continue
#todo:查询天眼查id
data
=
baseCore
.
getInfomation
(
social_code
)
if
len
(
data
)
!=
0
:
tycid
=
data
[
11
]
else
:
# 数据重新塞入redis
# log.info(f'数据库中无该企业{social_code}')
sql
=
f
"SELECT * FROM sys_base_enterprise WHERE social_credit_code = '{social_code}'"
cursor
.
execute
(
sql
)
data
=
cursor
.
fetchone
()
if
data
:
pass
else
:
# 数据库中并没有该企业 需要新增
pass
com_name_c
=
data
[
3
]
xydm
=
data
[
1
]
# 写入数据库
insert
=
"INSERT INTO EnterpriseInfo(CompanyName, SocialCode) VALUES (
%
s,
%
s)"
cursor_
.
execute
(
insert
,
(
com_name_c
,
xydm
))
cnx_
.
commit
()
tycid
=
''
if
tycid
==
None
or
tycid
==
''
:
count
=
redaytowork
(
com_name
,
social_code
,
file_name
)
else
:
company_url
=
'https://www.tianyancha.com/company/'
+
tycid
spiderinfo
(
company_url
,
com_name
,
file_name
)
time
.
sleep
(
10
)
# break
baseCore
.
close
()
\ No newline at end of file
编写
预览
Markdown
格式
0%
重试
或
添加新文件
添加附件
取消
您添加了
0
人
到此讨论。请谨慎行事。
请先完成此评论的编辑!
取消
请
注册
或者
登录
后发表评论