Skip to content

Z
zzsn_spider
提交 443a395e 作者: 薛凌堃
操作

跨国100大基本信息

上级 c1cac75e
正在显示 包含 193 行增加 和 66 行删除
comData/newlist/global100/baseinfo_start.bat 0 → 100644
title dujiaoshoubaseinfo
call activate
call conda activate zzsn@3.8.0
python baseinfo_dujiaoshou.py
pause
\ No newline at end of file
comData/newlist/global100/getQccId.py 0 → 100644
# -*- coding: utf-8 -*-
import time
from urllib.parse import quote
import requests
import urllib3
from BaseCore import BaseCore
baseCore = BaseCore()
log = baseCore.getLogger()
# headers = {
# 'Host': 'xcx.qcc.com',
# 'Connection': 'keep-alive',
# 'Qcc-Platform': 'mp-weixin',
# 'Qcc-Timestamp': '',
# 'Qcc-Version': '1.0.0',
# 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat',
# 'content-type': 'application/json',
# 'Referer': 'https://servicewechat.com/wx395200814fcd7599/166/page-frame.html',
# 'Accept-Encoding': 'gzip, deflate, br,'
# }
headers = {
'Host': 'xcx.qcc.com',
'Connection': 'keep-alive',
'x-request-device-type': 'Android',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 MicroMessenger/7.0.20.1781(0x6700143B) NetType/WIFI MiniProgramEnv/Windows WindowsWechat/WMPF XWEB/8391',
'Content-Type': 'application/json',
'Qcc-Version': '1.0.0',
'authMini': 'Bearer f51dae1a2fcb109fa9ec58bd4a85e5c5',
'xweb_xhr': '1',
'xcx-version': '2023.09.27',
'Qcc-Platform': 'mp-weixin',
'Qcc-CurrentPage': '/company-subpackages/business/index',
'Qcc-Timestamp': '1696661787803',
'Qcc-RefPage': '/company-subpackages/detail/index',
'Accept': '*/*',
'Sec-Fetch-Site': 'cross-site',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Dest': 'empty',
'Referer': 'https://servicewechat.com/wx395200814fcd7599/307/page-frame.html',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh'
}
# 通过企业名称或信用代码获取企查查id
def find_id_by_name(start,token,name):
urllib3.disable_warnings()
qcc_key = name
t = str(int(time.time()) * 1000)
headers['Qcc-Timestamp'] = t
url = f"https://xcx.qcc.com/mp-weixin/forwardApp/v3/base/advancedSearch?token={token}&t={t}&pageIndex=1&needGroup=yes&insuredCntStart=&insuredCntEnd=&startDateBegin=&startDateEnd=&registCapiBegin=&registCapiEnd=&countyCode=&province=&sortField=&isSortAsc=&searchKey={quote(qcc_key)}&searchIndex=default&industryV3="
for lll in range(1, 6):
try:
resp_dict = requests.get(url=url, headers=headers, verify=False).json()
break
except Exception as e:
print(f'{e}-------------重试')
time.sleep(5)
continue
time.sleep(2)
#{'status': 40101, 'message': '无效的sessionToken!'} {'status': 401, 'message': '您的账号访问超频，请升级小程序版本'}
if resp_dict['status']==40101:
KeyNo = False
log.info(f'====token失效====时间{baseCore.getTimeCost(start, time.time())}')
return KeyNo
if resp_dict['status']==401:
KeyNo = False
log.info(f'=======您的账号访问超频，请升级小程序版本=====时间{baseCore.getTimeCost(start, time.time())}')
return KeyNo
if resp_dict['status']==40102:
KeyNo = False
log.info(f'=======无效的session=====时间{baseCore.getTimeCost(start, time.time())}')
return KeyNo
try:
if resp_dict['result']['Result']:
result_dict = resp_dict['result']['Result'][0]
KeyNo = result_dict['KeyNo']
Name = result_dict['Name'].replace('<em>', '').replace('</em>', '').strip()
if Name == '':
KeyNo = 'null'
else:
KeyNo = 'null'
except:
KeyNo = False
log.info(f'====token失效====时间{baseCore.getTimeCost(start,time.time())}')
return KeyNo
log.info("{}，企业代码为:{}".format(qcc_key, KeyNo))
return KeyNo
\ No newline at end of file
comData/dfcfwGpdm/NQenterprise/NQgetid.py → comData/newlist/global100/global100.py
...@@ -7,16 +7,17 @@ import requests ...@@ -7,16 +7,17 @@ import requests
import json import json
from kafka import KafkaProducer from kafka import KafkaProducer
from base.BaseCore import BaseCore from BaseCore import BaseCore
from getQccId import find_id_by_name from getQccId import find_id_by_name
baseCore = BaseCore() baseCore = BaseCore()
cnx_ = baseCore.cnx cnx_ = baseCore.cnx
cursor_ = baseCore.cursor cursor_ = baseCore.cursor
log = baseCore.getLogger() log = baseCore.getLogger()
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# 通过企查查id获取企业基本信息 # 通过企查查id获取企业基本信息
def info_by_id(com_id,com_name,gpdm): def info_by_id(com_id,com_name):
aa_dict_list = [] aa_dict_list = []
t = str(int(time.time()) * 1000) t = str(int(time.time()) * 1000)
...@@ -31,7 +32,7 @@ def info_by_id(com_id,com_name,gpdm): ...@@ -31,7 +32,7 @@ def info_by_id(com_id,com_name,gpdm):
result_dict = resp_dict['result']['Company'] result_dict = resp_dict['result']['Company']
except: except:
log.info(com_name + ":获取失败===========重新放入redis") log.info(com_name + ":获取失败===========重新放入redis")
baseCore.rePutIntoR('EnterpriseIpo:nq_gpdm',gpdm) baseCore.rePutIntoR('global100:baseinfo',com_name)
return aa_dict_list return aa_dict_list
company_name = result_dict['Name'] company_name = result_dict['Name']
...@@ -306,12 +307,12 @@ def info_by_id(com_id,com_name,gpdm): ...@@ -306,12 +307,12 @@ def info_by_id(com_id,com_name,gpdm):
} }
aa_dict_list.append(aa_dict) aa_dict_list.append(aa_dict)
print(company_name + "：爬取完成") log.info(company_name + "：爬取完成")
return aa_dict_list return aa_dict_list
if __name__ == '__main__': if __name__ == '__main__':
taskType = '基本信息/企查查' taskType = '基本信息/企查查/中国100强'
headers = { headers = {
'Host': 'xcx.qcc.com', 'Host': 'xcx.qcc.com',
'Connection': 'keep-alive', 'Connection': 'keep-alive',
...@@ -323,81 +324,110 @@ if __name__ == '__main__': ...@@ -323,81 +324,110 @@ if __name__ == '__main__':
'Referer': 'https://servicewechat.com/wx395200814fcd7599/166/page-frame.html', 'Referer': 'https://servicewechat.com/wx395200814fcd7599/166/page-frame.html',
'Accept-Encoding': 'gzip, deflate, br,' 'Accept-Encoding': 'gzip, deflate, br,'
} }
list_weicha = []
name_list = []
#从redis里拿数据 #从redis里拿数据
while True: while True:
# TODO:需要隔两个小时左右抓包修改,token从数据库中获得 # TODO:需要隔两个小时左右抓包修改,token从数据库中获得
token = '83a9a9be4e9ecf3a8f8a20364227dc5d' token = baseCore.GetToken()
list_weicha = [] dataList = []
list_all_info = [] if token:
name_list = [] pass
else:
log.info('==========已无token==========')
time.sleep(30)
continue
# list_all_info = []
start_time = time.time() start_time = time.time()
# 获取企业信息 # 获取企业信息
com_code = baseCore.redicPullData('EnterpriseIpoqccid:nq_gpdm') social_code = baseCore.redicPullData('global100:baseinfo')
if '.NQ' in com_code:
com_code1 = com_code
else:
com_code1 = com_code + '.NQ'
company_id = find_id_by_name(start_time,token,com_code1) # com_name = '卓新市万达铸业有限公司'
if social_code == '' or social_code is None:
time.sleep(20)
continue
if '搜索不到' in social_code:
continue
else:
pass
dic_info = baseCore.getInfomation(social_code)
log.info(f'----当前企业{social_code}--开始处理---')
com_name = dic_info[1]
#企查查id
company_id = dic_info[3]
#如果没有信用代码 就通过名字搜索 如果有信用代码 就通过信用代码
if company_id == None or company_id == False:
if social_code:
company_id = find_id_by_name(start_time,token,social_code)
else:
company_id = find_id_by_name(start_time,token,com_name)
if company_id == 'null':
log.info('=====搜索不到该企业====')
#todo:搜不到的企业没有信用代码 传输不过去 生成一个信用代码
baseCore.rePutIntoR('global100:baseinfo', social_code + '：搜索不到')
continue
if not company_id: if not company_id:
log.info(com_code + "：企业ID获取失败===重新放入redis") log.info(com_name + "：企业ID获取失败===重新放入redis")
list_weicha.append(com_code + "：企业ID获取失败") list_weicha.append(com_name + "：企业ID获取失败")
baseCore.rePutIntoR('EnterpriseIpoqccid:nq_gpdm',com_code) baseCore.rePutIntoR('global100:baseinfo',com_name)
log.info('-----已重新放入redis-----') baseCore.delete_token(token)
log.info('=====已重新放入redis,失效token已删除======')
time.sleep(20) time.sleep(20)
continue continue
else: else:
log.info(f'====={com_code}===={company_id}=====获取企业id成功=====') log.info(f'====={com_name}===={company_id}=====获取企业id成功=====')
# todo:企查查id写入gpdm表中 # todo:写入数据库
updateSql = f"update gpdm set QCCID = '{company_id}' where gpdm = '{com_code}'" updateqccid = f"update Global100 set qccid = '{company_id}' where CompanyName = '{com_name}'"
cursor_.execute(updateSql) cursor_.execute(updateqccid)
cnx_.commit() cnx_.commit()
# try: try:
# post_data_list = info_by_id(company_id, '',com_code) post_data_list = info_by_id(company_id, com_name)
# except: except:
# log.info(f'====={com_code}=====获取基本信息失败，重新放入redis=====') log.info(f'====={social_code}=====获取基本信息失败，重新放入redis=====')
# baseCore.rePutIntoR('BaseInfoEnterprise:gnqy_social_code', com_code) baseCore.rePutIntoR('global100:baseinfo', com_name)
# continue baseCore.delete_token(token)
# if post_data_list: log.info('=====已重新放入redis,失效token已删除======')
# pass continue
# else: if post_data_list:
# log.info(f'======{com_code}====企查查token失效====') pass
# time.sleep(20) else:
# continue # log.info(f'======{social_code}====企查查token失效====')
# for post_data in post_data_list: time.sleep(20)
continue
for post_data in post_data_list:
# list_all_info.append(post_data) # list_all_info.append(post_data)
# if post_data is None: if post_data is None:
# print(com_code + "：企业信息获取失败") print(com_name + "：企业信息获取失败")
# list_weicha.append(com_code + "：企业信息获取失败") list_weicha.append(com_name + "：企业信息获取失败")
# continue continue
# get_name = post_data['name'] get_name = post_data['name']
# get_socialcode = post_data['socialCreditCode'] # get_socialcode = post_data['socialCreditCode']
# name_compile = { #todo:将信用代码更新到表中
# 'yuan_name':com_code, # updatesocialcode = f"update Global100 set SocialCode = '{get_socialcode}' where CompanyName = '{com_name}'"
# 'get_name':get_name # cursor_.execute(updatesocialcode)
# } # cnx_.commit()
# name_list.append(name_compile) name_compile = {
# 'yuan_name':com_name,
# log.info(f'采集{com_code}成功=======耗时{baseCore.getTimeCost(start_time,time.time())}') 'get_name':get_name
# try: }
# producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'], api_version=(2, 0, 2)) name_list.append(name_compile)
# kafka_result = producer.send("regionInfo", json.dumps(post_data, ensure_ascii=False).encode('utf8')) # dataList.append(post_data)
# print(kafka_result.get(timeout=10)) baseCore.writerToExcel(name_list,'跨国公司100大.xlsx')
# except: log.info(f'采集{com_name}成功=======耗时{baseCore.getTimeCost(start_time,time.time())}')
# exception = 'kafka传输失败' try:
# state = 0 producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'], api_version=(2, 0, 2))
# takeTime = baseCore.getTimeCost(start_time, time.time()) kafka_result = producer.send("regionInfo", json.dumps(post_data, ensure_ascii=False).encode('utf8'))
# baseCore.recordLog(get_socialcode, taskType, state, takeTime, '', exception) print(kafka_result.get(timeout=10))
# log.info(f"{get_name}--{get_socialcode}--kafka传输失败") except:
# # 信息采集完成后将该企业的采集次数更新 exception = 'kafka传输失败'
state = 0
takeTime = baseCore.getTimeCost(start_time, time.time())
baseCore.recordLog(get_socialcode, taskType, state, takeTime, '', exception)
log.info(f"{get_name}--{get_socialcode}--kafka传输失败")
# break
# nowtime = baseCore.getNowTime(1).replace('-','_')[:10]
# companyName = pd.DataFrame(name_list)
# companyName.to_excel(f'./data/企业名称对比_{nowtime}.xlsx',index=False)
# false_com = pd.DataFrame(list_weicha)
# false_com.to_excel(f'./data/采集失败企业名单_{nowtime}.xlsx',index=False)
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论