import hashlib
from datetime import datetime
import re
import time
import uuid
from urllib.parse import unquote

import numpy as np
import pandas as pd
import pymongo
import requests
import os
import json

from obs import ObsClient
from retry import retry

from base import BaseCore

obsClient = ObsClient(
    access_key_id='VEHN7D0TJ9316H8AHCAV',  # 你的华为云的ak码
    secret_access_key='heR353lvSWVPNU8pe2QxDtd8GDsO5L6PGH5eUoQY',  # 你的华为云的sk
    server='https://obs.cn-north-1.myhuaweicloud.com'  # 你的桶的地址
)
baseCore = BaseCore.BaseCore()
db_storage = pymongo.MongoClient('mongodb://114.115.221.202:27017', username='admin', password='ZZsn@9988').RESCenter[
    'REITsProjDynamics']
log = baseCore.getLogger()
cursor_ = baseCore.cursor_
cnx_ = baseCore.cnx_
headers = {
    'Accept': 'application/json, text/javascript, */*; q=0.01',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,zh-TW;q=0.9,zh;q=0.8',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive',
    'Content-Type': 'application/json;charset=utf-8',
    'Host': 'reits.szse.cn',
    'Pragma': 'no-cache',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0',
    'X-Requested-With': 'XMLHttpRequest',
}


class obsOperate():
    def __init__(self, cursor_, cnx_, log):
        self.headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'zh-CN,zh-TW;q=0.9,zh;q=0.8',
            'Cache-Control': 'no-cache',
            'Connection': 'keep-alive',
            'Host': 'reportdocs.static.szse.cn',
            'Pragma': 'no-cache',
            'Referer': 'http://reits.szse.cn/',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0',
        }
        self.cursor_ = cursor_
        self.cnx_ = cnx_
        self.log = log

    def secrchATT(self, item_id, file_name, type_id, order_by):
        sel_sql = '''select id from clb_sys_attachment where item_id = %s and name = %s and type_id=%s and order_by=%s '''
        self.cursor_.execute(sel_sql, (item_id, file_name, type_id, order_by))
        selects = self.cursor_.fetchone()
        return selects

    # 插入到att表 返回附件id
    def tableUpdate(self, retData, com_name, file_name, num, pub_time):
        item_id = retData['item_id']
        type_id = retData['type_id']
        group_name = retData['group_name']
        path = retData['path']
        full_path = retData['full_path']
        category = retData['category']
        file_size = retData['file_size']
        status = retData['status']
        create_by = retData['create_by']
        page_size = retData['page_size']
        create_time = retData['create_time']
        order_by = num

        Upsql = '''insert into clb_sys_attachment(name,type_id,item_id,group_name,path,full_path,category,file_size,order_by,status,create_by,create_time,object_key,bucket_name,publish_time) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''

        values = (
            file_name, type_id, item_id, group_name, path, full_path, category, file_size, order_by,
            status, create_by,
            create_time, path, 'zzsn', pub_time)

        self.cursor_.execute(Upsql, values)  # 插入
        self.cnx_.commit()  # 提交
        self.log.info("更新完成:{}".format(Upsql))
        selects = self.secrchATT(item_id, file_name, type_id, order_by)
        id = selects[0]
        return id, full_path

    def getuuid(self):
        get_timestamp_uuid = uuid.uuid1()  # 根据 时间戳生成 uuid , 保证全球唯一
        return get_timestamp_uuid

    # 获取文件大小
    def convert_size(self, size_bytes):
        # 定义不同单位的转换值
        units = ['bytes', 'KB', 'MB', 'GB', 'TB']
        i = 0
        while size_bytes >= 1024 and i < len(units) - 1:
            size_bytes /= 1024
            i += 1
        return f"{size_bytes:.2f} {units[i]}"

    @retry(tries=5, delay=10)
    def getRes(self, file_href):
        response = requests.get(file_href, headers=self.headers)
        if response.status_code != 200:
            raise
        return response

    @retry(tries=5, delay=10)
    def sendOBS(self, file_name, response):
        result = obsClient.putContent('zzsn', 'PolicyDocument/' + file_name, content=response.content)
        return result

    def uptoOBS(self, file_href, item_id, file_name):

        category = os.path.splitext(file_href)[1]
        retData = {'state': False, 'type_id': 15, 'item_id': item_id, 'group_name': '', 'path': '',
                   'full_path': '',
                   'category': category, 'file_size': '', 'status': 1, 'create_by': 'LiuLiYuan',
                   'create_time': '', 'page_size': '', 'content': ''}
        try:
            response = self.getRes(file_href)
        except:
            self.log.error('文件获取失败')
            return retData

        file_size = int(response.headers.get('Content-Length'))
        file_name = str(self.getuuid()) + category
        try:
            result = self.sendOBS(file_name, response)
        except:
            self.log.error(f'obs上传失败')
            return retData
        try:
            time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
            retData['state'] = True
            retData['path'] = result['body']['objectUrl'].split('.com')[1]
            retData['full_path'] = unquote(result['body']['objectUrl'])
            retData['file_size'] = self.convert_size(file_size)
            retData['create_time'] = time_now
        except Exception as e:
            print(f'error:{e}')
            return retData
        return retData


def md5_encrypt(text):
    # 创建 MD5 对象
    md5 = hashlib.md5()

    # 更新 MD5 对象内容
    md5.update(text.encode('utf-8'))

    # 获取加密结果
    encrypted_text = md5.hexdigest()

    return encrypted_text


@retry(tries=5, delay=10)
def getPageSize():
    # ip = baseCore.get_proxy()
    url = 'http://reits.szse.cn/api/reits/projectrends/query?biztypsb=21&bizType=2&pageIndex=0&pageSize=10'
    req = requests.get(url, headers=headers)
    req.encoding = req.apparent_encoding
    total = int(req.json()['totalSize'])
    if total % 10 == 0:
        pageSize = int(total / 10)
    else:
        pageSize = int(total / 10) + 1
    req.close()
    return pageSize


@retry(tries=5, delay=10)
def getDataJson(page):
    # ip = baseCore.get_proxy()
    url = f'http://reits.szse.cn/api/reits/projectrends/query?biztypsb=21&bizType=2&pageIndex={page}&pageSize=10'
    req = requests.get(url, headers=headers)
    req.encoding = req.apparent_encoding
    data_json = req.json()['data']
    req.close()
    return data_json


@retry(tries=5, delay=10)
def getDataJson_(url):
    # ip = baseCore.get_proxy()
    req = requests.get(url, headers=headers)
    req.encoding = req.apparent_encoding
    data_json = req.json()['data']
    req.close()
    return data_json


def getBaseInfo(data):
    cmpnm = data['cmpnm']
    specialPlanName = data['specialPlanName']
    issueTargetName = data['issueTargetName']
    primitiveInterestsor = data['primitiveInterestsor']
    acctfm = data['acctfm']
    sprinst = data['sprinst']
    lawfm = data['lawfm']
    biztypsbName = data['biztypsbName']
    prjst = data['prjst']
    updtdt = data['updtdt']
    acptdt = data['acptdt']
    sponsors = []
    sponsorMD5 = []
    if ',' in primitiveInterestsor:
        for sponsor in primitiveInterestsor.split(','):
            sponsors.append(sponsor)
    elif '，' in primitiveInterestsor:
        for sponsor in primitiveInterestsor.split('，'):
            sponsors.append(sponsor)
    elif ';' in primitiveInterestsor:
        for sponsor in primitiveInterestsor.split(';'):
            sponsors.append(sponsor)
    else:
        sponsors.append(primitiveInterestsor)
    for sponsor in sponsors:
        sponsorMD5.append(md5_encrypt(sponsor))
    baseInfo = {
        'name': cmpnm,
        'variety': issueTargetName,
        'sponsor': primitiveInterestsor,
        'sponsors':sponsors,
        'sponsorMD5':sponsorMD5,
        'caretaker': acctfm,
        'planName': specialPlanName,
        'planCaretaker': sprinst,
        'letterNum': '',
        'status': prjst,
        'updateDate': datetime.strptime(updtdt, '%Y-%m-%d'),
        'strUpdateDate': updtdt,
        'accDate': datetime.strptime(acptdt, '%Y-%m-%d'),
        'strAccDate': acptdt,
        'type': biztypsbName,
    }
    log.info(f'{cmpnm}===项目基本信息采集成功')
    return baseInfo


def getFile(obsOperate, data_json, mongoId):
    data = {
        'fundContracts': {'title': '基金合同', 'fdbkResp': '', 'fdbkRespTime': '', 'strFdbkRespTime': ''},
        'fundCustodianAgt': {'title': '基金托管协议', 'fdbkResp': '', 'fdbkRespTime': '', 'strFdbkRespTime': ''},
        'prospectus': {'title': '基金招募说明书', 'fdbkResp': '', 'fdbkRespTime': '', 'strFdbkRespTime': ''},
        'legalOpinion': {'title': '基金法律意见书', 'fdbkResp': '', 'fdbkRespTime': '', 'strFdbkRespTime': ''},
    }
    num = 1
    for data_json_ in data_json['disclosureMaterials']:
        type = int(data_json_['type'])
        path = 'http://reportdocs.static.szse.cn' + data_json_['dfpth']
        version = data_json_['matnm']
        time_ = data_json_['ddt']
        file_name = data_json_['dfnm']
        category = os.path.splitext(path)[1]
        if category not in file_name:
            file_name = file_name + category
        retData = obsOperate.uptoOBS(path, mongoId, file_name)
        time.sleep(2)
        if retData['state']:
            pass
        else:
            log.error(f'{id}===信息披露文件及备查文件下载obs失败')
            return 0
        att_id, full_path = obsOperate.tableUpdate(retData, 'RETIs文件', file_name, num, time_)
        num += 1
        if version == '基金招募说明书':
            if type == 1:
                data['prospectus']['declaration'] = full_path
                data['prospectus']['declarationTime'] = datetime.strptime(time_, '%Y-%m-%d')
                data['prospectus']['strDeclarationTime'] = time_
            elif type == 3:
                data['prospectus']['cover'] = full_path
                data['prospectus']['coverTime'] = datetime.strptime(time_, '%Y-%m-%d')
                data['prospectus']['strCoverTime'] = time_
        elif version == '基金合同':
            if type == 1:
                data['fundContracts']['declaration'] = full_path
                data['fundContracts']['declarationTime'] = datetime.strptime(time_, '%Y-%m-%d')
                data['fundContracts']['strDeclarationTime'] = time_
            elif type == 3:
                data['fundContracts']['cover'] = full_path
                data['fundContracts']['coverTime'] = datetime.strptime(time_, '%Y-%m-%d')
                data['fundContracts']['strCoverTime'] = time_
        elif version == '基金托管协议':
            if type == 1:
                data['fundCustodianAgt']['declaration'] = full_path
                data['fundCustodianAgt']['declarationTime'] = datetime.strptime(time_, '%Y-%m-%d')
                data['fundCustodianAgt']['strDeclarationTime'] = time_
            elif type == 3:
                data['fundCustodianAgt']['cover'] = full_path
                data['fundCustodianAgt']['coverTime'] = datetime.strptime(time_, '%Y-%m-%d')
                data['fundCustodianAgt']['strCoverTime'] = time_
        elif version == '基金法律意见书':
            if type == 1:
                data['legalOpinion']['declaration'] = full_path
                data['legalOpinion']['declarationTime'] = datetime.strptime(time_, '%Y-%m-%d')
                data['fundCustodianAgt']['strDeclarationTime'] = time_
            elif type == 3:
                data['legalOpinion']['cover'] = full_path
                data['legalOpinion']['coverTime'] = datetime.strptime(time_, '%Y-%m-%d')
                data['fundCustodianAgt']['strCoverTime'] = time_
    log.info(f'{mongoId}===信息披露文件及备查文件采集成功')
    return data


def getFeedback(obsOperate, data, mongoId):
    feedback = []
    data_json = data['enquiryResponseAttachment']
    num = 1
    for data_json_ in data_json:
        path = 'http://reportdocs.static.szse.cn' + data_json_['dfpth']
        time_ = data_json_['ddt']
        file_name = data_json_['dfnm']
        category = os.path.splitext(path)[1]
        if category not in file_name:
            file_name = file_name + category
        retData = obsOperate.uptoOBS(path, mongoId, file_name)
        time.sleep(2)
        if retData['state']:
            pass
        else:
            log.error(f'{id}===反馈意见及回复文件下载obs失败')
            return 0
        att_id, full_path = obsOperate.tableUpdate(retData, 'RETIs文件', file_name, num, time_)
        data_ = {
            'num': num,
            'title': file_name,
            'updateTime': datetime.strptime(time_, '%Y-%m-%d'),
            'strUpdateTime': time_,
            'filePath': full_path,
        }
        feedback.append(data_)
    log.info(f'{mongoId}===反馈意见及回复采集成功')
    return feedback


def updateFile(obsOperate, data_json, mongoId, mongoData):
    data = {
        'fundContracts': {'title': '基金合同', 'fdbkResp': '', 'fdbkRespTime': '', 'strFdbkRespTime': ''},
        'fundCustodianAgt': {'title': '基金托管协议', 'fdbkResp': '', 'fdbkRespTime': '', 'strFdbkRespTime': ''},
        'prospectus': {'title': '基金招募说明书', 'fdbkResp': '', 'fdbkRespTime': '', 'strFdbkRespTime': ''},
        'legalOpinion': {'title': '基金法律意见书', 'fdbkResp': '', 'fdbkRespTime': '', 'strFdbkRespTime': ''},
    }
    num = 1
    for data_json_ in data_json['disclosureMaterials']:
        type = int(data_json_['type'])
        path = 'http://reportdocs.static.szse.cn' + data_json_['dfpth']
        version = data_json_['matnm']
        time_ = data_json_['ddt']
        file_name = data_json_['dfnm']
        category = os.path.splitext(path)[1]
        if category not in file_name:
            file_name = file_name + category
        if version == '基金招募说明书':
            if datetime.strptime(time_, '%Y-%m-%d') == mongoData['file']['prospectus']['declarationTime']:
                if type == 1:
                    data['prospectus']['declaration'] = mongoData['file']['prospectus']['declaration']
                    data['prospectus']['declarationTime'] = mongoData['file']['prospectus']['declarationTime']
                    data['prospectus']['strDeclarationTime'] = mongoData['file']['prospectus']['strDeclarationTime']
                elif type == 3:
                    data['prospectus']['cover'] = mongoData['file']['prospectus']['cover']
                    data['prospectus']['coverTime'] = mongoData['file']['prospectus']['coverTime']
                    data['prospectus']['strCoverTime'] = mongoData['file']['prospectus']['strCoverTime']
            else:
                retData = obsOperate.uptoOBS(path, mongoId, file_name)
                time.sleep(2)
                if retData['state']:
                    pass
                else:
                    log.error(f'{id}===信息披露文件及备查文件下载obs失败')
                    return 0
                att_id, full_path = obsOperate.tableUpdate(retData, 'RETIs文件', file_name, num, time_)
                if type == 1:
                    data['prospectus']['declaration'] = full_path
                    data['prospectus']['declarationTime'] = datetime.strptime(time_, '%Y-%m-%d')
                    data['prospectus']['strDeclarationTime'] = time_
                elif type == 3:
                    data['prospectus']['cover'] = full_path
                    data['prospectus']['coverTime'] = datetime.strptime(time_, '%Y-%m-%d')
                    data['prospectus']['strCoverTime'] = time_
        elif version == '基金合同':
            if datetime.strptime(time_, '%Y-%m-%d') == mongoData['file']['fundContracts']['declarationTime']:
                if type == 1:
                    data['fundContracts']['declaration'] = mongoData['file']['fundContracts']['declaration']
                    data['fundContracts']['declarationTime'] = mongoData['file']['fundContracts']['declarationTime']
                    data['fundContracts']['strDeclarationTime'] = mongoData['file']['fundContracts'][
                        'strDeclarationTime']
                elif type == 3:
                    data['fundContracts']['cover'] = mongoData['file']['fundContracts']['cover']
                    data['fundContracts']['coverTime'] = mongoData['file']['fundContracts']['coverTime']
                    data['fundContracts']['strCoverTime'] = mongoData['file']['fundContracts']['strCoverTime']
            else:
                retData = obsOperate.uptoOBS(path, mongoId, file_name)
                time.sleep(2)
                if retData['state']:
                    pass
                else:
                    log.error(f'{id}===信息披露文件及备查文件下载obs失败')
                    return 0
                att_id, full_path = obsOperate.tableUpdate(retData, 'RETIs文件', file_name, num, time_)
                if type == 1:
                    data['fundContracts']['declaration'] = full_path
                    data['fundContracts']['declarationTime'] = datetime.strptime(time_, '%Y-%m-%d')
                    data['fundContracts']['strDeclarationTime'] = time_
                elif type == 3:
                    data['fundContracts']['cover'] = full_path
                    data['fundContracts']['coverTime'] = datetime.strptime(time_, '%Y-%m-%d')
                    data['fundContracts']['strCoverTime'] = time_
        elif version == '基金托管协议':
            if datetime.strptime(time_, '%Y-%m-%d') == mongoData['file']['fundCustodianAgt']['declarationTime']:
                if type == 1:
                    data['fundCustodianAgt']['declaration'] = mongoData['file']['fundContracts']['declaration']
                    data['fundCustodianAgt']['declarationTime'] = mongoData['file']['fundCustodianAgt'][
                        'declarationTime']
                    data['fundCustodianAgt']['strDeclarationTime'] = mongoData['file']['fundCustodianAgt'][
                        'strDeclarationTime']
                elif type == 3:
                    data['fundCustodianAgt']['cover'] = mongoData['file']['fundCustodianAgt']['cover']
                    data['fundCustodianAgt']['coverTime'] = mongoData['file']['fundCustodianAgt']['coverTime']
                    data['fundCustodianAgt']['strCoverTime'] = mongoData['file']['fundCustodianAgt']['strCoverTime']
            else:
                retData = obsOperate.uptoOBS(path, mongoId, file_name)
                time.sleep(2)
                if retData['state']:
                    pass
                else:
                    log.error(f'{id}===信息披露文件及备查文件下载obs失败')
                    return 0
                att_id, full_path = obsOperate.tableUpdate(retData, 'RETIs文件', file_name, num, time_)
                if type == 1:
                    data['fundCustodianAgt']['declaration'] = full_path
                    data['fundCustodianAgt']['declarationTime'] = datetime.strptime(time_, '%Y-%m-%d')
                    data['fundCustodianAgt']['strDeclarationTime'] = time_
                elif type == 3:
                    data['fundCustodianAgt']['cover'] = full_path
                    data['fundCustodianAgt']['coverTime'] = datetime.strptime(time_, '%Y-%m-%d')
                    data['fundCustodianAgt']['strCoverTime'] = time_
        elif version == '基金法律意见书':
            if datetime.strptime(time_, '%Y-%m-%d') == mongoData['file']['legalOpinion']['declarationTime']:
                if type == 1:
                    data['legalOpinion']['declaration'] = mongoData['file']['legalOpinion']['declaration']
                    data['legalOpinion']['declarationTime'] = mongoData['file']['legalOpinion']['declarationTime']
                    data['fundCustodianAgt']['strDeclarationTime'] = mongoData['file']['fundCustodianAgt'][
                        'strDeclarationTime']
                elif type == 3:
                    data['legalOpinion']['cover'] = mongoData['file']
                    data['legalOpinion']['coverTime'] = mongoData['file']
                    data['fundCustodianAgt']['strCoverTime'] = mongoData['file']
            else:
                retData = obsOperate.uptoOBS(path, mongoId, file_name)
                time.sleep(2)
                if retData['state']:
                    pass
                else:
                    log.error(f'{id}===信息披露文件及备查文件下载obs失败')
                    return 0
                att_id, full_path = obsOperate.tableUpdate(retData, 'RETIs文件', file_name, num, time_)
                if type == 1:
                    data['legalOpinion']['declaration'] = full_path
                    data['legalOpinion']['declarationTime'] = datetime.strptime(time_, '%Y-%m-%d')
                    data['fundCustodianAgt']['strDeclarationTime'] = time_
                elif type == 3:
                    data['legalOpinion']['cover'] = full_path
                    data['legalOpinion']['coverTime'] = datetime.strptime(time_, '%Y-%m-%d')
                    data['fundCustodianAgt']['strCoverTime'] = time_
        num += 1
    log.info(f'{mongoId}===信息披露文件及备查文件采集成功')
    return data


def updateFeedback(obsOperate, data, mongoId, mongoData):
    feedback = []
    data_json = data['enquiryResponseAttachment']
    num = 1
    for data_json_ in data_json:
        flg = True
        path = 'http://reportdocs.static.szse.cn' + data_json_['dfpth']
        time_ = data_json_['ddt']
        file_name = data_json_['dfnm']
        category = os.path.splitext(path)[1]
        if category not in file_name:
            file_name = file_name + category
        for feedback_ in mongoData['feedback']:
            if time_ == feedback_['strUpdateTime'] and file_name == feedback_['title']:
                feedback_['num'] = num
                feedback.append(feedback_)
                flg = False
                num += 1
                break
        if flg:
            retData = obsOperate.uptoOBS(path, mongoId, file_name)
            time.sleep(2)
            if retData['state']:
                pass
            else:
                log.error(f'{id}===反馈意见及回复文件下载obs失败')
                return 0
            att_id, full_path = obsOperate.tableUpdate(retData, 'RETIs文件', file_name, num, time_)
            data_ = {
                'num': num,
                'title': file_name,
                'updateTime': datetime.strptime(time_, '%Y-%m-%d'),
                'strUpdateTime': time_,
                'filePath': full_path,
            }
            feedback.append(data_)
    log.info(f'{mongoId}===反馈意见及回复采集成功')
    return feedback


def getInfo(obsOperate, id):
    url = f'http://reits.szse.cn/api/reits/projectrends/details?id={id}'
    try:
        data = getDataJson_(url)
    except:
        log.error(f'{id}===信息总列表获取失败')
        return {}
    baseInfo = getBaseInfo(data)
    is_insert = db_storage.find_one({'baseInfo.name': f"{baseInfo['name']}", 'exchange': '深圳证券交易所'})
    if is_insert:
        mongoId = is_insert['_id']
        strUpdateDate = is_insert['baseInfo']['strUpdateDate']
        if strUpdateDate == baseInfo['strUpdateDate']:
            log.info(f'{id}===已采集,不需要更新')
            return
        else:
            log.info(f'{id}===已采集,需要更新,正在更新中')
            createDate = is_insert['createDate']
            file = updateFile(obsOperate, data, str(mongoId), is_insert)
            feedback = updateFeedback(obsOperate, data, str(mongoId), is_insert)
            now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            dic_info = {
                'baseInfo': baseInfo,
                'file': file,
                'feedback': feedback,
                'exchange': '深圳证券交易所',
                'updateDate': now,
                'createDate': createDate,
            }
            try:
                db_storage.update_one({'_id': mongoId}, {'$set': dic_info})
                log.info(f'{id}===更新成功')
            except:
                log.error(f'{id}===更新失败')
    else:
        dic_info_ = {
            'baseInfo': baseInfo,
            'exchange': '深圳证券交易所'
        }
        try:
            db_storage.insert_one(dic_info_)
            log.error(f'{id}===初次保存成功')
        except:
            log.error(f'{id}===初次入库失败')
            return
        mongoId = db_storage.find_one({'baseInfo.name': f"{baseInfo['name']}", 'exchange': '深圳证券交易所'})['_id']
        file = getFile(obsOperate, data, str(mongoId))
        feedback = getFeedback(obsOperate, data, str(mongoId))
        if not baseInfo or file == 0 or feedback == 0:
            db_storage.delete_one({'_id': mongoId})
            log.error(f'{id}===数据采集不完整,初次入库数据已删除')
            return
        now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        dic_info = {
            'baseInfo': baseInfo,
            'file': file,
            'feedback': feedback,
            'exchange': '深圳证券交易所',
            'updateDate': now,  # 修改日期
            'createDate': now  # 创建日期
        }
        try:
            db_storage.update_one({'_id': mongoId}, {'$set': dic_info})
            log.info(f'{id}===保存成功')
        except:
            db_storage.delete_one({'_id': mongoId})
            log.error(f'{id}===二次入库失败,已删除')


def doJob(obsOperate):
    pageSize = getPageSize()
    log.info(f'共{pageSize}页')
    for page in range(pageSize):
        log.info(f'开始采集第{page + 1}页')
        try:
            data_json = getDataJson(page)
        except Exception as e:
            log.error(f'第{page + 1}页数据获取失败==={e}')
            continue
        for data_ in data_json:
            id = data_['prjid']
            getInfo(obsOperate, id)
            time.sleep(3)


if __name__ == '__main__':
    obsOperate = obsOperate(cursor_, cnx_, log)
    doJob(obsOperate)
    baseCore.close()
