import os
import re

import fitz
import openpyxl
import pandas as pd
import requests
from bs4 import BeautifulSoup
from datetime import datetime
import time

from openpyxl import load_workbook
from selenium import webdriver
from selenium.webdriver.chrome.service import Service

from urllib.parse import urljoin

import BaseCore
baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()
filepath = "data/"
topic = 'research_center_fourth'
class Policy():
    def getrequest_soup(self,headers,url):
        req = requests.get(headers=headers,url=url)
        result = BeautifulSoup(req.content,'html.parser')
        return result

    def getrequest_json(self,headers,url):
        req = requests.get(headers=headers,url=url)
        result = req.json()
        return result

    def requestPost(self,headers, url, payload):
        req = requests.post(headers=headers, url=url, data=payload)
        data_json = req.json()
        return data_json

    def requestPost_html(self,headers, url, payload):
        req = requests.post(headers=headers, url=url, data=payload)
        result = BeautifulSoup(req.content,'html.parser')
        return result

    def createDriver(self):
        # chrome_driver = r'D:\cmd100\chromedriver.exe'
        chrome_driver = r'F:\spider\cmd100\chromedriver.exe'
        path = Service(chrome_driver)
        chrome_options = webdriver.ChromeOptions()
        # chrome_options.binary_location = r'D:\Google\Chrome\Application\chrome.exe'
        chrome_options.binary_location = r'F:\spider\85\Google\Chrome\Application\chrome.exe'
        # 设置代理
        # proxy = "127.0.0.1:8080"  # 代理地址和端口
        # chrome_options.add_argument('--proxy-server=http://' + proxy)
        driver = webdriver.Chrome(service=path, chrome_options=chrome_options)
        return driver

    def deletep(self,soup,i,tag,attribute_to_delete,value_to_delete):
        # 查找带有指定属性的标签并删除
        tags = soup.find_all(tag, {attribute_to_delete: value_to_delete})
        for tag in tags[:i]:
            tag.decompose()

    def deletespan(self, td):
        spans = td.find_all('span')
        for span in spans:
            span.extract()  # 删除span标签

    def deletetag(self, td,tag):
        tags = td.find_all(tag)
        for tag_ in tags:
            tag_.extract()  # 删除指定标签

    def deletetext(self,soup, tag, text):# 删除带有特定内容的标签
        tags = soup.find_all(tag)[:10]
        for tag_ in tags:
            text_ = tag_.text
            if text in text_:
                tag_.extract()

    def deletek(self,soup):
        # 删除空白标签（例如<p></p>、<p><br></p>, img、video、hr除外）
        for i in soup.find_all(lambda tag: len(tag.get_text()) == 0 and tag.name not in ["img", "video", "br"] and tag.name != "br" or tag.get_text()==' '):
            for j in i.descendants:
                if j.name in ["img", "video", "br"]:
                    break
            else:
                i.decompose()

    def paserUrl(self,html, listurl):
        # 获取所有的<a>标签和<img>标签
        if isinstance(html, str):
            html = BeautifulSoup(html, 'html.parser')

        links = html.find_all(['a', 'img'])
        # 遍历标签，将相对地址转换为绝对地址
        for link in links:
            if 'href' in link.attrs:
                link['href'] = urljoin(listurl, link['href'])
            elif 'src' in link.attrs:
                link['src'] = urljoin(listurl, link['src'])
        return html

    def attuributefile(self,file_name,file_href,num,publishDate):
        # 下载附件到本地，并上传文件服务器
        if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xls' in file_href or '.zip' in file_href \
                or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
                or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:

            category = os.path.splitext(file_href)[1]
            if category not in file_name:
                file_name = file_name + category
            retData = baseCore.uptoOBS(file_href, '', file_name)
            if retData['state']:
                pass
            else:
                return '', ''
            att_id, full_path = baseCore.tableUpdate(retData, 'RETIs文件', file_name, num, publishDate)
            return att_id, full_path
        else:
            log.info('不是文件格式')
            return '', ''



    def downloadfile(self,file_href,path):
        response = requests.get(file_href)

        with open(path,"wb") as file:
            file.write(response.content)
        pass

    def createfile(self,file_path):
        file_exist = baseCore.check_excel_file(file_path)
        if file_exist:
            wb = load_workbook(file_path)
            return wb
        else:
            wb = openpyxl.Workbook()
            wb.save(file_path)
            log.info("Excel文件已创建")
            return wb

policy = Policy()
#国家发展和改革委员会 https://www.ndrc.gov.cn/xxgk/wjk/index.html?tab=all&qt=
def reform():
    headers = {
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Connection': 'keep-alive',
        'Host': 'fwfx.ndrc.gov.cn',
        'Origin': 'https://www.ndrc.gov.cn',
        'Referer': 'https://www.ndrc.gov.cn/',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-site',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
        'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"'
    }
    # DataList = []
    num = 0
    webname = '中华人民共和国国家发展和改革委员会'
    # path = 'data/国家改革发展委员会'
    # if not os.path.exists(path):
    #     os.makedirs(path)
    for page in range(1,3):
        url = f'https://fwfx.ndrc.gov.cn/api/query?qt=REITs&tab=all&page={page}&pageSize=20&siteCode=bm04000fgk&key=CAB549A94CF659904A7D6B0E8FC8A7E9&startDateStr=&endDateStr=&timeOption=0&sort=dateDesc'
        result = policy.getrequest_json(headers, url)
        data_list = result['data']['resultList']
        for info in data_list:
            num += 1
            id_list = []
            # info = data_list[1]
            publishDate_ = info['docDate']
            title = info['title']
            summary = info['summary'].replace('<em>','').replace('</em>','')
            newsUrl = info['url']

            # 根据链接判重
            is_member = baseCore.r.sismember('REITs::' + webname, newsUrl)
            if is_member:
                continue

            header = {
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
                'Accept-Encoding': 'gzip, deflate, br',
                'Accept-Language': 'zh-CN,zh;q=0.9',
                'Cache-Control': 'max-age=0',
                'Connection': 'keep-alive',
                'Cookie': 'Hm_lvt_6c8165462fd93121348afe212168341f=1699338341; yfx_c_g_u_id_10005970=_ck23110714254113251712738304141; http_waf_cookie=05e8486c-c47f-4927291823a10f5e24ceed45b1eaa3eb7354; SF_cookie_3=21321202; Hm_lpvt_6c8165462fd93121348afe212168341f=1699422316; yfx_f_l_v_t_10005970=f_t_1699338341317__r_t_1699412780356__v_t_1699422316031__r_c_1',
                'Host': 'www.ndrc.gov.cn',
                'Referer': 'https://www.ndrc.gov.cn/xxgk/wjk/index.html?tab=all&qt=',
                'Sec-Fetch-Dest': 'document',
                'Sec-Fetch-Mode': 'navigate',
                'Sec-Fetch-Site': 'same-origin',
                'Sec-Fetch-User': '?1',
                'Upgrade-Insecure-Requests': '1',
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
                'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
                'sec-ch-ua-mobile': '?0',
                'sec-ch-ua-platform': '"Windows"'
            }
            newssoup = policy.getrequest_soup(header, newsUrl)
            # print(newssoup)
            policy.paserUrl(newssoup, newsUrl)
            try:
                pubHao = ''
                source = ''

                try:
                    # article_con article_con_title
                    contentWithTag = newssoup.select('div[class="article_con article_con_notitle"]')[0]
                except:
                    try:
                        contentWithTag = newssoup.select('div[class="article_con article_con_title"]')[0]
                    except:
                        continue
                policy.deletek(contentWithTag)
                try:
                    for pubHao_ in newssoup.find_all('div', style='text-align: center;')[:10]:
                        # print(pubHao_)
                        if '〔' in pubHao_.text:
                            pubHao = pubHao_.text
                            print(pubHao)
                        else:
                            continue
                except:
                    pass
                try:
                    source = newssoup.select('div[class="ly laiyuantext"]>span')[0].text
                except:
                    pass

                policy.deletek(newssoup)
                try:
                    publishDate = newssoup.find('div',class_="article_con article_con_notitle").find_all('span')[-1].text
                except:
                    try:
                        publishDate = newssoup.find('div',class_="article_con article_con_title").find_all('span')[-1].text
                    except:
                        publishDate = None
                if publishDate:
                    pattern = r"\d{4}年\d{1,2}月\d{1,2}日"
                    match = re.match(pattern, publishDate)
                    if match:
                        date1 = datetime.strptime(publishDate, "%Y年%m月%d日")
                        publishDate = date1.strftime("%Y-%m-%d")
                        pass
                    else:
                        publishDate = None
                policy.deletep(contentWithTag, 3, 'div', 'style', 'text-align: center;')
                policy.deletek(contentWithTag)

                content = contentWithTag.text
                try:
                    policy.paserUrl(newssoup,newsUrl)
                    att = newssoup.find('div', class_='attachment_r')
                except:
                    att = ''
                if att:
                    for a in att.find_all('a'):
                        file_name = a.text.replace('.', '、')
                        if '<' in file_name or '>' in file_name:
                            file_name = file_name.replace('<', '').replace('>', '')
                        file_href = a['href']
                        category = os.path.splitext(file_href)[1]
                        if category in file_name:
                            pass
                        else:
                            file_name = file_name + category
                        att_id,full_path = policy.attuributefile(file_name,file_href,num,publishDate_)
                        if att_id:
                            id_list.append(att_id)
                            a['href'] = full_path

                    contentWithTag_str = str(contentWithTag) + str(newssoup.find('div', class_='attachment'))
                else:
                    contentWithTag_str = str(contentWithTag)

                time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                lang = baseCore.detect_language(content)
                dic_info = {
                    'attachmentIds': id_list,
                    'subjectId': '1729021803283533825',
                    'lang': lang,
                    'author': '',
                    'content': content,
                    'contentWithTag': contentWithTag_str,
                    'deleteFlag': 0,
                    'checkStatus': 1,
                    'id': '1729021803283533825'+str(int(time.time())),
                    'title': title,
                    'publishDate': publishDate_,
                    'origin': source,
                    'sourceAddress': newsUrl,
                    'writtenDate': publishDate,
                    'organ': '',
                    'topicClassification': '',
                    'issuedNumber': pubHao,
                    'summary': summary,
                    'createDate': time_now,
                    'sid': '1729029275400646658'
                }
                # DataList.append(dic_info)
                try:
                    baseCore.sendkafka(dic_info, topic)
                    baseCore.r.sadd('REITs::' + webname, newsUrl)
                    log.info(f'采集成功--{title}--{newsUrl}')
                except:
                    for att_id in id_list:
                        baseCore.deliteATT(att_id)
                # sheet_name = "国家发展和改革委员会"
                # if sheet_name in wb.sheetnames:
                #     log.info(f"{sheet_name}工作表已存在！")
                # else:
                #     # 创建新工作表
                #     wb.create_sheet(sheet_name)
                #     print(f"{sheet_name}新工作表创建完成！")
                # # 保存Excel文件
                # wb.save(file_path)
                #
                # baseCore.writerToExcel(DataList, file_path, sheet_name)
            except Exception as e:
                log.info(f"error！！！{newsUrl}")
                log.info(e)
        log.info(f'====第{page}页====处理结束，已采集{num}条数据=================')

#证券期货 https://neris.csrc.gov.cn/falvfagui/multipleFindController/indexJsp
def zhengquanqihuo():
    headers = {
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Connection': 'keep-alive',
        'Content-Length': '140',
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'Cookie': 'JSESSIONID=D18F2DF64366325AC0A50E09AA98EE84',
        'Host': 'neris.csrc.gov.cn',
        'Origin': 'https://neris.csrc.gov.cn',
        'Referer': 'https://neris.csrc.gov.cn/falvfagui/multipleFindController/indexJsp',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-origin',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
        'X-Requested-With': 'XMLHttpRequest',
        'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"'
    }
    url = 'https://neris.csrc.gov.cn/falvfagui/multipleFindController/solrSearch'
    payload = {
        'pageNo': '1',
        'secFutrsLawName': '',
        'body': '"REITs"',
        'lawPubOrgName': '',
        'titleQry': '',
        'keyQry': 'REITs、',
        'fileno': '',
        'pubDate_from': '',
        'pubDate_thru': '',
        'nbr': '1',
        'isLike': '0'
    }
    result = policy.requestPost(headers, url, payload)
    pageUtil = result['pageUtil']
    total = pageUtil['rowCount']
    page_size = pageUtil['pageSize']
    Max_page = int(total / page_size)
    # DataList = []
    num = 0
    webname = '证券期货法规数据库系统'
    # path = 'data/证监会'
    # if not os.path.exists(path):
    #     os.makedirs(path)
    for page in range(0, Max_page+1):
        payload_page = {
            'pageNo': page + 1,
            'secFutrsLawName': '',
            'body': '"REITs"',
            'lawPubOrgName': '',
            'titleQry': '',
            'keyQry': 'REITs、',
            'fileno': '',
            'pubDate_from': '',
            'pubDate_thru': '',
            'nbr': '1',
            'isLike': '0'
        }
        data_page = policy.requestPost(headers, url, payload_page)
        info_list = data_page['pageUtil']['pageList']
        for info in info_list:
            id_list = []
            num += 1
            try:
                title = info['secFutrsLawName']
                pubHao = info['fileno']
                source = info['lawPubOrgName']
                publish_ = datetime.strptime(info['secFutrsLawVersion'], "%Y%m%d")
                publishDate = datetime.strftime(publish_, "%Y-%m-%d")
                # print(publishDate)
                secFutrsLawId = info['secFutrsLawId']
                newsUrl = f'https://neris.csrc.gov.cn/falvfagui/rdqsHeader/mainbody?navbarId=3&secFutrsLawId={secFutrsLawId}&body=REITs'

                # 根据链接判重
                is_member = baseCore.r.sismember('REITs::' + webname, newsUrl)
                if is_member:
                    continue

                browser = policy.createDriver()
                browser.get(newsUrl)
                time.sleep(1)
                page_source = browser.page_source
                newssoup = BeautifulSoup(page_source, 'html.parser')
                policy.paserUrl(newssoup,newsUrl)
                # print(newssoup)
                contentWithTag = newssoup.find('div', class_='law_text mainBody catalog')
                content = contentWithTag.text.replace('显示注释', '')
                # print(content)
                contentWithTag_str = str(contentWithTag)

                time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                lang = baseCore.detect_language(content)
                dic_info = {
                    'attachmentIds': id_list,
                    'subjectId': '1729021803283533825',
                    'lang': lang,
                    'author': '',
                    'content': content,
                    'contentWithTag': contentWithTag_str,
                    'deleteFlag': 0,
                    'checkStatus': 1,
                    'id': '1729021803283533825'+str(int(time.time())),
                    'title': title,
                    'publishDate': publishDate,
                    'origin': source,
                    'sourceAddress': newsUrl,
                    'writtenDate': publishDate,
                    'organ': source,
                    'issuedNumber': pubHao,
                    'summary': '',
                    'topicClassification': '',
                    'createDate': time_now,
                    'sid': '1729030277461815298'
                }
                try:
                    baseCore.sendkafka(dic_info, topic)
                    baseCore.r.sadd('REITs::' + webname, newsUrl)
                    log.info(f'采集成功--{title}--{newsUrl}')
                except:
                    for att_id in id_list:
                        baseCore.deliteATT(att_id)
                # DataList.append(dic_info)
                # sheet_name = "证监会"
                # if sheet_name in wb.sheetnames:
                #     log.info(f"{sheet_name}工作表已存在！")
                # else:
                #     # 创建新工作表
                #     wb.create_sheet(sheet_name)
                #     print(f"{sheet_name}新工作表创建完成！")
                # # 保存Excel文件
                # wb.save(file_path)
                #
                # baseCore.writerToExcel(DataList, file_path, sheet_name)
            except Exception as e:
                log.info(f"error！！！{num}")
                log.info({e})
        log.info(f'====第{page}页====处理结束，已采集{num}条数据=================')

#深圳交易所 http://www.szse.cn/lawrules/index.html

#上海交易所 http://www.sse.com.cn/home/search/index.shtml?webswd=REITs
def sse():
    # url = 'http://query.sse.com.cn/search/getESSearchDoc.do?page=0&limit=10&publishTimeEnd=&publishTimeStart=&orderByDirection=DESC&orderByKey=score&searchMode=fuzzy&spaceId=3&keyword=REITs&siteName=sse&keywordPosition=title%2Cpaper_content&channelId=10001&channelCode=8640%2C8641%2C8642%2C8643%2C8644%2C8645%2C8646%2C8647%2C8648%2C8649%2C8650%2C8651%2C8652%2C8653%2C8654%2C8655%2C8656%2C8657%2C8658%2C8659%2C8660%2C8661%2C8685%2C9348%2C12632%2C12768%2C12769%2C12770%2C12771%2C12772%2C12773%2C12774%2C12775%2C12776%2C12777%2C12778%2C12779%2C12780%2C12781%2C12782%2C12783%2C12784%2C12785%2C12786%2C12787%2C12788%2C12789%2C12790%2C12791%2C12792%2C12793%2C12794%2C12795%2C12796%2C12797%2C12798%2C12799%2C12800%2C12801%2C12802%2C12803%2C12804%2C12805%2C12806%2C12807%2C12808%2C12809%2C12810%2C12811%2C12812%2C13061%2C13282%2C13283%2C13284%2C13285%2C13286%2C13287%2C13288%2C13289%2C13294%2C13364%2C13365%2C13366%2C13367%2C14595%2C14596%2C14597%2C14598%2C14599%2C14600%2C14601%2C14602%2C14603%2C14604%2C14605%2C14606&trackId=50619067167713018335655119683810&_=1699508921761'
    url = 'http://query.sse.com.cn/search/getESSearchDoc.do?page=0&limit=10&publishTimeEnd=&publishTimeStart=&orderByDirection=DESC&orderByKey=score&searchMode=fuzzy&spaceId=3&keyword=REITs&siteName=sse&keywordPosition=title%2Cpaper_content&channelId=10001&channelCode=8640%2C8641%2C8642%2C8643%2C8644%2C8645%2C8646%2C8647%2C8648%2C8649%2C8650%2C8651%2C8652%2C8653%2C8654%2C8655%2C8656%2C8657%2C8658%2C8659%2C8660%2C8661%2C12632&trackId=00752019013296307464953343505659&_=1703469889542'
    headers = {
        'Accept': '*/*',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Connection': 'keep-alive',
        'Cookie': 'ba17301551dcbaf9_gdp_user_key=; ba17301551dcbaf9_gdp_session_id=878c2669-93f0-43bd-91c1-cc30ca7136ef; gdp_user_id=gioenc-9a36dgc8%2C6b5d%2C5265%2Ccdc5%2C2ea193d9g222; ba17301551dcbaf9_gdp_session_id_878c2669-93f0-43bd-91c1-cc30ca7136ef=true; ba17301551dcbaf9_gdp_sequence_ids={%22globalKey%22:28%2C%22VISIT%22:2%2C%22PAGE%22:2%2C%22CUSTOM%22:17%2C%22VIEW_CLICK%22:10}',
        'Host': 'query.sse.com.cn',
        'Referer': 'http://www.sse.com.cn/',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'
    }
    result = policy.getrequest_json(headers,url)
    total_page = result['data']['totalPage']
    # DataList = []
    num = 0
    webname = '上海证券交易所'
    # path = 'data/上海交易所'
    # if not os.path.exists(path):
    #     os.makedirs(path)
    for page in range(0, int(total_page)):
        t = int(time.time())
        url_page = f'http://query.sse.com.cn/search/getESSearchDoc.do?page={page}&limit=10&publishTimeEnd=&publishTimeStart=&orderByDirection=DESC&orderByKey=score&searchMode=fuzzy&spaceId=3&keyword=REITs&siteName=sse&keywordPosition=title%2Cpaper_content&channelId=10001&channelCode=8640%2C8641%2C8642%2C8643%2C8644%2C8645%2C8646%2C8647%2C8648%2C8649%2C8650%2C8651%2C8652%2C8653%2C8654%2C8655%2C8656%2C8657%2C8658%2C8659%2C8660%2C8661%2C12632&trackId=00752019013296307464953343505659&_={t}'
        data = policy.getrequest_json(headers, url_page)
        newslist = data['data']['knowledgeList']
        # if newslist:
        #     pass
        # else:
        #     continue
        # print(newslist)
        for news in newslist:
            num += 1
            id_list = []
            title = news['title'].replace("<em>",'').replace('</em>','')
            publishDate = news['createTime']
            # print(newsUrl)
            summary = news['rtfContent']
            source = news['spaceName']
            header = {
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
                'Accept-Encoding': 'gzip, deflate',
                'Accept-Language': 'zh-CN,zh;q=0.9',
                'Cache-Control': 'max-age=0',
                'Connection': 'keep-alive',
                'Cookie': 'ba17301551dcbaf9_gdp_user_key=; ba17301551dcbaf9_gdp_session_id=878c2669-93f0-43bd-91c1-cc30ca7136ef; gdp_user_id=gioenc-9a36dgc8%2C6b5d%2C5265%2Ccdc5%2C2ea193d9g222; ba17301551dcbaf9_gdp_session_id_878c2669-93f0-43bd-91c1-cc30ca7136ef=true; VISITED_MENU=%5B%228307%22%5D; seecookie=REITs; home-search-scroll=; ba17301551dcbaf9_gdp_sequence_ids={%22globalKey%22:33%2C%22VISIT%22:2%2C%22PAGE%22:3%2C%22CUSTOM%22:18%2C%22VIEW_CLICK%22:13}',
                'Host': 'www.sse.com.cn',
                'Referer': 'http://www.sse.com.cn/home/search/index.shtml?webswd=REITs',
                'Upgrade-Insecure-Requests': '1',
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'
            }
            newsUrl = 'http://www.sse.com.cn' + news['extend'][4]['value']

            # 根据链接判重
            is_member = baseCore.r.sismember('REITs::' + webname, newsUrl)
            if is_member:
                continue
            try:
                if '.pdf' in newsUrl:
                    # pass
                    content = ''
                    response = requests.get(newsUrl, timeout=20)
                    with fitz.open(stream=response.content, filetype='pdf') as doc:
                        for page_ in doc.pages():
                            content += page_.get_text()
                    file_href = newsUrl
                    file_name = title

                    att_id, full_path = policy.attuributefile(title, newsUrl, num, publishDate)
                    if att_id:
                        id_list.append(att_id)
                    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                    lang = baseCore.detect_language(content)
                    dic_info = {
                        'attachmentIds':id_list,
                        'subjectId': '1729021803283533825',
                        'lang': lang,
                        'author': '',
                        'content': content,
                        'contentWithTag': '',
                        'deleteFlag': 0,
                        'checkStatus': 1,
                        'id': '1729021803283533825'+str(int(time.time())),
                        'title': title,
                        'publishDate': publishDate,
                        'origin': source,
                        'sourceAddress': newsUrl,
                        'writtenDate': None,
                        'organ': '',
                        'topicClassification': '',
                        'issuedNumber': '',
                        'summary': summary,
                        'createDate': time_now,
                        'sid': '1729035244826374145'
                    }
                    # DataList.append(dic_info)
                    try:
                        baseCore.sendkafka(dic_info, topic)
                        baseCore.r.sadd('REITs::' + webname, newsUrl)
                        log.info(f'采集成功--{title}--{newsUrl}')
                    except:
                        for att_id in id_list:
                            baseCore.deliteATT(att_id)
                else:
                    newssoup = policy.getrequest_soup(header, newsUrl)
                    # print(newssoup)
                    policy.paserUrl(newssoup, newsUrl)
                    content_ = newssoup.find('div', class_='allZoom')
                    # print(content_)
                    # #  将链接替换为绝对路径
                    contentWithTag = policy.paserUrl(content_, newsUrl)
                    try:
                        pubHao = contentWithTag.find('p',style='text-align: center;').text.strip(' ')
                        if '〔' in pubHao:
                            pass
                        else:
                            pubHao = ''
                    except:
                        pubHao = ''
                    # print(contentWithTag)
                    content = contentWithTag.text

                    fujian_list = contentWithTag.find_all('a')

                    for fujian in fujian_list:
                        try:
                            file_href = fujian['href']
                        except:
                            continue
                        file_name = fujian.text.strip(' ')
                        category = os.path.splitext(file_href)[1]
                        if category in file_name:
                            pass
                        else:
                            file_name = file_name + category

                        att_id, full_path = policy.attuributefile(file_name, file_href, num, publishDate)
                        if att_id:
                            id_list.append(att_id)
                            fujian['href'] = full_path

                    contentWithTag_str = str(contentWithTag)
                    # print(contentWithTag_str)
                    lang = baseCore.detect_language(content)
                    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                    dic_info = {
                        'attachmentIds':id_list,
                        'subjectId': '1729021803283533825',
                        'lang': lang,
                        'author': '',
                        'content': content,
                        'contentWithTag': contentWithTag_str,
                        'deleteFlag': 0,
                        'checkStatus': 1,
                        'id': '1729021803283533825'+str(int(time.time())),
                        'title': title,
                        'publishDate': publishDate,
                        'origin': source,
                        'sourceAddress': newsUrl,
                        'writtenDate': None,
                        'organ': '',
                        'issuedNumber': pubHao,
                        'summary': summary,
                        'createDate': time_now,
                        'sid': '1729035244826374145'
                    }
                    try:
                        baseCore.sendkafka(dic_info, topic)
                        baseCore.r.sadd('REITs::' + webname, newsUrl)
                        log.info(f'采集成功--{title}--{newsUrl}')
                    except:
                        for att_id in id_list:
                            baseCore.deliteATT(att_id)
            except Exception as e:
                log.info(f"error！！！{newsUrl}===={title}")
                log.info(e)
        log.info(f'====第{page}页====处理结束，================')


# 河北省人民政府
def hebei():
    # path = 'data/河北省人民政府'
    # if not os.path.exists(path):
    #     os.makedirs(path)
    num = 0
    webname = '河北省人民政府'
    url = "https://www.hebei.gov.cn/search/pcRender?pageId=b97a38833f7343cebc31dec44544f684"
    appNames = ['信息公开', '热点专题']
    for appName in appNames:
        payload = {'qAnd': ' ',
                   'qOr': ' ',
                   'qAll': ' ',
                   'qNot': ' ',
                   'startTime': ' ',
                   'endTime': ' ',
                   'advSearch': ' ',
                   'originalSearchUrl': ' /search/pcRender?pageId=b97a38833f7343cebc31dec44544f684',
                   'originalSearch': ' ',
                   'app': ' 20c723b3a36e4906b0d91e6950d3dc29,8b157f193fb54ea7837d6380a37bb84a,0ad7369c794e4b2fbd6a4e76f9b84e9c,47fb4bc5c08d49d3b937c56c7960a909,9f54f8001d8747e4826d542fedcc6abc,b42baf238f43435ea7f796bec4ef7592,c943f166fb9042d288743397b12978fc,4b2050e6bb5d48dc9b200385dd99b4e3,7b5b083a6d254960ab34e34009e7e8d7,aa9d0848dcb84e8b919fd02b2da090b4,54e1a38a0e2846a4bc60258af5ced450,b88b6ee476494a16b66ea9cacc0456ee,4d0e00783a2e4037a6d3bdcd1fe98fb1,a8cb58e7494e4ae4a682b0e79df63dc6,f70c53427500439cbdeee467c5a185a6,d3f6aaca16c54e7b8626993314ad27b7,4d63955d8ec441018e8fddc6131997b0',
                   'searchArea': ' ',
                   'appName': appName,
                   'sr': ' score desc',
                   'advtime': ' ',
                   'advrange': ' ',
                   'articleType': ' ',
                   'siteId': ' ',
                   'siteName': ' ',
                   'ext': ' ',
                   'pNo': ' 1',
                   'deviceType': ' pc',
                   'q2': ' ',
                   'q': ' REITs'}

        headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Cache-Control': 'max-age=0',
            'Connection': 'keep-alive',
            'Content-Length': '907',
            'Content-Type': 'application/x-www-form-urlencoded',
            'Cookie': 'aisearchbehavior=42b33c1f2d22475bb571093346193219; JSESSIONID=251311215A6447AE509141936F4569D4; arialoadData=true',
            'Host': 'www.hebei.gov.cn',
            'Origin': 'https://www.hebei.gov.cn',
            'Referer': 'https://www.hebei.gov.cn/search/pcRender?pageId=b97a38833f7343cebc31dec44544f684',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'same-origin',
            'Sec-Fetch-User': '?1',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
            'sec-ch-ua': '"Google Chrome";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"Windows"'
        }
        soup_ = policy.requestPost_html(headers, url, payload)
        # 第一次请求获取页数
        pages = int(soup_.find('span',class_='default-result-tolal-records').find('span').text)
        # DataList = []
        for page in range(1, pages+1):
            payload_page = {
                    'qAnd': ' ',
                   'qOr': ' ',
                   'qAll': ' ',
                   'qNot': ' ',
                   'startTime': ' ',
                   'endTime': ' ',
                   'advSearch': ' ',
                   'originalSearchUrl': ' /search/pcRender?pageId=b97a38833f7343cebc31dec44544f684',
                   'originalSearch': ' ',
                   'app': ' 20c723b3a36e4906b0d91e6950d3dc29,8b157f193fb54ea7837d6380a37bb84a,0ad7369c794e4b2fbd6a4e76f9b84e9c,47fb4bc5c08d49d3b937c56c7960a909,9f54f8001d8747e4826d542fedcc6abc,b42baf238f43435ea7f796bec4ef7592,c943f166fb9042d288743397b12978fc,4b2050e6bb5d48dc9b200385dd99b4e3,7b5b083a6d254960ab34e34009e7e8d7,aa9d0848dcb84e8b919fd02b2da090b4,54e1a38a0e2846a4bc60258af5ced450,b88b6ee476494a16b66ea9cacc0456ee,4d0e00783a2e4037a6d3bdcd1fe98fb1,a8cb58e7494e4ae4a682b0e79df63dc6,f70c53427500439cbdeee467c5a185a6,d3f6aaca16c54e7b8626993314ad27b7,4d63955d8ec441018e8fddc6131997b0',
                   'searchArea': ' ',
                   'appName': appName,
                   'sr': ' score desc',
                   'advtime': ' ',
                   'advrange': ' ',
                   'articleType': ' ',
                   'siteId': ' ',
                   'siteName': ' ',
                   'ext': ' ',
                   'pNo': str(page),
                   'deviceType': ' pc',
                   'q2': ' ',
                   'q': ' REITs'}
            soup = policy.requestPost_html(headers, url, payload_page)

            list_news = soup.find_all('div',class_='szf-data-tpl1-item')
            for news in list_news:
                num += 1
                id_list = []
                title = news.find('h3').text

                summary = news.find('div').find('p', class_='txtCon').text
                publishDate_ = news.find('div').find('p', class_='dates').text.replace('发布日期：', '').replace('\n', '')
                date1 = datetime.strptime(publishDate_, "%Y年%m月%d日")
                publishDate = date1.strftime("%Y-%m-%d")
                news_href = news.find('div').find('p', class_='txtCon').find('a')['href']
                # news_href = 'http://info.hebei.gov.cn//hbszfxxgk/6898876/7026469/7026511/7026506/7033297/index.html'
                try:
                    # 根据链接判重
                    is_member = baseCore.r.sismember('REITs::' + webname, news_href)
                    if is_member:
                        continue

                    news_req = requests.get(news_href, headers)
                    news_soup = BeautifulSoup(news_req.content, 'html.parser')
                    policy.paserUrl(news_soup, news_href)

                    writeDate = None
                    pub_hao = ''
                    source = ''
                    content = ''
                    pub_origin = ''
                    try:
                        content = news_soup.find('div', id='zoom').text
                        contentWithTag = news_soup.find('div', id='zoom')
                        try:
                            source = news_soup.find('div', class_='article_tit').find('li', class_='xl_laiyuan').text
                        except:
                            source = ''

                        try:
                            info_ = news_soup.find('div',class_='xxgk_bmxl')
                            policy.deletetag(info_, 'strong')
                            policy.deletek(info_)
                            info_list = info_.find_all('td')
                            pub_origin = info_list[1].text
                            pub_hao = info_list[2].text
                        except:
                            # 处理空标签
                            policy.deletek(news_soup)
                            p_list = news_soup.find_all('p')
                            for p in p_list:
                                text_pubhao = p.text
                                if '号' in text_pubhao and '〔' in text_pubhao:
                                    pattern = r"冀政办字〔\d+〕\d+号"
                                    match = re.search(pattern, text_pubhao)
                                    if match:
                                        pub_hao = match.group(0)
                                        break
                                else:
                                    continue

                                writeDate_ = p.text
                                pattern = r"\d{4}年\d{1,2}月\d{1,2}日"
                                match = re.search(pattern, writeDate_)
                                if match:
                                    writeDate1 = match.group(0)
                                    date2 = datetime.strptime(writeDate1, "%Y年%m月%d日")
                                    writeDate = date2.strftime("%Y-%m-%d")
                                    break
                                else:
                                    continue
                    except:
                        try:
                            contentWithTag = news_soup.find('div',class_='xxgk_gfxwjk_xqy-wznr')
                            content = news_soup.find('div',class_='xxgk_gfxwjk_xqy-wznr').text
                            info = news_soup.find('div', class_='xxgk_gfxwjk-xqy-touxx')
                            policy.deletespan(info)
                            pub_hao = info.find('p', class_='xxgk_gfxwjk-xqy-touxx4').text
                            pub_origin = info.find('p', class_='xxgk_gfxwjk-xqy-touxx3').text
                            writeDate = info.find('p', class_='xxgk_gfxwjk-xqy-touxx5').text
                        except:
                            pass
                    # 附件：
                    try:
                        fujian_href = contentWithTag.find_all('a')

                        for file_href_ in fujian_href:
                            file_href = file_href_['href']
                            file_name = file_href_.text
                            category = os.path.splitext(file_href)[1]
                            if category in file_name:
                                pass
                            else:
                                file_name = file_name + category
                            att_id, full_path = policy.attuributefile(file_name, file_href, num, publishDate)
                            if att_id:
                                id_list.append(att_id)
                                file_href_['href'] = full_path
                        contentWithTag_str = str(contentWithTag)
                    except Exception as e:
                        contentWithTag_str = str(contentWithTag)

                    if content == '':
                        continue
                    time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                    lang = baseCore.detect_language(content)
                    dic_info = {
                        'attachmentIds':id_list,
                        'subjectId': '1729315113088765953',
                        'lang': lang,
                        'author': '',
                        'content': content,
                        'checkStatus': 1,
                        'deleteFlag': 0,
                        'id': '1729315113088765953'+str(int(time.time())),
                        'contentWithTag': contentWithTag_str,
                        'title': title.replace('\n', ''),
                        'publishDate': publishDate,
                        'origin': source,
                        'sourceAddress': news_href,
                        'writtenDate': writeDate,
                        'organ': pub_origin,
                        'issuedNumber': pub_hao,
                        'summary': summary.replace('\n', ''),
                        'createDate': time_now,
                        'sid': '1729041576348274689',
                    }
                    # print(dic_info)
                    try:
                        baseCore.sendkafka(dic_info, topic)
                        baseCore.r.sadd('REITs::' + webname, news_href)
                        log.info(f'采集成功--{title}--{news_href}')
                    except:
                        for att_id in id_list:
                            baseCore.deliteATT(att_id)
                except Exception as e:
                    log.info(f"error！！！{news_href}")
                    log.info(e)
            log.info(f'====第{page}页====处理结束，已采集{num}条数据=================')

# 贵州省人民政府
def guizhou():
    url = "https://www.guizhou.gov.cn/irs/front/search"
    num = 0
    # path = 'data/贵州省人民政府'
    # if not os.path.exists(path):
    #     os.makedirs(path)
    # DataList = []
    webname = '贵州省人民政府'
    payload = "{\"tenantId\":\"186\",\"configTenantId\":\"\",\"tenantIds\":\"\",\"searchWord\":\"REITs\",\"historySearchWords\":[\"REITs\"],\"dataTypeId\":\"965\",\"orderBy\":\"related\",\"searchBy\":\"all\",\"appendixType\":\"\",\"granularity\":\"ALL\",\"beginDateTime\":\"\",\"endDateTime\":\"\",\"isSearchForced\":0,\"filters\":[],\"pageNo\":1,\"pageSize\":9}"
    headers = {
        'Accept': 'application/json, text/javascript, */*; q=0.01',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
        'Connection': 'keep-alive',
        'Content-Length': '291',
        'Content-Type': 'application/json',
        'Cookie': 'SESSION=MGY2NWQ3NjctZTNhZC00OTJhLWIzNGQtMDI1MmQ5MWVlZmNm; _trs_uv=lp15qktj_367_a56u; _trs_ua_s_1=lp15qktj_367_lac; yfx_c_g_u_id_10000921=_ck23111620182819813554574558557; yfx_f_l_v_t_10000921=f_t_1700137108976__r_t_1700137108976__v_t_1700137108976__r_c_0; arialoadData=false',
        'Host': 'www.guizhou.gov.cn',
        'Origin': 'https://www.guizhou.gov.cn',
        'Referer': 'https://www.guizhou.gov.cn/so/search.shtml?tenantId=186&tenantIds=&configTenantId=&searchWord=REITs&dataTypeId=965&sign=6bd8592c-2e19-4f22-ae6d-f129f729e795',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-origin',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0',
        'X-Requested-With': 'XMLHttpRequest',
        'sec-ch-ua': '"Microsoft Edge";v="119", "Chromium";v="119", "Not?A_Brand";v="24"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"'
    }
    jsonData = policy.requestPost(headers, url, payload)
    result_list = jsonData['data']['middle']["list"]
    for datainfo in result_list:
        num += 1
        id_list = []
        title = datainfo['title']
        publishDate = datainfo['time']
        source = datainfo['source']
        summary = datainfo['content']
        newsUrl = datainfo['url']

        # 根据链接判重
        is_member = baseCore.r.sismember('REITs::' + webname, newsUrl)
        if is_member:
            continue
        try:
            soup = policy.getrequest_soup(headers, newsUrl)
            # print(soup)
            policy.paserUrl(soup, newsUrl)
            pub_hao_ = soup.find('head').find('title').text
            start_index = pub_hao_.find("(") + 1
            end_index = pub_hao_.find(")")
            pub_hao = pub_hao_[start_index:end_index]
            # print(pub_hao)
            # 删除包含特定字段的标签

            contentWithTag = soup.find('div', class_='Zoom Box')
            policy.deletetext(contentWithTag, 'p', title)
            policy.deletetext(contentWithTag, 'p', pub_hao)
            content = contentWithTag.text

            # 附件：
            try:
                fujian_href = contentWithTag.find_all('a')

                for file_href_ in fujian_href:
                    file_href = file_href_['href']
                    file_name = file_href_.text
                    category = os.path.splitext(file_href)[1]
                    if category in file_name:
                        pass
                    else:
                        file_name = file_name + category
                    att_id, full_path = policy.attuributefile(file_name, file_href, num, publishDate)
                    if att_id:
                        id_list.append(att_id)
                        file_href_['href'] = full_path
            except:
                pass
            contentWithTag_str = str(contentWithTag)

            time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
            lang = baseCore.detect_language(content)
            dic_info = {
                'attachmentIds':id_list,
                'subjectId': '1729315113088765953',
                'lang': lang,
                'author': '',
                'content': content,
                'contentWithTag': contentWithTag_str,
                'deleteFlag': 0,
                'checkStatus': 1,
                'id': '1729315113088765953'+str(int(time.time())),
                'title': title.replace('\n', ''),
                'publishDate': publishDate,
                'source': source,
                'sourceAddress': newsUrl,
                'writtenDate': publishDate,
                'organ': '',
                'issuedNumber': pub_hao,
                'summary': summary.replace('\n', '').replace('<em>', '').replace('</em>', ''),
                'createDate': time_now,
                'sid': '1729046185945182210',
            }
            # print(dic_info)
            try:
                baseCore.sendkafka(dic_info, topic)
                baseCore.r.sadd('REITs::' + webname, newsUrl)
                log.info(f'采集成功--{title}--{newsUrl}')
            except:
                for att_id in id_list:
                    baseCore.deliteATT(att_id)
        except Exception as e:
            log.info(f"error！！！{newsUrl}")
            log.info(e)
    log.info(f'====处理结束，已采集{num}条数据=================')



if __name__=="__main__":
    # file_path = f'data/REITs贵州省人民政府.xlsx'
    # wb = policy.createfile(file_path)
    reform()
    # shenzhen()
    zhengquanqihuo()
    try:
        sse()
    except:
        pass
    hebei()
    guizhou()

# zhengquanqihuo()