import os
import time
from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup
from retry import retry

import BaseCore

baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()
headers = {
    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0',
    'X-Requested-With': 'XMLHttpRequest',
}

topic = 'research_center_fourth'
webname = '浙江省人民政府'

class Policy():
    def getrequest_soup(self, headers, url):
        req = requests.get(headers=headers, url=url)
        result = BeautifulSoup(req.content, 'html.parser')
        return result

    def getrequest_json(self, headers, url):
        req = requests.get(headers=headers, url=url)
        result = req.json()
        return result

    def requestPost(self, headers, url, payload):
        req = requests.post(headers=headers, url=url, data=payload)
        data_json = req.json()
        return data_json

    def requestPost_html(self, headers, url, payload):
        req = requests.post(headers=headers, url=url, data=payload)
        result = BeautifulSoup(req.content, 'html.parser')
        return result

    def deletep(self, soup, i, tag, attribute_to_delete, value_to_delete):
        # 查找带有指定属性的标签并删除
        tags = soup.find_all(tag, {attribute_to_delete: value_to_delete})
        for tag in tags[:i]:
            tag.decompose()

    def deletespan(self, td):
        spans = td.find_all('span')
        for span in spans:
            span.extract()  # 删除span标签

    def deletetag(self, td, tag):
        tags = td.find_all(tag)
        for tag_ in tags:
            tag_.extract()  # 删除指定标签

    def deletetext(self, soup, tag, text):  # 删除带有特定内容的标签
        tags = soup.find_all(tag)[:10]
        for tag_ in tags:
            text_ = tag_.text
            if text in text_:
                tag_.extract()

    def deletek(self, soup):
        # 删除空白标签（例如<p></p>、<p><br></p>, img、video、hr除外）
        for i in soup.find_all(lambda tag: len(tag.get_text()) == 0 and tag.name not in ["img", "video",
                                                                                         "br"] and tag.name != "br" or tag.get_text() == ' '):
            for j in i.descendants:
                if j.name in ["img", "video", "br"]:
                    break
            else:
                i.decompose()

    def paserUrl(self, html, listurl):
        # 获取所有的<a>标签和<img>标签
        if isinstance(html, str):
            html = BeautifulSoup(html, 'html.parser')

        links = html.find_all(['a', 'img'])
        # 遍历标签，将相对地址转换为绝对地址
        for link in links:
            if 'href' in link.attrs:
                link['href'] = urljoin(listurl, link['href'])
            elif 'src' in link.attrs:
                link['src'] = urljoin(listurl, link['src'])
        return html

    def attuributefile(self, file_name, file_href, num, publishDate):
        # 下载附件到本地，并上传文件服务器
        if '.pdf' in file_href or '.docx' in file_href or '.doc' in file_href or 'xls' in file_href or '.zip' in file_href \
                or '.rar' in file_href or '.ppt' in file_href or '.PDF' in file_href or '.DOC' in file_href \
                or '.XLS' in file_href or '.ZIP' in file_href or '.RAR' in file_href:

            category = os.path.splitext(file_href)[1]
            if category not in file_name:
                file_name = file_name + category
            try:
                retData = baseCore.uptoOBS(file_href, '', file_name)
            except:
                return '', ''
            if retData['state']:
                pass
            else:
                return '', ''
            try:
                att_id, full_path = baseCore.tableUpdate(retData, 'RETIs文件', file_name, num, publishDate)
                return att_id, full_path
            except:
                return '', ''
        else:
            return '', ''


policy = Policy()


def paserUrl(html, listurl):
    # 获取所有的<a>标签和<img>标签
    if isinstance(html, str):
        html = BeautifulSoup(html, 'html.parser')

    links = html.find_all(['a', 'img'])
    # 遍历标签，将相对地址转换为绝对地址
    for link in links:
        if 'href' in link.attrs:
            link['href'] = urljoin(listurl, link['href'])
        elif 'src' in link.attrs:
            link['src'] = urljoin(listurl, link['src'])
    return html


@retry(tries=3, delay=10)
def getPageSize():
    ip = baseCore.get_proxy()
    url = 'https://search.zj.gov.cn/jsearchfront/interfaces/cateSearch.do'
    data_post = {
        'websiteid': '330000000000000',
        'pg': '10',
        'p': '1',
        'tpl': '1569',
        'cateid': '372',
        'word': 'REITs',
        'checkError': '1',
        'isContains': '1',
        'q': 'REITs',
        'pos': 'content,filenumber',
        'sortType': '1',
    }
    req = requests.post(url, headers=headers, data=data_post, proxies=ip)
    req.encoding = req.apparent_encoding
    total = req.json()['total']
    if total % 10 == 0:
        pageSize = total // 10
    else:
        pageSize = total // 10 + 1
    req.close()
    return pageSize


@retry(tries=3, delay=10)
def getDataJson(page):
    # ip = baseCore.get_proxy()
    url = 'https://search.zj.gov.cn/jsearchfront/interfaces/cateSearch.do'
    data_post = {
        'websiteid': '330000000000000',
        'pg': '10',
        'p': f'{page}',
        'tpl': '1569',
        'cateid': '372',
        'word': 'REITs',
        'checkError': '1',
        'isContains': '1',
        'q': 'REITs',
        'pos': 'content,filenumber',
        'sortType': '1',
    }
    req = requests.post(url, headers=headers, data=data_post)
    req.encoding = req.apparent_encoding
    data_json = req.json()['result']
    req.close()
    return data_json


def getContent(url, publishDate, num):
    id_list = []
    req = requests.get(url, headers=headers)
    if 'weixin' in url:
        req.encoding = 'utf-8'
    else:
        req.encoding = req.apparent_encoding
    soup = BeautifulSoup(req.text, 'lxml')
    soup = paserUrl(soup, url)
    contentWithTag = soup.find('div', class_='box_wzy_ys')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='oh_main_cont_flbox_show_cont')
    if not contentWithTag:
        contentWithTag = soup.find('div',class_='article-content')
        try:
            contentWithTag.find('table',class_='xxgk_table').decompose()
        except:
            pass
    if not contentWithTag:
        contentWithTag = soup.find('div', attrs={'id': 'zoom'})
        try:
            contentWithTag.find('div', class_='audioBox').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='zcbox').decompose()
            div_list = soup.find_all('div', class_='yybb')
            for div in div_list:
                div.decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='fz_xx').decompose()
        except:
            pass
        try:
            contentWithTag.find('a', class_='zcjdlj').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='fenxiang').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='Interpretation').decompose()
        except:
            pass
        try:
            contentWithTag.find('a', class_='bmjd').decompose()
        except:
            pass
        try:
            contentWithTag.find('a', class_='tjlj').decompose()
        except:
            pass
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='g_content')
    if not contentWithTag:
        contentWithTag = soup.find('span', class_='zcjdlink')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='main_section')
        try:
            contentWithTag = contentWithTag.find('div', class_='main_section')
        except:
            pass
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='zoomnr')
    if not contentWithTag:
        contentWithTag = soup.find('div', attrs={'id': 'mainText'})
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='text')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='wz')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='news_content')
        try:
            contentWithTag.find('div', class_='ywlj').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='zcjd').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='tpjd').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='spjd').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='jgfzr').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='fzr').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='jgdz').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='lxfs').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='gkdh').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='zipcode').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='fax').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='mail').decompose()
        except:
            pass
        try:
            contentWithTag.find('div',class_='bgsj').decompose()
        except:
            pass
    if not contentWithTag:
        try:
            contentWithTag = soup.find('div', class_='mian').find('div', class_='article_text')
        except:
            contentWithTag = None
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='wenz')
    if not contentWithTag:
        # contentWithTag = soup.find('table', attrs={'id': 'word'})
        contentWithTag = soup.find('table', attrs={'id': 'inside'})
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='ewb-content')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='content-info-content')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='main-txt')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='zoom')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='showPage')
    if not contentWithTag:
        try:
            contentWithTag = soup.find_all('div', class_='content')[1]
            try:
                contentWithTag.find('div', class_='linke').decompose()
            except:
                contentWithTag = None
        except:
            pass
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='article')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='content')
        try:
            contentWithTag.find('div', class_='dy').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='con_top').decompose()
            contentWithTag.find('div', class_='flex_between').decompose()
        except:
            pass
        try:
            contentWithTag.find('div', class_='dqwz').decompose()
            contentWithTag.find('div', class_='top').decompose()
        except:
            pass
        try:
            contentWithTag.find('h4', class_='fr').decompose()
        except:
            pass
        try:
            contentWithTag.find('ul', class_='Fileclass').decompose()
            contentWithTag.find('h4').decompose()
        except:
            pass
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='main-body')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='articlePage_content')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='Gbc_Cm')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='zhengw')
    if not contentWithTag:
        contentWithTag = soup.find('div', attrs={'id': 'zhengw'})
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='xy-detail')
    if not contentWithTag:
        contentWithTag = soup.find('td', class_='bt_content')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='xy-detail')
    if not contentWithTag:
        contentWithTag = soup.find('div', attrs={'id': 'js_content'})
    if not contentWithTag:
        contentWithTag = soup.find('div', attrs={'id': 'cr'})
    if not contentWithTag:
        contentWithTag = soup.find('div', attrs={'id': 'art_c'})
    if not contentWithTag:
        contentWithTag = soup.find('article', class_='content_main')
    if not contentWithTag:
        contentWithTag = soup.find('div', attrs={'id': 'ivs_content'})
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='con_con')
        try:
            div_list = contentWithTag.find('div', class_='yybb')
            for div in div_list:
                div.decompose()
        except:
            pass
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='pic')
    if not contentWithTag:
        contentWithTag = soup.find('td', class_='bt_content')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='rich_media_content')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='xl_main_con')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='jh_xl_m2')
        try:
            contentWithTag.find('span', class_='jiedu-link-box').decompose()
        except:
            pass
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='nrEmit')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='details-content')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='zf-jd-nr')
    if not contentWithTag:
        contentWithTag = soup.find('div', class_='article-conter')
    if not contentWithTag:
        contentWithTag = soup.find('div',class_='class="rich_media_area_primary"')
    if not contentWithTag:
        contentWithTag = soup.find('body > div:nth-of-type(2) > div:nth-of-type(3) > div:nth-of-type(3)')
    if not contentWithTag:
        contentWithTag = soup.find('div',class_='detail-pic')
    if not contentWithTag:
        contentWithTag = soup.find('div',class_='mian')
    try:
        contentWithTag.find('video').decompose()
        contentWithTag = None
    except:
        pass
    try:
        scripts = contentWithTag.find_all('script')
        for script in scripts:
            script.decompose()
    except:
        pass
    try:
        styles = contentWithTag.find_all('style')
        for style in styles:
            style.decompose()
    except:
        pass
    if contentWithTag:
        pass
    else:
        log.info(f"内容未解析出来===={url}")
        return '','',[]
    a_list = contentWithTag.find_all('a')
    for a in a_list:
        href = a.get('href')
        if href:
            pass
        else:
            continue
        fj_title = a.text.strip().lstrip()
        category = os.path.splitext(href)[1]
        if category not in fj_title:
            fj_title = fj_title + category
        att_id, full_path = policy.attuributefile(fj_title, href, num, publishDate)
        if att_id:
            id_list.append(att_id)
            a['href'] = full_path
    content = contentWithTag.text
    return str(contentWithTag), content, id_list


def getDatas(page):
    data_json = getDataJson(page)
    num = 1
    for data_ in data_json:
        soup = BeautifulSoup(data_, 'lxml')
        title = soup.find('div', class_='titleWrapper').find('a').text.lstrip().strip().replace(' ',
                                                                                                '').replace(
            '\r\n', ' ')
        href = soup.find('div', class_='titleWrapper').find('a').get('href')
        href = href.split('url=')[1].split('.html')[0].replace('%3A', ':').replace('%2F', '/') + '.html'
        # 根据链接判重
        is_member = baseCore.r.sismember('REITs::' + webname, href)
        if is_member:
            continue
        try:
            info = soup.find('table', class_='fgwj_table_list').text
            organ = info.split('发布机构：')[1].split('成文日期：')[0].lstrip().strip()
            writtenDate = info.split('成文日期：')[1].lstrip().strip()
        except:
            organ = ''
            writtenDate = None
        origin = soup.find('div', class_='sourceTime').text.split('来源:')[1].split('时间:')[0].lstrip().strip().replace(
            ' ', '').replace(' ', '').replace('\r\n', '')
        publishDate = soup.find('div', class_='sourceTime').text.split('时间:')[1].lstrip().strip()
        contentWithTag, content, id_list = getContent(href, publishDate, num)
        if contentWithTag:
            pass
        else:
            continue
        num += 1
        time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        lang = baseCore.detect_language(content)
        dic_info = {
            'attachmentIds': id_list,
            'subjectId': '1729315113088765953',
            'lang': lang,
            'author': '',
            'content': content,
            'contentWithTag': contentWithTag,
            'deleteFlag': 0,
            'checkStatus': 1,
            'id': '1729315113088765953'+str(int(time.time())),
            'title': title,
            'publishDate': publishDate,
            'origin': origin,
            'sourceAddress': href,
            'writtenDate': writtenDate,
            'organ': organ,
            'topicClassification': '',
            'issuedNumber': '',
            'summary': '',
            'createDate': time_now,
            'sid': '1730472253306552321'
        }
        try:
            baseCore.sendkafka(dic_info, topic)
            baseCore.r.sadd('REITs::' + webname, href)
            log.info(f'{title}===完成')
        except:
            for att_id in id_list:
                baseCore.deliteATT(att_id)
            log.error(f'第{page}页==={title}===失败')
        time.sleep(5)


def doJob():
    pageSize = getPageSize()
    for page in range(1, pageSize + 1):
        getDatas(page)


if __name__ == '__main__':
    doJob()
    baseCore.close()
