import os
import time
from urllib.parse import urljoin

import requests
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

import BaseCore
baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()

from reits import Policy
policy = Policy()


topic = 'research_center_fourth'
webname = '深圳证券交易所'
headers = {
    'Accept': 'application/json, text/javascript, */*; q=0.01',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,zh-TW;q=0.9,zh;q=0.8',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive',
    'Content-Type': 'application/x-www-form-urlencoded',
    'Host': 'www.szse.cn',
    'Origin': 'http://www.szse.cn',
    'Pragma': 'no-cache',
    # 'Referer': 'http://www.szse.cn/lawrules/search/index.html?rulekeyword=REITs&channelCode=["rules","csrcrules","szseBussrules","memorandumServicedirect","publicadvice","lawruleSearch"]&range=content&searchtype=0',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0',
    'X-Request-Type': 'ajax',
    'X-Requested-With': 'XMLHttpRequest',
}


def paserUrl(html, listurl):
    # 获取所有的<a>标签和<img>标签
    if isinstance(html, str):
        html = BeautifulSoup(html, 'html.parser')

    links = html.find_all(['a', 'img'])
    # 遍历标签，将相对地址转换为绝对地址
    for link in links:
        if 'href' in link.attrs:
            link['href'] = urljoin(listurl, link['href'])
        elif 'src' in link.attrs:
            link['src'] = urljoin(listurl, link['src'])
    return html


def getFjContent(url):
    ip = baseCore.get_proxy()
    session = requests.session()
    session.get('http://www.szse.cn/',headers=headers,proxies=ip)
    req = session.get(url)
    req.encoding = req.apparent_encoding
    content = req.content
    session.close()
    return content


def getContent(url, publishDate, num, id_list):
    num += 1
    ip = baseCore.get_proxy()
    req = requests.get(url, headers=headers, proxies=ip)
    req.encoding = req.apparent_encoding
    soup = BeautifulSoup(req.text, 'html.parser')
    soup = paserUrl(soup, 'http://www.szse.cn/')
    contentWithTag = soup.find('div', class_='des-content')
    a_list = contentWithTag.find_all('a')

    for a in a_list:
        fj_href = a.get('href')
        if not fj_href:
            continue

        fj_title = a.text.lstrip().strip()
        category = os.path.splitext(fj_href)[1]
        if category not in fj_title:
            fj_title = fj_title + category

        # 上传附件至obs
        att_id,full_path = policy.attuributefile(fj_title, fj_href, num, publishDate)
        if att_id:
            id_list.append(att_id)
            a['href'] = full_path

    try:
        scripts = contentWithTag.find_all('script')
        for script in scripts:
            script.decompose()
    except:
        pass
    try:
        styles = contentWithTag.find_all('style')
        for style in styles:
            style.decompose()
    except:
        pass
    pub_hao = contentWithTag.find('p').text.lstrip().strip()
    content = contentWithTag.text.lstrip().strip()
    return pub_hao, content, id_list,contentWithTag


def doJob():
    # if not os.path.exists('./相关政策/深圳证券交易所/政策文件'):
    #     os.makedirs('./相关政策/深圳证券交易所/政策文件')
    url = 'http://www.szse.cn/lawrules/search/index.html?rulekeyword=REITs&channelCode=%5B%22rules%22,%22csrcrules%22,%22szseBussrules%22,%22memorandumServicedirect%22,%22publicadvice%22,%22lawruleSearch%22%5D&range=content&searchtype=0'
    driver = policy.createDriver()
    driver.get(url)
    WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.CLASS_NAME, 'article-item'))
    )
    div_list = driver.find_elements(By.CLASS_NAME, 'article-item')
    num = 0
    data_list = []
    for div in div_list:
        id_list = []
        title = div.find_element(By.TAG_NAME, 'a').text.lstrip().strip()
        href = div.find_element(By.TAG_NAME, 'a').get_attribute('href')
        publishDate = div.find_element(By.CLASS_NAME, 'pull-right').text.lstrip().strip()
        writtenDate = publishDate
        origin = '深圳证券交易所'
        organ = origin
        # 文章就为pdf
        # 根据链接判重
        is_member = baseCore.r.sismember('REITs::' + webname, href)
        if is_member:
            continue
        if '.pdf' in href:

            summary = ''
            pub_hao = ''
            fjcontent = getFjContent(href)
            time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
            lang = baseCore.detect_language(content)
            dic_info = {
                'attachmentIds': id_list,
                'subjectId': '1729315113088765953',
                'lang': lang,
                'author': '',
                'content': fjcontent,
                'contentWithTag': '',
                'deleteFlag': 0,
                'checkStatus': 1,
                'id': '1729315113088765953'+str(int(time.time())),
                'title': title,
                'publishDate': publishDate,
                'origin': origin,
                'sourceAddress': href,
                'writtenDate': writtenDate,
                'organ': organ,
                'topicClassification': '',
                'issuedNumber': pub_hao,
                'summary': summary,
                'createDate': time_now,
                'sid': '1729029275400646658',
            }
            try:
                baseCore.sendkafka(dic_info, topic)
                baseCore.r.sadd('REITs::' + webname, href)
                log.info(f'采集成功--{title}--{href}')
            except:
                for att_id in id_list:
                    baseCore.deliteATT(att_id)
        else:
            summary = div.find_element(By.CLASS_NAME, 'item-content').text.lstrip().strip()
            pub_hao, content, id_list, contentWithTag = getContent(href, publishDate, num, id_list)
            time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
            dic_info = {
                'attachmentIds': id_list,
                'author': '',
                'content': content,
                'contentWithTag': str(contentWithTag),
                'deleteFlag': 0,
                'checkStatus': 1,
                'id': '',
                'title': title,
                'publishDate': publishDate,
                'origin': origin,
                'sourceAddress': href,
                'writtenDate': writtenDate,
                'organ': organ,
                'topicClassification': '',
                'issuedNumber': pub_hao,
                'summary': summary,
                'createDate': time_now,
                'sid': '1729029275400646658',
            }
            try:
                baseCore.sendkafka(dic_info, topic)
                baseCore.r.sadd('REITs::' + webname, href)
                log.info(f'采集成功--{title}--{href}')
            except:
                for att_id in id_list:
                    baseCore.deliteATT(att_id)
    driver.close()
    # df = pd.DataFrame(np.array(data_list))
    # df.columns = ['序号', '标题', '发布时间', '来源', '原文链接', '发文时间', '发文机构', '发文字号', '摘要', '正文', '附件名称', '附件连接']
    # df.to_excel('./相关政策/深圳证券交易所/深圳证券交易所政策文件.xlsx', index=False)


if __name__ == '__main__':
    doJob()
    baseCore.close()
