import json
import re
import time

import fitz
import pymongo
import requests
from bs4 import BeautifulSoup
from kafka import KafkaProducer
from pyquery import PyQuery as pq
from requests.packages import urllib3
from urllib.parse import urljoin

from BaseCore import BaseCore
baseCore = BaseCore()

urllib3.disable_warnings()
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from lxml import etree
from random import choice
def paserUrl(html,listurl):
    # soup = BeautifulSoup(html, 'html.parser')
    # 获取所有的<a>标签和<img>标签
    links = html.find_all(['a', 'img'])
    # 遍历标签，将相对地址转换为绝对地址
    for link in links:
        if 'href' in link.attrs:
            link['href'] = urljoin(listurl, link['href'])
        elif 'src' in link.attrs:
            link['src'] = urljoin(listurl, link['src'])
    return html
real_href = 'http://gzw.fujian.gov.cn/ztzl/gzjgfzjs/gfxwj_7426/201809/t20180911_4492105.htm'
href_text = requests.get(url=real_href, verify=False)
href_text.encoding = href_text.apparent_encoding
i_html = href_text.text
i_soup = BeautifulSoup(i_html, 'html.parser')
#相对路径转化为绝对路径
i_soup = paserUrl(i_soup,real_href)
# print(i_soup)
source_ = str(i_soup.find('div', attrs={'class': 'xl_tit2_l'}).text)
pub_source = source_.split('来源：')[1].split('发布时间：')[0].strip().lstrip()
pub_time = source_.split('发布时间：')[1].split('浏览量：')[0].strip().lstrip()
content = i_soup.find('div', attrs={'class': 'xl_con1'})
pub_hao = ''
print(real_href)
id_list = []
#todo:获取附件地址
fu_jian_list = content.find('ul',class_='clearflx myzj_xl_list').find_all('a')
for fu_jian in fu_jian_list:

    fj_href = fu_jian['href']
    if '.doc' in fj_href or '.pdf' in fj_href or '.xls' in fj_href or '.zip' in fj_href \
            or '.rar' in fj_href or '.ppt' in fj_href or '.PDF' in fj_href or '.DOC' in fj_href \
            or '.XLS' in fj_href or '.ZIP' in fj_href or '.RAR' in fj_href:
        #找到附件后 上传至文件服务器
        retData = baseCore.uploadToserver(fj_href,'1673')
        if retData['state']:
            pass
        else:
            continue
        att_id, full_path = baseCore.tableUpdate(retData, '福建省国资委', 'aaa', 1)
        id_list.append(att_id)

        fu_jian['href'] = 'http://114.115.215.96/' + full_path
        print(f'-----{content}')