
"""sorted(title_extracted_by_h,key = lambda x:similarity(x,title_extracted_by_title),reverse=True)

etree.strip_elements(elements,tag) #tag是要删除的标签名
for child in children(element):  #children(element)是一个函数，获取当前HTML元素的所有子元素
    child.tag.lower() #获取标签名称
    etree.strip_tags(child,'span') #删除标签对

element.getparent() #获取给定元素的父元素


"""
# from io import StringIO
#
# import pandas as pd
# data = '"1234","456\r7","897"'
# print(data)
# aa = pd.read_csv(StringIO(data),escapechar='\r')
# print(aa)

# import pandas as pd
#
# # 读取txt文件
# data = pd.read_csv('D:\\美国证券交易委员会\\2022q4\\sub.txt', delimiter='\t')  # 根据实际情况选择正确的分隔符
#
# # 将数据保存为csv文件
# data.to_csv('D:\\美国证券交易委员会\\2022q4\\sub.csv', index=False)  # index=False表示不保存行索引
"""验证码识别测试"""
# import ddddocr
#
# ocr = ddddocr.DdddOcr()
#
# with open("D:\\kkwork\\captchaNew (3).jfif", 'rb') as f:
#     image = f.read()
#
# res = ocr.classification(image)
# print(res)
"""测试中国执行信息公开网  模拟浏览器"""

# import ddddocr
from PIL import Image
import re

import requests, time, random, json, pymysql, redis

import urllib3
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from obs import ObsClient
from kafka import KafkaProducer

from base.BaseCore import BaseCore
# baseCore = BaseCore()
# log = baseCore.getLogger()
# cnx_ = baseCore.cnx
# cursor_ = baseCore.cursor
#
# def use_ocr(img):
#     ocr = ddddocr.DdddOcr()
#     with open(img, 'rb') as f:
#         image = f.read()
#     res = ocr.classification(image)
#     print(res)
#     return res
#
# if __name__=="__main__":
#     requests.DEFAULT_RETRIES = 5
#     time_start = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
#     log.info(f'开始时间为：{time_start}')
#
#     requests.adapters.DEFAULT_RETRIES = 3
#     headers = {
#         'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
#     }
#
#     opt = webdriver.ChromeOptions()
#     opt.add_argument(
#         'user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36')
#
#     opt.add_argument("--ignore-certificate-errors")
#     opt.add_argument("--ignore-ssl-errors")
#     opt.add_experimental_option("excludeSwitches", ["enable-automation"])
#     opt.add_experimental_option('excludeSwitches', ['enable-logging'])
#     opt.add_experimental_option('useAutomationExtension', False)
#     opt.binary_location = r'D:/Google/Chrome/Application/chrome.exe'
#     chromedriver = r'D:/cmd100/chromedriver.exe'
#     browser = webdriver.Chrome(chrome_options=opt, executable_path=chromedriver)
#     url = "http://zxgk.court.gov.cn/shixin/"
#     browser.get(url)
#     # 可改动
#     time.sleep(20)
#     page_source = browser.page_source
#     soup = BeautifulSoup(page_source, 'html.parser')
#     img_url = soup.select('img[id="captchaImg"]')[0]['src']
#
#     browser.find_element(By.ID, 'pName').send_keys('北京远翰国际教育咨询有限责任公司')
#
#
#     browser.find_element(By.ID, 'yzm').send_keys(yzm)
#     browser.find_element(By.ID, 'searchForm').click()
#     wait = WebDriverWait(browser, 30)
#     wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))

# screen_img_path = "D:/screen/xxx.png"
# out_img_path = "D:/out/xxx.png"
# ele = driver.find_element(By.ID, 'XXXX')
#
# code = use_ocr(out_img_path)
# 验证码输入框元素.send_keys(code)
# import requests
# headers = {
#     # 'Accept': '*/*',
#     # 'Accept-Encoding': 'gzip, deflate, br',
#     # 'Accept-Language': 'zh-CN,zh-TW;q=0.9,zh;q=0.8',
#     # 'Cache-Control': 'no-cache',
#     # 'Connection': 'keep-alive',
#     # 'Host': 'search-api-web.eastmoney.com',
#     # 'Pragma': 'no-cache',
#     # 'Sec-Fetch-Dest': 'script',
#     # 'Sec-Fetch-Mode': 'no-cors',
#     # 'Sec-Fetch-Site': 'same-site',
#     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0',
#     # 'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Microsoft Edge";v="120"',
#     # 'sec-ch-ua-mobile': '?0',
#     # 'sec-ch-ua-platform': '"Windows"'
# }
# url = "https://www-private-oss.mob.com/academy_reports/2023/03/31/Mob%E7%A0%94%E7%A9%B6%E9%99%A2%E3%80%8A2023%E5%B9%B4%E4%B8%AD%E5%9B%BD%E6%96%87%E6%97%85%E4%BA%A7%E4%B8%9A%E5%8F%91%E5%B1%95%E8%B6%8B%E5%8A%BF%E6%8A%A5%E5%91%8A%E3%80%8B.pdf?response-content-disposition=attachment&OSSAccessKeyId=LTAI5t5mdPuMS9gNj93RPowJ&Expires=1703839064&Signature=X1mpakYGVaBffNokvhvW917UH%2Fk%3D"
#
#
# # res = requests.get(url).text[1:-1]
# res = requests.get(url=url, headers=headers)
# with open('./a.pdf','wb') as f:
#     f.write(res.content)


import datetime
import json

import requests
import pymongo
from base import BaseCore
# baseCore = BaseCore.BaseCore()
# log = baseCore.getLogger()
#
#
# db_storage = pymongo.MongoClient('mongodb://114.115.221.202:27017/', username='admin', password='ZZsn@9988').中科软[
#     '数据源_0504']
#
# datas = db_storage.find({'postCode':'2'}).limit(1)
# for data in datas:
#     title = data['titleForeign']
#     contentWithTag = data['richTextForeign']
#     summary = data['contentForeign']
#     dic_info = {
#         'title':title,
#         # 'summary':summary,
#         'contentWithTag':contentWithTag
#     }
#     headers = {
#         'Content-Type': 'application/json',
#     }
#     dic_info_ = json.dumps(dic_info)
#     # print(dic_info_)
#     # with open('./data.json','w') as f:
#     #     f.write(dic_info_)
#     # break
#     # req = requests.post('http://192.168.1.236:5000/translate',data=dic_info_,headers=headers)
#     req = requests.post('http://117.78.23.14:5000/translate',data=dic_info_,headers=headers)
#     log.info(req.text)

# import re, datetime
#
#
# def paserTime(publishtime):
#     timeType = ['年前', '月前', '周前', '前天', '昨天', '天前', '今天', '小时前', '分钟前']
#     current_datetime = datetime.datetime.now()
#     publishtime = publishtime.strip()
#     print(publishtime)
#
#     try:
#         if '年前' in publishtime:
#             numbers = re.findall(r'\d+', publishtime)
#             day = int(numbers[0])
#             delta = datetime.timedelta(days=365 * day)
#             publishtime = current_datetime - delta
#         elif '月前' in publishtime:
#             numbers = re.findall(r'\d+', publishtime)
#             day = int(numbers[0])
#             delta = datetime.timedelta(months=day)
#             publishtime = current_datetime - delta
#         elif '周前' in publishtime:
#             numbers = re.findall(r'\d+', publishtime)
#             day = int(numbers[0])
#             delta = datetime.timedelta(weeks=day)
#             publishtime = current_datetime - delta
#         elif '天前' in publishtime:
#             numbers = re.findall(r'\d+', publishtime)
#             day = int(numbers[0])
#             delta = datetime.timedelta(days=day)
#             publishtime = current_datetime - delta
#         elif '前天' in publishtime:
#             delta = datetime.timedelta(days=2)
#             publishtime = current_datetime - delta
#         elif '昨天' in publishtime:
#             current_datetime = datetime.datetime.now()
#             delta = datetime.timedelta(days=1)
#             publishtime = current_datetime - delta
#         elif '今天' in publishtime or '小时前' in publishtime or '分钟前' in publishtime:
#             if '小时' in publishtime:
#                 hour = publishtime.split("小时")[0]
#             else:
#                 hour = 0
#             if hour != 0:
#                 min = publishtime.split("小时")[1].split("分钟")[0]
#             else:
#                 min = publishtime.split("分钟")[0]
#
#             delta = datetime.timedelta(hours=int(hour), minutes=int(min))
#             publishtime = current_datetime - delta
#         elif '年' in publishtime and '月' in publishtime:
#             time_format = '%Y年%m月%d日'
#             publishtime = datetime.datetime.strptime(publishtime, time_format)
#         elif '月' in publishtime and '日' in publishtime:
#             current_year = current_datetime.year
#             time_format = '%Y年%m月%d日'
#             publishtime = str(current_year) + '年' + publishtime
#             publishtime = datetime.datetime.strptime(publishtime, time_format)
#     except Exception as e:
#         print('时间解析异常！！')
#     return publishtime
#
# if __name__ == "__main__":
#     publishtime_ = '1小时17分钟前'
#     publish_time = paserTime(publishtime_).strftime("%Y-%m-%d")
#     print(publish_time)

# import pandas as pd
#
# # 创建一个示例DataFrame
# df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]})
#
# # 要追加的行
# new_rows = pd.DataFrame({'A': [4, 5], 'B': [7, 8]})
#
# # 追加行到原DataFrame
# df = pd.concat([df, new_rows], ignore_index=True)
#
# print(df)

# import pandas as pd
#
# # 假设我们有两个DataFrame
# df1 = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]})
# df2 = pd.DataFrame({'A': [7, 8, 9], 'B': [10, 11, 12]})
#
# # 从df1中取出第一行数据
# row_to_append = df1.iloc[0].to_frame().T
# # print(row_to_append)
#
# # 将这一行数据追加到df2中
# # 注意：这里使用ignore_index=True来忽略索引，并重新设置索引
# # df2 = df2.append(row_to_append, ignore_index=True)
# df2 = pd.concat([df2, row_to_append], ignore_index=True)
#
# # 打印结果
# print(df2)


# import openpyxl
# import redis
#
# # 连接到Redis服务器
# redis_client = redis.Redis(host="114.116.90.53", port=6380, password='clbzzsn', db=6)
#
# # 打开Excel文件
# workbook = openpyxl.load_workbook('D:\\kkwork\\企业数据\\年报数据任务.xlsx')
#
# # 选择要读取的工作表
# worksheet = workbook['系统批量导出年报名单']
#
# # 选择要读取的列
# column_index = 0  # 选择第2列
#
# # 遍历指定列的单元格，并将值放入Redis列表
# for row in worksheet.iter_rows(values_only=True):
#     try:
#         cell_value = row[1] + "|" + row[2]
#     except:
#         print(row[1])
#         continue
#     # cell_value = row[1]
#     redis_client.rpush('NianBao:socialcode', cell_value)
#
# # 关闭Excel文件
# workbook.close()


# def find_empty_id_sequences(id_list):
#     empty_id_count = 0
#     empty_id_sequences = []
#     current_sequence = []
#
#     for id in id_list:
#         if id is None:  # 假设 id 为 None 表示空
#             empty_id_count += 1
#             current_sequence.append(empty_id_count)
#         else:
#             if empty_id_count > 0:
#                 empty_id_sequences.append(current_sequence)
#                 current_sequence = []
#             empty_id_count = 0
#
#     if empty_id_count > 0:
#         empty_id_sequences.append(current_sequence)
#
#     return empty_id_sequences
#
# # 示例使用
# id_list = [1, None, 2, None, None, 3, None, None, None, 4]
# empty_id_sequences = find_empty_id_sequences(id_list)
# print(empty_id_sequences)


# def process_results(results):
#     # 初始化输出字符串和上一个结果
#     final_output = ""
#     previous_res = None
#
#     # 初始化连续空的结果索引列表
#     empty_indices = []
#     empty_id_sequences = []
#     all_result = []
#     # 对结果进行排序
#     sorted_results = sorted(results.items(), key=lambda x: int(x[0].split(".")[0]))
#
#     # 遍历排序后的结果
#     for index, (_, res) in enumerate(sorted_results):
#         if res == '':
#             # 如果是空结果，将索引添加到空结果索引列表中
#             empty_indices.append(_)
#
#         else:
#             # 如果结果非空，并且连续空的结果列表不为空，
#             # 则添加连续空的结果索引列表到输出中
#             if empty_indices:
#                 # final_output += " ".join(map(str, empty_indices)) + '\n'
#                 empty_id_sequences.append(empty_indices)
#                 all_result.append(empty_indices)
#                 empty_indices = []
#             # 如果是非空结果并且与上一个结果不同，则添加到输出中
#             if res != previous_res:
#                 final_output += res + '\n'
#                 all_result.append(res)
#                 previous_res = res
#
#     return all_result, empty_id_sequences
#
# # 示例使用
# results = {
#     "1.0": "",
#     "2.0": "",
#     "3.0": "result1",
#     "4.0": "",
#     "5.0": "",
#     "6.0": "result2",
#     "7.0": "result2",
#     "8.0": "",
#     "9.0": "result3"
# }
#
# all_result, empty_id_sequences, dic_index = process_results(results)
# print(all_result)
# print(empty_id_sequences)
# print(dic_index)
import Levenshtein
def same_rule(same_list):
    # 记录最长的一个
    max_len = 0

    for item in same_list:
        if len(item) > max_len:
            max_len = len(item)

    char_map_list = []
    char_map_score = []
    for i in range(max_len):
        char_map_list.append(dict())
        char_map_score.append(dict())

    for index in range(max_len):
        for i in range(len(same_list)):
            if index < len(same_list[i]):
                char = same_list[i][index]
                score = same_list[i][index]
            else:
                char = ""
                score = 1

            if char not in char_map_list[index]:
                char_map_list[index][char] = 1
            else:
                char_map_list[index][char] += 1

            if char not in char_map_score[index]:
                char_map_score[index][char] = score
            else:
                char_map_score[index][char] += score

    print(char_map_list)
    print(char_map_score)

    #返回个数多的；当个数相同时，返回打分高的
    result = []
    for i in range(max_len):
        print('---------------')
        print(char_map_list[i].items())
        print(sorted(char_map_list[i].items(), key=lambda item:item[1], reverse=True))
        result.append(sorted(char_map_list[i].items(), key=lambda item:item[1],reverse=True)[0][0])
    return "".join(result)

def aaaaa(final_output):
    finall_list = []
    same_list = []
    # 处理相似的
    for result in final_output:
        print(f"result:{result}")
        if len(same_list) > 0:
            ratio = Levenshtein.ratio(result, same_list[-1])
            if ratio < 0.5:
                # 差异大于0.5
                # 对相似的做处理
                if len(same_list) > 1:
                    result_ = max(same_list, key=lambda x: (len(x), x))
                    same_list = [result_]
                    finall_list[-1] = result_
                    finall_list.append(result)

                else:
                    same_list = [result]
                    finall_list.append(result)
            else:
                same_list.append(result)

        else:
            same_list.append(result)
            finall_list.append(result)
    print(finall_list)

if __name__ == '__main__':
    same_list = ['让我们从一次时光旅行', '开启植物天堂的故事', '地球的午夜', '是在火山喷发中度过的', '到了凌晨三四点', '在海洋深处有了生命的迹象', '清晨6点多', '更加壮丽的生命乐章开始了', '更加壮丽的生命乐草开始了', '更加壮丽的生命乐章开始了', '更加壮丽的生命乐草开始了', '更加壮丽的生命乐章开始了', '种蓝藻细菌', '一种蓝藻细菌', '学会利用二氧化碳水和阳光', '制造生命所需能量', '同时释放出了氧气', '这个被称为光合作用的过程', '为植物世界打开了大门', '此时', '中国的陆地', '也逐渐从海洋露出形成岛屿', '但在相当长的时间里', '陆地十分荒凉没有生机', '这些岩石坚硬', '无法储存水分', '是当时陆地环境的写照', '直到晚上九点多', '也就是四亿年前左右', '些矮小的生命', '开始征服陆地', '她们用一种近似于根的构造', '固定在岩石上', '苔藓', '是陆地最早的拓荒者之', '小', '她们死后的身体', '形成了肥沃的土壤', '让更多的植物可以在这里生存', '从此', '绿色成为植物天堂的底色']


    # aaa = aaaaa(same_list)

    #
    # for i in range(len(same_list)):
    #     print(i, same_list[i])
    #

