import datetime
import json
import random
import re

import numpy as np
import pandas as pd
import pymongo
import requests
import time

from retry import retry
from selenium.webdriver.common.by import By

from base import BaseCore

db_storage = pymongo.MongoClient('mongodb://192.168.1.36:27017', username='admin', password='zzsn@9988').RESCenter[
    'RETIsProdQuot']
baseCore = BaseCore.BaseCore()

log = baseCore.getLogger()

headers = {
    'Accept': '*/*',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,zh-TW;q=0.9,zh;q=0.8',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive',
    'Pragma': 'no-cache',
    'Referer': 'http://www.sse.com.cn/',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0'
}


# 获取基金代码列表
@retry(tries=3, delay=10)
def getCode():
    code_list = []
    url = f'http://yunhq.sse.com.cn:32041/v1/sh1/list/exchange/reits?callback=jQuery112407214866998855156_1699360786929&select=code%2Cname%2Clast%2Cchg_rate%2Cchange%2Cvolume%2Camount%2Cprev_close%2Copen%2Chigh%2Clow%2Camp_rate%2Ccpxxtype%2Ccpxxsubtype%2Ccpxxextendname&order=code%2Case&begin=0&end=25&_={int(time.time())}'
    ip = baseCore.get_proxy()
    req = requests.get(url, headers=headers, proxies=ip)
    req.encoding = req.apparent_encoding
    data_json = re.findall('\((.*)\)', req.text)[0]
    data_json = json.loads(data_json)['list']
    for data in data_json:
        code_list.append([data[0], data[-1]])
    req.close()
    return code_list


@retry(tries=4, delay=20)
def getDataJson(code):
    url = f'http://yunhq.sse.com.cn:32041/v1/sh1/dayk/{code}?callback=jQuery1124021168281852977966_1699359286492&begin=-1000&end=-1&period=day&_={int(time.time())}'
    # ip = baseCore.get_proxy()
    req = requests.get(url, headers=headers)
    req.encoding = req.apparent_encoding
    data_json = re.findall('\((.*)\)', req.text)[0]
    data_json = json.loads(data_json)['kline']
    req.close()
    return data_json


@retry(tries=5, delay=10)
def getDataB(code, date_):
    # ip = baseCore.get_proxy()
    date = str(date_)[:4] + '-' + str(date_)[4:6] + '-' + str(date_)[6:]
    url = f'http://query.sse.com.cn/commonQuery.do?jsonCallBack=jsonpCallback99984&sqlId=COMMON_SSE_SJ_JJSJ_JJGM_REITSGM_L&FUND_CODE={code}&TRADE_DATE={date_}&SEARCH_DAY={date}&FUND_TYPE=01&_={int(time.time())}'
    req = requests.get(url, headers=headers)
    req.encoding = req.apparent_encoding
    data_json_ = re.findall('\((.*)\)', req.text)[0]
    data_json_ = json.loads(data_json_)['result'][0]
    totalValue = data_json_['TOTAL_VALUE']
    negoValue = data_json_['NEGO_VALUE']
    toRate = data_json_['TO_RATE']
    req.close()
    time.sleep(2)
    return totalValue, negoValue, toRate


# 获取基金交易信息
def getData():
    codes_list = getCode()
    for codes in codes_list:
        start_time = time.time()
        code = codes[0]
        name = codes[1]
        log.info(f'{code}==={name}===开始采集')
        try:
            data_json = getDataJson(code)
            del (data_json)[-1]
            num = 1
            for data_ in data_json:
                year = str(data_[0])[:4]
                month = str(data_[0])[4:6]
                day = str(data_[0])[6:]
                date = datetime.datetime(int(year), int(month), int(day))
                date_ytd = date - datetime.timedelta(days=1)
                if num != 1:
                    while True:
                        beforeClosed = db_storage.find_one({'code':code,'date':date_ytd,'exchange':'上海证券交易所'})
                        if beforeClosed:
                            beforeClosed = beforeClosed['closed']
                            break
                        else:
                            date_ytd = date_ytd - datetime.timedelta(days=1)
                    num += 1
                else:
                    beforeClosed = 0
                    num +=1
                is_insert = db_storage.find_one({'code': code, 'date': date, 'exchange': '上海证券交易所'})
                if is_insert:
                    log.info(f'{code}==={date}===已采集')
                    time.sleep(1)
                    continue
                try:
                    totalValue, negoValue, toRate = getDataB(code, data_[0])
                except Exception as e:
                    log.error(e)
                    log.error(f'{code}==={date}===采集失败')
                    continue
                info_dic = {
                    'code': code,  # 代码
                    'shortName': name,  # 简称
                    'opening': float(data_[1]),  # 开盘价
                    'max': float(data_[2]),  # 最高价
                    'min': float(data_[3]),  # 最低价
                    'closed': float(data_[4]),  # 收盘价
                    'ytdClosed':float(beforeClosed), # 前收价
                    'volume': float(data_[5]),  # 交易量
                    'amount': float(data_[6]),  # 交易金额
                    'totalValue': float(totalValue),  # 市价总值
                    'negoValue': float(negoValue),  # 流通总值
                    'toRate': float(toRate),  # 换手率
                    'date': date,  # 时间
                    'country': '中国',  # 国家
                    'exchange': '上海证券交易所'  # 交易所
                }
                db_storage.insert_one(info_dic)
                time.sleep(2)
                log.info(f'{date}===采集成功')
        except Exception as e:
            log.error(e)
            log.error(f'{code}===采集失败')
        time.sleep(5)
        log.info(f'{code}==={name}===记录完成===耗时{baseCore.getTimeCost(start_time, time.time())}')





if __name__ == '__main__':
    getData()
    # getInfo()
    baseCore.close()
