import json
import time

import pymongo


# url = "https://web.archive.org/web/20230702131549/https://www.forbes.com/lists/global2000/"
url = "https://web.archive.org/web/20220929184024/https://www.forbes.com/lists/global2000/"
db_storage = pymongo.MongoClient('mongodb://114.115.221.202:27017/', username='admin', password='ZZsn@9988').ZZSN[
    '福布斯企业人数']
headers = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
    'Accept-Encoding': 'gzip, deflate, br, zstd',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
    'Cookie': 'client_id=53d840cb729d7420b1443fea045a4b1c71c; _gcl_au=1.1.934727975.1688303765; _li_dcdm_c=.com; _lc2_fpi=5fb552a76ef3--01h4bby2zps7e7m7k4va1nmp0x; _swb=8e76883c-49d5-497f-ba08-91910e3f0394; _gid=GA1.2.1372767781.1688303769; notice_gdpr_prefs=0,1,2; _swb_consent_=eyJvcmdhbml6YXRpb25Db2RlIjoiZm9yYmVzIiwicHJvcGVydHlDb2RlIjoid2Vic2l0ZV9zbWFydF90YWciLCJlbnZpcm9ubWVudENvZGUiOiJwcm9kdWN0aW9uIiwiaWRlbnRpdGllcyI6eyJzd2Jfd2Vic2l0ZV9zbWFydF90YWciOiI4ZTc2ODgzYy00OWQ1LTQ5N2YtYmEwOC05MTkxMGUzZjAzOTQifSwianVyaXNkaWN0aW9uQ29kZSI6ImNjcGEiLCJwdXJwb3NlcyI6eyJiZWhhdmlvcmFsX2FkdmVydGlzaW5nIjp7ImFsbG93ZWQiOiJ0cnVlIiwibGVnYWxCYXNpc0NvZGUiOiJjb25zZW50X29wdG91dCJ9LCJhbmFseXRpY3MiOnsiYWxsb3dlZCI6InRydWUiLCJsZWdhbEJhc2lzQ29kZSI6ImNvbnNlbnRfb3B0b3V0In0sImZ1bmN0aW9uYWwiOnsiYWxsb3dlZCI6InRydWUiLCJsZWdhbEJhc2lzQ29kZSI6ImRpc2Nsb3N1cmUifSwicmVxdWlyZWQiOnsiYWxsb3dlZCI6InRydWUiLCJsZWdhbEJhc2lzQ29kZSI6ImRpc2Nsb3N1cmUifX0sImNvbGxlY3RlZEF0IjoxNjg4MzAzNzY5fQ%3D%3D; _ketch_consent_v1_=eyJiZWhhdmlvcmFsX2FkdmVydGlzaW5nIjp7InN0YXR1cyI6ImdyYW50ZWQiLCJjYW5vbmljYWxQdXJwb3NlcyI6WyJiZWhhdmlvcmFsX2FkdmVydGlzaW5nIl19LCJhbmFseXRpY3MiOnsic3RhdHVzIjoiZ3JhbnRlZCIsImNhbm9uaWNhbFB1cnBvc2VzIjpbImFuYWx5dGljcyJdfSwiZnVuY3Rpb25hbCI6eyJzdGF0dXMiOiJncmFudGVkIiwiY2Fub25pY2FsUHVycG9zZXMiOlsicHJvZF9lbmhhbmNlbWVudCJdfSwicmVxdWlyZWQiOnsic3RhdHVzIjoiZ3JhbnRlZCIsImNhbm9uaWNhbFB1cnBvc2VzIjpbImVzc2VudGlhbF9zZXJ2aWNlcyJdfX0%3D; _fbp=fb.0.1688303769818.1241836180; ln_or=eyI0NDUxMDYiOiJkIn0%3D; __qca=P0-1546201114-1688303767140; _cb=CILAkNBqefvRLy7_N; _chartbeat2=.1688303782194.1688303782194.1.CU0yf7BtnWSMBnjN8tDChNq8CLaEIA.1; _hp2_ses_props.657665248=%7B%22z%22%3A1%2C%22ts%22%3A1688307007339%2C%22d%22%3A%22www.forbes.com%22%2C%22h%22%3A%22%2Flists%2Fglobal2000%2F%22%2C%22t%22%3A%22The%20Global%202000%202023%22%2C%22q%22%3A%22%3Fsh%3D7b5ee1385ac0%22%7D; lux_uid=168830375963815056; gaWelcomPageTracked=true; _hp2_id.657665248=%7B%22userId%22%3A%227866808861400171%22%2C%22pageviewId%22%3A%224987331001554425%22%2C%22sessionId%22%3A%222988129651120211%22%2C%22identity%22%3Anull%2C%22trackerVersion%22%3A%224.0%22%7D; AMP_TOKEN=%24NOT_FOUND; _ga=GA1.2.14802614.1688303763; _dc_gtm_UA-5883199-3=1; _gat_UA-5883199-3=1; _ga_DLD85VJ5QY=GS1.1.1688307006.2.1.1688303798.37.0.0; donation-identifier=aab33e1c4e293a8fcd5490465688bb01',
    'Priority': 'u=0, i',
    'Sec-Ch-Ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Microsoft Edge";v="126"',
    'Sec-Ch-Ua-Mobile': '?0',
    'Sec-Ch-Ua-Platform': '"Windows"',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'none',
    'Sec-Fetch-User': '?1',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0'
}

import requests
from bs4 import BeautifulSoup
requests.adapters.DEFAULT_RETRIES = 5
proxies = {
    'https': 'http://127.0.0.1:1080',
    'http': 'http://127.0.0.1:1080',
}

# req = requests.get(headers=headers, url=url, verify=False, proxies=proxies)
# # with open('./a.html','w',encoding='utf-8') as f:
# #     f.write(req.text)
# soup = BeautifulSoup(req.text, 'lxml')
# scripts = soup.find_all('script')
# for script in scripts:
#     if 'window["f' in script.text and 'ICBC' in script.text:
#         break
# with open('./a.txt','w',encoding='utf-8')as f:
#     f.write(script.text)

with open('./a.txt', 'r', encoding='utf-8') as f:
    dataJson = f.read()
dataJson = json.loads(dataJson)
tableDates = dataJson['tableData']
for tableDate in tableDates:
    uri = tableDate['uri']
    rank = tableDate['rank']

    organizationName = tableDate['organizationName']
    print(uri, rank, organizationName)
    href = f'https://web.archive.org/web/20230909091923/https://www.forbes.com/companies/{uri}/?list=global2000'
    req = requests.get(headers=headers, url=href, verify=False, proxies=proxies)
    soup = BeautifulSoup(req.text,'lxml')
    scripts = soup.find_all('script')
    # print(scripts)
    for script in scripts:
        if 'numberOfEmployees' in script.text:
            break
        else:
            continue
            # print(f'{rank}--{uri}---not found')
    try:
        employeesJson = script.text
        # print(employeesJson)

        employeesJson = json.loads(employeesJson)
        numberOfEmployees = employeesJson['numberOfEmployees'].replace(',', '')
    except:
        numberOfEmployees = '--'
    dic = {
        '排名':rank,
        '企业名称':organizationName,
        '员工人数':numberOfEmployees,
    }
    db_storage.insert_one(dic)
    print(f'{rank}==={organizationName}===已入库')
    req.close()
    # time.sleep(1)
    break