68定时任务脚本维护

comData/68定时任务/10_交易明细.py

+import requests,random,time,json,re,pymysql,datetime
+import requests,random,time,json,re,pymysql,datetime
+import pandas as pd
+
+
+cnx = pymysql.connect(host='114.115.159.144',user='caiji', password='zzsn9988', db='clb_project', charset='utf8mb4')
+cnx_in = pymysql.connect(host='114.116.44.11',user='caiji', password='f7s0&7qqtK', db='clb_project', charset='utf8mb4')
+
+list_all_info = []
+list_code = []
+list_code_no = []
+
+now = datetime.datetime.now()
+yes1 = now + datetime.timedelta(days = -1)
+date_yes = yes1.strftime('%Y-%m-%d')
+
+print(date_yes)
+for page in range(21,200):
+    print(page)
+    url = f'https://datacenter-web.eastmoney.com/api/data/v1/get?reportName=RPTA_WEB_RZRQ_GGMX&columns=ALL&source=WEB&pageNumber={page}&pageSize=50&sortColumns=RZJME&sortTypes=-1&filter=(date%3D%27{date_yes}%27)'
+
+    res = requests.get(url).json()
+    try:
+        list_all = res['result']['data']
+    except:
+        print('无数据')
+        break
+
+    for one_com in list_all:
+        com_code = one_com['SCODE']
+        list_code.append(com_code)
+        with cnx.cursor() as cursor:
+            sql_sel = '''select social_credit_code from sys_base_enterprise_ipo where securities_code = %s '''
+            cursor.execute(sql_sel,com_code)
+            select = cursor.fetchone()
+            if select:
+                com_social = select[0]
+            else:
+                print(f'数据库中无此股票代码：{com_code}')
+                list_code_no.append(com_code)
+                com_social = ''
+        if com_social:
+            with cnx.cursor() as cursor:
+                sql_sel = '''select name from sys_base_enterprise where social_credit_code = %s '''
+                cursor.execute(sql_sel,com_social)
+                select = cursor.fetchone()
+                if select:
+                    com_name = select[0]
+                else:
+                    print(f'数据库中无此信用代码：{com_social}')
+                    com_name = ''
+        else:
+            com_name = ''
+
+        com_short = one_com['SECNAME']
+        securities_code = com_code
+        closing_price = str(one_com['SPJ'])
+        fluctuation_range = str(one_com['ZDF'])
+        financing_balance = str(one_com['RZYE'])
+        financing_balance_circulating_ratio = str(one_com['RZYEZB'])
+        financing_balance_purchase_amount = str(one_com['RZMRE'])
+        financing_balance_refund = str(one_com['RZCHE'])
+        financing_balance_net_purchases = str(one_com['RZJME'])
+        short_sale_balance = str(one_com['RQYE'])
+        short_sale_margin = str(one_com['RQYL'])
+        short_sale_selling_volume = str(one_com['RQMCL'])
+        short_sale_repayment_amount = str(one_com['RQCHL'])
+        short_sale_net_sales = str(one_com['RQJMG'])
+        financing_short_sale_balance = str(one_com['RZRQYE'])
+        financing_short_sale_balance_difference = str(one_com['RZRQYECZ'])
+        create_by = 'TangYuHang'
+        create_time = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
+
+        with cnx_in.cursor() as cursor:
+            sql_sel = '''select id from ranklist_market_value where stock_code = %s and data_time = %s '''
+            cursor.execute(sql_sel,(com_code,date_yes))
+            select = cursor.fetchone()
+            if select:
+                continue
+
+        tuple_info = (com_social,com_code,com_name,com_short,securities_code,
+                     closing_price,fluctuation_range,financing_balance,
+                     financing_balance_circulating_ratio,financing_balance_purchase_amount,
+                     financing_balance_refund,financing_balance_net_purchases,
+                     short_sale_balance,short_sale_margin,short_sale_selling_volume,
+                     short_sale_repayment_amount,short_sale_net_sales,
+                     financing_short_sale_balance,financing_short_sale_balance_difference,
+                     create_by,create_time,date_yes)
+
+        list_all_info.append(tuple_info)
+
+with cnx_in.cursor() as cursor:
+    sql_in = '''insert into ranklist_market_value(social_credit_code,stock_code,enterprise_name,securities_abbreviation,securities_code,closing_price,fluctuation_range,financing_balance,financing_balance_circulating_ratio,financing_balance_purchase_amount,financing_balance_refund,financing_balance_net_purchases,short_sale_balance,short_sale_margin,short_sale_selling_volume,short_sale_repayment_amount,short_sale_net_sales,financing_short_sale_balance,financing_short_sale_balance_difference,create_by,create_time,data_time) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''
+    cursor.executemany(sql_in,list_all_info)
+    cnx_in.commit()

comData/68定时任务/4_企业市值.py

+import requests,pymysql,time,re,json
+import requests,pymysql,time,re,json
+from datetime import datetime
+import locale
+locale.setlocale(locale.LC_CTYPE,"chinese")
+# 近期市值
+#cnx = pymysql.connect(host='114.115.159.144',user='root', password='zzsn9988', db='clb_project', charset='utf8mb4')  # mysql数据库链接
+cnx = pymysql.connect(host='114.116.44.11',user='caiji', password='f7s0&7qqtK', db='clb_project', charset='utf8mb4')  # mysql数据库链接
+def tableUpdate(social_id, v):
+
+    current_date = datetime.now().date()
+
+    # 格式化日期为 %年%月%日 格式
+    formatted_date = current_date.strftime("%Y年%m月%d日")
+    with cnx.cursor() as cursor:
+        Upsql = ''' update ranklist_state_holding set recent_market_value = %s, recent_date = %s where social_credit_code = "{}" '''.format(social_id)
+        cursor.execute(Upsql,v,formatted_date)  # 更新
+        cnx.commit()  # 提交
+        print("更新完成:{}".format(social_id))
+
+with cnx.cursor() as cursor:
+    querySql = '''select stock_code,social_credit_code,listed_exchange from ranklist_state_holding'''
+    cursor.execute(querySql)
+    selects = cursor.fetchall()
+    
+    for row in selects:
+        code = row[0]
+        social_id = row[1]
+        try:
+            if 'HK' in code:
+                continue
+        except:
+            continue
+        code1 = str(code)
+        while True:
+            if len(code1)<6:
+                code1 = '0'+code1
+            else:
+                break
+        if code1[0] == '0' or code1[0] == '3':
+            com_code = 'SZ'+code1
+        elif code1[0] == '6':
+            com_code = 'SH'+code1
+        elif code1[0] == '8':
+            com_code = 'BJ'+code1
+        
+        t = int(time.time()*1000)
+        if 'SZ' in com_code:
+            code_sz = com_code[2:]
+            url2 = f'https://push2.eastmoney.com/api/qt/stock/get?invt=2&fltt=1&cb=jQuery35109980141635843787_1669189162845&fields=f58%2Cf734%2Cf107%2Cf57%2Cf43%2Cf59%2Cf169%2Cf170%2Cf152%2Cf177%2Cf111%2Cf46%2Cf60%2Cf44%2Cf45%2Cf47%2Cf260%2Cf48%2Cf261%2Cf279%2Cf277%2Cf278%2Cf288%2Cf19%2Cf17%2Cf531%2Cf15%2Cf13%2Cf11%2Cf20%2Cf18%2Cf16%2Cf14%2Cf12%2Cf39%2Cf37%2Cf35%2Cf33%2Cf31%2Cf40%2Cf38%2Cf36%2Cf34%2Cf32%2Cf211%2Cf212%2Cf213%2Cf214%2Cf215%2Cf210%2Cf209%2Cf208%2Cf207%2Cf206%2Cf161%2Cf49%2Cf171%2Cf50%2Cf86%2Cf84%2Cf85%2Cf168%2Cf108%2Cf116%2Cf167%2Cf164%2Cf162%2Cf163%2Cf92%2Cf71%2Cf117%2Cf292%2Cf51%2Cf52%2Cf191%2Cf192%2Cf262%2Cf294%2Cf295%2Cf269%2Cf270%2Cf256%2Cf257%2Cf285%2Cf286&secid=0.{code_sz}&ut=fa5fd1943c7b386f172d6893dbfba10b&wbp2u=%7C0%7C0%7C0%7Cweb&_={t}'
+        if 'SH' in com_code:
+            code_sh = com_code[2:]
+            url2 = f'https://push2.eastmoney.com/api/qt/stock/get?invt=2&fltt=1&cb=jQuery35109980141635843787_1669189162845&fields=f58%2Cf734%2Cf107%2Cf57%2Cf43%2Cf59%2Cf169%2Cf170%2Cf152%2Cf177%2Cf111%2Cf46%2Cf60%2Cf44%2Cf45%2Cf47%2Cf260%2Cf48%2Cf261%2Cf279%2Cf277%2Cf278%2Cf288%2Cf19%2Cf17%2Cf531%2Cf15%2Cf13%2Cf11%2Cf20%2Cf18%2Cf16%2Cf14%2Cf12%2Cf39%2Cf37%2Cf35%2Cf33%2Cf31%2Cf40%2Cf38%2Cf36%2Cf34%2Cf32%2Cf211%2Cf212%2Cf213%2Cf214%2Cf215%2Cf210%2Cf209%2Cf208%2Cf207%2Cf206%2Cf161%2Cf49%2Cf171%2Cf50%2Cf86%2Cf84%2Cf85%2Cf168%2Cf108%2Cf116%2Cf167%2Cf164%2Cf162%2Cf163%2Cf92%2Cf71%2Cf117%2Cf292%2Cf51%2Cf52%2Cf191%2Cf192%2Cf262%2Cf294%2Cf295%2Cf269%2Cf270%2Cf256%2Cf257%2Cf285%2Cf286&secid=1.{code_sh}&ut=fa5fd1943c7b386f172d6893dbfba10b&wbp2u=%7C0%7C0%7C0%7Cweb&_={t}'
+        
+        for i in range(0,5):
+            try:
+                res2 = requests.get(url2)
+                break
+            except:
+                if i >=3:
+                    time.sleep(100)
+                    continue
+                time.sleep(3)
+                continue
+        ttt = re.findall('\((.*?)\)',res2.text)[0]
+        try:
+            zongshizhi = json.loads(ttt)['data']['f116']
+        except:
+            continue
+        
+        tableUpdate(social_id,zongshizhi)
+        print(com_code)
\ No newline at end of file

comData/68定时任务/7_500强公告.py

+import requests,re,time,random,pymysql,json
+import requests,re,time,random,pymysql,json
+import pandas as pd
+from bs4 import BeautifulSoup
+from selenium import webdriver
+
+cnx = pymysql.connect(host='114.115.159.144',user='caiji', password='zzsn9988', db='dbScore', charset='utf8mb4')
+
+df_all = pd.read_excel('500强股票代码-信用代码.xlsx')
+
+
+for num in range(len(df_all)):
+    com_code = df_all['股票代码'][num]
+    social_code = df_all['信用代码'][num]
+    print(num,social_code)
+    if com_code != '--' and social_code != '--':
+        pass
+    else:
+        continue
+    print(num)
+
+    page = 1
+
+    t = int(time.time()*1000)
+    url = f'https://np-anotice-stock.eastmoney.com/api/security/ann?sr=-1&page_size=50&page_index={page}&ann_type=U%2CU_Pink&client_source=web&stock_list={com_code}'
+    res_json = requests.get(url).json()
+    #res_json = json.loads(re.findall('\((.*?)\)',res_text)[0])
+    list_all = res_json['data']['list']
+
+    for one_info in list_all:
+
+        news_title = one_info['title']
+
+        news_date = one_info['notice_date'][:10]
+        art_code = one_info['art_code']
+        news_url = f'https://data.eastmoney.com/notices/detail/{com_code}/{art_code}.html'
+
+        t = int(time.time()*1000)
+        url_news = f'https://np-cnotice-stock.eastmoney.com/api/content/ann?art_code={art_code}&client_source=web&page_index=1&_={t}'  
+        json_news = requests.get(url_news).json()
+        #json_news = json.loads(re.findall('\((.*?)\)',res_news)[0])
+
+        news_html = json_news['data']['attach_url']
+
+        list_info = [
+            social_code,
+            news_title,
+            news_date,
+            news_url,
+            news_html,
+            '东方财富网',
+            '1'
+        ]
+
+        # list_all_info.append(tuple(list_info))
+        with cnx.cursor() as cursor:
+            sel_sql = '''select social_credit_code from brpa_source_article where source_address = %s ''' 
+
+            cursor.execute(sel_sql,news_url)
+            selects = cursor.fetchall()
+            if selects:
+                break
+
+            insert_sql = '''insert into brpa_source_article(social_credit_code,title,publish_date,source_address,pdf_address,origin,type) values(%s,%s,%s,%s,%s,%s,%s)'''      
+            cursor.execute(insert_sql,tuple(list_info))
+            cnx.commit()
+            print(news_title)
+

comData/68定时任务/9_股本结构.py

+import requests,random,time,json,re,pymysql
+import requests,random,time,json,re,pymysql
+import pandas as pd
+from bs4 import BeautifulSoup
+
+cnx = pymysql.connect(host='114.115.159.144',user='caiji', password='zzsn9988', db='clb_project', charset='utf8mb4')
+cnx_in = pymysql.connect(host='114.116.44.11',user='caiji', password='f7s0&7qqtK', db='clb_project', charset='utf8mb4')
+
+list_com = []
+
+with cnx_in.cursor() as cursor:
+    sql_sel = '''select securities_code,social_credit_code,exchange,securities_short_name from sys_base_enterprise_ipo where category = 1 '''
+    cursor.execute(sql_sel)
+    selects = cursor.fetchall()
+
+    for rows in selects:
+        try:
+            com_code = rows[0]
+            social_code = rows[1]
+            com_type = rows[2]
+            short_name = rows[3]
+
+            if com_type == 1:
+                com_code = 'BJ'+str(com_code)
+            elif com_type == 2:
+                com_code = 'SH'+str(com_code)
+            elif com_type == 3:
+                com_code = 'SZ'+str(com_code)
+            else:
+                print(f'{com_code}:!!!!!!!!!!!!!!!!!')
+            sql_sel2 = '''select name from sys_base_enterprise where social_credit_code = %s '''
+            cursor.execute(sql_sel2,social_code)
+            select = cursor.fetchone()
+            com_name = select[0]
+
+            list_com.append([com_code,social_code,com_name,short_name])
+        except:
+            continue
+        # print(com_code)
+
+
+for com_num in range(len(list_com)):
+    com = list_com[com_num]
+    com_code = com[0]
+    print(com_num)
+    print(com_code)
+    code = com_code[2:]
+    social_code = com[1]
+    com_name = com[2]
+    short_name = com[3]
+    url = f'https://emweb.securities.eastmoney.com/PC_HSF10/CapitalStockStructure/PageAjax?code={com_code}'
+
+    res = requests.get(url).json()
+    list_all = res['lngbbd']
+
+    for one_info in list_all:
+        total_share_capital = one_info['TOTAL_SHARES']
+        timeline = one_info['END_DATE'][:10]
+        change_reason = one_info['CHANGE_REASON']
+        listed_circulating_a_shares = one_info['LISTED_A_SHARES']
+        unit = '万股'
+        create_by = 'TangYuHang'
+        create_time = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
+
+        with cnx_in.cursor() as cursor:
+            sql_sel = '''select id from ranklist_market_equity where social_credit_code = %s and timeline = %s '''
+            cursor.execute(sql_sel,(social_code,timeline))
+            select = cursor.fetchone()
+            if select:
+                break
+            else:
+                tuple_info = (social_code,code,com_name,short_name,code,
+                             total_share_capital,timeline,change_reason,
+                             listed_circulating_a_shares,unit,create_by,
+                             create_time)
+                sql_in = '''insert into ranklist_market_equity(social_credit_code,stock_code,enterprise_name,securities_abbreviation,securities_code,total_share_capital,timeline,change_reason,listed_circulating_a_shares,unit,create_by,create_time) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''
+                cursor.execute(sql_in,tuple_info)
+                cnx_in.commit()
+