Merge remote-tracking branch 'origin/master'

base/RedisPPData.py

@@ -297,6 +297,38 @@ def FBS():
             r.rpush('NewsEnterpriseFbs:gnqy_socialCode', item)
     closeSql(cnx,cursor)
 
+#将IPO的国外股票代码放到redis中
+def yahooCodeFromSql():
+    cnx_,cursor_=cnn11()
+    try:
+        gn_query = "select securities_code from sys_base_enterprise_ipo where category in ('4','5','6') "
+        cursor_.execute(gn_query)
+        gn_result = cursor_.fetchall()
+        gn_social_list = [item[0] for item in gn_result]
+        print('=======')
+        for item in gn_social_list:
+            r.rpush('NoticeEnterprise:securities_code', item)
+    except Exception as e:
+        log.info("数据查询异常")
+    finally:
+        close11(cnx_,cursor_)
+
+#雅虎财务数据3天一次
+def yahooCode_task():
+    # 实例化一个调度器
+    scheduler = BlockingScheduler()
+    # 每天执行一次
+    # scheduler.add_job(yahooCodeFromSql, 'cron', hour=0,minute=0)
+    #3天执行一次
+    scheduler.add_job(yahooCodeFromSql, 'interval', days=3)
+    try:
+        yahooCodeFromSql()  # 定时开始前执行一次
+        scheduler.start()
+    except Exception as e:
+        print('定时采集异常', e)
+        pass
+
+
 if __name__ == "__main__":
     start = time.time()
     # NoticeEnterprise()

comData/caiwushuju/RedisPPData.py

+import time
+
+from base import BaseCore
+from apscheduler.schedulers.blocking import BlockingScheduler
+import pymysql
+basecore = BaseCore.BaseCore()
+log = basecore.getLogger()
+r = basecore.r
+
+
+def conn11():
+    conn = pymysql.Connect(host='114.116.44.11', port=3306, user='root', passwd='f7s0&7qqtK', db='clb_project',
+                           charset='utf8')
+    cursor = conn.cursor()
+    return conn,cursor
+
+#企业公告
+def yahooCodeFromSql():
+    conn,cursor=conn11()
+    try:
+        gn_query = "select securities_code from sys_base_enterprise_ipo where category in ('4','5','6') "
+        cursor.execute(gn_query)
+        gn_result = cursor.fetchall()
+        gn_social_list = [item[0] for item in gn_result]
+        print('=======')
+        for item in gn_social_list:
+            r.rpush('NoticeEnterprise:securities_code', item)
+    except Exception as e:
+        log.info("数据查询异常")
+    finally:
+        cursor.close()
+        conn.close()
+
+def yahooCode_task():
+    # 实例化一个调度器
+    scheduler = BlockingScheduler()
+    # 每天执行一次
+    # scheduler.add_job(yahooCodeFromSql, 'cron', hour=0,minute=0)
+    #3天执行一次
+    scheduler.add_job(yahooCodeFromSql, 'interval', days=3)
+    try:
+        yahooCodeFromSql()  # 定时开始前执行一次
+        scheduler.start()
+    except Exception as e:
+        print('定时采集异常', e)
+        pass
+
+if __name__ == "__main__":
+    start = time.time()
+    # NoticeEnterprise()
+    # AnnualEnterpriseIPO()
+    # AnnualEnterprise()
+    # BaseInfoEnterpriseAbroad()
+    # NewsEnterprise_task()
+    # NewsEnterprise()
+    # BaseInfoEnterprise()
+    # FBS()
+    # NoticeEnterprise_task()
+    # AnnualEnterprise_task()
+    # NoticeEnterprise()
+    yahooCode_task()
+    log.info(f'====={basecore.getNowTime(1)}=====添加数据成功======耗时：{basecore.getTimeCost(start,time.time())}===')
+    # cnx.close()
+    # cursor.close()
+    # basecore.close()

comData/caiwushuju/config.ini

+[redis]
+host=114.115.236.206
+port=6379
+pass=clbzzsn
+
+[mysql]
+host=114.115.159.144
+username=root
+password=zzsn9988
+database=caiji
+url=jdbc:mysql://114.115.159.144:3306/caiji?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&useSSL=false
+
+[kafka]
+bootstrap_servers=114.115.159.144:9092
+topic=keyWordsInfo
+groupId=python_baidu
+
+[selenium]
+chrome_driver=C:\Users\WIN10\DataspellProjects\crawlerProjectDemo\tmpcrawler\cmd100\chromedriver.exe
+binary_location=D:\crawler\baidu_crawler\tool\Google\Chrome\Application\chrome.exe
+