新三板财务数据采集维护

comData/NQenterprise/finance_xq.py

@@ -48,6 +48,18 @@ def getrequests(url):
     return a_infoData
 
 def getdriver(url_name):
+    opt = webdriver.ChromeOptions()
+    opt.add_argument(
+        'user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36')
+
+    opt.add_argument("--ignore-certificate-errors")
+    opt.add_argument("--ignore-ssl-errors")
+    opt.add_experimental_option("excludeSwitches", ["enable-automation"])
+    opt.add_experimental_option('excludeSwitches', ['enable-logging'])
+    opt.add_experimental_option('useAutomationExtension', False)
+    opt.binary_location = r'D:/Google/Chrome/Application/chrome.exe'
+    chromedriver = r'D:/cmd100/chromedriver.exe'
+    browser = webdriver.Chrome(chrome_options=opt, executable_path=chromedriver)
     # 使用模拟浏览器打开
     browser.get(url_name)
     time.sleep(3)
@@ -244,18 +256,7 @@ if __name__ == '__main__':
                 continue
             list_date.append(date)
 
-    opt = webdriver.ChromeOptions()
-    opt.add_argument(
-        'user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36')
 
-    opt.add_argument("--ignore-certificate-errors")
-    opt.add_argument("--ignore-ssl-errors")
-    opt.add_experimental_option("excludeSwitches", ["enable-automation"])
-    opt.add_experimental_option('excludeSwitches', ['enable-logging'])
-    opt.add_experimental_option('useAutomationExtension', False)
-    opt.binary_location = r'D:/Google/Chrome/Application/chrome.exe'
-    chromedriver = r'D:/cmd100/chromedriver.exe'
-    browser = webdriver.Chrome(chrome_options=opt, executable_path=chromedriver)
     headers = {
         'authority': 'stock.xueqiu.com',
         'method': 'GET',

comData/YanBao/resentYanbao.py

@@ -1436,37 +1436,37 @@ if __name__ == '__main__':
     # except:
     #     pass
 
-    try:
-        log.info('dongfangcaifu3')
-        dongfangcaifu3()
-    except Exception as e:
-        log.info(e)
-        pass
-
     # try:
-    #     log.info('dongfangcaifu4')
-    #     dongfangcaifu4()
+    #     log.info('dongfangcaifu3')
+    #     dongfangcaifu3()
     # except Exception as e:
     #     log.info(e)
     #     pass
 
     # try:
-    #     log.info('dongfangcaifu5')
-    #     dongfangcaifu5()
+    #     log.info('dongfangcaifu4')
+    #     dongfangcaifu4()
     # except Exception as e:
     #     log.info(e)
     #     pass
 
-    # try:
-    #     log.info('dongfangcaifu6')
-    #     dongfangcaifu6()
-    # except Exception as e:
-    #     log.info(e)
-    #     pass
+    try:
+        log.info('dongfangcaifu5')
+        dongfangcaifu5()
+    except Exception as e:
+        log.info(e)
+        pass
 
-    # try:
-    #     log.info('dongfangcaifu7')
-    #     dongfangcaifu7()
-    # except Exception as e:
-    #     log.info(e)
-    #     pass
+    try:
+        log.info('dongfangcaifu6')
+        dongfangcaifu6()
+    except Exception as e:
+        log.info(e)
+        pass
+
+    try:
+        log.info('dongfangcaifu7')
+        dongfangcaifu7()
+    except Exception as e:
+        log.info(e)
+        pass

comData/annualReport_US/uploadfile.py

@@ -14,7 +14,8 @@ from obs import ObsClient
 from retry import retry
 from elasticsearch import Elasticsearch
 from base import BaseCore
-
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 baseCore = BaseCore.BaseCore()
 log = baseCore.getLogger()
 baseCore = BaseCore.BaseCore()
@@ -144,7 +145,7 @@ def main(esMethod):
     content = soup.text.strip()
 
     esMethod.updateaunn(esMethod.index_name, str(id), content, str(soup))
-
+    return
 
 
 def run_threads(num_threads,esMethod):