lly 2024-07-25

jixie/script/.idea/.gitignore

+# 默认忽略的文件
+/shelf/
+/workspace.xml
+# 基于编辑器的 HTTP 客户端请求
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

jixie/script/.idea/inspectionProfiles/Project_Default.xml

+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="E501" />
+        </list>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="SqlNoDataSourceInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+  </profile>
+</component>
\ No newline at end of file

jixie/script/.idea/inspectionProfiles/profiles_settings.xml

+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>
\ No newline at end of file

jixie/script/.idea/misc.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file

jixie/script/.idea/modules.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/script.iml" filepath="$PROJECT_DIR$/.idea/script.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file

jixie/script/.idea/script.iml

+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>
\ No newline at end of file

jixie/script/BaseCore.py

+# 核心工具包
+import os
+import random
+import socket
+import sys
+import time
+
+import langid
+import logbook
+import logbook.more
+import zhconv
+
+
+class BaseCore:
+    # 计算耗时
+    def getTimeCost(self, start, end):
+        seconds = int(end - start)
+        m, s = divmod(seconds, 60)
+        h, m = divmod(m, 60)
+        if (h > 0):
+            return "%d小时%d分钟%d秒" % (h, m, s)
+        elif (m > 0):
+            return "%d分钟%d秒" % (m, s)
+        elif (seconds > 0):
+            return "%d秒" % (s)
+        else:
+            ms = int((end - start) * 1000)
+            return "%d毫秒" % (ms)
+
+    # 当前时间格式化
+    # 1 : 2001-01-01 12:00:00 %Y-%m-%d %H:%M:%S
+    # 2 : 010101120000 %y%m%d%H%M%S
+    # 时间戳 3:1690179526555  精确到秒
+    def getNowTime(self, type):
+        now_time = ""
+        if type == 1:
+            now_time = time.strftime("%Y-%m-%d %H:%M:%S")
+        if type == 2:
+            now_time = time.strftime("%y%m%d%H%M%S")
+        if type == 3:
+            now_time = int(time.time() * 1000)
+        return now_time
+
+    # 日志格式
+    def logFormate(self, record, handler):
+        formate = "[{date}] [{level}] [{filename}] [{func_name}] [{lineno}] {msg}".format(
+            date=record.time,  # 日志时间
+            level=record.level_name,  # 日志等级
+            filename=os.path.split(record.filename)[-1],  # 文件名
+            func_name=record.func_name,  # 函数名
+            lineno=record.lineno,  # 行号
+            msg=record.message  # 日志内容
+        )
+        return formate
+
+    # 获取logger
+    def getLogger(self, fileLogFlag=True, stdOutFlag=True):
+        dirname, filename = os.path.split(os.path.abspath(sys.argv[0]))
+        dirname = os.path.join(dirname, "logs")
+        filename = filename.replace(".py", "") + ".log"
+        if not os.path.exists(dirname):
+            os.mkdir(dirname)
+
+        logbook.set_datetime_format('local')
+        logger = logbook.Logger(filename)
+        logger.handlers = []
+
+        if fileLogFlag:  # 日志输出到文件
+            logFile = logbook.TimedRotatingFileHandler(os.path.join(dirname, filename), date_format='%Y-%m-%d',
+                                                       bubble=True, encoding='utf-8')
+            logFile.formatter = self.logFormate
+            logger.handlers.append(logFile)
+
+        if stdOutFlag:  # 日志打印到屏幕
+            logStd = logbook.more.ColorizedStderrHandler(bubble=True)
+            logStd.formatter = self.logFormate
+            logger.handlers.append(logStd)
+        return logger
+
+    # 获取随机的userAgent
+    def getRandomUserAgent(self):
+        return random.choice(self.__USER_AGENT_LIST)
+
+    # 字符串截取
+    def getSubStr(self, str, beginStr, endStr):
+        if beginStr == '':
+            pass
+        else:
+            begin = str.rfind(beginStr)
+            if begin == -1:
+                begin = 0
+            str = str[begin:]
+        if endStr == '':
+            pass
+        else:
+            end = str.rfind(endStr)
+            if end == -1:
+                pass
+            else:
+                str = str[0:end + 1]
+        return str
+
+    # 繁体字转简体字
+    def hant_2_hans(self, hant_str: str):
+        '''
+        Function: 将 hant_str 由繁体转化为简体
+        '''
+        return zhconv.convert(hant_str, 'zh-hans')
+
+    # 判断字符串里是否含数字
+    def str_have_num(self, str_num):
+        panduan = False
+
+        for str_1 in str_num:
+            ppp = str_1.isdigit()
+            if ppp:
+                panduan = ppp
+        return panduan
+
+    # 获得脚本进程PID
+    def getPID(self):
+        PID = os.getpid()
+        return PID
+
+    # 获取本机IP
+    def getIP(self):
+        IP = socket.gethostbyname(socket.gethostname())
+        return IP
+
+    # 检测语言
+    def detect_language(self, text):
+        # 使用langid.py判断文本的语言
+        result = langid.classify(text)
+        if result == '':
+            return 'cn'
+        if result[0] == '':
+            return 'cn'
+        return result[0]

jixie/script/config.ini

+[mysql]
+host=114.115.159.144
+port=3306
+username=caiji
+password=zzsn9988
+database=jx_enterprise
+
+
+# 基本信息采集时间设定
+[taskA]
+# 每月的几号开始采集
+# 格式:1,2,3/1
+#每月的1、2、3号开始执行 或 每月的1号执行
+day=15
+# 几点开始执行
+# 格式:12,13/12
+# 12、13各执行一次 或 12点执行一次
+hour=12
+# 几分开始执行
+#格式:0,30/0
+# 整点、半点各执行一次 或 整点执行一次
+minute=0
+
+# 其它数据采集时间设定
+[taskB]
+# 每月的几号开始采集
+# 格式:1,2,3/1
+#每月的1、2、3号开始执行 或 每月的1号执行
+day=15
+# 几点开始执行
+# 格式:12,13/12
+# 12、13各执行一次 或 12点执行一次
+hour=12
+# 几分开始执行
+#格式:0,30/0
+# 整点、半点各执行一次 或 整点执行一次
+minute=0
\ No newline at end of file

jixie/script/invInfo.sh

+export PATH="/root/anaconda3/bin:/home/root/anaconda3/bin:/root/anaconda3/condabin:/usr/local/jdk/bin:/usr/lib64/qt-3.3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/usr/local/bin:/data/zookeeper/bin:/data/kafka/bin:/opt/google/chrome:/root/bin"
+nohup python invInfo.py > ./logs/invInfo_output.log 2>&1 &
\ No newline at end of file

jixie/script/logs/negativeInfo-2024-06-19.log

+[2024-06-19 14:07:43.002166] [INFO] [negativeInfo.py] [AbnormalEnterprise] [640] 开始采集企业非正常户信息===接口63.2
+[2024-06-19 14:07:44.969510] [ERROR] [negativeInfo.py] [dataRequest] [69] 中国机械工业联合会===出错状态码104
+[2024-06-19 14:07:48.759553] [ERROR] [negativeInfo.py] [dataRequest] [69] 中国农业机械工业协会===出错状态码104

jixie/script/negativeInfo.sh

+export PATH="/root/anaconda3/bin:/home/root/anaconda3/bin:/root/anaconda3/condabin:/usr/local/jdk/bin:/usr/lib64/qt-3.3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/usr/local/bin:/data/zookeeper/bin:/data/kafka/bin:/opt/google/chrome:/root/bin"
+nohup python negativeInfo.py > ./logs/negativeInfo_outpu.log 2>&1 &
\ No newline at end of file

jixie/script/organization.sh

+export PATH="/root/anaconda3/bin:/home/root/anaconda3/bin:/root/anaconda3/condabin:/usr/local/jdk/bin:/usr/lib64/qt-3.3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/usr/local/bin:/data/zookeeper/bin:/data/kafka/bin:/opt/google/chrome:/root/bin"
+nohup python organization.py > ./logs/organization_outpu.log 2>&1 &
\ No newline at end of file

jixie/script/start.sh

+nohup python doJob.py > ./logs/outpu.log 2>&1 &
\ No newline at end of file