Merge remote-tracking branch 'origin/master'

base/BaseCore.py

@@ -369,7 +369,7 @@ class BaseCore:
         if beginStr=='':
             pass
         else:
-            begin=str.find(beginStr)
+            begin=str.rfind(beginStr)
             if begin==-1:
                 begin=0
             str=str[begin:]
@@ -425,11 +425,18 @@ class BaseCore:
         IP = socket.gethostbyname(socket.gethostname())
         return IP
 
+    def mkPath(self,path):
+        folder = os.path.exists(path)
+        if not folder:  # 判断是否存在文件夹如果不存在则创建为文件夹
+            os.makedirs(path)  # makedirs 创建文件时如果路径不存在会创建这个路径
+        else:
+            pass
     # 生成google模拟浏览器  必须传入值为googledriver位置信息
     # headless用于决定是否为无头浏览器,初始默认为无头浏览器
     # 正常浏览器可用于开始对页面解析使用或一些网站无头时无法正常采集
     # 无头浏览器用于后续对信息采集时不会有浏览器一直弹出，
     def buildDriver(self, path, headless=True):
+
         service = Service(path)
         chrome_options = webdriver.ChromeOptions()
         if headless:
@@ -442,7 +449,7 @@ class BaseCore:
 
         chrome_options.add_argument('user-agent=' + self.getRandomUserAgent())
         # 'user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36')
-        driver = webdriver.Chrome(chrome_options=chrome_options, service=service)
+        driver = webdriver.Chrome(options=chrome_options, service=service)
         # with open(r'F:\zzsn\zzsn_spider\base\stealth.min.js') as f:
         #     js = f.read()
         #
@@ -586,3 +593,4 @@ class BaseCore:
 
 
 
+

google_comm/googleSpider.py

+from urllib.parse import urljoin
 
 import langid
 import pymysql
@@ -407,12 +408,45 @@ class GoogleSpider(object):
             else:
                 break
                 # time.sleep(5)
+
+    def rmTagattr(self,html,url):
+        # 使用BeautifulSoup解析网页内容
+        # soup = BeautifulSoup(html, 'html.parser')
+        soup = self.paserUrl(html,url)
+        # 遍历所有标签，并去掉属性
+        for tag in soup.find_all(True):
+            if tag.name == 'img':
+                tag.attrs = {key: value for key, value in tag.attrs.items() if key == 'src'}
+            elif tag.name !='img':
+                tag.attrs = {key: value for key, value in tag.attrs.items() if key == 'src'}
+            else:
+                tag.attrs = {key: value for key, value in tag.attrs.items()}
+        # 打印去掉属性后的网页内容
+        # print(soup.prettify())
+        html=soup.prettify()
+        return html
+
+    # 将html中的相对地址转换成绝对地址
+    def paserUrl(self,html,listurl):
+        soup = BeautifulSoup(html, 'html.parser')
+        # 获取所有的<a>标签和<img>标签
+        links = soup.find_all(['a', 'img'])
+        # 遍历标签，将相对地址转换为绝对地址
+        for link in links:
+            if 'href' in link.attrs:
+                link['href'] = urljoin(listurl, link['href'])
+            elif 'src' in link.attrs:
+                link['src'] = urljoin(listurl, link['src'])
+
+        return soup
+
     #获取资讯内容信息
     def getDetailmsg(self,detailmsg):
         try:
             detailurl=detailmsg['detailUrl']
             title = detailmsg['title']
             content,contentWithTag=self.extractorMsg(detailurl,title)
+            contentWithTag=self.rmTagattr(contentWithTag)
         except Exception as e:
             content=''
             contentWithTag=''

sougou_comm/sougouSpider.py

@@ -122,6 +122,7 @@ class SougouSpider(object):
                 "user-agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36"
             }
             # url = 'https://www.sogou.com/link?url=hedJjaC291NbWrwHYHKCyPQj_ei8OKC13fJZ5YRQyvgjcXe6RUhCEXfbi95UdEys0ztd7q5nl6o.'
+            url=f"https://www.sogou.com{url}"
             res = requests.get(url,headers=header)
             text=res.text
             # 定义正则表达式