采集代码更新6

baidu_search/src/main/java/com/zzsn/search/MetaBaiduSearchThread.java

 package com.zzsn.search;
 
+import cn.hutool.core.date.DateUtil;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.zzsn.cache.JedisUtil;
 import com.zzsn.cache.MemcachedUtils;
@@ -649,6 +650,8 @@ public class MetaBaiduSearchThread implements Runnable {
         clbAnsProcessitem.setOrigin(docInfo.getOrigin());
         clbAnsProcessitem.setPublishDate(docInfo.getPublishDate());
         clbAnsProcessitem.setSourceAddress(docInfo.getSourceaddress());
+        String now = DateUtil.now();
+        clbAnsProcessitem.setCreateDate(now);
         clbAnsProcessitem.setSource("3");
         return clbAnsProcessitem;
     }

baidu_search/src/main/java/com/zzsn/search/util/RecorderUtil.java

@@ -337,7 +337,7 @@ public class RecorderUtil {
 //					docstr = seleniumTime.getScopehtmlByProxy(urlList.get(i));
 					if(StringUtils.isEmpty(docstr)){
 						try {
-							Thread.sleep(1000*5);
+							Thread.sleep(1000*2);
 							SeleniumTime seleniumTime2=new SeleniumTime();
 							docstr=seleniumTime2.getScopehtml(urlList.get(i));
 						} catch (InterruptedException e) {

sougou_crawler/src/main/java/com/zzsn/search/MetaSouGouSearchThread.java

@@ -90,14 +90,14 @@ public class MetaSouGouSearchThread implements Runnable {
             return;
         }
         for (String kWord :keyWords) {
-            try {
-                boolean sismember = JedisUtil.sismember(cache_key, kWord);
-                if (sismember) {
-                    continue;
-                }
-            }catch (Exception e){
-                log.info("缓存出问题");
-            }
+//            try {
+//                boolean sismember = JedisUtil.sismember(cache_key, kWord);
+//                if (sismember) {
+//                    continue;
+//                }
+//            }catch (Exception e){
+//                log.info("缓存出问题");
+//            }
             String url1= Constants.META_SEARCH_URL;
 //            String[] kwords=kWord.split("\\+");
             String url="";
@@ -362,6 +362,7 @@ public class MetaSouGouSearchThread implements Runnable {
 
                     // 加入缓存池中
                     System.out.println("加入缓存池");
+//                    JedisUtil.sadd(rediskey, cwbm.getSourceaddress());
 //                    addressList.add(docInfo.getSourceaddress());
 //                    MemcachedUtils.set(key,addressList,60*60*24);
                     JedisUtil.setString(Constants.SOURCEADDRESS+"_"+orgId+"_"+cwbm.getSourceaddress(),"1",-1);