专题数据同步任务优化

src/main/java/com/zzsn/knowbase/entity/KbKnowledges.java

@@ -66,5 +66,15 @@ public class KbKnowledges extends Model<KbKnowledges> {
      */
     @TableField("sort")
     private Integer sort;
+    /**
+     * 数据同步开始时间
+     */
+    @TableField(exist = false)
+    private String syncStart;
+    /**
+     * 数据同步结束时间
+     */
+    @TableField(exist = false)
+    private String syncEnd;
 
 }

src/main/java/com/zzsn/knowbase/task/SubjectDataSyncTask.java

@@ -48,16 +48,31 @@ public class SubjectDataSyncTask {
 	private ProduceInfo produceInfo;
 
 	@Scheduled(cron = "0 */10 * * * ?")
-	@GetMapping("startTask")
 	public void startTask() {
-		List<KbKnowledges> confList = kbKnowledgesService.syncSubjectConf();
-		for (KbKnowledges kbKnowledges : confList) {
-			sync(kbKnowledges);
-		}
+		manualRunTask(null,null,null);
 	}
-	@GetMapping("syncById")
-	public void syncById(String id) {
-		sync(kbKnowledgesService.getById(id));
+
+	/**
+	 * 任务开始执行
+	 * @param syncStart 更新时间-开始
+	 * @param syncEnd 更新时间-结束
+	 * @param knowId 知识库id
+	 */
+	@GetMapping("manualRunTask")
+	public void manualRunTask(String syncStart,String syncEnd,String knowId) {
+		if(StringUtils.isNotEmpty(knowId)){
+			KbKnowledges byId = kbKnowledgesService.getById(knowId);
+			byId.setSyncStart(syncStart);
+			byId.setSyncEnd(syncEnd);
+			sync(byId);
+		}else{
+			List<KbKnowledges> confList = kbKnowledgesService.syncSubjectConf();
+			for (KbKnowledges kbKnowledges : confList) {
+				kbKnowledges.setSyncStart(syncStart);
+				kbKnowledges.setSyncEnd(syncEnd);
+				sync(kbKnowledges);
+			}
+		}
 	}
 	public void sync(KbKnowledges kbKnowledges) {
 		//按审核状态分组
@@ -199,8 +214,15 @@ public class SubjectDataSyncTask {
 		//查询待同步数据id
 		BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
 		boolQuery.must(QueryBuilders.termsQuery("subjectId", kbKnowledges.getSubjectId().split(",")));
-		//只查询更新时间在最近一周的数据
-		boolQuery.filter(QueryBuilders.rangeQuery("updateDate").gte(DateUtil.getStringDate(new Date(System.currentTimeMillis() - 7 * 24 * 60 * 60 * 1000L))));
+		if(StringUtils.isNotEmpty(kbKnowledges.getSyncStart())){
+			boolQuery.filter(QueryBuilders.rangeQuery("updateDate").gte(kbKnowledges.getSyncStart()));
+		}else{
+			//只查询更新时间在最近一周的数据
+			boolQuery.filter(QueryBuilders.rangeQuery("updateDate").gte(DateUtil.getStringDate(new Date(System.currentTimeMillis() - 7 * 24 * 60 * 60 * 1000L))));
+		}
+		if(StringUtils.isNotEmpty(kbKnowledges.getSyncEnd())){
+			boolQuery.filter(QueryBuilders.rangeQuery("updateDate").lte(kbKnowledges.getSyncEnd()));
+		}
 		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
 		//更新时间倒序
 		searchSourceBuilder.sort("updateDate", SortOrder.DESC);
@@ -209,7 +231,7 @@ public class SubjectDataSyncTask {
 		//查询指定字段
 		searchSourceBuilder.fetchSource(new String[]{"id","deleteFlag","checkStatus","publishStatus","updateDate"}, null);
 		Map<String, Object> map = esOpUtil.searchByQuery(Constants.ES_SUBJECT_DATA ,
-				0, 10000, searchSourceBuilder);
+				0, 100000, searchSourceBuilder);
 		List<Map<String, Object>> list = (List<Map<String, Object>>) map.get("data");
 		if(list != null && !list.isEmpty()) {
 			//分组提取id列表
@@ -230,7 +252,15 @@ public class SubjectDataSyncTask {
 		BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
 		boolQuery.must(QueryBuilders.termsQuery("subjectId", kbKnowledges.getSubjectId().split(",")));
 		//只查询更新时间在最近一周的数据
-		boolQuery.filter(QueryBuilders.rangeQuery("updateDate").gte(DateUtil.getStringDate(new Date(System.currentTimeMillis() - 7 * 24 * 60 * 60 * 1000L))));
+		if(StringUtils.isNotEmpty(kbKnowledges.getSyncStart())){
+			boolQuery.filter(QueryBuilders.rangeQuery("updateDate").gte(kbKnowledges.getSyncStart()));
+		}else{
+			//只查询更新时间在最近一周的数据
+			boolQuery.filter(QueryBuilders.rangeQuery("updateDate").gte(DateUtil.getStringDate(new Date(System.currentTimeMillis() - 7 * 24 * 60 * 60 * 1000L))));
+		}
+		if(StringUtils.isNotEmpty(kbKnowledges.getSyncEnd())){
+			boolQuery.filter(QueryBuilders.rangeQuery("updateDate").lte(kbKnowledges.getSyncEnd()));
+		}
 		SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
 		//更新时间倒序
 		searchSourceBuilder.sort("updateDate", SortOrder.DESC);
@@ -239,7 +269,7 @@ public class SubjectDataSyncTask {
 		//查询指定字段
 		searchSourceBuilder.fetchSource(new String[]{"id","deleteFlag","verifyStatus","updateDate"}, null);
 		Map<String, Object> map = esOpUtil.searchByQuery(Constants.ES_DATA_FOR_KNOWLEDGE ,
-				0, 10000, searchSourceBuilder);
+				0, 100000, searchSourceBuilder);
 		List<Map<String, Object>> list = (List<Map<String, Object>>) map.get("data");
 		if(list != null || !list.isEmpty()) {
 			//分组提取id列表

src/main/resources/application.yml

@@ -28,7 +28,7 @@ spring:
           zkNodes: 114.116.36.231:2181
           requiredAcks: 1
   redis:
-    database: 0
+    database: 9
     host: 114.116.90.53
     lettuce:
       pool:
@@ -37,8 +37,8 @@ spring:
         max-wait: -1ms  #最大建立连接等待时间。如果超过此时间将接到异常。设为-1表示无限制。
         min-idle: 0     #最小等待连接中的数量,设 0 为没有限制
       shutdown-timeout: 100ms
-    port: 6380
-    password: clbzzsn
+    port: 6379
+    password: zzsn9988
 
   thymeleaf:
     prefix: classpath:/templates