Skip to content

E
event
提交 1820fab7 作者: 925993793@qq.com
操作

【fix】导出时数据查询bug修改

上级 b3c4febe
正在显示 包含 83 行增加 和 49 行删除
src/main/java/com/zzsn/event/controller/FileController.java
......@@ -360,7 +360,7 @@ public class FileController {
fetchFields = new String[]{"id", "title", "content", "origin", "publishDate", "sourceAddress"};
}
searchCondition.setFetchFields(fetchFields);
List<SpecialInformation> informationList = esService.informationList(searchCondition);
List<SpecialInformation> informationList = informationService.informationAllList(searchCondition);
Map<String, Object> map = formatDocData(informationList, null, exportType);
Template template = FreeMarkerConfiguration.get().getTemplate("EVENT_DATA_REPORT.ftl", "UTF-8");
bos = new ByteArrayOutputStream();
......@@ -433,7 +433,7 @@ public class FileController {
SXSSFWorkbook workbook = new SXSSFWorkbook();
for (int i = 1; ; i++) {
searchCondition.setPageNo(i);
List<SpecialInformation> informationList = esService.informationList(searchCondition);
List<SpecialInformation> informationList = informationService.informationAllList(searchCondition);
log.info("本次循环-{}，数据量为-{}", i, informationList.size());
if (CollectionUtils.isEmpty(informationList)) {
break;
......
src/main/java/com/zzsn/event/controller/common/StatisticalAnalysisController.java
......@@ -8,10 +8,7 @@ import com.zzsn.event.entity.InfoSource;
import com.zzsn.event.entity.Subject;
import com.zzsn.event.entity.SysDictItem;
import com.zzsn.event.es.EsService;
import com.zzsn.event.service.ClbLabelService;
import com.zzsn.event.service.IInfoSourceService;
import com.zzsn.event.service.SubjectService;
import com.zzsn.event.service.SysDictItemService;
import com.zzsn.event.service.*;
import com.zzsn.event.util.DateUtil;
import com.zzsn.event.vo.*;
import com.zzsn.event.vo.es.Label;
......@@ -47,6 +44,8 @@ public class StatisticalAnalysisController {
private SysDictItemService sysDictItemService;
@Autowired
private ClbLabelService clbLabelService;
@Autowired
private InformationService informationService;
/**
* 概述分析
......@@ -76,7 +75,7 @@ public class StatisticalAnalysisController {
} else {
String[] fetchFields = new String[]{"id", "sid", "labels"};
searchCondition.setFetchFields(fetchFields);
List<SpecialInformation> specialInformationList = esService.informationList(searchCondition);
List<SpecialInformation> specialInformationList = informationService.informationAllList(searchCondition);
totalCount = specialInformationList.size();
Map<String, List<SpecialInformation>> map = buildMap(specialInformationList, labelMap);
for (Map.Entry<String, List<SpecialInformation>> entry : map.entrySet()) {
......@@ -114,7 +113,7 @@ public class StatisticalAnalysisController {
} else {
String[] fetchFields = new String[]{"id", "classificationType", "publishDate"};
searchCondition.setFetchFields(fetchFields);
List<SpecialInformation> informationList = esService.informationList(searchCondition);
List<SpecialInformation> informationList = informationService.informationAllList(searchCondition);
String groupType = searchCondition.getGroupType();
String format = null;
if ("year".equalsIgnoreCase(groupType)) {
......@@ -174,7 +173,7 @@ public class StatisticalAnalysisController {
} else {
String[] fetchFields = new String[]{"id", "classificationType"};
searchCondition.setFetchFields(fetchFields);
List<SpecialInformation> informationList = esService.informationList(searchCondition);
List<SpecialInformation> informationList = informationService.informationAllList(searchCondition);
Map<Integer, List<SpecialInformation>> map = informationList.stream()
.collect(Collectors.groupingBy(SpecialInformation::getClassificationType));
for (Map.Entry<Integer, List<SpecialInformation>> entry : map.entrySet()) {
......@@ -208,7 +207,7 @@ public class StatisticalAnalysisController {
} else {
String[] fetchFields = new String[]{"id", "labels"};
searchCondition.setFetchFields(fetchFields);
List<SpecialInformation> specialInformationList = esService.informationList(searchCondition);
List<SpecialInformation> specialInformationList = informationService.informationAllList(searchCondition);
Map<String, List<SpecialInformation>> map = buildMap(specialInformationList, labelMap);
for (Map.Entry<String, List<SpecialInformation>> entry : map.entrySet()) {
CountVO countVO = new CountVO();
......@@ -245,7 +244,7 @@ public class StatisticalAnalysisController {
} else {
String[] fetchFields = new String[]{"id", "sid"};
searchCondition.setFetchFields(fetchFields);
List<SpecialInformation> informationList = esService.informationList(searchCondition);
List<SpecialInformation> informationList = informationService.informationAllList(searchCondition);
Map<String, List<SpecialInformation>> map = informationList.stream()
.collect(Collectors.groupingBy(SpecialInformation::getSid));
for (Map.Entry<String, List<SpecialInformation>> entry : map.entrySet()) {
......
src/main/java/com/zzsn/event/es/EsService.java
......@@ -1158,27 +1158,24 @@ public class EsService {
}
//默认最大数量是10000，设置为true后，显示准确数量
searchSourceBuilder.trackTotalHits(true);
//设置分页参数
Integer pageNo = searchCondition.getPageNo();
Integer pageSize = searchCondition.getPageSize();
//创建查询对象
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
//判断ids字段是否为空，若不为空，则表示按照id勾选
List<String> ids = searchCondition.getIds();
if (CollectionUtils.isNotEmpty(ids)) {
searchSourceBuilder.size(ids.size());
boolQuery.must(QueryBuilders.termsQuery("id", ids));
List<String> sourceIdList = searchCondition.getSourceIdList();
if (CollectionUtils.isNotEmpty(sourceIdList)) {
boolQuery.must(QueryBuilders.termsQuery("sid.keyword", sourceIdList));
}
} else {
//设置分页参数
Integer pageNo = searchCondition.getPageNo();
Integer pageSize = searchCondition.getPageSize();
Integer num = searchCondition.getNum();
if (num != null) {
pageSize = num;
}
searchSourceBuilder.size(pageSize);
searchSourceBuilder.from((pageNo - 1) * pageSize);
List<String> subjectIdList = new ArrayList<>();
//判断是否是专题
if ("1".equals(searchCondition.getIsSubject())) {
......@@ -1199,6 +1196,8 @@ public class EsService {
}
boolQuery = buildQuery(searchCondition, subjectIdList);
}
searchSourceBuilder.from((pageNo - 1) * pageSize);
searchSourceBuilder.size(pageSize);
searchSourceBuilder.query(boolQuery);
searchRequest.source(searchSourceBuilder);
List<SpecialInformation> dataList = new ArrayList<>();
......
src/main/java/com/zzsn/event/service/InformationService.java
......@@ -7,6 +7,7 @@ import com.zzsn.event.entity.ClbFileOperationLog;
import com.zzsn.event.util.user.UserVo;
import com.zzsn.event.vo.*;
import com.zzsn.event.vo.es.DisplayInfo;
import com.zzsn.event.vo.es.SpecialInformation;
import java.util.HashMap;
import java.util.List;
......@@ -218,4 +219,13 @@ public interface InformationService {
* @date 2025/2/20
*/
void supplyByCondition(String subjectId, List<String> themeIds);
/**
* 获取所有资讯列表
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/2/28
*/
List<SpecialInformation> informationAllList(InfoDataSearchCondition searchCondition);
}
src/main/java/com/zzsn/event/service/impl/InformationServiceImpl.java
......@@ -289,40 +289,46 @@ public class InformationServiceImpl implements InformationService {
public void saveAsDataSet(InfoDataSearchCondition searchCondition) {
String[] fetchFields = new String[]{"id", "labels"};
searchCondition.setFetchFields(fetchFields);
searchCondition.setPageSize(300);
Label dataSet = new Label();
String dataSetId = searchCondition.getDataSetId();
dataSet.setRelationId(dataSetId);
Map<String, List<SpecialInformation>> map = new HashMap<>();
for (int i = 1; ; i++) {
searchCondition.setPageNo(i);
List<SpecialInformation> informationList = esService.informationList(searchCondition);
log.info("保存数据集：本次循环-{}，数据量为-{}", i, informationList.size());
if (CollectionUtils.isEmpty(informationList)) {
break;
}
for (SpecialInformation information : informationList) {
String index = information.getDbIndex();
List<Label> labels = information.getLabels();
if (CollectionUtils.isNotEmpty(labels)) {
boolean present = labels.stream().anyMatch(label -> label.getRelationId().equals(dataSetId));
if (!present) {
labels.add(dataSet);
}
} else {
Integer num = searchCondition.getNum();
List<SpecialInformation> informationList = this.informationAllList(searchCondition);
formatUpdateMap(map,informationList,dataSet);
map.forEach((k, v) -> esOpUtil.docUpdateBulk(k, v));
}
/**
* 格式化数据
*
* @param map 待更新的数据map集合
* @param informationList 查询到的数据集合
* @param dataSet 数据集标签信息
* @author lkg
* @date 2025/2/28
*/
private void formatUpdateMap(Map<String, List<SpecialInformation>> map,List<SpecialInformation> informationList,Label dataSet){
for (SpecialInformation information : informationList) {
String index = information.getDbIndex();
List<Label> labels = information.getLabels();
if (CollectionUtils.isNotEmpty(labels)) {
boolean present = labels.stream().anyMatch(label -> label.getRelationId().equals(dataSet.getRelationId()));
if (!present) {
labels.add(dataSet);
}
information.setLabels(labels);
if (map.containsKey(index)) {
map.get(index).add(information);
} else {
List<SpecialInformation> list = new ArrayList<>();
list.add(information);
map.put(index, list);
}
} else {
labels.add(dataSet);
}
information.setLabels(labels);
if (map.containsKey(index)) {
map.get(index).add(information);
} else {
List<SpecialInformation> list = new ArrayList<>();
list.add(information);
map.put(index, list);
}
}
map.forEach((k, v) -> esOpUtil.docUpdateBulk(k, v));
}
@Override
......@@ -923,7 +929,27 @@ public class InformationServiceImpl implements InformationService {
}
redisUtil.rpushMultipleValues(Constants.HISTORY_SUBJECT_DATE_QUEUE + subject.getSubjectCode(),redisCacheList.toArray(new String[0]));
}
}
@Override
public List<SpecialInformation> informationAllList(InfoDataSearchCondition searchCondition) {
List<SpecialInformation> informationList = new ArrayList<>();
Integer num = searchCondition.getNum();
if (num == null) {
searchCondition.setPageSize(300);
for (int i = 1; ; i++) {
searchCondition.setPageNo(i);
List<SpecialInformation> list = esService.informationList(searchCondition);
log.info("保存数据集：本次循环-{}，数据量为-{}", i, list.size());
if (CollectionUtils.isEmpty(list)) {
break;
}
informationList.addAll(list);
}
} else {
informationList = esService.informationList(searchCondition);
}
return informationList;
}
private void formatLabel(List<LabelModelVo> labelModelVos, DisplayInfo info) {
......
src/main/java/com/zzsn/event/service/impl/SubjectSimpleServiceImpl.java
......@@ -71,9 +71,9 @@ public class SubjectSimpleServiceImpl implements SubjectSimpleService {
private ScoreModelService scoreModelService;
@Autowired
private ConfigurationMessageService configurationMessageService;
@Autowired
private EsService esService;
private InformationService informationService;
@Autowired
private EsOpUtil esOpUtil;
@Autowired
......@@ -371,7 +371,7 @@ public class SubjectSimpleServiceImpl implements SubjectSimpleService {
String[] fetchFields = new String[]{"id", "checkStatus"};
searchCondition.setFetchFields(fetchFields);
Map<String, List<SpecialInformation>> updateMap = new HashMap<>();
List<SpecialInformation> informationList = esService.informationList(searchCondition);
List<SpecialInformation> informationList = informationService.informationAllList(searchCondition);
for (SpecialInformation information : informationList) {
String index = information.getDbIndex();
information.setCheckStatus(1);
......@@ -391,7 +391,7 @@ public class SubjectSimpleServiceImpl implements SubjectSimpleService {
String[] fetchFields = new String[]{"id", "checkStatus"};
searchCondition.setFetchFields(fetchFields);
Map<String, List<SpecialInformation>> updateMap = new HashMap<>();
List<SpecialInformation> informationList = esService.informationList(searchCondition);
List<SpecialInformation> informationList = informationService.informationAllList(searchCondition);
for (SpecialInformation information : informationList) {
String index = information.getDbIndex();
information.setCheckStatus(0);
......@@ -411,7 +411,7 @@ public class SubjectSimpleServiceImpl implements SubjectSimpleService {
String[] fetchFields = new String[]{"id", "deleteFlag"};
searchCondition.setFetchFields(fetchFields);
Map<String, List<SpecialInformation>> updateMap = new HashMap<>();
List<SpecialInformation> informationList = esService.informationList(searchCondition);
List<SpecialInformation> informationList = informationService.informationAllList(searchCondition);
for (SpecialInformation information : informationList) {
String index = information.getDbIndex();
information.setDeleteFlag(1);
......
src/main/java/com/zzsn/event/vo/InfoDataSearchCondition.java
......@@ -125,7 +125,7 @@ public class InfoDataSearchCondition {
//选择前num条数据
private Integer num;
//资讯id集合-研究中心
//资讯id集合
private List<String> ids;
//关联标签名称
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论