Skip to content

E
event
提交 db56138c 作者: 925993793@qq.com
操作

bug修改

上级 5c773684
正在显示 包含 91 行增加 和 64 行删除
src/main/java/com/zzsn/event/controller/SubjectManageController.java
......@@ -86,6 +86,10 @@ public class SubjectManageController {
@Resource
private KafkaTemplate<String, String> kafkaTemplate;
@Value("${kafka.topic.subject.run:}")
private String SUBJECT_MODEL_KAFKA_CHANNEL;
/**
* 专题列表-资讯转换时使用
*
......@@ -282,7 +286,7 @@ public class SubjectManageController {
.eq(XxlJobInfo::getInfoSourceCode, subject.getSubjectCode())
.set(XxlJobInfo::getTriggerStatus,subject.getStatus()));
if (subject.getStatus() == 1) {
kafkaTemplate.send("subjectModel", subject.getSubjectCode());
kafkaTemplate.send(SUBJECT_MODEL_KAFKA_CHANNEL, subject.getSubjectCode());
}
return Result.OK();
}
......
src/main/java/com/zzsn/event/controller/common/StatisticalAnalysisController.java
......@@ -65,15 +65,15 @@ public class StatisticalAnalysisController {
List<CountVO> countList = new ArrayList<>();
int totalCount;
List<Label> bindLabelList = bindLabelList(searchCondition.getSubjectId());
Map<String, Label> labelMap = bindLabelList.stream().collect(Collectors.toMap(label -> label.getLabelMark() + "-" + label.getRelationId(), label -> label));
if (CollectionUtils.isEmpty(searchCondition.getIds()) && searchCondition.getNum() == null) {
List<String> includeValues = new ArrayList<>();
bindLabelList.forEach(e -> includeValues.add(e.getRelationId()));
bindLabelList.forEach(e -> includeValues.add(e.getLabelMark() + "-" + e.getRelationId()));
searchCondition.setIncludeValues(includeValues.toArray(new String[0]));
countList = esService.overView(searchCondition);
totalCount = countList.get(0).getValue().intValue();
countList.remove(0);
} else {
Map<String, Label> labelMap = bindLabelList.stream().collect(Collectors.toMap(Label::getRelationId, label -> label));
String[] fetchFields = new String[]{"id", "sid", "labels"};
searchCondition.setFetchFields(fetchFields);
List<SpecialInformation> specialInformationList = esService.informationList(searchCondition);
......@@ -86,8 +86,11 @@ public class StatisticalAnalysisController {
List<SpecialInformation> value = entry.getValue();
Map<String, List<SpecialInformation>> sidMap = value.stream().collect(Collectors.groupingBy(SpecialInformation::getSid));
countVO.setValue((long) sidMap.size());
countList.add(countVO);
}
}
//补充数据
supplyData(countList,bindLabelList,labelMap);
overviewAnalysisVO.setInfoCount(totalCount);
overviewAnalysisVO.setCountList(countList);
return Result.OK(overviewAnalysisVO);
......@@ -109,7 +112,7 @@ public class StatisticalAnalysisController {
if (CollectionUtils.isEmpty(searchCondition.getIds()) && searchCondition.getNum() == null) {
dataList = esService.groupByDate(searchCondition);
} else {
String[] fetchFields = new String[]{"id", "classificationType","publishDate"};
String[] fetchFields = new String[]{"id", "classificationType", "publishDate"};
searchCondition.setFetchFields(fetchFields);
List<SpecialInformation> informationList = esService.informationList(searchCondition);
String groupType = searchCondition.getGroupType();
......@@ -196,10 +199,10 @@ public class StatisticalAnalysisController {
public Result<?> dataSource(@RequestBody InfoDataSearchCondition searchCondition) {
List<CountVO> dataList = new ArrayList<>();
List<Label> bindLabelList = bindLabelList(searchCondition.getSubjectId());
Map<String, Label> labelMap = bindLabelList.stream().collect(Collectors.toMap(Label::getRelationId, label -> label));
Map<String, Label> labelMap = bindLabelList.stream().collect(Collectors.toMap(label-> label.getLabelMark() + "-"+label.getRelationId(), label -> label));
if (CollectionUtils.isEmpty(searchCondition.getIds()) && searchCondition.getNum() == null) {
List<String> includeValues = new ArrayList<>();
bindLabelList.forEach(e -> includeValues.add(e.getRelationId()));
bindLabelList.forEach(e -> includeValues.add(e.getLabelMark() + "-" +e.getRelationId()));
searchCondition.setIncludeValues(includeValues.toArray(new String[0]));
dataList = esService.groupByInfoSourceTag(searchCondition);
} else {
......@@ -212,36 +215,17 @@ public class StatisticalAnalysisController {
String key = entry.getKey();
countVO.setName(key);
countVO.setValue((long) entry.getValue().size());
}
}
if (CollectionUtils.isNotEmpty(dataList)) {
for (CountVO countVO : dataList) {
String name = countVO.getName();
Label label = labelMap.get(name);
if (label != null) {
countVO.setName(label.getRelationName());
labelMap.remove(name);
}
}
if (MapUtils.isNotEmpty(labelMap)) {
dataList.addAll(labelMap.values().stream().map(e -> {
CountVO countVO = new CountVO();
countVO.setName(e.getRelationName());
countVO.setValue(0L);
return countVO;
}).collect(Collectors.toList()));
}
} else {
for (Label label : bindLabelList) {
CountVO countVO = new CountVO();
countVO.setName(label.getRelationName());
countVO.setValue(0L);
dataList.add(countVO);
}
}
//补充数据
supplyData(dataList, bindLabelList, labelMap);
return Result.OK(dataList);
}
/**
* 资讯数量top的信息源统计
*
......@@ -304,9 +288,11 @@ public class StatisticalAnalysisController {
List<InfoSourceLabelVO> infoSourceLabelInfos = clbLabelService.bindInfoSourceLabelInfo(subjectId);
if (ObjectUtil.isNotEmpty(infoSourceLabelInfos)) {
for (InfoSourceLabelVO infoSourceLabel : infoSourceLabelInfos) {
String labelCode = infoSourceLabel.getLabelCode();
List<InfoSourceLabelItemVO> infoSourceLabelItemList = infoSourceLabel.getInfoSourceLabelItemList();
for (InfoSourceLabelItemVO infoSourceLabelItem : infoSourceLabelItemList) {
Label label = new Label();
label.setLabelMark(labelCode);
label.setRelationName(infoSourceLabelItem.getLabelItemName());
label.setRelationId(infoSourceLabelItem.getLabelItemCode());
labels.add(label);
......@@ -341,11 +327,50 @@ public class StatisticalAnalysisController {
List<SpecialInformation> list = map.get(labelId);
if (CollectionUtils.isEmpty(list)) {
list = new ArrayList<>();
list.add(information);
map.put(labelId, list);
} else {
list.add(information);
}
list.add(information);
}
}
return map;
}
/**
* 补充数据
*
* @param dataList 聚合出来的数据
* @param bindLabelList 绑定的标签信息-list
* @param labelMap 绑定的标签信息-map
* @author lkg
* @date 2025/1/16
*/
private void supplyData(List<CountVO> dataList, List<Label> bindLabelList, Map<String, Label> labelMap) {
if (CollectionUtils.isNotEmpty(dataList)) {
for (CountVO countVO : dataList) {
String name = countVO.getName();
Label label = labelMap.get(name);
if (label != null) {
countVO.setName(label.getRelationName());
labelMap.remove(name);
}
}
if (MapUtils.isNotEmpty(labelMap)) {
dataList.addAll(labelMap.values().stream().map(e -> {
CountVO countVO = new CountVO();
countVO.setName(e.getRelationName());
countVO.setValue(0L);
return countVO;
}).collect(Collectors.toList()));
}
} else {
for (Label label : bindLabelList) {
CountVO countVO = new CountVO();
countVO.setName(label.getRelationName());
countVO.setValue(0L);
dataList.add(countVO);
}
}
}
}
src/main/java/com/zzsn/event/es/EsService.java
......@@ -41,6 +41,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.Cardinality;
import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.collapse.CollapseBuilder;
import org.elasticsearch.search.sort.SortBuilders;
......@@ -758,6 +759,7 @@ public class EsService {
if (fetchFields != null && fetchFields.length > 0) {
searchSourceBuilder.fetchSource(fetchFields, null);
}
//构建es查询条件
BoolQueryBuilder boolQuery = buildQuery(searchCondition, subjectIdList);
searchSourceBuilder.query(boolQuery);
searchRequest.source(searchSourceBuilder);
......@@ -1127,6 +1129,9 @@ public class EsService {
searchSourceBuilder.sort("publishDate", SortOrder.ASC);
}
break;
default:
searchSourceBuilder.sort("publishDate", SortOrder.DESC);
break;
}
String[] fetchFields = searchCondition.getFetchFields();
if (fetchFields != null && fetchFields.length > 0) {
......@@ -1425,9 +1430,9 @@ public class EsService {
NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("labels", "labels")
.subAggregation(AggregationBuilders.terms("groupTag")
.field("labels.relationId")
.size(10)
.size(searchCondition.getIncludeValues().length)
.includeExclude(new IncludeExclude(searchCondition.getIncludeValues(), searchCondition.getExcludeValues()))
.subAggregation(AggregationBuilders.cardinality("sidCount").field("sid.keyword")));
.subAggregation(AggregationBuilders.cardinality("sidCount").field("labels.sourceId.keyword")));
searchSourceBuilder.aggregation(nestedAggregationBuilder);
searchRequest.source(searchSourceBuilder);
long totalCount = 0;
......@@ -1722,6 +1727,12 @@ public class EsService {
if (StringUtils.isNotEmpty(searchCondition.getOrigin())) {
boolQuery.must(QueryBuilders.matchPhraseQuery("origin", searchCondition.getOrigin()));
}
if (searchCondition.getLabelName() != null) {
BoolQueryBuilder nestedBoolQueryBuilder = QueryBuilders.boolQuery();
MatchPhraseQueryBuilder relationNameQuery = QueryBuilders.matchPhraseQuery("labels.relationName", searchCondition.getLabelName());
nestedBoolQueryBuilder.should(QueryBuilders.nestedQuery("labels", relationNameQuery, ScoreMode.None));
boolQuery.must(nestedBoolQueryBuilder);
}
}
Integer checkStatus = searchCondition.getCheckStatus();
if (checkStatus != null) {
......@@ -1767,27 +1778,14 @@ public class EsService {
List<String> labelIds = searchCondition.getLabelIds();
if (CollectionUtils.isNotEmpty(labelIds)) {
Set<String> relationIdSet = new HashSet<>(labelIds);
BoolQueryBuilder nestedBoolQueryBuilder = QueryBuilders.boolQuery();
for (String relationId : relationIdSet) {
TermQueryBuilder relationIdQuery = QueryBuilders.termQuery("labels.relationId", relationId);
nestedBoolQueryBuilder.should(QueryBuilders.nestedQuery("labels", relationIdQuery, ScoreMode.None));
}
boolQuery.must(nestedBoolQueryBuilder);
}
if (StringUtils.isNotEmpty(searchCondition.getArea())) {
String[] regionArr = searchCondition.getArea().split(",");
for (String regionId : regionArr) {
BoolQueryBuilder nestedRegionBoolQueryBuilder = QueryBuilders.boolQuery();
TermQueryBuilder relationIdQuery = QueryBuilders.termQuery("labels.relationId", regionId);
nestedRegionBoolQueryBuilder.should(QueryBuilders.nestedQuery("labels", relationIdQuery, ScoreMode.None));
boolQuery.must(nestedRegionBoolQueryBuilder);
}
}
if (searchCondition.getLabelName() != null) {
BoolQueryBuilder nestedBoolQueryBuilder = QueryBuilders.boolQuery();
MatchPhraseQueryBuilder relationNameQuery = QueryBuilders.matchPhraseQuery("labels.relationName", searchCondition.getLabelName());
nestedBoolQueryBuilder.should(QueryBuilders.nestedQuery("labels", relationNameQuery, ScoreMode.None));
boolQuery.must(nestedBoolQueryBuilder);
TermsQueryBuilder relationIdQuery = QueryBuilders.termsQuery("labels.relationId", relationIdSet);
boolQuery.must(QueryBuilders.nestedQuery("labels", relationIdQuery, ScoreMode.None));
}
List<Label> labelList = searchCondition.getLabelList();
if (CollectionUtils.isNotEmpty(labelList)) {
List<String> collect = labelList.stream().map(label -> label.getLabelMark() + "-" + label.getRelationId()).collect(Collectors.toList());
TermsQueryBuilder relationIdQuery = QueryBuilders.termsQuery("labels.relationId", collect);
boolQuery.must(QueryBuilders.nestedQuery("labels", relationIdQuery, ScoreMode.None));
}
String sourceId = searchCondition.getSourceId();
if (StringUtils.isNotBlank(sourceId)) {
......
src/main/java/com/zzsn/event/mapper/xml/ClbLabelMapper.xml
......@@ -23,6 +23,6 @@
from clb_label label
left join clb_label_item item on label.id = item.label_id
where label.label_type_id = #{labelTypeId}
order by label.create_time
order by label.order_no,item.order_no
</select>
</mapper>
\ No newline at end of file
src/main/java/com/zzsn/event/mapper/xml/SubjectMapper.xml
......@@ -78,6 +78,7 @@
<select id="researchCenterPageList" resultType="com.zzsn.event.vo.SubjectPage">
SELECT d.id,
d.subject_name,
d.subject_code,
d.remark,
d.create_time,
d.status,
......@@ -116,6 +117,7 @@
<select id="researchCenterFacePageList" resultType="com.zzsn.event.vo.SubjectPage">
SELECT d.id,
d.subject_name,
d.subject_code,
d.remark,
d.create_time,
d.status,
......
src/main/java/com/zzsn/event/vo/InfoDataSearchCondition.java
package com.zzsn.event.vo;
import com.zzsn.event.vo.es.Label;
import lombok.Data;
import java.util.List;
......@@ -28,9 +29,6 @@ public class InfoDataSearchCondition {
//来源
private String origin;
//区域
private String area;
//开始时间
private String startTime;
......@@ -60,10 +58,12 @@ public class InfoDataSearchCondition {
private String isSubject = "1";
//爬虫类型
private String crawler;
//标签id集合
//企业类标签id集合
private List<String> labelTypeIds;
//关联标签值id集合
private List<String> labelIds;
//关联标签集合
private List<Label> labelList;
//信息源id
private String sourceId;
//搜索词信息集合-研究中心
......
src/main/java/com/zzsn/event/vo/es/Label.java
......@@ -16,15 +16,12 @@ public class Label {
//标签备注
private String labelRemarks;
//项目标签id
private String projectLabelId;
//信息源id
private String sourceId;
//关联标签id
private String relationId;
//关联标签名称
private String relationName;
//审核状态
private Integer status;
}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论