Skip to content

E
event
提交 50551025 作者: 925993793@qq.com
操作

新版事件分析

上级 f6a3ba48
正在显示 包含 1354 行增加 和 290 行删除
pom.xml
...@@ -227,6 +227,14 @@ ...@@ -227,6 +227,14 @@
<artifactId>neo4j-java-driver</artifactId> <artifactId>neo4j-java-driver</artifactId>
<version>4.1.2</version> <version>4.1.2</version>
</dependency> </dependency>
<!-- 通义千问 -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>dashscope-sdk-java</artifactId>
<!-- 请将 'the-latest-version' 替换为最新版本号：https://mvnrepository.com/artifact/com.alibaba/dashscope-sdk-java -->
<version>2.18.2</version>
</dependency>
</dependencies> </dependencies>
<build> <build>
......
src/main/java/com/zzsn/event/controller/EventAnalysisController.java
...@@ -9,26 +9,20 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; ...@@ -9,26 +9,20 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.zzsn.event.constant.Constants; import com.zzsn.event.constant.Constants;
import com.zzsn.event.constant.Result; import com.zzsn.event.constant.Result;
import com.zzsn.event.entity.Event;
import com.zzsn.event.entity.EventAnalysisReport; import com.zzsn.event.entity.EventAnalysisReport;
import com.zzsn.event.entity.LabelEntity; import com.zzsn.event.entity.LabelEntity;
import com.zzsn.event.entity.SubjectAnalysis;
import com.zzsn.event.es.EsService; import com.zzsn.event.es.EsService;
import com.zzsn.event.service.AnalysisService; import com.zzsn.event.llm.LlmService;
import com.zzsn.event.service.EsStatisticsService; import com.zzsn.event.service.*;
import com.zzsn.event.service.EventAnalysisReportService;
import com.zzsn.event.service.LabelEntityService;
import com.zzsn.event.util.CalculateUtil; import com.zzsn.event.util.CalculateUtil;
import com.zzsn.event.util.HttpUtil;
import com.zzsn.event.util.RedisUtil; import com.zzsn.event.util.RedisUtil;
import com.zzsn.event.vo.CountVO; import com.zzsn.event.vo.*;
import com.zzsn.event.vo.EventViewVO;
import com.zzsn.event.vo.SubjectDataVo;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import java.math.RoundingMode; import java.math.RoundingMode;
...@@ -51,6 +45,8 @@ import java.util.stream.Collectors; ...@@ -51,6 +45,8 @@ import java.util.stream.Collectors;
public class EventAnalysisController { public class EventAnalysisController {
@Autowired @Autowired
private IEventService eventService;
@Autowired
private EsStatisticsService esStatisticsService; private EsStatisticsService esStatisticsService;
@Autowired @Autowired
private LabelEntityService labelEntityService; private LabelEntityService labelEntityService;
...@@ -62,10 +58,31 @@ public class EventAnalysisController { ...@@ -62,10 +58,31 @@ public class EventAnalysisController {
private EsService esService; private EsService esService;
@Autowired @Autowired
private RedisUtil redisUtil; private RedisUtil redisUtil;
@Autowired
private LlmService llmService;
@Value(("${serviceProject.url:}")) /**
private String SERVICE_PROJECT_URL; * 核心摘要
*
* @param eventId 事件id
* @param model LLM模型
* @author lkg
* @date 2025/7/10
*/
@GetMapping("/coreSummary")
public Result<?> coreSummary(@RequestParam String eventId, @RequestParam(required = false) String model) {
EventVO eventVO = eventService.queryInfo(eventId);
String eventDescribe = eventVO.getEventDescribe();
if (StringUtils.isEmpty(eventDescribe)) {
String prompt = "根据提供的事件名称【name】和对应的关键词【keyword】,关键词【keyword】中'|'表示或;'+'表示与;括号表示优先计算，再结合网络上的相关资讯，生成一段200字左右的核心摘要";
JSONObject jsonObject = new JSONObject();
jsonObject.put("name", eventVO.getEventName());
jsonObject.put("keyword", eventVO.getKeywordsVO().getKeyword());
eventDescribe = llmService.model(model, prompt, jsonObject.toJSONString());
}
return Result.OK(eventDescribe);
}
/** /**
...@@ -74,7 +91,7 @@ public class EventAnalysisController { ...@@ -74,7 +91,7 @@ public class EventAnalysisController {
* @param subjectId 专题id * @param subjectId 专题id
* @param startTime 开始时间 * @param startTime 开始时间
* @param endTime 结束时间 * @param endTime 结束时间
* @param type 结束时间 * @param type 1-按小时;2-按天
* @param category 自定义门户使用(自定义组件) * @param category 自定义门户使用(自定义组件)
* @author lkg * @author lkg
* @date 2024/1/24 * @date 2024/1/24
...@@ -103,7 +120,7 @@ public class EventAnalysisController { ...@@ -103,7 +120,7 @@ public class EventAnalysisController {
if ("custom".equalsIgnoreCase(category)) { if ("custom".equalsIgnoreCase(category)) {
List<Map<String, Object>> dataList = new ArrayList<>(); List<Map<String, Object>> dataList = new ArrayList<>();
Map<String, Object> totalCountMap = new HashMap<>(); Map<String, Object> totalCountMap = new HashMap<>();
totalCountMap.put("name", "信息总量"); totalCountMap.put("name", "事件资讯数量");
totalCountMap.put("value", count); totalCountMap.put("value", count);
totalCountMap.put("unit", "条"); totalCountMap.put("unit", "条");
dataList.add(totalCountMap); dataList.add(totalCountMap);
...@@ -115,12 +132,12 @@ public class EventAnalysisController { ...@@ -115,12 +132,12 @@ public class EventAnalysisController {
Map<String, Object> spreadMap = new HashMap<>(); Map<String, Object> spreadMap = new HashMap<>();
spreadMap.put("name", "平均传播速度"); spreadMap.put("name", "平均传播速度");
spreadMap.put("value", divide); spreadMap.put("value", divide);
spreadMap.put("unit", "条/小时"); spreadMap.put("unit", type == 1 ? "条/小时" : "条/天");
dataList.add(spreadMap); dataList.add(spreadMap);
Map<String, Object> mainReportMap = new HashMap<>(); /*Map<String, Object> mainReportMap = new HashMap<>();
mainReportMap.put("name", "主流报道"); mainReportMap.put("name", "主流报道");
mainReportMap.put("value", mainReport); mainReportMap.put("value", mainReport);
dataList.add(mainReportMap); dataList.add(mainReportMap);*/
return Result.OK(dataList); return Result.OK(dataList);
} else { } else {
return Result.OK(map); return Result.OK(map);
...@@ -182,13 +199,14 @@ public class EventAnalysisController { ...@@ -182,13 +199,14 @@ public class EventAnalysisController {
@GetMapping("/eventContext") @GetMapping("/eventContext")
public Result<?> eventContext(@RequestParam String subjectId, public Result<?> eventContext(@RequestParam String subjectId,
@RequestParam(value = "fakeNum", required = false) Integer fakeNum) { @RequestParam(value = "fakeNum", required = false) Integer fakeNum) {
if (fakeNum == null) { /*if (fakeNum == null) {
fakeNum = Constants.FAKE_NUM; fakeNum = Constants.FAKE_NUM;
} }
List<SubjectAnalysis> eventContextList = analysisService.eventContext(subjectId, fakeNum); List<SubjectAnalysis> eventContextList = analysisService.eventContext(subjectId, fakeNum);
if (CollectionUtils.isNotEmpty(eventContextList)) { if (CollectionUtils.isNotEmpty(eventContextList)) {
eventContextList.forEach(e -> e.setOrigin(Constants.getRealOrigin(e.getOrigin()))); eventContextList.forEach(e -> e.setOrigin(Constants.getRealOrigin(e.getOrigin())));
} }*/
List<JSONObject> eventContextList = analysisService.eventContext(subjectId);
return Result.OK(eventContextList); return Result.OK(eventContextList);
} }
...@@ -210,22 +228,46 @@ public class EventAnalysisController { ...@@ -210,22 +228,46 @@ public class EventAnalysisController {
} }
/** /**
* 3.3 热词查询 * 热词分析
* *
* @return * @param subjectId 事件id
* @param pageSize 返回个数
* @author lkg
* @date 2025/7/8
*/ */
@ApiOperation(value = "热词查询", notes = "热词查询") @ApiOperation(value = "热词查询", notes = "热词查询")
@GetMapping(value = "/keywordsCount") @GetMapping(value = "/keywordsCount")
public Object keywordsCount(@RequestParam String subjectId, @RequestParam Integer pageSize) { public Result<?> keywordsCount(@RequestParam String subjectId, @RequestParam(defaultValue = "150") Integer pageSize) {
try { List<CountVO> countVOS = analysisService.wordCloud(subjectId, pageSize);
JSONObject params = new JSONObject(); return Result.OK(countVOS);
params.put("subjectId", subjectId); }
params.put("pageSize", pageSize);
String url = SERVICE_PROJECT_URL + "event/keywordsCount"; /**
return HttpUtil.doPost(url, params, 10000); * 热词趋势分析
} catch (Exception e) { *
return null; * @param eventId 事件id
} * @author lkg
* @date 2025/7/8
*/
@GetMapping("/keywordTrend")
public Result<?> keywordTrend(@RequestParam String eventId, @RequestParam(required = false) String endTime) {
List<CountVO> countVOS = analysisService.wordTrend(eventId, endTime);
return Result.OK(countVOS);
}
/**
* 热词共现
*
* @param eventId 事件id
* @param endTime 结束时间
* @author lkg
* @date 2025/7/9
*/
@GetMapping("/coOccurrence")
public Result<?> coOccurrence(String eventId, @RequestParam(required = false) String endTime) {
List<CoOccurrenceVO> coOccurrenceVOS = analysisService.coOccurrence(eventId, endTime);
return Result.OK(coOccurrenceVOS);
} }
/** /**
...@@ -337,6 +379,12 @@ public class EventAnalysisController { ...@@ -337,6 +379,12 @@ public class EventAnalysisController {
return Result.OK(list); return Result.OK(list);
} }
public Result<?> measureSuggest() {
//todo
return Result.OK();
}
/** /**
* 获取事件分析报告详细信息 * 获取事件分析报告详细信息
* *
......
src/main/java/com/zzsn/event/controller/EventManageController.java
...@@ -62,7 +62,6 @@ public class EventManageController { ...@@ -62,7 +62,6 @@ public class EventManageController {
private ConfigurationMessageService configurationMessageService; private ConfigurationMessageService configurationMessageService;
@Autowired @Autowired
private IKeyWordsService keyWordsService; private IKeyWordsService keyWordsService;
@Autowired @Autowired
private RedisUtil redisUtil; private RedisUtil redisUtil;
@Resource @Resource
......
src/main/java/com/zzsn/event/controller/SubjectManageController.java
...@@ -1569,7 +1569,7 @@ public class SubjectManageController { ...@@ -1569,7 +1569,7 @@ public class SubjectManageController {
searchWordList.add(searchWord); searchWordList.add(searchWord);
} }
searchCondition.setSearchWordList(searchWordList); searchCondition.setSearchWordList(searchWordList);
List<CountVO> countVOS = esService.groupBySourceId(searchCondition); List<CountVO> countVOS = esService.groupByTerm(searchCondition, "groupSid", "sid.keyword", false,10);
if (CollectionUtils.isNotEmpty(countVOS)) { if (CollectionUtils.isNotEmpty(countVOS)) {
List<String> sourceIdList = new ArrayList<>(); List<String> sourceIdList = new ArrayList<>();
for (CountVO countVO : countVOS) { for (CountVO countVO : countVOS) {
......
src/main/java/com/zzsn/event/controller/common/StatisticalAnalysisController.java
...@@ -175,7 +175,7 @@ public class StatisticalAnalysisController { ...@@ -175,7 +175,7 @@ public class StatisticalAnalysisController {
List<String> includeValues = dictItemList.stream().map(SysDictItem::getItemValue).collect(Collectors.toList()); List<String> includeValues = dictItemList.stream().map(SysDictItem::getItemValue).collect(Collectors.toList());
searchCondition.setIncludeValues(includeValues.toArray(new String[0])); searchCondition.setIncludeValues(includeValues.toArray(new String[0]));
if (CollectionUtils.isEmpty(searchCondition.getIds()) && searchCondition.getNum() == null) { if (CollectionUtils.isEmpty(searchCondition.getIds()) && searchCondition.getNum() == null) {
dataList = esService.groupByClassificationType(searchCondition); dataList = esService.groupByTerm(searchCondition, "groupType", "classificationType", false,null);
} else { } else {
String[] fetchFields = new String[]{"id", "classificationType"}; String[] fetchFields = new String[]{"id", "classificationType"};
searchCondition.setFetchFields(fetchFields); searchCondition.setFetchFields(fetchFields);
...@@ -244,7 +244,7 @@ public class StatisticalAnalysisController { ...@@ -244,7 +244,7 @@ public class StatisticalAnalysisController {
if (CollectionUtils.isNotEmpty(sourceIdList)) { if (CollectionUtils.isNotEmpty(sourceIdList)) {
searchCondition.setIncludeValues(sourceIdList.toArray(new String[0])); searchCondition.setIncludeValues(sourceIdList.toArray(new String[0]));
} }
dataList = esService.groupBySourceId(searchCondition); dataList = esService.groupByTerm(searchCondition, "groupSid", "sid.keyword", false,null);
} else { } else {
String[] fetchFields = new String[]{"id", "sid"}; String[] fetchFields = new String[]{"id", "sid"};
searchCondition.setFetchFields(fetchFields); searchCondition.setFetchFields(fetchFields);
......
src/main/java/com/zzsn/event/controller/yjzx/EventHomeController.java
package com.zzsn.event.controller.yjzx; package com.zzsn.event.controller.yjzx;
import cn.hutool.core.date.DateUtil;
import cn.hutool.http.HttpUtil; import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson2.JSONArray; import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject; import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.zzsn.event.constant.Result; import com.zzsn.event.constant.Result;
import com.zzsn.event.entity.EventNetwork;
import com.zzsn.event.service.EsStatisticsService; import com.zzsn.event.service.EsStatisticsService;
import com.zzsn.event.service.EventNetworkService;
import com.zzsn.event.service.IEventService; import com.zzsn.event.service.IEventService;
import com.zzsn.event.util.user.UserUtil; import com.zzsn.event.util.user.UserUtil;
import com.zzsn.event.util.user.UserVo; import com.zzsn.event.util.user.UserVo;
...@@ -45,6 +51,8 @@ public class EventHomeController { ...@@ -45,6 +51,8 @@ public class EventHomeController {
private EsStatisticsService esStatisticsService; private EsStatisticsService esStatisticsService;
@Autowired @Autowired
private IEventService eventService; private IEventService eventService;
@Autowired
private EventNetworkService eventNetworkService;
@Value(("${yjzx.url.checkToken:http://sasac-rc.com/api/v4/user/urUser/getUserByToken?token=}")) @Value(("${yjzx.url.checkToken:http://sasac-rc.com/api/v4/user/urUser/getUserByToken?token=}"))
private String checkTokenProd; private String checkTokenProd;
...@@ -64,7 +72,7 @@ public class EventHomeController { ...@@ -64,7 +72,7 @@ public class EventHomeController {
UserVo loginUser = UserUtil.getLoginUser(); UserVo loginUser = UserUtil.getLoginUser();
String username = loginUser.getUsername(); String username = loginUser.getUsername();
Map<String, Integer> map = new HashMap<>(); Map<String, Integer> map = new HashMap<>();
List<EventExcelVO> eventList = eventService.frontAllList(projectId, username, null, null,null,null,null,null); List<EventExcelVO> eventList = eventService.frontAllList(projectId, username, null, null, null, null, null, null);
map.put("eventCount", eventList.size()); map.put("eventCount", eventList.size());
if (!CollectionUtils.isEmpty(eventList)) { if (!CollectionUtils.isEmpty(eventList)) {
List<String> eventIdList = eventList.stream().map(EventExcelVO::getId).collect(Collectors.toList()); List<String> eventIdList = eventList.stream().map(EventExcelVO::getId).collect(Collectors.toList());
...@@ -123,7 +131,7 @@ public class EventHomeController { ...@@ -123,7 +131,7 @@ public class EventHomeController {
UserVo loginUser = UserUtil.getLoginUser(); UserVo loginUser = UserUtil.getLoginUser();
String username = loginUser.getUsername(); String username = loginUser.getUsername();
List<Map<String, Object>> dataList = new ArrayList<>(); List<Map<String, Object>> dataList = new ArrayList<>();
List<EventRegionVO> eventRegionVOList = eventService.listByRegion(projectId,username,type, regionName); List<EventRegionVO> eventRegionVOList = eventService.listByRegion(projectId, username, type, regionName);
Map<String, List<EventRegionVO>> map = eventRegionVOList.stream().collect(Collectors.groupingBy(EventRegionVO::getRegionName)); Map<String, List<EventRegionVO>> map = eventRegionVOList.stream().collect(Collectors.groupingBy(EventRegionVO::getRegionName));
for (Map.Entry<String, List<EventRegionVO>> entry : map.entrySet()) { for (Map.Entry<String, List<EventRegionVO>> entry : map.entrySet()) {
Map<String, Object> regionMap = new HashMap<>(); Map<String, Object> regionMap = new HashMap<>();
...@@ -134,28 +142,50 @@ public class EventHomeController { ...@@ -134,28 +142,50 @@ public class EventHomeController {
} }
return Result.OK(dataList); return Result.OK(dataList);
} }
/**
* 今日网络事件列表
*
* @param type 来源类型(1-百度;2-新浪新闻)
* @author lkg
* @date 2025/7/4
*/
@GetMapping("/networkEventList")
public Result<?> networkEventList(@RequestParam Integer type) {
String today = DateUtil.today();
LambdaQueryWrapper<EventNetwork> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.eq(EventNetwork::getPublishDate, today)
.eq(EventNetwork::getType, type)
.eq(EventNetwork::getLatest, 1)
.eq(EventNetwork::getYnNeed, 1)
.orderByDesc(EventNetwork::getHot);
return Result.OK(eventNetworkService.list(queryWrapper));
}
/** /**
* 权限校验 * 权限校验
*
* @param token 研究中心token * @param token 研究中心token
* @author yanxin * @author yanxin
* @date 2025/2/20 * @date 2025/2/20
*/ */
@GetMapping("/checkAuth") @GetMapping("/checkAuth")
public Result<?> checkAuth(@RequestParam String token,@RequestParam String key) { public Result<?> checkAuth(@RequestParam String token, @RequestParam String key) {
try{ try {
String checkToken = checkTokenProd; String checkToken = checkTokenProd;
if("test".equals(UserUtil.getLoginUser().getTenant())){ if ("test".equals(UserUtil.getLoginUser().getTenant())) {
checkToken = checkTokenTest; checkToken = checkTokenTest;
} }
String res = HttpUtil.get(checkToken + token); String res = HttpUtil.get(checkToken + token);
JSONObject data = JSONObject.parseObject(res).getJSONObject("data"); JSONObject data = JSONObject.parseObject(res).getJSONObject("data");
if(data == null){ if (data == null) {
return Result.FAIL("token失效"); return Result.FAIL("token失效");
} }
JSONArray powerCodeSet = data.getJSONArray("powerCodeSet"); JSONArray powerCodeSet = data.getJSONArray("powerCodeSet");
return Result.OK(powerCodeSet.contains(key)); return Result.OK(powerCodeSet.contains(key));
}catch (Exception e){ } catch (Exception e) {
log.error("研究中心用户权限校验异常,e:{}",e.getMessage(),e); log.error("研究中心用户权限校验异常,e:{}", e.getMessage(), e);
} }
return Result.OK(false); return Result.OK(false);
} }
......
src/main/java/com/zzsn/event/controller/yjzx/EventMaintenanceController.java
package com.zzsn.event.controller.yjzx; package com.zzsn.event.controller.yjzx;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.zzsn.event.constant.Result; import com.zzsn.event.constant.Result;
import com.zzsn.event.entity.SubjectAnalysis; import com.zzsn.event.entity.SubjectAnalysis;
import com.zzsn.event.service.AnalysisService;
import com.zzsn.event.service.SubjectAnalysisService; import com.zzsn.event.service.SubjectAnalysisService;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
...@@ -22,6 +24,8 @@ public class EventMaintenanceController { ...@@ -22,6 +24,8 @@ public class EventMaintenanceController {
@Autowired @Autowired
private SubjectAnalysisService subjectAnalysisService; private SubjectAnalysisService subjectAnalysisService;
@Autowired
private AnalysisService analysisService;
/** /**
* 专题事件脉络列表 * 专题事件脉络列表
...@@ -32,15 +36,12 @@ public class EventMaintenanceController { ...@@ -32,15 +36,12 @@ public class EventMaintenanceController {
*/ */
@GetMapping("/eventContextList") @GetMapping("/eventContextList")
public Result<?> eventContextList(@RequestParam String subjectId){ public Result<?> eventContextList(@RequestParam String subjectId){
LambdaQueryWrapper<SubjectAnalysis> queryWrapper = Wrappers.lambdaQuery(); List<JSONObject> eventContextList = analysisService.eventContext(subjectId);
queryWrapper.eq(SubjectAnalysis::getSubjectId, subjectId).eq(SubjectAnalysis::getCategory, 2) return Result.OK(eventContextList);
.orderByAsc(SubjectAnalysis::getPublishDate);
List<SubjectAnalysis> list = subjectAnalysisService.list(queryWrapper);
return Result.OK(list);
} }
/** /**
* 事件脉络编辑 * 事件脉络新增/编辑
* *
* @param subjectAnalysis 事件脉络信息 * @param subjectAnalysis 事件脉络信息
* @author lkg * @author lkg
......
src/main/java/com/zzsn/event/entity/Event.java
package com.zzsn.event.entity; package com.zzsn.event.entity;
import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.*;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiModelProperty;
...@@ -225,6 +222,13 @@ public class Event { ...@@ -225,6 +222,13 @@ public class Event {
* 访问环境，prod：正式环境 test：测试环境 * 访问环境，prod：正式环境 test：测试环境
*/ */
private String tenant; private String tenant;
/**预估状态*/
private String estimateStatus;
/** 环境 1-测试 2-正式 */
private String environment;
/**数据范围（是否是全库） - 采集库全库-1，企业库全库-2，政策库全库-3*/
@TableField(updateStrategy = FieldStrategy.IGNORED) // 忽略更新策略
private String dataScope;
/** /**
* 关键词信息 * 关键词信息
*/ */
......
src/main/java/com/zzsn/event/entity/EventNetwork.java 0 → 100644
package com.zzsn.event.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.experimental.Accessors;
import org.springframework.format.annotation.DateTimeFormat;
import java.io.Serializable;
import java.util.Date;
/**
* 网络事件表
* @TableName event_network
*/
@Data
@TableName("event_network")
@EqualsAndHashCode(callSuper = false)
@Accessors(chain = true)
@ApiModel(value="event_network对象", description="网络事件")
public class EventNetwork implements Serializable {
/**
* 主键id
*/
@TableId(value = "id",type = IdType.ASSIGN_ID)
private String id;
/**
* 标题
*/
private String title;
/**
* 图片地址
*/
private String picture;
/**
* 热度
*/
private Integer hot;
/**
* 发布日期
*/
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd")
@DateTimeFormat(pattern="yyyy-MM-dd")
private Date publishDate;
/**
* 是否最新(1-是;0-否)
*/
private Integer latest;
/**
* 是否所要事件(1-是;0-否)[政治经济领域相关]
*/
private Integer ynNeed;
/**
* 来源类型(1-百度;2-新浪)
*/
private Integer type;
/**
* 创建时间
*/
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
private Date createTime;
/**
* 创建人
*/
private String createBy;
/**
* 更新时间
*/
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
private Date updateTime;
/**
* 更新人
*/
private String updateBy;
}
src/main/java/com/zzsn/event/entity/SubjectAnalysis.java
...@@ -48,4 +48,8 @@ public class SubjectAnalysis implements Serializable { ...@@ -48,4 +48,8 @@ public class SubjectAnalysis implements Serializable {
private Date analysisDate; private Date analysisDate;
/**专家名称*/ /**专家名称*/
private String professionName; private String professionName;
/**是否主要资讯(新版事件脉络)*/
private Integer isMain;
/**生成法方式(1-自动;2-人工)(新版事件脉络)*/
private Integer createWay;
} }
src/main/java/com/zzsn/event/es/EsService.java
...@@ -37,6 +37,7 @@ import org.elasticsearch.common.text.Text; ...@@ -37,6 +37,7 @@ import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.*;
import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.BucketOrder;
...@@ -57,6 +58,7 @@ import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; ...@@ -57,6 +58,7 @@ import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.sort.*; import org.elasticsearch.search.sort.*;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import javax.annotation.Resource; import javax.annotation.Resource;
...@@ -1701,27 +1703,6 @@ public class EsService { ...@@ -1701,27 +1703,6 @@ public class EsService {
return list; return list;
} }
/**
* 按资讯类型分组统计
*
* @param searchCondition 筛选条件
* @author lkg
* @date 2024/12/26
*/
public List<CountVO> groupByClassificationType(InfoDataSearchCondition searchCondition) {
return groupByTerm(searchCondition, "group", "classificationType", false);
}
/**
* 按信息源分组统计
*
* @param searchCondition 筛选条件
* @author lkg
* @date 2024/12/26
*/
public List<CountVO> groupBySourceId(InfoDataSearchCondition searchCondition) {
return groupByTerm(searchCondition, "groupSid", "sid.keyword", false);
}
/** /**
* 按信息源标签分组统计 * 按信息源标签分组统计
...@@ -2025,10 +2006,11 @@ public class EsService { ...@@ -2025,10 +2006,11 @@ public class EsService {
* @param groupName 分组名称 * @param groupName 分组名称
* @param field 分组字段 * @param field 分组字段
* @param sort 排序方式 true正序；false倒序 * @param sort 排序方式 true正序；false倒序
* @param size 返回数量
* @author lkg * @author lkg
* @date 2024/12/27 * @date 2024/12/27
*/ */
private List<CountVO> groupByTerm(InfoDataSearchCondition searchCondition, String groupName, String field, boolean sort) { public List<CountVO> groupByTerm(InfoDataSearchCondition searchCondition, String groupName, String field, boolean sort, Integer size) {
List<CountVO> list = new ArrayList<>(); List<CountVO> list = new ArrayList<>();
SearchRequest searchRequest = new SearchRequest(Constants.SUBJECT_INDEX); SearchRequest searchRequest = new SearchRequest(Constants.SUBJECT_INDEX);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
...@@ -2046,8 +2028,14 @@ public class EsService { ...@@ -2046,8 +2028,14 @@ public class EsService {
if (!Objects.isNull(searchCondition.getIncludeValues()) || !Objects.isNull(searchCondition.getExcludeValues())) { if (!Objects.isNull(searchCondition.getIncludeValues()) || !Objects.isNull(searchCondition.getExcludeValues())) {
if (!Objects.isNull(searchCondition.getIncludeValues())) { if (!Objects.isNull(searchCondition.getIncludeValues())) {
aggregationBuilder.size(searchCondition.getIncludeValues().length); aggregationBuilder.size(searchCondition.getIncludeValues().length);
} else {
aggregationBuilder.size(size);
} }
aggregationBuilder.includeExclude(new IncludeExclude(searchCondition.getIncludeValues(), searchCondition.getExcludeValues())); aggregationBuilder.includeExclude(new IncludeExclude(searchCondition.getIncludeValues(), searchCondition.getExcludeValues()));
} else {
if (size != null) {
aggregationBuilder.size(size);
}
} }
aggregationBuilder.order(BucketOrder.count(sort)); aggregationBuilder.order(BucketOrder.count(sort));
searchSourceBuilder.aggregation(aggregationBuilder); searchSourceBuilder.aggregation(aggregationBuilder);
...@@ -2071,6 +2059,122 @@ public class EsService { ...@@ -2071,6 +2059,122 @@ public class EsService {
return list; return list;
} }
//热词趋势
public List<CountVO> keywordTrend(InfoDataSearchCondition searchCondition, String groupDateType) {
List<CountVO> list = new ArrayList<>();
SearchRequest searchRequest = new SearchRequest(Constants.SUBJECT_INDEX);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.trackTotalHits(true);
searchSourceBuilder.size(0);
//构建查询语句
List<String> subjectIds = new ArrayList<>();
if (StringUtils.isNotBlank(searchCondition.getSubjectId())) {
subjectIds.add(searchCondition.getSubjectId());
}
BoolQueryBuilder boolQuery = buildQuery(searchCondition, subjectIds);
searchSourceBuilder.query(boolQuery);
//按时间分组
DateHistogramAggregationBuilder dateHistogramAggregationBuilder = AggregationBuilders.dateHistogram("groupDate").field("publishDate");
switch (groupDateType) {
case "day":
dateHistogramAggregationBuilder.calendarInterval(DateHistogramInterval.DAY).format("yyyy-MM-dd");
break;
case "week":
dateHistogramAggregationBuilder.calendarInterval(DateHistogramInterval.WEEK).format("yyyy-MM-dd");
break;
case "month":
dateHistogramAggregationBuilder.calendarInterval(DateHistogramInterval.MONTH).format("yyyy-MM");
break;
}
//按热词分组
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("groupKeyword");
aggregationBuilder.field("keyWordsList.keyword");
if (!Objects.isNull(searchCondition.getIncludeValues()) || !Objects.isNull(searchCondition.getExcludeValues())) {
if (!Objects.isNull(searchCondition.getIncludeValues())) {
aggregationBuilder.size(searchCondition.getIncludeValues().length);
}
aggregationBuilder.includeExclude(new IncludeExclude(searchCondition.getIncludeValues(), searchCondition.getExcludeValues()));
}
aggregationBuilder.order(BucketOrder.count(false));
aggregationBuilder.subAggregation(dateHistogramAggregationBuilder);
searchSourceBuilder.aggregation(aggregationBuilder);
searchRequest.source(searchSourceBuilder);
try {
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
Aggregations aggregations = searchResponse.getAggregations();
Terms groupSource = aggregations.get("groupKeyword");
List<? extends Terms.Bucket> buckets = groupSource.getBuckets();
for (Terms.Bucket bucket : buckets) {
CountVO countVO = new CountVO();
countVO.setName(bucket.getKeyAsString());
countVO.setValue(bucket.getDocCount());
List<CountVO> children = new ArrayList<>();
ParsedDateHistogram groupDate = bucket.getAggregations().get("groupDate");
List<? extends Histogram.Bucket> dateBuckets = groupDate.getBuckets();
for (Histogram.Bucket dateBucket : dateBuckets) {
CountVO child = new CountVO();
child.setName(dateBucket.getKeyAsString());
child.setValue(dateBucket.getDocCount());
children.add(child);
}
countVO.setChildren(children);
list.add(countVO);
}
} catch (Exception e) {
e.printStackTrace();
}
return list;
}
public List<Map<String, String>> co_occurrence(InfoDataSearchCondition searchCondition) {
List<Map<String, String>> list = new ArrayList<>();
SearchRequest searchRequest = new SearchRequest(Constants.SUBJECT_INDEX);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.trackTotalHits(true);
searchSourceBuilder.size(0);
//构建查询语句
List<String> subjectIds = new ArrayList<>();
if (StringUtils.isNotBlank(searchCondition.getSubjectId())) {
subjectIds.add(searchCondition.getSubjectId());
}
BoolQueryBuilder boolQuery = buildQuery(searchCondition, subjectIds);
searchSourceBuilder.query(boolQuery);
//按热词分组
TermsAggregationBuilder subAggregationBuilder = AggregationBuilders.terms("groupId")
.field("id").size(10000);
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("groupKeyword")
.field("keyWordsList.keyword")
.size(searchCondition.getPageSize())
.order(BucketOrder.count(false));
searchSourceBuilder.aggregation(aggregationBuilder);
searchRequest.source(searchSourceBuilder);
try {
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
Aggregations aggregations = searchResponse.getAggregations();
Terms groupSource = aggregations.get("groupKeyword");
List<? extends Terms.Bucket> buckets = groupSource.getBuckets();
for (Terms.Bucket bucket : buckets) {
String keyword = bucket.getKeyAsString();
long count = bucket.getDocCount();
Terms groupId = bucket.getAggregations().get("groupId");
List<? extends Terms.Bucket> idBuckets = groupId.getBuckets();
for (Terms.Bucket idBucket : idBuckets) {
Map<String,String> map = new HashMap<>();
map.put("keyword",keyword);
map.put("num", String.valueOf(count));
map.put("id",idBucket.getKeyAsString());
list.add(map);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return list;
}
/** /**
* 构建通用的es查询语句 * 构建通用的es查询语句
* *
...@@ -2830,8 +2934,8 @@ public class EsService { ...@@ -2830,8 +2934,8 @@ public class EsService {
} }
public List<Map<String,Object>> singleCountryStatistic(String subjectId,String country, String startTime, String endTime) { public List<Map<String, Object>> singleCountryStatistic(String subjectId, String country, String startTime, String endTime) {
List<Map<String,Object>> dataList = new ArrayList<>(); List<Map<String, Object>> dataList = new ArrayList<>();
//根据地区的id从ES查询相关的记录 //根据地区的id从ES查询相关的记录
SearchRequest searchRequest = new SearchRequest(Constants.SUBJECT_INDEX); SearchRequest searchRequest = new SearchRequest(Constants.SUBJECT_INDEX);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
......
src/main/java/com/zzsn/event/llm/LlmService.java 0 → 100644
package com.zzsn.event.llm;
/**
* 模型调用
*/
public interface LlmService {
/**
* glm模型调用
*
* @param model 模型类型
* @param system 提示词
* @param content 引用内容
* @return 大模型响应结果
*/
String model(String model, String system, String content);
}
src/main/java/com/zzsn/event/llm/LlmServiceImpl.java 0 → 100644
package com.zzsn.event.llm;
import com.alibaba.dashscope.aigc.generation.Generation;
import com.alibaba.dashscope.aigc.generation.GenerationParam;
import com.alibaba.dashscope.aigc.generation.GenerationResult;
import com.alibaba.dashscope.common.Message;
import com.alibaba.dashscope.common.Role;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.*;
/**
* 大模型调用
*
* @author lkg
* @date 2025/7/4
*/
@Service
public class LlmServiceImpl implements LlmService {
private static final Logger log = LoggerFactory.getLogger(LlmServiceImpl.class);
@Value("${model.aichain.url:}")
private String AICHAIN_URL;
@Value("${model.aichain.authorization:}")
private String AICHAIN_AUTHOR;
@Value("${model.aichain.cookie:}")
private String AICHAIN_COOKIE;
@Value("${model.qwen.apiKey:}")
private String QWEN_APIKEY;
@Override
public String model(String model, String system, String content) {
if (StringUtils.isNotEmpty(model)) {
if (model.startsWith("glm")) {
return glmModel(model.replace("glm:", ""), system, content);
} else if (model.startsWith("qwen")) {
return qwenModel(model.replace("qwen:", ""), system, content);
}
}
//默认调用模型
return glmModel("glm-4-flash", system, content);
}
/**
* glm模型调用
*
* @param model 模型
* @param system 提示词
* @param content 引用内容
* @return 大模型响应结果
*/
public String glmModel(String model, String system, String content) {
Map<String, Object> params = new HashMap<>();
params.put("model", model);
List<Map<String, Object>> messageList = new ArrayList<>();
Map<String, Object> systemMap = new HashMap<>();
systemMap.put("role", "system");
systemMap.put("content", system);
messageList.add(systemMap);
Map<String, Object> messagesMap = new HashMap<>();
messagesMap.put("role", "user");
messagesMap.put("content", content);
messageList.add(messagesMap);
params.put("messages", messageList);
params.put("history", "[]");
params.put("top_p", 0.7);
params.put("temperature", 0.01);
String responseStr = glmPost(AICHAIN_URL, new JSONObject(params));
String choices = JSONObject.parseObject(responseStr).getString("choices");
JSONObject choicesObject = (JSONObject) JSON.parseArray(choices).get(0);
String messageStr = choicesObject.getString("message");
return JSONObject.parseObject(messageStr).getString("content");
}
/**
* 千问模型调用
*
* @param model 模型
* @param system 提示词
* @param content 引用内容
* @return 大模型响应结果
*/
public String qwenModel(String model, String system, String content) {
try {
Generation gen = new Generation();
Message systemMsg = Message.builder()
.role(Role.SYSTEM.getValue())
.content(system)
.build();
Message userMsg = Message.builder()
.role(Role.USER.getValue())
.content(content)
.build();
GenerationParam param = GenerationParam.builder()
// 若没有配置环境变量，请用阿里云百炼API Key将下行替换为：.apiKey("sk-xxx")
.apiKey(QWEN_APIKEY)
// 模型列表：https://help.aliyun.com/zh/model-studio/getting-started/models
.model(model)
.messages(Arrays.asList(systemMsg, userMsg))
.resultFormat(GenerationParam.ResultFormat.MESSAGE)
.build();
GenerationResult call = gen.call(param);
return call.getOutput().getChoices().get(0).getMessage().getContent();
} catch (Exception e) {
log.error("qwenModel error:{}", e.getMessage(), e);
}
return "";
}
public String glmPost(String urlstr, JSONObject jsonObject) {
String responseStr = null;
try {
URL url = new URL(urlstr);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("POST");
conn.setRequestProperty("Content-Type", "application/json; utf-8");
conn.setRequestProperty("Accept", "application/json");
conn.setRequestProperty("Authorization", AICHAIN_AUTHOR);
conn.setRequestProperty("Cookie", AICHAIN_COOKIE);
conn.setDoOutput(true);
String jsonInputString = jsonObject.toJSONString();
try (OutputStream os = conn.getOutputStream()) {
byte[] input = jsonInputString.getBytes(StandardCharsets.UTF_8);
os.write(input, 0, input.length);
}
// 检查响应状态
if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
throw new RuntimeException("Failed : HTTP error code : " + conn.getResponseCode());
}
// 读取服务器响应
BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()));
responseStr = br.readLine();
conn.disconnect();
} catch (Exception e) {
e.printStackTrace();
}
return responseStr;
}
}
src/main/java/com/zzsn/event/mapper/EventNetworkMapper.java 0 → 100644
package com.zzsn.event.mapper;
import com.zzsn.event.entity.EventNetwork;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Mapper;
/**
* @author lenovo
* @description 针对表【event_network(网络事件表)】的数据库操作Mapper
* @createDate 2025-07-04 11:00:17
* @Entity com.zzsn.event.entity.EventNetwork
*/
@Mapper
public interface EventNetworkMapper extends BaseMapper<EventNetwork> {
}
src/main/java/com/zzsn/event/mapper/xml/EventNetworkMapper.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.zzsn.event.mapper.EventNetworkMapper">
<resultMap id="BaseResultMap" type="com.zzsn.event.entity.EventNetwork">
<id property="id" column="id" jdbcType="VARCHAR"/>
<result property="title" column="title" jdbcType="VARCHAR"/>
<result property="picture" column="picture" jdbcType="VARCHAR"/>
<result property="hot" column="hot" jdbcType="VARCHAR"/>
<result property="publishDate" column="publish_date" jdbcType="DATE"/>
<result property="latest" column="latest" jdbcType="INTEGER"/>
<result property="ynNeed" column="yn_need" jdbcType="INTEGER"/>
<result property="type" column="type" jdbcType="INTEGER"/>
<result property="createTime" column="create_time" jdbcType="TIMESTAMP"/>
<result property="createBy" column="create_by" jdbcType="VARCHAR"/>
<result property="updateTime" column="update_time" jdbcType="TIMESTAMP"/>
<result property="updateBy" column="update_by" jdbcType="VARCHAR"/>
</resultMap>
<sql id="Base_Column_List">
id,title,picture,
hot,publish_date,latest,
yn_need,type,create_time,
create_by,update_time,update_by
</sql>
</mapper>
src/main/java/com/zzsn/event/service/AnalysisService.java
package com.zzsn.event.service; package com.zzsn.event.service;
import com.alibaba.fastjson2.JSONObject;
import com.zzsn.event.entity.SubjectAnalysis; import com.zzsn.event.entity.SubjectAnalysis;
import com.zzsn.event.vo.CoOccurrenceVO;
import com.zzsn.event.vo.CountVO;
import com.zzsn.event.vo.EventContextVO;
import com.zzsn.event.vo.PropagationPathVo; import com.zzsn.event.vo.PropagationPathVo;
import org.springframework.web.bind.annotation.RequestParam;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map;
/** /**
* @author lkg * @author lkg
...@@ -15,16 +22,57 @@ public interface AnalysisService { ...@@ -15,16 +22,57 @@ public interface AnalysisService {
/** /**
* 事件脉络 * 事件脉络
* @param subjectId 专题id *
* @param eventId 专题id
* @param fakeNum 专题事件脉络展示 伪事件脉络 的资讯数量阈值 * @param fakeNum 专题事件脉络展示 伪事件脉络 的资讯数量阈值
* @return java.util.List<com.zzsn.subjectAnalysis.entity.SubjectAnalysis> * @return java.util.List<com.zzsn.subjectAnalysis.entity.SubjectAnalysis>
*/ */
List<SubjectAnalysis> eventContext(String subjectId, int fakeNum); List<SubjectAnalysis> eventContext(String eventId, int fakeNum);
/**
* 事件脉络-新版
*
* @param eventId 专题id
* @author lkg
* @date 2025/7/7
*/
List<JSONObject> eventContext(String eventId);
/**
* 词云
*
* @param eventId 专题id
* @param size 返回热词个数
* @author lkg
* @date 2025/7/7
*/
List<CountVO> wordCloud(String eventId, Integer size);
/**
* 关键词时间趋势
*
* @param eventId 事件id
* @param endTime 结束时间
* @author lkg
* @date 2025/7/9
*/
List<CountVO> wordTrend(String eventId, String endTime);
/**
* 关键词共现
*
* @param eventId 事件id
* @param endTime 结束时间
* @author lkg
* @date 2025/7/9
*/
List<CoOccurrenceVO> coOccurrence(String eventId, String endTime);
/** /**
* 传播路径 * 传播路径
* @param subjectId 专题id *
* @param eventId 专题id
* @return com.zzsn.subjectAnalysis.vo.PropagationPathVo * @return com.zzsn.subjectAnalysis.vo.PropagationPathVo
*/ */
PropagationPathVo propagationPath(String subjectId); PropagationPathVo propagationPath(String eventId);
} }
src/main/java/com/zzsn/event/service/EventNetworkService.java 0 → 100644
package com.zzsn.event.service;
import com.zzsn.event.entity.EventNetwork;
import com.baomidou.mybatisplus.extension.service.IService;
/**
* @author lenovo
* @description 针对表【event_network(网络事件表)】的数据库操作Service
* @createDate 2025-07-04 11:00:17
*/
public interface EventNetworkService extends IService<EventNetwork> {
}
src/main/java/com/zzsn/event/service/impl/AnalysisServiceImpl.java
package com.zzsn.event.service.impl; package com.zzsn.event.service.impl;
import cn.hutool.core.date.DateUnit;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.zzsn.event.constant.Constants; import com.zzsn.event.constant.Constants;
import com.zzsn.event.entity.Event; import com.zzsn.event.entity.Event;
import com.zzsn.event.entity.SubjectAnalysis; import com.zzsn.event.entity.SubjectAnalysis;
import com.zzsn.event.service.AnalysisService;
import com.zzsn.event.es.EsService; import com.zzsn.event.es.EsService;
import com.zzsn.event.service.AnalysisService;
import com.zzsn.event.service.CommonService;
import com.zzsn.event.service.IEventService; import com.zzsn.event.service.IEventService;
import com.zzsn.event.service.SubjectAnalysisService; import com.zzsn.event.service.SubjectAnalysisService;
import com.zzsn.event.util.DateUtil; import com.zzsn.event.util.DateUtil;
import com.zzsn.event.vo.PropagationPathVo; import com.zzsn.event.util.HotWordUtil;
import com.zzsn.event.vo.SubjectDataVo; import com.zzsn.event.util.SimilarityUtil;
import com.zzsn.event.vo.*;
import com.zzsn.event.vo.es.SpecialInformation;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
...@@ -35,6 +40,8 @@ public class AnalysisServiceImpl implements AnalysisService { ...@@ -35,6 +40,8 @@ public class AnalysisServiceImpl implements AnalysisService {
private IEventService eventService; private IEventService eventService;
@Autowired @Autowired
private EsService esService; private EsService esService;
@Autowired
private CommonService commonService;
/* /*
* 优先级：事件脉络 > 伪事件脉络 > 资讯 * 优先级：事件脉络 > 伪事件脉络 > 资讯
...@@ -47,16 +54,16 @@ public class AnalysisServiceImpl implements AnalysisService { ...@@ -47,16 +54,16 @@ public class AnalysisServiceImpl implements AnalysisService {
* 伪事件脉络/资讯作为事件脉络的情况一般是资讯数量少/刚建立/时间脉络抽取失败的事件专题，才会触发。 * 伪事件脉络/资讯作为事件脉络的情况一般是资讯数量少/刚建立/时间脉络抽取失败的事件专题，才会触发。
*/ */
@Override @Override
public List<SubjectAnalysis> eventContext(String subjectId, int fakeNum) { public List<SubjectAnalysis> eventContext(String eventId, int fakeNum) {
//专题下的事件脉络 //专题下的事件脉络
List<SubjectAnalysis> list = getList(subjectId, 2); List<SubjectAnalysis> list = getList(eventId, 2);
if (list.size() < fakeNum) { if (list.size() < fakeNum) {
//专题下的伪事件脉络 //专题下的伪事件脉络
List<SubjectAnalysis> fakeList = getList(subjectId, 3); List<SubjectAnalysis> fakeList = getList(eventId, 3);
if (CollectionUtils.isEmpty(fakeList)) { if (CollectionUtils.isEmpty(fakeList)) {
if (CollectionUtils.isEmpty(list)) { if (CollectionUtils.isEmpty(list)) {
List<SubjectAnalysis> finalList = new ArrayList<>(); List<SubjectAnalysis> finalList = new ArrayList<>();
List<SubjectDataVo> dataList = esService.pageList(subjectId, null, null, Constants.FETCH_FIELDS_STATISTIC, 2,1, 15); List<SubjectDataVo> dataList = esService.pageList(eventId, null, null, Constants.FETCH_FIELDS_STATISTIC, 2, 1, 15);
dataList.forEach(e -> { dataList.forEach(e -> {
String dataId = e.getId(); String dataId = e.getId();
SubjectAnalysis subjectAnalysis = new SubjectAnalysis(); SubjectAnalysis subjectAnalysis = new SubjectAnalysis();
...@@ -75,11 +82,135 @@ public class AnalysisServiceImpl implements AnalysisService { ...@@ -75,11 +82,135 @@ public class AnalysisServiceImpl implements AnalysisService {
return list; return list;
} }
@Override
public List<JSONObject> eventContext(String eventId) {
List<JSONObject> dataList = new ArrayList<>();
LambdaQueryWrapper<SubjectAnalysis> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.eq(SubjectAnalysis::getSubjectId, eventId).eq(SubjectAnalysis::getCategory, 4);
List<SubjectAnalysis> list = subjectAnalysisService.list(queryWrapper);
if (CollectionUtils.isNotEmpty(list)) {
Map<String, List<SubjectAnalysis>> map = list.stream().collect(Collectors.groupingBy(e -> DateUtil.format(e.getPublishDate(), "yyyy-MM-dd")));
for (Map.Entry<String, List<SubjectAnalysis>> entry : map.entrySet()) {
JSONObject jsonObject = new JSONObject();
jsonObject.put("date", entry.getKey());
jsonObject.put("dataList", entry.getValue());
dataList.add(jsonObject);
}
}
return dataList;
}
@Override
public List<CountVO> wordCloud(String eventId, Integer size) {
InfoDataSearchCondition searchCondition = new InfoDataSearchCondition();
searchCondition.setCategory(2);
searchCondition.setSubjectId(eventId);
//排除词
Set<String> excludeKeywords = commonService.getExcludeKeywords(eventId);
searchCondition.setExcludeValues(excludeKeywords.toArray(new String[0]));
return esService.groupByTerm(searchCondition, "groupKeyWord", "keyWordsList.keyword", false, size);
}
@Override
public List<CountVO> wordTrend(String eventId, String endTime) {
Event event = eventService.getById(eventId);
Date startTime = event.getStartTime();
Date finishTime;
if (StringUtils.isEmpty(endTime)) {
finishTime = event.getEndTime();
if (Objects.isNull(event.getEndTime())) {
finishTime = new Date();
}
} else {
finishTime = DateUtil.stringToDate(endTime, "yyyy-mm-dd HH:mm:ss");
}
long between = DateUtil.betweenTwoDate(startTime, finishTime);
String groupType;
if (between <= 30) {
groupType = "day";
} else if (between <= 180) {
groupType = "week";
} else {
groupType = "month";
}
InfoDataSearchCondition searchCondition = new InfoDataSearchCondition();
searchCondition.setCategory(2);
searchCondition.setSubjectId(eventId);
searchCondition.setStartTime(DateUtil.dateToString(startTime));
searchCondition.setEndTime(DateUtil.dateToString(finishTime));
//排除词
Set<String> excludeKeywords = commonService.getExcludeKeywords(eventId);
searchCondition.setExcludeValues(excludeKeywords.toArray(new String[0]));
return esService.keywordTrend(searchCondition, groupType);
}
@Override @Override
public PropagationPathVo propagationPath(String subjectId) { public List<CoOccurrenceVO> coOccurrence(String eventId, String endTime) {
List<CoOccurrenceVO> list = new ArrayList<>();
InfoDataSearchCondition searchCondition = new InfoDataSearchCondition();
searchCondition.setCategory(2);
searchCondition.setSubjectId(eventId);
String[] fetchFields = new String[]{"id", "keyWordsList"};
searchCondition.setFetchFields(fetchFields);
int pageNo = 1;
int size = 300;
searchCondition.setPageSize(size);
boolean flag = true;
//排除词
Set<String> excludeKeywords = commonService.getExcludeKeywords(eventId);
List<List<String>> wordList = new ArrayList<>();
do {
searchCondition.setPageNo(pageNo);
List<SpecialInformation> informationList = esService.informationList(searchCondition);
if (CollectionUtils.isNotEmpty(informationList)) {
for (SpecialInformation information : informationList) {
List<String> keyWordsList = information.getKeyWordsList();
if (CollectionUtils.isNotEmpty(keyWordsList)) {
continue;
}
keyWordsList = keyWordsList.stream().filter(e -> !excludeKeywords.contains(e)).collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(keyWordsList) && keyWordsList.size() > 2) {
wordList.add(keyWordsList);
}
}
if (informationList.size() < size) {
flag = false;
} else {
pageNo++;
}
} else {
flag = false;
}
} while (flag);
Set<String> keyWordSet = new HashSet<>();
wordList.forEach(keyWordSet::addAll);
searchCondition.setIncludeValues(keyWordSet.toArray(new String[0]));
List<CountVO> groupKeyWord = esService.groupByTerm(searchCondition, "groupKeyWord", "keyWordsList.keyword", false, size);
Map<String, Long> wordFrequencyMap = groupKeyWord.stream().collect(Collectors.toMap(CountVO::getName, CountVO::getValue));
Map<String, Integer> map = HotWordUtil.calculateCoOccurrence_list(wordList);
Map<String, Integer> filterMap = map.entrySet().stream().filter(e -> e.getValue() > 10).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
for (Map.Entry<String, Integer> entry : filterMap.entrySet()) {
CoOccurrenceVO coOccurrenceVO = new CoOccurrenceVO();
String key = entry.getKey();
Integer coOccurrenceNum = entry.getValue();
coOccurrenceVO.setCoOccurrenceNum((long) coOccurrenceNum);
CountVO word_1 = new CountVO();
word_1.setName(key.split("&")[0]);
word_1.setValue(wordFrequencyMap.getOrDefault(key.split("&")[0], 0L));
coOccurrenceVO.setWord_1(word_1);
CountVO word_2 = new CountVO();
word_2.setName(key.split("&")[1]);
word_2.setValue(wordFrequencyMap.getOrDefault(key.split("&")[1], 0L));
coOccurrenceVO.setWord_2(word_2);
list.add(coOccurrenceVO);
}
return list;
}
@Override
public PropagationPathVo propagationPath(String eventId) {
PropagationPathVo top = null; PropagationPathVo top = null;
Event event = eventService.getById(subjectId); Event event = eventService.getById(eventId);
String subjectName = event.getEventName(); String subjectName = event.getEventName();
//获取专题数据 //获取专题数据
String startDate = null; String startDate = null;
...@@ -90,7 +221,7 @@ public class AnalysisServiceImpl implements AnalysisService { ...@@ -90,7 +221,7 @@ public class AnalysisServiceImpl implements AnalysisService {
if (event.getEndTime() != null) { if (event.getEndTime() != null) {
endDate = DateUtil.dateToString(event.getEndTime(), "yyyy-MM-dd HH:mm:ss"); endDate = DateUtil.dateToString(event.getEndTime(), "yyyy-MM-dd HH:mm:ss");
} }
List<SubjectDataVo> specialDataList = esService.getDataBySubjectId(subjectId, startDate, endDate, Constants.FETCH_FIELDS_STATISTIC); List<SubjectDataVo> specialDataList = esService.getDataBySubjectId(eventId, startDate, endDate, Constants.FETCH_FIELDS_STATISTIC);
if (CollectionUtils.isNotEmpty(specialDataList)) { if (CollectionUtils.isNotEmpty(specialDataList)) {
//用于来源去重 //用于来源去重
List<String> allOriginList = new ArrayList<>(); List<String> allOriginList = new ArrayList<>();
...@@ -99,8 +230,8 @@ public class AnalysisServiceImpl implements AnalysisService { ...@@ -99,8 +230,8 @@ public class AnalysisServiceImpl implements AnalysisService {
top.setTime(DateUtil.dateToString(event.getStartTime())); top.setTime(DateUtil.dateToString(event.getStartTime()));
//获取发布时间最早的前10的资讯(来源不重复) //获取发布时间最早的前10的资讯(来源不重复)
List<PropagationPathVo> secondList = new ArrayList<>(); List<PropagationPathVo> secondList = new ArrayList<>();
List<SubjectDataVo> earlyList = topN(specialDataList,Constants.FAKE_NUM); List<SubjectDataVo> earlyList = topN(specialDataList, Constants.FAKE_NUM);
earlyList.forEach(e->allOriginList.add(Constants.getRealOrigin(e.getOrigin()))); earlyList.forEach(e -> allOriginList.add(Constants.getRealOrigin(e.getOrigin())));
for (SubjectDataVo subjectDataVo : earlyList) { for (SubjectDataVo subjectDataVo : earlyList) {
String origin = subjectDataVo.getOrigin(); String origin = subjectDataVo.getOrigin();
String time = subjectDataVo.getPublishDate(); String time = subjectDataVo.getPublishDate();
...@@ -113,7 +244,7 @@ public class AnalysisServiceImpl implements AnalysisService { ...@@ -113,7 +244,7 @@ public class AnalysisServiceImpl implements AnalysisService {
thirdList.removeAll(allOriginList); thirdList.removeAll(allOriginList);
List<PropagationPathVo> lastList = new ArrayList<>(); List<PropagationPathVo> lastList = new ArrayList<>();
if (thirdList.size() > 3) { if (thirdList.size() > 3) {
thirdList = thirdList.subList(0,3); thirdList = thirdList.subList(0, 3);
} }
for (String s : thirdList) { for (String s : thirdList) {
PropagationPathVo third = new PropagationPathVo(); PropagationPathVo third = new PropagationPathVo();
...@@ -128,8 +259,9 @@ public class AnalysisServiceImpl implements AnalysisService { ...@@ -128,8 +259,9 @@ public class AnalysisServiceImpl implements AnalysisService {
} }
return top; return top;
} }
//获取发布时间最早的前N条资讯(来源不重复) //获取发布时间最早的前N条资讯(来源不重复)
private List<SubjectDataVo> topN(List<SubjectDataVo> list,Integer num){ private List<SubjectDataVo> topN(List<SubjectDataVo> list, Integer num) {
List<SubjectDataVo> collect = list.stream().filter(e -> StringUtils.isNotEmpty(e.getOrigin())). List<SubjectDataVo> collect = list.stream().filter(e -> StringUtils.isNotEmpty(e.getOrigin())).
sorted(Comparator.comparing(SubjectDataVo::getPublishDate)).collect(Collectors.toList()); sorted(Comparator.comparing(SubjectDataVo::getPublishDate)).collect(Collectors.toList());
TreeSet<SubjectDataVo> subjectDataVos = new TreeSet<>(Comparator.comparing(SubjectDataVo::getOrigin)); TreeSet<SubjectDataVo> subjectDataVos = new TreeSet<>(Comparator.comparing(SubjectDataVo::getOrigin));
...@@ -148,4 +280,54 @@ public class AnalysisServiceImpl implements AnalysisService { ...@@ -148,4 +280,54 @@ public class AnalysisServiceImpl implements AnalysisService {
return subjectAnalysisService.list(queryWrapper); return subjectAnalysisService.list(queryWrapper);
} }
//补充是否主要资讯标识以及排序后截取
private List<EventContextVO> supplyMain(List<EventContextVO> list) {
EventContextVO dataVo = null;
double minDistance = Double.MAX_VALUE;
for (EventContextVO eventContextVO : list) {
Double similarity = eventContextVO.getDistance();
if (similarity == 0) {
continue;
}
if (similarity == 1.0) {
dataVo = eventContextVO;
break;
} else if (similarity < minDistance) {
minDistance = similarity;
dataVo = eventContextVO;
}
}
if (dataVo == null) {
list.get(0).setIsMain(true);
} else {
for (EventContextVO eventContextVO : list) {
if (eventContextVO.getId().equals(dataVo.getId())) {
eventContextVO.setIsMain(true);
break;
}
}
}
//先按是否是主事件排序，再按相似度算法(编辑距离)返回值正序排序，最后按时间倒序排序
list.sort(Comparator.comparing(EventContextVO::getIsMain, Comparator.reverseOrder())
.thenComparing((o1, o2) -> {
Double distance1 = o1.getDistance();
Double distance2 = o2.getDistance();
if (distance1 == 0 && distance2 == 0) {
return 0;
}
if (distance1 == 0) {
return 1;
}
if (distance2 == 0) {
return -1;
}
return distance1.compareTo(distance2);
})
.thenComparing(EventContextVO::getPublishDate, Comparator.reverseOrder()));
if (list.size() > 3) {
list = list.subList(0, 3);
}
return list;
}
} }
src/main/java/com/zzsn/event/service/impl/EventNetworkServiceImpl.java 0 → 100644
package com.zzsn.event.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.event.entity.EventNetwork;
import com.zzsn.event.service.EventNetworkService;
import com.zzsn.event.mapper.EventNetworkMapper;
import org.springframework.stereotype.Service;
/**
* @author lenovo
* @description 针对表【event_network(网络事件表)】的数据库操作Service实现
* @createDate 2025-07-04 11:00:17
*/
@Service
public class EventNetworkServiceImpl extends ServiceImpl<EventNetworkMapper, EventNetwork> implements EventNetworkService{
}
src/main/java/com/zzsn/event/service/impl/SubjectAnalysisServiceImpl.java
...@@ -20,10 +20,11 @@ public class SubjectAnalysisServiceImpl extends ServiceImpl<SubjectAnalysisMappe ...@@ -20,10 +20,11 @@ public class SubjectAnalysisServiceImpl extends ServiceImpl<SubjectAnalysisMappe
@Override @Override
public void eventContextModify(SubjectAnalysis subjectAnalysis) { public void eventContextModify(SubjectAnalysis subjectAnalysis) {
String id = subjectAnalysis.getId(); String id = subjectAnalysis.getId();
subjectAnalysis.setCreateWay(2);
if (StringUtils.isNotEmpty(id)) { if (StringUtils.isNotEmpty(id)) {
baseMapper.updateById(subjectAnalysis); baseMapper.updateById(subjectAnalysis);
} else { } else {
subjectAnalysis.setCategory(2); subjectAnalysis.setCategory(4);
subjectAnalysis.setAnalysisDate(new Date()); subjectAnalysis.setAnalysisDate(new Date());
baseMapper.insert(subjectAnalysis); baseMapper.insert(subjectAnalysis);
} }
......
src/main/java/com/zzsn/event/task/AnalysisTask.java
package com.zzsn.event.task; package com.zzsn.event.task;
import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
import com.alibaba.fastjson2.JSONWriter; import com.alibaba.fastjson2.JSONWriter;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.zzsn.event.constant.Constants; import com.zzsn.event.constant.Constants;
import com.zzsn.event.entity.SubjectAnalysis; import com.zzsn.event.entity.SubjectAnalysis;
...@@ -12,10 +14,9 @@ import com.zzsn.event.service.IEventService; ...@@ -12,10 +14,9 @@ import com.zzsn.event.service.IEventService;
import com.zzsn.event.service.SubjectAnalysisService; import com.zzsn.event.service.SubjectAnalysisService;
import com.zzsn.event.util.DateUtil; import com.zzsn.event.util.DateUtil;
import com.zzsn.event.util.RedisUtil; import com.zzsn.event.util.RedisUtil;
import com.zzsn.event.vo.KafkaDataVo; import com.zzsn.event.util.SimilarityUtil;
import com.zzsn.event.vo.PropagationPathVo; import com.zzsn.event.vo.*;
import com.zzsn.event.vo.SubjectDataVo; import com.zzsn.event.vo.es.SpecialInformation;
import com.zzsn.event.vo.SubjectKafkaVo;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.ListUtils; import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
...@@ -54,13 +55,146 @@ public class AnalysisTask { ...@@ -54,13 +55,146 @@ public class AnalysisTask {
@Value("${scheduling.yjzxEnable:false}") @Value("${scheduling.yjzxEnable:false}")
Boolean yjzxEnable; Boolean yjzxEnable;
/**
* 补充事件当天的脉络
* 每3小时执行一次
*
* @author lkg
* @date 2025/7/9
*/
@Scheduled(cron = "0 0 0/3 * * ?")
public void eventContext() {
Date today = new Date();
Date disableDate = DateUtil.addDate(today, -1);
List<SubjectKafkaVo> subjectList = eventService.progressList(disableDate);
InfoDataSearchCondition searchCondition = new InfoDataSearchCondition();
searchCondition.setCategory(2);
String startTime = DateUtil.format(today, "yyyy-MM-dd 00:00:00");
searchCondition.setStartTime(startTime);
String endTime = DateUtil.format(today, "yyyy-MM-dd 23:59:59");
searchCondition.setEndTime(endTime);
String[] fetchFields = new String[]{"id", "title", "origin", "publishDate", "sourceAddress"};
searchCondition.setFetchFields(fetchFields);
for (SubjectKafkaVo subjectKafkaVo : subjectList) {
String eventId = subjectKafkaVo.getId();
searchCondition.setSubjectId(eventId);
int pageNo = 1;
int size = 100;
searchCondition.setPageSize(size);
boolean flag = true;
List<EventContextVO> dataList = new ArrayList<>();
do {
searchCondition.setPageNo(pageNo);
List<SpecialInformation> informationList = esService.informationList(searchCondition);
if (CollectionUtils.isNotEmpty(informationList)) {
informationList.forEach(e -> {
double similarity = SimilarityUtil.levenshteinSimilarity(subjectKafkaVo.getSubjectName(), e.getTitle());
if (similarity > 0) {
EventContextVO eventContextVO = new EventContextVO();
BeanUtils.copyProperties(e, eventContextVO);
eventContextVO.setDistance(similarity);
dataList.add(eventContextVO);
}
});
if (informationList.size() < size) {
flag = false;
} else {
pageNo++;
}
} else {
flag = false;
}
} while (flag);
if (CollectionUtils.isEmpty(dataList)) {
continue;
}
List<EventContextVO> list = supplyMain(dataList);
LambdaQueryWrapper<SubjectAnalysis> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.eq(SubjectAnalysis::getSubjectId, eventId)
.eq(SubjectAnalysis::getCategory, 4)
.between(SubjectAnalysis::getPublishDate, startTime, endTime);
int count = subjectAnalysisService.count(queryWrapper);
if (count > 0) {
subjectAnalysisService.remove(queryWrapper);
}
List<SubjectAnalysis> analysisList = new ArrayList<>();
for (EventContextVO eventContextVO : list) {
SubjectAnalysis subjectAnalysis = new SubjectAnalysis();
subjectAnalysis.setSubjectId(eventId);
subjectAnalysis.setDataId(eventContextVO.getId());
subjectAnalysis.setTitle(eventContextVO.getTitle());
subjectAnalysis.setOrigin(eventContextVO.getOrigin());
subjectAnalysis.setPublishDate(DateUtil.stringToDate(eventContextVO.getPublishDate(), "yyyy-MM-dd HH:mm:ss"));
subjectAnalysis.setSourceAddress(eventContextVO.getSourceAddress());
subjectAnalysis.setIsMain(eventContextVO.getIsMain() ? 1 : 0);
subjectAnalysis.setCategory(4);
subjectAnalysis.setAnalysisDate(new Date());
analysisList.add(subjectAnalysis);
}
subjectAnalysisService.saveBatch(analysisList);
}
}
//补充是否主要资讯标识以及排序后截取
private List<EventContextVO> supplyMain(List<EventContextVO> list) {
EventContextVO dataVo = null;
double minDistance = Double.MAX_VALUE;
for (EventContextVO eventContextVO : list) {
Double similarity = eventContextVO.getDistance();
if (similarity == 0) {
continue;
}
if (similarity == 1.0) {
dataVo = eventContextVO;
break;
} else if (similarity < minDistance) {
minDistance = similarity;
dataVo = eventContextVO;
}
}
if (dataVo == null) {
list.get(0).setIsMain(true);
} else {
for (EventContextVO eventContextVO : list) {
if (eventContextVO.getId().equals(dataVo.getId())) {
eventContextVO.setIsMain(true);
break;
}
}
}
//先按是否是主事件排序，再按相似度算法(编辑距离)返回值正序排序，最后按时间倒序排序
list.sort(Comparator.comparing(EventContextVO::getIsMain, Comparator.reverseOrder())
.thenComparing((o1, o2) -> {
Double distance1 = o1.getDistance();
Double distance2 = o2.getDistance();
if (distance1 == 0 && distance2 == 0) {
return 0;
}
if (distance1 == 0) {
return 1;
}
if (distance2 == 0) {
return -1;
}
return distance1.compareTo(distance2);
})
.thenComparing(EventContextVO::getPublishDate, Comparator.reverseOrder()));
if (list.size() > 3) {
list = list.subList(0, 3);
}
return list;
}
/** /**
* 每天凌晨0点20分执行一次 * 每天凌晨0点20分执行一次
* 发送 伪事件脉络 所需信息到kafka对应的topic * 发送 伪事件脉络 所需信息到kafka对应的topic
*/ */
@Scheduled(cron = "0 20 0 * * ?") @Scheduled(cron = "0 20 0 * * ?")
public void eventContext_fake() { public void eventContext_fake() {
if(yjzxEnable){ if (yjzxEnable) {
//研究中心不需要此任务 //研究中心不需要此任务
return; return;
} }
...@@ -80,7 +214,7 @@ public class AnalysisTask { ...@@ -80,7 +214,7 @@ public class AnalysisTask {
List<KafkaDataVo> kafkaDataVoList = new ArrayList<>(); List<KafkaDataVo> kafkaDataVoList = new ArrayList<>();
String subjectId = subjectKafkaVo.getId(); String subjectId = subjectKafkaVo.getId();
String keyWord = subjectKafkaVo.getKeyWord(); String keyWord = subjectKafkaVo.getKeyWord();
List<SubjectDataVo> dataList = esService.getDataBySubjectId(subjectId, null, null, Constants.FETCH_FIELDS_DATA); List<SubjectDataVo> dataList = esService.getDataBySubjectId(subjectId, null, null, Constants.FETCH_FIELDS_DATA);
format(subjectId, kafkaDataVoList, dataList); format(subjectId, kafkaDataVoList, dataList);
splitSend(Constants.FAKE_EVENT_CONTEXT_SEND_TOPIC, subjectId, kafkaDataVoList, keyWord); splitSend(Constants.FAKE_EVENT_CONTEXT_SEND_TOPIC, subjectId, kafkaDataVoList, keyWord);
} }
...@@ -124,7 +258,7 @@ public class AnalysisTask { ...@@ -124,7 +258,7 @@ public class AnalysisTask {
Date deadlineDate = DateUtil.addDate(today, -1); Date deadlineDate = DateUtil.addDate(today, -1);
List<SubjectKafkaVo> subjects = eventService.eventSubjectList(); List<SubjectKafkaVo> subjects = eventService.eventSubjectList();
for (SubjectKafkaVo subject : subjects) { for (SubjectKafkaVo subject : subjects) {
CompletableFuture.runAsync(()->{ CompletableFuture.runAsync(() -> {
String subjectId = subject.getId(); String subjectId = subject.getId();
int count = esService.count(subjectId, null, null); int count = esService.count(subjectId, null, null);
if (count > 0) { if (count > 0) {
......
src/main/java/com/zzsn/event/util/HotWordUtil.java 0 → 100644
package com.zzsn.event.util;
import java.util.*;
import java.util.Map.Entry;
/**
* 基于词典文章热词推荐
* @author kongliufeng
*
*/
public class HotWordUtil {
public static void main(String[] args) {
List<Entry<String, String>> entries = new ArrayList<>();
entries.add(new AbstractMap.SimpleEntry<>("1", "关键词1"));
entries.add(new AbstractMap.SimpleEntry<>("1", "关键词2"));
entries.add(new AbstractMap.SimpleEntry<>("2", "关键词1"));
entries.add(new AbstractMap.SimpleEntry<>("2", "关键词2"));
entries.add(new AbstractMap.SimpleEntry<>("1", "关键词3"));
Map<String, Integer> coOccurrenceMap = calculateCoOccurrence_map(entries);
for (Entry<String, Integer> coOccurrence : coOccurrenceMap.entrySet()) {
System.out.println(coOccurrence.getKey() + ": " + coOccurrence.getValue());
}
}
/**
* 计算关键词共现次数
* @author liuxiaopeng
* @data 2024-05-31 下午5:18:21
*/
public static Map<String, Integer> calculateCoOccurrence_map(List<Entry<String, String>> entries) {
Map<String, Integer> coOccurrenceMap = new HashMap<>();
// 先将文章ID分组，便于后续处理
Map<String, List<String>> articleKeywords = new HashMap<>();
for (Entry<String, String> entry : entries) {
articleKeywords.putIfAbsent(entry.getKey(), new ArrayList<>());
articleKeywords.get(entry.getKey()).add(entry.getValue());
}
// 计算共现次数
for (List<String> keywords : articleKeywords.values()) {
for (int i = 0; i < keywords.size(); i++) {
for (int j = i + 1; j < keywords.size(); j++) {
String pair = sortKeywords(keywords.get(i), keywords.get(j));
coOccurrenceMap.put(pair, coOccurrenceMap.getOrDefault(pair, 0) + 1);
}
}
}
return coOccurrenceMap;
}
/**
* 计算关键词共现次数
* @author liuxiaopeng
* @data 2024-05-31 下午5:18:21
*/
public static Map<String, Integer> calculateCoOccurrence_list(List<List<String>> wordList) {
Map<String, Integer> coOccurrenceMap = new HashMap<>();
// 计算共现次数
for (List<String> keywords : wordList) {
for (int i = 0; i < keywords.size(); i++) {
for (int j = i + 1; j < keywords.size(); j++) {
String pair = sortKeywords(keywords.get(i), keywords.get(j));
coOccurrenceMap.put(pair, coOccurrenceMap.getOrDefault(pair, 0) + 1);
}
}
}
return coOccurrenceMap;
}
// 辅助方法，确保关键词对以固定的顺序排列，避免("关键词1", "关键词2")和("关键词2", "关键词1")被视为两个不同的键
private static String sortKeywords(String keyword1, String keyword2) {
return keyword1.compareTo(keyword2) < 0 ? (keyword1 + "&" + keyword2) : (keyword2 + "&" + keyword1);
}
}
src/main/java/com/zzsn/event/util/SimilarityUtil.java
package com.zzsn.event.util; package com.zzsn.event.util;
import org.apache.lucene.search.spell.LevenshteinDistance;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
...@@ -8,204 +10,230 @@ import java.util.regex.Pattern; ...@@ -8,204 +10,230 @@ import java.util.regex.Pattern;
/** /**
* 编辑距离的两字符串相似度 * 编辑距离的两字符串相似度
* *
* @author jianpo.mo * @author jianpo.mo
*/ */
public class SimilarityUtil { public class SimilarityUtil {
private static int min(int one, int two, int three) { private static int min(int one, int two, int three) {
int min = one; int min = one;
if(two < min) { if (two < min) {
min = two; min = two;
} }
if(three < min) { if (three < min) {
min = three; min = three;
} }
return min; return min;
} }
public static int ld(String str1, String str2) { public static int ld(String str1, String str2) {
int d[][]; //矩阵 int d[][]; //矩阵
int n = str1.length(); int n = str1.length();
int m = str2.length(); int m = str2.length();
int i; //遍历str1的 int i; //遍历str1的
int j; //遍历str2的 int j; //遍历str2的
char ch1; //str1的 char ch1; //str1的
char ch2; //str2的 char ch2; //str2的
int temp; //记录相同字符,在某个矩阵位置值的增量,不是0就是1 int temp; //记录相同字符,在某个矩阵位置值的增量,不是0就是1
if(n == 0) { if (n == 0) {
return m; return m;
} }
if(m == 0) { if (m == 0) {
return n; return n;
} }
d = new int[n+1][m+1]; d = new int[n + 1][m + 1];
for(i=0; i<=n; i++) { //初始化第一列 for (i = 0; i <= n; i++) { //初始化第一列
d[i][0] = i; d[i][0] = i;
} }
for(j=0; j<=m; j++) { //初始化第一行 for (j = 0; j <= m; j++) { //初始化第一行
d[0][j] = j; d[0][j] = j;
} }
for(i=1; i<=n; i++) { //遍历str1 for (i = 1; i <= n; i++) { //遍历str1
ch1 = str1.charAt(i-1); ch1 = str1.charAt(i - 1);
//去匹配str2 //去匹配str2
for(j=1; j<=m; j++) { for (j = 1; j <= m; j++) {
ch2 = str2.charAt(j-1); ch2 = str2.charAt(j - 1);
if(ch1 == ch2) { if (ch1 == ch2) {
temp = 0; temp = 0;
} else { } else {
temp = 1; temp = 1;
} }
//左边+1,上边+1, 左上角+temp取最小 //左边+1,上边+1, 左上角+temp取最小
d[i][j] = min(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]+temp); d[i][j] = min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + temp);
} }
} }
return d[n][m]; return d[n][m];
} }
public static double sim(String str1, String str2) { public static double sim(String str1, String str2) {
int ld = ld(str1, str2); int ld = ld(str1, str2);
return 1 - (double) ld / Math.min(str1.length(), str2.length()); return 1 - (double) ld / Math.min(str1.length(), str2.length());
} }
public static int ldforcatlcn(String str1, String str2) {
public static int ldforcatlcn(String str1, String str2) { if (str1 == null || str2 == null) {
if(str1==null||str2==null){ return 0;
return 0; }
} int d = 0; //相同数
int d=0; //相同数 int f = 0; //相同数
int f=0; //相同数 int n = str1.length();
int n = str1.length(); int m = str2.length();
int m = str2.length(); int i; //遍历str1的
int i; //遍历str1的 int j; //遍历str2的
int j; //遍历str2的 String ch1; //str1的
String ch1; //str1的 String ch2; //str2的
String ch2; //str2的 if (n == 0) {
if(n == 0) { return 0;
return 0; }
} if (m == 0) {
if(m == 0) { return 0;
return 0; }
}
for (i = 0; i < n; i++) { //遍历str1
for(i=0; i<n; i++) { //遍历str1 ch1 = str1.substring(i, i + 1);
ch1 = str1.substring(i, i+1); //去匹配str2
//去匹配str2 if (str2.contains(ch1)) {
if(str2.contains(ch1)){ d++;
d++;
} }
} }
for(i=0; i<m; i++) { //遍历str1 for (i = 0; i < m; i++) { //遍历str1
ch2 = str2.substring(i, i+1); ch2 = str2.substring(i, i + 1);
//去匹配str2 //去匹配str2
if(str1.contains(ch2)){ if (str1.contains(ch2)) {
f++; f++;
} }
} }
return Math.min(d,f); return Math.min(d, f);
} }
public static double ldforcatlen(String str1, String str2) {
if(str1==null||str2==null){ public static double ldforcatlen(String str1, String str2) {
return 0; if (str1 == null || str2 == null) {
} return 0;
}
String[] a = str1.split("[^a-zA-Z]+");
String[] b = str2.split("[^a-zA-Z]+"); String[] a = str1.split("[^a-zA-Z]+");
int d=0; //相同数 String[] b = str2.split("[^a-zA-Z]+");
int f=0; //相同数 int d = 0; //相同数
int n = a.length; int f = 0; //相同数
int m = b.length; int n = a.length;
int i; //遍历str1的 int m = b.length;
String ch1; //str1的 int i; //遍历str1的
String ch2; //str2的 String ch1; //str1的
if(n == 0) { String ch2; //str2的
return 0; if (n == 0) {
} return 0;
if(m == 0) { }
return 0; if (m == 0) {
} return 0;
}
for(i=0; i<n; i++) { //遍历str1
for (i = 0; i < n; i++) { //遍历str1
ch1 = a[i]; ch1 = a[i];
//去匹配str2 //去匹配str2
if(str2.contains(ch1)){ if (str2.contains(ch1)) {
d++; d++;
} }
} }
for(i=0; i<m; i++) { //遍历str1 for (i = 0; i < m; i++) { //遍历str1
ch2 = b[i]; ch2 = b[i];
//去匹配str2 //去匹配str2
if(str1.contains(ch2)){ if (str1.contains(ch2)) {
f++; f++;
} }
}
return (double)(Math.min(d,f))/(Math.min(m,n));
}
public static double simforcatl(String str1, String str2) {
if(isChinese(str1)&&isChinese(str2)){
int ld = ldforcatlcn(str1, str2);
return (double) ld / Math.min(str1.length(), str2.length());
}
if(isEnglish(str1)&&isEnglish(str2)){
double ld = ldforcatlen(str1, str2);
return ld;
}
return 0;
}
public static boolean isEnglish(String charaString){
return charaString.getBytes().length == charaString.length();
}
public static boolean isChinese(String str){
String regEx = "[\\u4e00-\\u9fa5]+"; }
return (double) (Math.min(d, f)) / (Math.min(m, n));
}
public static double simforcatl(String str1, String str2) {
if (isChinese(str1) && isChinese(str2)) {
int ld = ldforcatlcn(str1, str2);
return (double) ld / Math.min(str1.length(), str2.length());
}
if (isEnglish(str1) && isEnglish(str2)) {
double ld = ldforcatlen(str1, str2);
return ld;
}
return 0;
}
Pattern p = Pattern.compile(regEx);
public static boolean isEnglish(String charaString) {
return charaString.getBytes().length == charaString.length();
}
public static boolean isChinese(String str) {
String regEx = "[\\u4e00-\\u9fa5]+";
Pattern p = Pattern.compile(regEx);
Matcher m = p.matcher(str); Matcher m = p.matcher(str);
if (m.find()) {
return true;
} else {
return false;
}
}
/**
* 编辑距离算法[值越小，相似度越高]
*
* @param s1 字符串1
* @param s2 字符串2
* @author lkg
* @date 2025/7/5
*/
public static double levenshteinSimilarity(String s1, String s2) {
int distance = levenshteinDistance(s1, s2);
int maxLength = Math.max(s1.length(), s2.length());
return maxLength == 0 ? 1.0 : (1.0 - (double) distance / maxLength);
}
/**
* 编辑距离算法
*
* @param s1 源字符串
* @param s2 目标字符串
* @author lkg
* @date 2025/7/5
*/
private static int levenshteinDistance(String s1, String s2) {
int[][] dp = new int[s1.length() + 1][s2.length() + 1];
for (int i = 0; i <= s1.length(); i++) {
dp[i][0] = i;
}
for (int j = 0; j <= s2.length(); j++) {
dp[0][j] = j;
}
for (int i = 1; i <= s1.length(); i++) {
for (int j = 1; j <= s2.length(); j++) {
int cost = (s1.charAt(i - 1) == s2.charAt(j - 1)) ? 0 : 1;
dp[i][j] = Math.min(
Math.min(dp[i - 1][j] + 1, dp[i][j - 1] + 1),
dp[i - 1][j - 1] + cost
);
}
}
return dp[s1.length()][s2.length()];
}
if(m.find())
return true;
else
return false;
}
public static void main(String[] args) { public static void main(String[] args) {
String str6 = "国产商用大飞机";
String str1 = "【观察】 三年行动 国企改革再升级"; String str7 = "“她”力量丨赵春玲：矢志蓝天梦 C929总设计师的巾帼风采";
String str2 = "三年行动 国企改革再升级"; String str8 = "【中证快报】3月2日中证投资资讯";
String str3 = "【国企改革】三年行动, 国企改革再升级"; System.out.println("sim67=" + levenshteinSimilarity(str6, str7));
System.out.println("sim68=" + levenshteinSimilarity(str6, str8));
System.out.println("sim12="+simforcatl(str1, str2));
System.out.println("sim13="+simforcatl(str1, str3)); }
System.out.println("sim23="+simforcatl(str2, str3)); }
String str4="混合所有制改革";
String str5="混改";
System.out.println("sim45="+simforcatl(str4, str5));
System.out.println("sim55="+simforcatl(str5, str5));
String str6="【国企混改】国企混合所有制改革实务（上）";
String str7="【国企混改】国企混合所有制改革实务（下）";
System.out.println("sim67="+simforcatl(str5, str5));
}
}
\ No newline at end of file
src/main/java/com/zzsn/event/vo/CoOccurrenceVO.java 0 → 100644
package com.zzsn.event.vo;
import lombok.Data;
/**
* 关键词共现【同时出现在一篇资讯】实体对象
*
* @author lkg
* @date 2025/7/9
*/
@Data
public class CoOccurrenceVO {
//关键词1
private CountVO word_1;
//共现次数
private Long coOccurrenceNum;
//关键词2
private CountVO word_2;
}
src/main/java/com/zzsn/event/vo/EventContextVO.java 0 → 100644
package com.zzsn.event.vo;
import lombok.Data;
/**
* todo
*
* @author lkg
* @date 2025/7/5
*/
@Data
public class EventContextVO {
//资讯id
private String id;
//资讯标题
private String title;
//来源
private String origin;
//发布时间
private String publishDate;
//原文链接
private String sourceAddress;
//相似度算法【编辑距离】返回的值
private Double distance;
//是否关联度最高
private Boolean isMain = false;
//索引名称
private String dbIndex;
}
src/main/resources/application-dev.yml
...@@ -129,6 +129,7 @@ obs: ...@@ -129,6 +129,7 @@ obs:
endPoint: https://obs.cn-north-1.myhuaweicloud.com endPoint: https://obs.cn-north-1.myhuaweicloud.com
scheduling: scheduling:
enable: false enable: false
yjzxEnable: false
serviceProject: serviceProject:
url: https://clb.ciglobal.cn/clb-api/datapull/ url: https://clb.ciglobal.cn/clb-api/datapull/
scoreRule: scoreRule:
......
src/main/resources/application-test.yml
...@@ -131,6 +131,7 @@ obs: ...@@ -131,6 +131,7 @@ obs:
endPoint: https://obs.cn-north-1.myhuaweicloud.com endPoint: https://obs.cn-north-1.myhuaweicloud.com
scheduling: scheduling:
enable: false enable: false
yjzxEnable: false
serviceProject: serviceProject:
url: https://clb.ciglobal.cn/clb-api/datapull/ url: https://clb.ciglobal.cn/clb-api/datapull/
scoreRule: scoreRule:
...@@ -151,6 +152,7 @@ python: ...@@ -151,6 +152,7 @@ python:
keyWordsExtractUrl: http://1.95.91.200:7006/get_phrase/ keyWordsExtractUrl: http://1.95.91.200:7006/get_phrase/
#清空去重服务历史数据 #清空去重服务历史数据
clearDuplicateHistoryUrl: http://1.95.13.40:8080/subject/delete_history_data clearDuplicateHistoryUrl: http://1.95.13.40:8080/subject/delete_history_data
hot-crawler: http://139.9.144.106:5000/data?option=OPTION
jeecg: jeecg:
shiro: shiro:
excludeUrls: info/subjectPageListGroupByLabel,/subject/simple/queryInfoNoSign,/subject/manage/visiblePageListNoSign excludeUrls: info/subjectPageListGroupByLabel,/subject/simple/queryInfoNoSign,/subject/manage/visiblePageListNoSign
...@@ -177,3 +179,9 @@ infoSource: ...@@ -177,3 +179,9 @@ infoSource:
waitInfoRemove: http://1.95.79.85:8823/baseSourceInfo/api/infoSource/waitInfoRemove waitInfoRemove: http://1.95.79.85:8823/baseSourceInfo/api/infoSource/waitInfoRemove
columnListByWait: http://1.95.79.85:8823/baseSourceInfo/api/infoSource/columnListByWait columnListByWait: http://1.95.79.85:8823/baseSourceInfo/api/infoSource/columnListByWait
columnDetail: http://1.95.79.85:8823/baseSourceInfo/api/infoSource/columnDetail columnDetail: http://1.95.79.85:8823/baseSourceInfo/api/infoSource/columnDetail
model:
aichain:
url: https://open.bigmodel.cn/api/paas/v4/chat/completions
authorization: Bearer 3262d41a17a9490a8528eaf52a2be6a0.LNbHPCRmnzFgPHRp
cookie: acw_tc=2760822c17247248406261468e6c541507ba9035f95078363549469047ee74
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论