Skip to content

E
event
提交 4fc6b221 作者: 925993793@qq.com
操作

事件服务功能开发,联调修改

上级 14f24f28
正在显示 包含 469 行增加 和 104 行删除
pom.xml
......@@ -193,7 +193,17 @@
<artifactId>dynamic-datasource-spring-boot-starter</artifactId>
<version>3.2.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.freemarker/freemarker -->
<dependency>
<groupId>org.freemarker</groupId>
<artifactId>freemarker</artifactId>
<version>2.3.31</version>
</dependency>
<dependency>
<groupId>com.aspose</groupId>
<artifactId>aspose-words</artifactId>
<version>19.1</version>
</dependency>
</dependencies>
<build>
......
src/main/java/com/zzsn/event/config/FreeMarkerConfiguration.java 0 → 100644
package com.zzsn.event.config;
import freemarker.template.Configuration;
import java.util.Locale;
public class FreeMarkerConfiguration {
private static Configuration freemarkerConfig;
static {
freemarkerConfig = new Configuration(Configuration.VERSION_2_3_22);
freemarkerConfig.setEncoding(Locale.getDefault(), "UTF-8");
freemarkerConfig.setClassForTemplateLoading(FreeMarkerConfiguration.class, "/template");
}
public static Configuration get() {
return freemarkerConfig;
}
}
src/main/java/com/zzsn/event/controller/EventDataController.java
package com.zzsn.event.controller;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.zzsn.event.constant.Constants;
import com.zzsn.event.constant.Result;
import com.zzsn.event.service.EsService;
import com.zzsn.event.service.IEventService;
......@@ -63,7 +64,7 @@ public class EventDataController {
@RequestParam(name = "orderType", required = false) String orderType,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize) {
IPage<EventFrontVO> pageList = eventService.frontPageList(eventName, eventType, labelField,labelName,order,orderType,pageNo, pageSize);
IPage<EventFrontVO> pageList = eventService.frontPageList(eventName, eventType, labelField, labelName, order, orderType, pageNo, pageSize);
return Result.OK(pageList);
}
......@@ -91,7 +92,7 @@ public class EventDataController {
@RequestParam(name = "column", defaultValue = "publishDate") String column,
@RequestParam(name = "order", defaultValue = "desc") String order,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize) throws Exception {
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize) {
List<String> subjectIdList = new ArrayList<>();
if (StringUtils.isNotEmpty(subjectId)) {
subjectIdList.add(subjectId);
......@@ -111,7 +112,10 @@ public class EventDataController {
* @return
*/
@GetMapping(value = "/articleDetail")
public Result<?> articleDetail(@RequestParam String index, @RequestParam String id) {
public Result<?> articleDetail(@RequestParam(required = false) String index, @RequestParam String id) {
if (StringUtils.isEmpty(index)) {
index = Constants.SUBJECT_INDEX;
}
SubjectDataVo subjectDataVo = esService.queryInfo(index, id);
return Result.OK(subjectDataVo);
}
......@@ -127,8 +131,11 @@ public class EventDataController {
*/
@ApiOperation(value = "单篇文章热词", notes = "单篇文章热词")
@GetMapping(value = "/hotWords")
public Result<?> articleList(@RequestParam("index") String index, @RequestParam("id") String id,
public Result<?> articleList(@RequestParam(value = "index",required = false) String index, @RequestParam("id") String id,
@RequestParam(name = "number", defaultValue = "200") Integer number) {
if (StringUtils.isEmpty(index)) {
index = Constants.SUBJECT_INDEX;
}
List<StatisticsKeyWordVo> words = eventService.hotWords(index, id, number);
return Result.OK(words);
}
......@@ -136,18 +143,20 @@ public class EventDataController {
/**
* 相关推荐
*
* @param id 资讯id
* @param title 标题
* @param pageSize 返回条数
* @param subjectId 专题id
* @param id 资讯id
* @param title 标题
* @param pageSize 返回条数
* @author lkg
* @date 2024/4/10
*/
@GetMapping(value = "/recommendList")
public Result<?> recommendList(@RequestParam(name = "id") String id,
public Result<?> recommendList(@RequestParam(name = "subjectId") String subjectId,
@RequestParam(name = "id") String id,
@RequestParam(name = "title") String title,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize) throws Exception {
List<SubjectDataVo> recommendList = esService.queryRecommendList(id, title, pageNo, pageSize);
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize) {
List<SubjectDataVo> recommendList = esService.queryRecommendList(subjectId, id, title, pageNo, pageSize);
return Result.OK(recommendList);
}
......
src/main/java/com/zzsn/event/controller/EventExportController.java
package com.zzsn.event.controller;
import com.aspose.words.Document;
import com.aspose.words.SaveFormat;
import com.zzsn.event.config.FreeMarkerConfiguration;
import com.zzsn.event.service.EsService;
import com.zzsn.event.service.IEventService;
import com.zzsn.event.util.DateUtil;
import com.zzsn.event.util.ExcelExportUtil;
import com.zzsn.event.util.Utility;
import com.zzsn.event.vo.EventExcelVO;
import com.zzsn.event.vo.ExportParam;
import com.zzsn.event.vo.SubjectDataVo;
import freemarker.template.Template;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.io.*;
import java.net.URLEncoder;
import java.util.*;
/**
* 导出功能
......@@ -29,21 +39,20 @@ public class EventExportController {
@Autowired
private IEventService eventService;
@Autowired
private EsService esService;
/**
* 导出事件列表
*
* @param eventIdList 事件id集合
* @param size 导出数量
* @param exportParam 导出参数封装
* @author lkg
* @date 2024/4/10
*/
@GetMapping("/eventList")
public void exportEventList(@RequestParam(required = false) List<String> eventIdList,
@RequestParam(required = false) Integer size,
HttpServletResponse response) {
@PostMapping("/eventList")
public void exportEventList(@RequestBody ExportParam exportParam, HttpServletResponse response) {
String[] headers = new String[]{"id", "事件名称", "事件描述", "发布时间", "热度"};
List<EventExcelVO> eventList = eventService.frontList(eventIdList, size);
List<EventExcelVO> eventList = eventService.frontList(exportParam.getEventIdList(), exportParam.getSize());
if (CollectionUtils.isNotEmpty(eventList)) {
String name = "event.xlsx";
List<List<String>> dataList = new ArrayList<>();
......@@ -61,6 +70,126 @@ public class EventExportController {
}
}
/**
* 导出事件资讯列表
*
* @param exportParam 参数封装
* @author lkg
* @date 2024/4/11
*/
@PostMapping("/dataList")
public void exportDataList(@RequestBody ExportParam exportParam, HttpServletResponse response) {
OutputStream outputstream = null;
ByteArrayOutputStream bos = null;
try {
List<String> eventIdList = exportParam.getEventIdList();
if (CollectionUtils.isEmpty(eventIdList)) {
List<EventExcelVO> frontList = eventService.frontList(null, null);
frontList.forEach(e -> eventIdList.add(e.getId()));
}
List<SubjectDataVo> exportDataList = esService.exportDataList(eventIdList, exportParam.getSearchWord(), exportParam.getPosition(), exportParam.getCategory(),
exportParam.getArticleIdList(), exportParam.getColumn(), exportParam.getOrder(), exportParam.getType(), exportParam.getSize());
Map<String, Object> map = formatDocData(exportDataList, exportParam.getSearchWord(), exportParam.getType());
Template template = FreeMarkerConfiguration.get().getTemplate("EVENT_DATA_REPORT.ftl", "UTF-8");
bos = new ByteArrayOutputStream();
template.process(map, new OutputStreamWriter(bos));
Document document = new Document(new ByteArrayInputStream(bos.toByteArray()));
bos.reset();
document.updateFields();
document.save(bos, SaveFormat.DOCX);
String fileName = URLEncoder.encode("事件资讯", "UTF-8").replace("+", "-");
response.setHeader("content-Type", "application/msword");
response.setHeader("Content-Disposition", "attachment;filename=" + fileName);
response.setContentLength(bos.size());
outputstream = response.getOutputStream();
bos.writeTo(outputstream);
bos.close();
outputstream.flush();
outputstream.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
if (bos != null) {
try {
bos.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (outputstream != null) {
try {
outputstream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
* 格式化ftl文件所需要的数据格式
*
* @param exportDataList 资讯列表
* @param searchWord 搜索词
* @param type 导出方式(1-摘要;2-正文)
* @author lkg
* @date 2024/4/11
*/
private Map<String, Object> formatDocData(List<SubjectDataVo> exportDataList, String searchWord, Integer type) {
String docTitle = "事件资讯";
//遍历取到的文章
List<Map<String, Object>> contents = new ArrayList<>();
//文档结构图
Map<String, List<Map<String, Object>>> contentSortMap = new LinkedHashMap<>();//文档结构图
//目录
Map<String, List<Map<String, Object>>> catalogSortMap = new LinkedHashMap<String, List<Map<String, Object>>>();//目录
List<Map<String, Object>> caList = new ArrayList<Map<String, Object>>();
catalogSortMap.put(docTitle, caList);//目录
int num = 1;
for (SubjectDataVo subjectDataVo : exportDataList) {
String rid = subjectDataVo.getId();
String title = Utility.getValueAfterReplaceSpecialWordNotEnter(subjectDataVo.getTitle());
String sourceAddress = subjectDataVo.getSourceAddress();
String origin = subjectDataVo.getOrigin();
String summary = subjectDataVo.getSummary();
if (StringUtils.isNotEmpty(summary)) {
summary = Utility.getValueAfterReplaceSpecialWordNotEnter(Utility.TransferHTML2Text(summary));
}
String publishDate = subjectDataVo.getPublishDate();
String contentStr = subjectDataVo.getContent();
if (StringUtils.isNotEmpty(contentStr)) {
contentStr = Utility.getValueAfterReplaceSpecialWordNotEnter(Utility.TransferHTML2Text(contentStr));
}
String info = StringUtils.isNotEmpty(origin) ? "（来源：" + origin + "，发布时间：" + publishDate + "）" : "（发布时间：" + publishDate + "）";
Map<String, Object> content = new HashMap<>();
content.put("id", rid);
content.put("title", title);
content.put("url", Utility.getValueAfterReplaceSpecialWordNotEnter(sourceAddress));
if (type == 1) {
content.put("info", summary + info);
} else {
content.put("info", info);
content.put("contentStr", contentStr);
}
contents.add(content);
contentSortMap.put(docTitle, contents);//文档结构图中，文章标题
Map<String, Object> catalog = new HashMap();
catalog.put("index", num);
catalog.put("title", title);
catalog.put("id", rid + "-" + (num - 1));
caList.add(catalog);
catalogSortMap.put(docTitle, caList);
num++;
}
Map<String, Object> map = new HashMap<>();
map.put("docTitle", docTitle);
map.put("publishDate", DateUtil.dateToString(new Date(), "yyyy-MM-dd"));
map.put("keyWords", StringUtils.isEmpty(searchWord) ? "" : searchWord);
map.put("catalogMap", catalogSortMap);
map.put("contentMap", contentSortMap);
return map;
}
private void setResponseHeader(HttpServletResponse response, String name) {
try {
try {
......
src/main/java/com/zzsn/event/controller/EventManageController.java
......@@ -79,11 +79,38 @@ public class EventManageController {
@RequestParam(name = "orderType", defaultValue = "asc") String orderType,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize) {
IPage<EventManageVO> pageList = eventService.pageList(eventName,eventType,startTime,endTime, order, orderType, pageNo, pageSize);
IPage<EventManageVO> pageList = eventService.pageList(eventName, eventType, startTime, endTime, order, orderType, pageNo, pageSize);
return Result.OK(pageList);
}
/**
* 地域信息-树型结构
*
* @param type 类别(1-国际;2-国内)
* @author lkg
* @date 2024/4/10
*/
@GetMapping("/regionTree")
public Result<?> regionTree(@RequestParam Integer type) {
List<Node> nodes = labelEntityService.regionTree(type);
return Result.OK(nodes);
}
/**
* 2.17 上传icon
*
* @return
*/
@PostMapping(value = "/upload")
@ResponseBody
public Result<?> uploadKnowledge(HttpServletRequest request) {
MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
MultipartFile file = multipartRequest.getFile("file");// 获取上传文件对象
String url = eventService.upload(file);
return Result.OK(url);
}
/**
* 1.2 添加
*
* @param eventParam
......@@ -167,7 +194,7 @@ public class EventManageController {
.last(" limit 1"));
event.setEventTag(one);
AddEventParam eventParam = new AddEventParam();
BeanUtils.copyProperties(event,eventParam);
BeanUtils.copyProperties(event, eventParam);
List<RegionVO> regionList = eventRegionMapService.regionList(event.getId());
eventParam.setRegionList(regionList);
return Result.OK(eventParam);
......@@ -406,7 +433,6 @@ public class EventManageController {
public Object deleteKeyWordsBind(@RequestBody SubjectPage subjectPage) {
try {
JSONObject params = ObjectUtil.objectToJSONObject(subjectPage);
;
String url = SERVICE_PROJECT_URL + "event/deleteKeyWordsBind";
return HttpUtil.doPost(url, params, 10000);
} catch (Exception e) {
......@@ -445,29 +471,49 @@ public class EventManageController {
}
/**
* 地域信息-树型结构
* 模型信息列表
*
* @param type 类别(1-国际;2-国内)
* @author lkg
* @date 2024/4/10
* @date 2024/4/11
*/
@GetMapping("/regionTree")
public Result<?> regionTree(@RequestParam Integer type) {
List<Node> nodes = labelEntityService.regionTree(type);
return Result.OK(nodes);
@GetMapping("/modelList")
public Result<?> modelList() {
List<ModelVO> modelVOS = eventService.modelList();
return Result.OK(modelVOS);
}
/**
* 2.17 上传icon
* 算法模型信息列表
*
* @return
* @param subjectId 专题id
* @param type 类型id
* @author lkg
* @date 2024/4/11
*/
@PostMapping(value = "/upload")
@ResponseBody
public Result<?> uploadKnowledge(HttpServletRequest request) {
MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
MultipartFile file = multipartRequest.getFile("file");// 获取上传文件对象
String url = eventService.upload(file);
return Result.OK(url);
@GetMapping("/algorithmModelList")
public Object algorithmModelList(@RequestParam String subjectId, @RequestParam Integer type) {
String url = SERVICE_PROJECT_URL + "event/listNoPage";
Map<String, String> params = new HashMap<>();
params.put("subjectId", subjectId);
params.put("type", type.toString());
return HttpUtil.doGet(url, params, "utf-8");
}
/**
* 模型绑定
*
* @author lkg
* @date 2024/4/11
*/
@PostMapping("/modelBind")
public Object modelBind(@RequestBody SubjectPage subjectPage) {
try {
JSONObject params = ObjectUtil.objectToJSONObject(subjectPage);
String url = SERVICE_PROJECT_URL + "event/modelBind";
return HttpUtil.doPost(url, params, 10000);
} catch (Exception e) {
return null;
}
}
}
src/main/java/com/zzsn/event/entity/SubjectAnalysis.java
......@@ -38,7 +38,7 @@ public class SubjectAnalysis implements Serializable {
private String sourceAddress;
/*重复数*/
private Integer repeatNum;
/*观点分析下的类型(1-新闻；2-论坛；3-微博)*/
/*观点分析下的类型(1-新闻；2-论坛；3-微博；4-专家)*/
private Integer type;
/*分类(1-观点分析；2-事件脉络；3-伪事件脉络)*/
//伪事件脉络 即当事件脉络资讯数量少于约定数量，通过python算法生成临时的资讯数量大于/等于约定数量的事件脉络。
......@@ -46,5 +46,6 @@ public class SubjectAnalysis implements Serializable {
/*分析时间*/
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
private Date analysisDate;
/**专家名称*/
private String professionName;
}
src/main/java/com/zzsn/event/mapper/EventMapper.java
......@@ -123,4 +123,12 @@ public interface EventMapper extends BaseMapper<Event> {
* @date 2024/4/10
*/
List<EventExcelVO> frontList(@Param("eventIdList") List<String> eventIdList, @Param("size") Integer size);
/**
* 模型信息列表
*
* @author lkg
* @date 2024/4/11
*/
List<ModelVO> modelList();
}
src/main/java/com/zzsn/event/mapper/xml/EventMapper.xml
......@@ -125,7 +125,8 @@
</select>
<select id="topEventList" resultType="com.zzsn.event.vo.EventTopVO">
select t.id,t.event_name,t.publish_date,t.total_hot,ec.type_name from event t
select t.id,t.event_name,t.start_time,t.end_time,t.publish_date,t.total_hot,ec.type_name
from event t
inner join event_category ec on t.event_type = ec.id
where t.publish_status = 1 and t.face_public = 1
<if test="startTime!=null and startTime != ''">
......@@ -162,9 +163,11 @@
SELECT
e.id AS eventId,
e.event_name,
e.start_time,
e.end_time,
r.name_cn AS regionName
FROM
( SELECT id, event_name FROM event WHERE publish_status = 1 AND face_public = 1 ) e
( SELECT id, event_name,start_time,end_time FROM event WHERE publish_status = 1 AND face_public = 1 ) e
INNER JOIN event_region_map m ON e.id = m.event_id
INNER JOIN sys_base_region r ON m.top_region_id = r.id
WHERE m.type = #{type}
......@@ -174,8 +177,8 @@
</select>
<select id="frontList" resultType="com.zzsn.event.vo.EventExcelVO">
select id,event_name,event_describe,publish_date,total_hot from event where publish_status = 1 and face_public =
1
select id,event_name,event_describe,publish_date,total_hot from event
where publish_status = 1 and face_public = 1
<if test="eventIdList != null and eventIdList.size() > 0">
and id in
<foreach collection="eventIdList" open="(" separator="," close=")" item="item">
......@@ -187,4 +190,8 @@
limit #{size}
</if>
</select>
<select id="modelList" resultType="com.zzsn.event.vo.ModelVO">
select id,model_name,type from model where pid = '0' and type is not null
</select>
</mapper>
\ No newline at end of file
src/main/java/com/zzsn/event/service/EsStatisticsService.java
......@@ -20,10 +20,11 @@ public interface EsStatisticsService {
* @param subjectId 专题id
* @param startTime 开始时间
* @param endTime 结束时间
* @param type 1-按小时;2-按天
* @author lkg
* @date 2024/1/25
*/
Map<String, Object> totalAndMax(String subjectId, String startTime, String endTime);
Map<String, String> totalAndMax(String subjectId, String startTime, String endTime, Integer type);
/**
......@@ -65,11 +66,12 @@ public interface EsStatisticsService {
*
* @param subjectId 专题id
* @param startTime 开始时间
* @param endTime 结束时间
* @param endTime 结束时间
* @param type 1-按小时;2-按天
* @author lkg
* @date 2024/4/10
*/
CountVO flowData(String subjectId, String startTime, String endTime);
List<CountVO> flowData(String subjectId, String startTime, String endTime, Integer type);
/**
* 时间段内事件的信息总数
......
src/main/java/com/zzsn/event/service/IEventService.java
......@@ -102,4 +102,12 @@ public interface IEventService extends IService<Event> {
* @date 2024/4/10
*/
List<EventExcelVO> frontList(List<String> eventIdList, Integer size);
/**
* 模型信息列表
*
* @author lkg
* @date 2024/4/11
*/
List<ModelVO> modelList();
}
src/main/java/com/zzsn/event/service/impl/AnalysisServiceImpl.java
......@@ -17,10 +17,7 @@ import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.TreeSet;
import java.util.*;
/**
* @author lkg
......@@ -59,12 +56,18 @@ public class AnalysisServiceImpl implements AnalysisService {
List<SubjectAnalysis> finalList = new ArrayList<>();
List<SubjectDataVo> dataList = esService.pageList(subjectId, null, null, Constants.FETCH_FIELDS_STATISTIC, 1,1, 15);
dataList.forEach(e -> {
String dataId = e.getId();
SubjectAnalysis subjectAnalysis = new SubjectAnalysis();
BeanUtils.copyProperties(e, subjectAnalysis);
subjectAnalysis.setPublishDate(DateUtil.stringToDate(e.getPublishDate(), "yyyy-MM-dd HH:mm:ss"));
//todo 重复数
// List<SubjectDataVo> subjectDataVoList = esService.dataById(subjectId, e.getId());
// subjectAnalysis.setRepeatNum(subjectDataVoList.size());
List<String> idList = new ArrayList<>();
idList.add(dataId);
Map<String, Integer> similarNumber = esService.getSimilarNumber(subjectId, idList);
Integer count = similarNumber.get(dataId);
if (count == null) {
count = 0;
}
subjectAnalysis.setRepeatNum(count);
finalList.add(subjectAnalysis);
});
list = finalList;
......
src/main/java/com/zzsn/event/service/impl/EsStatisticsServiceImpl.java
......@@ -48,30 +48,36 @@ public class EsStatisticsServiceImpl implements EsStatisticsService {
private LabelEntityService labelEntityService;
@Override
public Map<String, Object> totalAndMax(String subjectId, String startTime, String endTime) {
Map<String, Object> map = new HashMap<>();
public Map<String, String> totalAndMax(String subjectId, String startTime, String endTime, Integer type) {
Map<String, String> map = new HashMap<>();
SearchRequest searchRequest = new SearchRequest(Constants.ES_DATA_FOR_SUBJECT);
SearchSourceBuilder searchSourceBuilder = formatSourceBuilder(subjectId, null, startTime, endTime);
searchSourceBuilder.size(0);
searchSourceBuilder.trackTotalHits(true);
DateHistogramAggregationBuilder aggregation = AggregationBuilders.dateHistogram("group_hour")
.field("publishDate")
.calendarInterval(DateHistogramInterval.HOUR)
.format("yyyy-MM-dd HH")
.order(BucketOrder.count(false));
DateHistogramAggregationBuilder aggregation = AggregationBuilders.dateHistogram("group_hour").field("publishDate");
if (type == 1) {
aggregation.calendarInterval(DateHistogramInterval.HOUR)
.format("yyyy-MM-dd HH")
.order(BucketOrder.count(false));
} else if (type == 2) {
aggregation.calendarInterval(DateHistogramInterval.DAY)
.format("yyyy-MM-dd")
.order(BucketOrder.count(false));
}
searchSourceBuilder.aggregation(aggregation);
searchRequest.source(searchSourceBuilder);
try {
SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
long value = response.getHits().getTotalHits().value;
map.put("totalCount", value);
map.put("totalCount", String.valueOf(value));
Aggregations aggregations = response.getAggregations();
ParsedDateHistogram groupHour = aggregations.get("group_hour");
List<? extends Histogram.Bucket> buckets = groupHour.getBuckets();
if (CollectionUtils.isNotEmpty(buckets)) {
Histogram.Bucket bucket = buckets.get(0);
long count = bucket.getDocCount();
map.put("max", count);
map.put("max", String.valueOf(count));
map.put("maxTime", bucket.getKeyAsString());
}
} catch (Exception e) {
e.printStackTrace();
......@@ -244,22 +250,23 @@ public class EsStatisticsServiceImpl implements EsStatisticsService {
}
@Override
public CountVO flowData(String subjectId, String startTime, String endTime) {
CountVO countVO = new CountVO();
public List<CountVO> flowData(String subjectId, String startTime, String endTime,Integer type) {
SearchRequest searchRequest = new SearchRequest(Constants.ES_DATA_FOR_SUBJECT);
SearchSourceBuilder searchSourceBuilder = formatSourceBuilder(subjectId, null, startTime, endTime);
searchSourceBuilder.size(0);
searchSourceBuilder.trackTotalHits(true);
DateHistogramAggregationBuilder aggregation = AggregationBuilders.dateHistogram("group_day")
.field("publishDate");
aggregation.calendarInterval(DateHistogramInterval.DAY).format("yyyy-MM-dd");
if (type == 1) {
aggregation.calendarInterval(DateHistogramInterval.HOUR).format("yyyy-MM-dd HH");
} else if (type == 2) {
aggregation.calendarInterval(DateHistogramInterval.DAY).format("yyyy-MM-dd");
}
searchSourceBuilder.aggregation(aggregation);
searchRequest.source(searchSourceBuilder);
List<CountVO> list = new ArrayList<>();
try {
SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
long value = response.getHits().getTotalHits().value;
countVO.setValue(value);
List<CountVO> list = new ArrayList<>();
Aggregations aggregations = response.getAggregations();
ParsedDateHistogram groupHour = aggregations.get("group_day");
List<? extends Histogram.Bucket> buckets = groupHour.getBuckets();
......@@ -271,11 +278,10 @@ public class EsStatisticsServiceImpl implements EsStatisticsService {
list.add(vo);
}
}
countVO.setChildren(list);
} catch (Exception e) {
e.printStackTrace();
}
return countVO;
return list;
}
@Override
......@@ -311,7 +317,7 @@ public class EsStatisticsServiceImpl implements EsStatisticsService {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//创建查询对象
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.termsQuery("subjectId", Arrays.asList(subjectId.split(","))));
boolQuery.must(QueryBuilders.termsQuery("subjectId.keyword", Arrays.asList(subjectId.split(","))));
//标签id
if (StringUtils.isNotEmpty(relationId)) {
BoolQueryBuilder relationIdQuery = QueryBuilders.boolQuery();
......
src/main/java/com/zzsn/event/service/impl/EventServiceImpl.java
......@@ -118,13 +118,15 @@ public class EventServiceImpl extends ServiceImpl<EventMapper, Event> implements
type = 1;
}
List<EventFrontVO> pageList = baseMapper.frontPageList(eventName, eventType,labelField,labelName,type,order,orderType, offset, pageSize);
//获取专题资讯的首发来源
Map<String, String> map = getFirstMap(pageList);
if (MapUtil.isNotEmpty(map)) {
pageList.forEach(e -> {
String firstOrigin = map.get(e.getId());
e.setFirstOrigin(firstOrigin);
});
if (CollectionUtils.isNotEmpty(pageList)) {
//获取专题资讯的首发来源
Map<String, String> map = getFirstMap(pageList);
if (MapUtil.isNotEmpty(map)) {
pageList.forEach(e -> {
String firstOrigin = map.get(e.getId());
e.setFirstOrigin(firstOrigin);
});
}
}
//获取总条数
Integer count = baseMapper.frontTotalCount(eventName, eventType,labelField,labelName,type);
......@@ -155,10 +157,12 @@ public class EventServiceImpl extends ServiceImpl<EventMapper, Event> implements
searchSourceBuilder.sort("publishDate", SortOrder.ASC);
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
boolQueryBuilder.must(QueryBuilders.termsQuery("subjectId.keyword", eventIdList));
// 聚合搜索
//聚合搜索
TermsAggregationBuilder one = AggregationBuilders.terms("one").field("subjectId.keyword").size(eventIdList.size());
//ES分组取每组第一条Java写法
TopHitsAggregationBuilder topHitsAggregationBuilder = AggregationBuilders.topHits("top_docs").size(1);
TopHitsAggregationBuilder topHitsAggregationBuilder = AggregationBuilders.topHits("top_docs")
.sort("publishDate",SortOrder.ASC)
.size(1);
one.subAggregation(topHitsAggregationBuilder);
searchSourceBuilder.aggregation(one);
searchSourceBuilder.query(boolQueryBuilder);
......@@ -263,6 +267,11 @@ public class EventServiceImpl extends ServiceImpl<EventMapper, Event> implements
}
@Override
public List<ModelVO> modelList() {
return baseMapper.modelList();
}
@Override
public List<EventExcelVO> frontList(List<String> eventIdList,Integer size) {
return baseMapper.frontList(eventIdList,size);
}
......
src/main/java/com/zzsn/event/task/AnalysisTask.java
......@@ -172,11 +172,17 @@ public class AnalysisTask {
private void format(String subjectId, List<KafkaDataVo> kafkaDataVoList, List<SubjectDataVo> dataList) {
dataList.forEach(e -> {
String dataId = e.getId();
KafkaDataVo kafkaDataVo = new KafkaDataVo();
BeanUtils.copyProperties(e, kafkaDataVo);
//todo
// List<SubjectDataVo> subjectDataVoList = esService.dataById(subjectId, e.getId());
// kafkaDataVo.setRepeatNum(subjectDataVoList.size());
List<String> idList = new ArrayList<>();
idList.add(dataId);
Map<String, Integer> similarNumber = esService.getSimilarNumber(subjectId, idList);
Integer count = similarNumber.get(dataId);
if (count == null) {
count = 0;
}
kafkaDataVo.setRepeatNum(count);
kafkaDataVoList.add(kafkaDataVo);
});
}
......
src/main/java/com/zzsn/event/task/EventHotTask.java
......@@ -20,7 +20,7 @@ import java.util.List;
import java.util.concurrent.CompletableFuture;
/**
* todo
*
*
* @author lkg
* @date 2024/4/10
......
src/main/java/com/zzsn/event/util/DateUtils.java
......@@ -193,7 +193,6 @@ public class DateUtils extends PropertyEditorSupport {
try {
_date = sformat.parse(date);
} catch (ParseException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return sformat.format(_date);
......
src/main/java/com/zzsn/event/util/HanlpUtil.java
......@@ -4,9 +4,7 @@ import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.seg.common.Term;
import org.springframework.util.StringUtils;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
......@@ -61,7 +59,7 @@ public class HanlpUtil {
}
//去重
List<String> distinctList = phraseList.stream().distinct().collect(Collectors.toList());
Map<String, Integer> map = StringUtil.getHitWordsAndTimes(distinctList, text);
Map<String, Integer> map = getHitWordsAndTimes(distinctList, text);
//根据频次排序
List<Map.Entry<String, Integer>> list = SortUtil.sortMap(map);
if (limitNo > list.size()) {
......@@ -85,4 +83,54 @@ public class HanlpUtil {
Matcher m = p.matcher(str);
return m.find();
}
/**
* @Description 获取srcList中在text存在的集合(包含频次)
* @author kongliufeng
* @创建时间 2020/9/3 18:41
* @Version 1.0
*/
private static Map<String, Integer> getHitWordsAndTimes(Collection<String> srcList, String text){
Map<String, Integer> map = new HashMap<>();
if(srcList==null || StringUtils.isEmpty(text)){
return map;
}
for (String s : srcList) {
int i = countKeyWordInContent(s, text);
if(i>0){
map.put(s,i);
}
}
return map;
}
/**
* @Description 计算一个词在一个文本中的次数
* @author kongliufeng
* @创建时间 2020/8/27 19:56
* @Version 1.0
*/
private static int countKeyWordInContent(String keyword, String srcContent){
if(keyword==null ||keyword.trim().equals("")){
return 0;
}
int count = 0;
int leng = srcContent.length();
int j = 0;
for (int i = 0; i < leng; i++){
if (srcContent.charAt(i) == keyword.charAt(j)){
j++;
if (j == keyword.length()){
count++;
j = 0;
}
}
else{
i = i - j;
j = 0;
}
}
return count;
}
}
src/main/java/com/zzsn/event/util/HttpUtil.java
......@@ -256,7 +256,6 @@ public class HttpUtil {
public static CloseableHttpResponse getProxyHttpClient(String url) {
//获取代理ip信息
//TODO
String proxyHost = "";
int proxyPort = 0;
......
src/main/java/com/zzsn/event/util/SortUtil.java
......@@ -6,7 +6,7 @@ import java.util.*;
/**
* @author kongliufeng
* @Description TODO: 自定义排序
* @Description 自定义排序
* @create 2020-09-03 19:11
* @Version 1.0
*/
......
src/main/java/com/zzsn/event/vo/EventRegionVO.java
......@@ -13,5 +13,7 @@ public class EventRegionVO {
private String eventId;
private String eventName;
private String startTime;
private String endTime;
private String regionName;
}
src/main/java/com/zzsn/event/vo/EventTopVO.java
......@@ -13,6 +13,8 @@ public class EventTopVO {
private String id;
private String eventName;
private String startTime;
private String endTime;
private String publishDate;
private Integer totalHot;
private String typeName;
......
src/main/java/com/zzsn/event/vo/ExportParam.java 0 → 100644
package com.zzsn.event.vo;
import lombok.Data;
import java.util.List;
/**
* 导出参数封装
*
* @author lkg
* @date 2024/4/11
*/
@Data
public class ExportParam {
//事件id集合
private List<String> eventIdList;
//搜索词
private String searchWord;
//搜索位置(title-标题;content-内容)
private String position;
//匹配度(1-模糊;2-精确)
private Integer category;
//资讯id集合
private List<String> articleIdList;
//排序字段
private String column;
//排序方式(asc-正序;desc-倒序)
private String order;
//导出条数
private Integer size;
//导出方式(1-摘要;2-正文)
private Integer type;
}
src/main/java/com/zzsn/event/vo/ModelVO.java 0 → 100644
package com.zzsn.event.vo;
import lombok.Data;
/**
* 模型
*
* @author lkg
* @date 2024/4/11
*/
@Data
public class ModelVO {
private String id;
private String modelName;
private String type;
}
src/main/resources/application-dev.yml
......@@ -48,7 +48,7 @@ spring:
url: jdbc:mysql://114.116.44.11:3306/clb_project?useUnicode=true&characterEncoding=utf-8&AllowPublicKeyRetrieval=True&serverTimezone=Asia/Shanghai&autoReconnect=true&rewriteBatchedStatements=true
username: ciglobal
password: qwer@9988&zzsn
# 多数据源配置
# 多数据源配置
multi-datasource1:
url: jdbc:mysql://114.116.44.11:3306/clb_xxl_job?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
username: ciglobal
......@@ -56,7 +56,7 @@ spring:
driver-class-name: com.mysql.cj.jdbc.Driver
elasticsearch:
rest:
uris: ["114.115.215.250:9700","114.116.19.92:9700","114.116.54.108:9200"]
uris: [ "114.115.215.250:9700","114.116.19.92:9700","114.116.54.108:9200" ]
username: elastic
password: zzsn9988
connection-timeout: 300000
......@@ -106,8 +106,6 @@ spring:
key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
#值的反序列化器类，实现类实现了接口org.apache.kafka.common.serialization.Deserializer
value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
thymeleaf:
prefix: classpath:/templates
mybatis-plus:
mapper-locations: classpath*:com/zzsn/event/mapper/xml/*.xml,classpath*:com/zzsn/event/xxljob/mapper/xml/*.xml
......
src/main/resources/application-pro.yml
......@@ -106,8 +106,6 @@ spring:
key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
#值的反序列化器类，实现类实现了接口org.apache.kafka.common.serialization.Deserializer
value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
thymeleaf:
prefix: classpath:/templates
mybatis-plus:
mapper-locations: classpath*:com/zzsn/event/mapper/xml/*.xml,classpath*:com/zzsn/event/xxljob/mapper/xml/*.xml
......
src/main/resources/application.yml
spring:
profiles:
# active: dev
active: pro
\ No newline at end of file
active: dev
# active: pro
\ No newline at end of file
src/main/resources/template/EVENT_DATA_REPORT.ftl 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论