Skip to content

E
event
提交 00256774 作者: chenshiqiang
操作

commit

上级 7fd0fc67
正在显示 包含 989 行增加 和 118 行删除
pom.xml
......@@ -50,6 +50,12 @@
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
<!--hanlp-->
<dependency>
<groupId>com.hankcs</groupId>
<artifactId>hanlp</artifactId>
<version>portable-1.8.3</version>
</dependency>
<!-- json-->
<dependency>
......
src/main/java/com/zzsn/event/constant/Constants.java
......@@ -29,7 +29,7 @@ public class Constants {
//专题事件脉络展示 伪事件脉络 的资讯数量阈值
public static final int FAKE_NUM = 6;
//kafka 发送分析命令 主题
public static final String EVENT_ANALYSIS_TOPIC = "event-analysis";
public static final String EVENT_VIEWPOINT_SEND_DATA = "event_viewpoint_send_data";
//kafka 发送 事件脉络所需信息 主题
public static final String EVENT_CONTEXT_SEND_TOPIC = "event_context_send_topic";
//kafka 发送 伪事件脉络所需信息 主题
......
src/main/java/com/zzsn/event/consumer/KafkaConsumer.java
......@@ -46,6 +46,7 @@ public class KafkaConsumer {
@KafkaListener(topics = {Constants.VIEWPOINT_RECEIVE_TOPIC})
public void viewPointAnalysis(ConsumerRecord<String, String> record) {
String value = record.value();
log.info("viewpointMessage:{}",value);
if (StringUtils.isNotEmpty(value)) {
String subjectId = null;
try {
......
src/main/java/com/zzsn/event/controller/ReportApiController.java
......@@ -604,13 +604,8 @@ public class ReportApiController {
* @param pageNo 当前页
* @param pageSize 返回条数
* @param column 排序字段
* @param isCustomer 是否为客户 1是 0 否
* @param order 排序方式
* @param crawler
* @param objectType
* @param objectId
* @param isSubject
* @param subjectType
* @param labelIds
* @param sourceId
* @author lkg
......@@ -622,22 +617,19 @@ public class ReportApiController {
@RequestParam(name = "subjectId", defaultValue = "") String subjectId,
@RequestParam(name = "video", defaultValue = "") String video,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "isCustomer", defaultValue = "0") Integer isCustomer,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
@RequestParam(name = "column", defaultValue = "common") String column,
@RequestParam(name = "order", defaultValue = "desc") String order,
@RequestParam(name = "crawler", defaultValue = "") String crawler,
@RequestParam(name = "objectType", defaultValue = "", required = false) String objectType,
@RequestParam(name = "objectId", defaultValue = "", required = false) String objectId,
@RequestParam(name = "isSubject", defaultValue = "1") String isSubject,
@RequestParam(name = "subjectType", defaultValue = "1") String subjectType,
@RequestParam(name = "labelIds", required = false) String labelIds,
@RequestParam(name = "sourceId", required = false) String sourceId) throws Exception {
SubjectInfoVo subjectInfoVo=new SubjectInfoVo();
subjectInfoVo.setSubjectId(subjectId);
List<String> socialCreditCodeList = new ArrayList<>();
//获取数据
IPage<DisplayInfo> pageList = subjectDisplayServive.frontListByPage(objectType, objectId, null, subjectInfoVo, video, pageNo, pageSize, column, order, crawler, isSubject, subjectType, labelIds, socialCreditCodeList, sourceId, isCustomer);
IPage<DisplayInfo> pageList = subjectDisplayServive.frontListByPage(subjectInfoVo, video, pageNo, pageSize, column, order, crawler, labelIds, socialCreditCodeList, sourceId);
return Result.OK(pageList);
}
}
src/main/java/com/zzsn/event/entity/CmsExpert.java 0 → 100644
package com.zzsn.event.entity;
import com.baomidou.mybatisplus.annotation.TableName;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* @Description: 专家
* @Author: jeecg-boot
* @Date: 2024-03-14
* @Version: V1.0
*/
@Data
@TableName("cms_expert")
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class CmsExpert {
/**id*/
private String expertId;
private String headPic;
}
src/main/java/com/zzsn/event/entity/Event.java
......@@ -127,6 +127,7 @@ public class Event {
private Integer otherHot;
private Integer publishStatus;
private String relationEvents;
private String publishDate;
@TableField(exist = false)
......@@ -137,4 +138,18 @@ public class Event {
private List<Event> relatedEventList;
@TableField(exist = false)
private EventTag eventTag;
@TableField(exist = false)
private String extractIndustryTag;
@TableField(exist = false)
private String extractCompanyTag;
@TableField(exist = false)
private String extractPersonTag;
@TableField(exist = false)
private String extractSentimentTag;
@TableField(exist = false)
private String extractKeywordsTag;
@TableField(exist = false)
private String extractLocationTag;
@TableField(exist = false)
private String extractTimeTag;
}
src/main/java/com/zzsn/event/entity/SubjectAnalysis.java
......@@ -46,4 +46,5 @@ public class SubjectAnalysis implements Serializable {
/*分析时间*/
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
private Date analysisDate;
private String professionName;
}
src/main/java/com/zzsn/event/entity/SysBaseDomesticRegion.java 0 → 100644
package com.zzsn.event.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.jeecgframework.poi.excel.annotation.Excel;
import org.springframework.format.annotation.DateTimeFormat;
import java.io.Serializable;
import java.util.Date;
/**
* @Description: 国内地域表
* @Author: jeecg-boot
* @Date: 2022-03-09
* @Version: V1.0
*/
@Data
@TableName("sys_base_domestic_region")
@ApiModel(value="sys_base_domestic_region对象", description="国内地域表")
public class SysBaseDomesticRegion implements Serializable {
private static final long serialVersionUID = 1L;
/**主键*/
@TableId(type = IdType.ASSIGN_ID)
@ApiModelProperty(value = "主键")
private String id;
/**名称*/
@Excel(name = "名称", width = 15)
@ApiModelProperty(value = "名称")
private String name;
/**别名*/
@Excel(name = "别名", width = 15)
@ApiModelProperty(value = "别名")
private String alias;
/**行政编码*/
@Excel(name = "行政编码", width = 15)
@ApiModelProperty(value = "行政编码")
private String code;
/**所属省或直辖市*/
@Excel(name = "所属省或直辖市", width = 15)
@ApiModelProperty(value = "所属省或直辖市")
private String topId;
/**地理区*/
@Excel(name = "地理区", width = 15)
@ApiModelProperty(value = "地理区")
private String area;
/**经济区*/
@Excel(name = "经济区", width = 15)
@ApiModelProperty(value = "经济区")
private String economy;
/**所有pid*/
@Excel(name = "所有pid", width = 15)
@ApiModelProperty(value = "所有pid")
private String pathIds;
/**组合词*/
@Excel(name = "组合词", width = 15)
@ApiModelProperty(value = "组合词")
private String composeWords;
/**层级*/
@Excel(name = "层级", width = 15)
@ApiModelProperty(value = "层级")
private Integer level;
/**创建人*/
@ApiModelProperty(value = "创建人")
private String createBy;
/**创建日期*/
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
@ApiModelProperty(value = "创建日期")
private Date createTime;
/**更新人*/
@ApiModelProperty(value = "更新人")
private String updateBy;
/**更新日期*/
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
@ApiModelProperty(value = "更新日期")
private Date updateTime;
/**所属部门*/
@ApiModelProperty(value = "所属部门")
private String sysOrgCode;
/**父级节点*/
@Excel(name = "父级节点", width = 15)
@ApiModelProperty(value = "父级节点")
private String pid;
/**是否有子节点*/
@Excel(name = "是否有子节点", width = 15, dicCode = "yn")
@ApiModelProperty(value = "是否有子节点")
private String hasChild;
/**经度*/
private String longitude;
/**纬度*/
private String latitude;
}
src/main/java/com/zzsn/event/mapper/EventMapper.java
......@@ -21,7 +21,7 @@ public interface EventMapper extends BaseMapper<Event> {
List<SubjectKafkaVo> eventSubjectList(int i);
List<Event> pageList(@Param("event")Event event, Integer offset, Integer pageSize);
List<Event> pageList(@Param("event")Event event, Integer offset, Integer pageSize,String order,String orderType);
Integer totalCount(@Param("event") Event event);
......
src/main/java/com/zzsn/event/mapper/SysBaseDomesticRegionMapper.java 0 → 100644
package com.zzsn.event.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zzsn.event.entity.SysBaseDomesticRegion;
import org.apache.ibatis.annotations.Mapper;
/**
* @Description: 国内地域表
* @Author: jeecg-boot
* @Date: 2022-03-09
* @Version: V1.0
*/
@Mapper
public interface SysBaseDomesticRegionMapper extends BaseMapper<SysBaseDomesticRegion> {
}
src/main/java/com/zzsn/event/mapper/xml/EventMapper.xml
......@@ -10,8 +10,9 @@
</select>
<select id="pageList" resultType="com.zzsn.event.entity.Event">
select t2.type_name,t1.* from event t1
select t2.type_name,t1.*,t3.* from event t1
left join event_category t2 on t1.event_type =t2.id
left join event_tag t3 on t1.id=t3.event_id
where 1=1
<if test="event.eventName!=null and event.eventName != ''">
and t1.event_name like CONCAT('%',#{event.eventName},'%')
......@@ -25,7 +26,12 @@
<if test="event.endDate!=null and event.endDate != ''">
and t1.create_time <![CDATA[ <= ]]> #{event.endDate}
</if>
order by t1.create_time desc
<if test="order!=null and order != ''">
order by t1.${order}
<if test="orderType!=null and orderType != ''">
${orderType}
</if>
</if>
limit #{offset}, #{pageSize}
</select>
<select id="totalCount" resultType="java.lang.Integer">
......@@ -50,7 +56,7 @@
select s.id,s.event_name as subject_name,s.start_time as time_enable,s.end_time as time_disable,s.incre_ana_rule,
s.total_ana_rule,s.time_ana_rule,s.analysis_time,s.event_time
from event s
where 1=1 and id='1702575766171996162'
where 1=1
<if test="endDate != null">
and (s.end_time is null or s.end_time <![CDATA[ >= ]]> #{endDate})
</if>
......
src/main/java/com/zzsn/event/service/ColumnService.java
......@@ -5,6 +5,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.zzsn.event.constant.Constants;
import com.zzsn.event.entity.SysBaseDomesticRegion;
import com.zzsn.event.enums.InfluenceEnum;
import com.zzsn.event.enums.SourceEnum;
import com.zzsn.event.util.CalculateUtil;
......@@ -57,6 +58,8 @@ public class ColumnService {
private RestHighLevelClient client;
@Autowired
private IInfoSourceService infoSourceService;
@Autowired
private ISysBaseDomesticRegionService domesticRegionService;
/**
* 获取栏目或栏目组下的信息数量
......@@ -494,7 +497,7 @@ public class ColumnService {
dataTypeBoolQuery.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("dataType")));
boolQuery.must(dataTypeBoolQuery);
if (StringUtils.isNotEmpty(subjectId)) {
boolQuery.must(QueryBuilders.termQuery("subjectId.keyword", subjectId));
boolQuery.must(QueryBuilders.termsQuery("subjectId.keyword", subjectId.split(",")));
}
if (StringUtils.isNotEmpty(startTime)) {
boolQuery.filter(QueryBuilders.rangeQuery("publishDate").gte(EsDateUtil.esFieldDateFormat(startTime)));
......@@ -528,4 +531,86 @@ public class ColumnService {
return percentage;
}
/**
* 地域分布
*
* @param column 栏目id或栏目组id,多个逗号隔开
* @param ynGroup 是否栏目组
* @param subjectId 专题id
* @param startTime 开始时间
* @param endTime 结束时间
* @author lkg
* @date 2024/2/2
*/
public List<CountVO> region(String column, Boolean ynGroup, String subjectId, String startTime, String endTime) {
List<CountVO> list = new ArrayList<>();
SearchRequest searchRequest = new SearchRequest(Constants.ES_DATA_FOR_SUBJECT);
SearchSourceBuilder searchSourceBuilder = getSearchSourceBuilder(column, ynGroup, subjectId, startTime, endTime, null);
searchSourceBuilder.size(0);
searchSourceBuilder.trackTotalHits(true);
NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("labels", "labels")
.subAggregation(AggregationBuilders.terms("group_type")
.field("labels.labelMark.keyword")
.subAggregation(AggregationBuilders.terms("group_code")
.field("labels.relationId")
.order(BucketOrder.count(false))
.size(10)));
searchSourceBuilder.aggregation(nestedAggregationBuilder);
searchRequest.source(searchSourceBuilder);
try {
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
//获取分组桶
Aggregations aggregations = searchResponse.getAggregations();
ParsedNested labels = aggregations.get("labels");
Aggregations labelsAggregations = labels.getAggregations();
//获取按企业信用代码分组集合
Terms groupType = labelsAggregations.get("group_type");
List<? extends Terms.Bucket> typeBuckets = groupType.getBuckets();
Map<String, Long> regionMap = new HashMap<>();
for (Terms.Bucket type : typeBuckets) {
String key = type.getKeyAsString();
if ("region_out".equals(key)) {
Aggregations codeAggregations = type.getAggregations();
Terms groupCode = codeAggregations.get("group_code");
List<? extends Terms.Bucket> codeBuckets = groupCode.getBuckets();
for (Terms.Bucket code : codeBuckets) {
regionMap.put(code.getKeyAsString(), code.getDocCount());
}
break;
}
}
List<SysBaseDomesticRegion> regionList = provinceList();
for (SysBaseDomesticRegion region : regionList) {
if (list.size() < 10) {
CountVO countVO = new CountVO();
String id = region.getId();
String name = region.getName();
countVO.setKey(name);
if (regionMap.containsKey(id)) {
Long count = regionMap.get(id);
countVO.setCount(String.valueOf(count));
list.add(countVO);
} else {
countVO.setCount("0");
}
} else {
break;
}
}
list.sort((o1, o2) -> o2.getCount().compareTo(o1.getCount()));
for (int i = 0; i < list.size(); i++) {
CountVO countVO = list.get(i);
countVO.setOrder(i + 1);
}
} catch (Exception e) {
e.printStackTrace();
}
return list;
}
private List<SysBaseDomesticRegion> provinceList() {
LambdaQueryWrapper<SysBaseDomesticRegion> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.select(SysBaseDomesticRegion::getId, SysBaseDomesticRegion::getName, SysBaseDomesticRegion::getLevel);
queryWrapper.eq(SysBaseDomesticRegion::getLevel, 1);
return domesticRegionService.list(queryWrapper);
}
}
src/main/java/com/zzsn/event/service/EsService.java
......@@ -18,6 +18,11 @@ import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -291,4 +296,37 @@ public class EsService {
}
return null;
}
public List<String> groupByOrigin(String title, String publishDate) {
List<String> originList = new ArrayList<>();
SearchRequest searchRequest = new SearchRequest(Constants.COLLECT_INDEX);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//只返回分组聚合结果，不返回具体数据
searchSourceBuilder.size(0);
//创建查询对象
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
String[] arr = new String[]{"title"};
boolQuery.must(QueryBuilders.multiMatchQuery(title,arr));
boolQuery.filter(QueryBuilders.rangeQuery("publishDate").gt(EsDateUtil.esFieldDateFormat(publishDate)));
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("group_origin")
.field("origin.keyword")
.size(20)
.order(BucketOrder.count(false));
searchSourceBuilder.query(boolQuery);
searchSourceBuilder.aggregation(aggregationBuilder);
searchRequest.source(searchSourceBuilder);
try {
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
Aggregations aggregations = searchResponse.getAggregations();
Terms groupOrigin = aggregations.get("group_origin");
List<? extends Terms.Bucket> buckets = groupOrigin.getBuckets();
for (Terms.Bucket bucket : buckets) {
String origin = bucket.getKeyAsString();
originList.add(origin);
}
} catch (Exception e) {
e.printStackTrace();
}
return originList;
}
}
src/main/java/com/zzsn/event/service/EsStatisticsService.java
......@@ -146,4 +146,6 @@ public interface EsStatisticsService {
Page<NegativeDataVO> labelPageList(String labelId, String startTime, String endTime, Integer pageNo, Integer pageSize);
CountVO flowData(String subjectId, String startTime, String endTime, Integer type);
long totalCount(List<String> eventIdList, String startTime, String endTime);
}
src/main/java/com/zzsn/event/service/IEventService.java
......@@ -4,9 +4,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
import com.zzsn.event.entity.Event;
import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.event.entity.LabelEntity;
import com.zzsn.event.vo.AddEventParam;
import com.zzsn.event.vo.KeyWordsPage;
import com.zzsn.event.vo.SubjectKafkaVo;
import com.zzsn.event.vo.*;
import org.springframework.web.multipart.MultipartFile;
import java.util.Date;
......@@ -29,7 +27,7 @@ public interface IEventService extends IService<Event> {
List<SubjectKafkaVo> eventSubjectList();
IPage<Event> pageList(Event event, Integer pageNo, Integer pageSize);
IPage<Event> pageList(Event event, Integer pageNo, Integer pageSize,String order,String orderType);
void extractHotWords(AddEventParam event);
......@@ -46,4 +44,8 @@ public interface IEventService extends IService<Event> {
String upload(MultipartFile file);
List<LabelEntity> listByType(String labelTypeId);
DisplayInfo articleDetail(String articleId);
Result<?> hotWords(String id, Integer number);
}
src/main/java/com/zzsn/event/service/ISysBaseDomesticRegionService.java 0 → 100644
package com.zzsn.event.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.event.entity.SysBaseDomesticRegion;
/**
* @Description: 国内地域表
* @Author: jeecg-boot
* @Date: 2022-03-09
* @Version: V1.0
*/
public interface ISysBaseDomesticRegionService extends IService<SysBaseDomesticRegion> {
}
src/main/java/com/zzsn/event/service/SubjectDisplayServive.java
package com.zzsn.event.service;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
......@@ -9,6 +10,8 @@ import com.zzsn.event.util.DateUtil;
import com.zzsn.event.util.EsDateUtil;
import com.zzsn.event.util.EsIndexUtil;
import com.zzsn.event.vo.DisplayInfo;
import com.zzsn.event.vo.RepeatHold;
import com.zzsn.event.vo.SpecialInformation;
import com.zzsn.event.vo.SubjectInfoVo;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
......@@ -21,12 +24,21 @@ import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.collapse.CollapseBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
@Service
......@@ -47,9 +59,8 @@ public class SubjectDisplayServive {
* @param sourceId
* @throws Exception 异常
*/
public IPage<DisplayInfo> frontListByPage(String objectType, String objectId, String userId, SubjectInfoVo subjectInfo, String video, int offset, int pageSize,
String column, String order, String crawler, String isSubject, String subjectType, String labelIds, List<String> socialCreditCodeList, String sourceId, Integer isCustomer) throws Exception {
String[] indexs = EsIndexUtil.getIndexIntervalYear(Constants.ES_DATA_FOR_SUBJECT, subjectInfo.getStartTime(), subjectInfo.getEndTime());
public IPage<DisplayInfo> frontListByPage(SubjectInfoVo subjectInfo, String video, int offset, int pageSize,
String column, String order, String crawler, String labelIds, List<String> socialCreditCodeList, String sourceId) throws Exception {
SearchRequest searchRequest = new SearchRequest(Constants.ES_DATA_FOR_SUBJECT);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//设置分页参数
......@@ -175,21 +186,137 @@ public class SubjectDisplayServive {
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
SearchHit[] searchHits = searchResponse.getHits().getHits();
List<DisplayInfo> list = new ArrayList<>();
int i = 0;
for (SearchHit hit : searchHits) {
i++;
String queryInfo = hit.getSourceAsString();
DisplayInfo info = JSONUtil.toBean(queryInfo, DisplayInfo.class);
info.setPublishDate(EsDateUtil.esFieldDateMapping(info.getPublishDate()));
String index = hit.getIndex();
info.setIndex(index);
info.setSimilarNumber(0);
list.add(info);
}
Map<String, Integer> map = getSimilarNumber(list.stream().map(DisplayInfo::getId).collect(Collectors.toList()));
for (DisplayInfo displayInfo : list) {
displayInfo.setSimilarNumber(map.get(displayInfo.getId()));
}
IPage<DisplayInfo> pageData = new Page<>(offset, pageSize, searchResponse.getHits().getTotalHits().value);
pageData.setRecords(list);
return pageData;
}
/**
* 获取相似文章数
*
* @param articleIdList 资讯id
* @return
*/
private Map<String, Integer> getSimilarNumber(List<String> articleIdList) {
Map<String, Integer> map = new HashMap<>();
articleIdList.add("70694089423478814");
articleIdList.add("71056913337311317");
articleIdList.add("71056913337311282");
Map<String, String> markmap = getMark(articleIdList);
List<String> markList = new ArrayList<>(markmap.keySet());
SearchRequest searchRequest = new SearchRequest(Constants.ES_REPEAT_OLD);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//创建查询对象
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.termsQuery("repeatMark", markList));
searchSourceBuilder.size(0);
searchSourceBuilder.trackTotalHits(true);
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("group_mark")
.field("repeatMark")
.order(BucketOrder.count(false))
.size(1);
searchSourceBuilder.aggregation(aggregationBuilder);
searchSourceBuilder.query(boolQuery);
searchRequest.source(searchSourceBuilder);
try {
SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
Aggregations aggregations = response.getAggregations();
Terms groupSource = aggregations.get("group_mark");
List<? extends Terms.Bucket> buckets = groupSource.getBuckets();
if (org.apache.commons.collections4.CollectionUtils.isNotEmpty(buckets)) {
for (Terms.Bucket bucket : buckets) {
map.put(markmap.get(bucket.getKeyAsString()), (int) bucket.getDocCount());
}
}
} catch (Exception e) {
e.printStackTrace();
}
return map;
}
private Map<String, String> getMark(List<String> articleIdList) {
Map<String, String> map = new HashMap<>();
String mark = null;
SearchRequest searchRequest = new SearchRequest(Constants.ES_REPEAT_OLD);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//创建查询对象
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.termsQuery("articleId", articleIdList));
searchSourceBuilder.query(boolQuery);
searchRequest.source(searchSourceBuilder);
try {
SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
SearchHit[] hits = response.getHits().getHits();
if (hits != null && hits.length != 0) {
for (SearchHit hit : hits) {
String queryInfo = hit.getSourceAsString();
RepeatHold info = JSONUtil.toBean(queryInfo, RepeatHold.class);
map.put(info.getRepeatMark(), info.getArticleId());
}
}
} catch (Exception e) {
e.printStackTrace();
}
return map;
}
/**
* 根据标题获取相关推荐文章
*
* @param title
* @return
*/
public IPage<SpecialInformation> queryRecommendList(String title, Integer pageNo, Integer pageSize) throws IOException {
String[] indexs = EsIndexUtil.getIndexLatelyTwoYear(Constants.ES_DATA_FOR_SUBJECT);
SearchRequest searchRequest = new SearchRequest(indexs);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//设置分页参数
searchSourceBuilder.size(pageSize);
searchSourceBuilder.from((pageNo - 1) * pageSize);
searchSourceBuilder.sort("score", SortOrder.DESC);
searchSourceBuilder.sort("publishDate", SortOrder.DESC);
//默认最大数量是10000，设置为true后，显示准确数量
searchSourceBuilder.trackTotalHits(true);
//创建查询对象
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
List<Integer> checkStatus = new ArrayList<>();
checkStatus.add(1);
boolQuery.must(QueryBuilders.termsQuery("checkStatus", checkStatus));
boolQuery.must(QueryBuilders.matchQuery("title", title));
boolQuery.mustNot(QueryBuilders.matchQuery("type", "video"));
searchSourceBuilder.query(boolQuery);
searchRequest.source(searchSourceBuilder);
searchSourceBuilder.collapse(new CollapseBuilder("sourceAddress.keyword"));
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
SearchHit[] searchHits = searchResponse.getHits().getHits();
List<SpecialInformation> list = new ArrayList<>();
for (SearchHit hit : searchHits) {
String queryInfo = hit.getSourceAsString();
SpecialInformation info = JSON.parseObject(queryInfo, SpecialInformation.class);
info.setPublishDate(EsDateUtil.esFieldDateMapping(info.getPublishDate()));
list.add(info);
}
IPage<SpecialInformation> pageData = new Page<>(pageNo, pageSize, searchResponse.getHits().getTotalHits().value);
pageData.setRecords(list);
return pageData;
}
}
src/main/java/com/zzsn/event/service/impl/AnalysisServiceImpl.java
......@@ -10,6 +10,7 @@ import com.zzsn.event.service.*;
import com.zzsn.event.util.DateUtil;
import com.zzsn.event.vo.PropagationPathVo;
import com.zzsn.event.vo.SubjectDataVo;
import io.netty.util.Constant;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.BeanUtils;
......@@ -104,83 +105,57 @@ public class AnalysisServiceImpl implements AnalysisService {
@Override
public PropagationPathVo propagationPath(String subjectId) {
PropagationPathVo top = null;
Event event = eventService.getById(subjectId);
String subjectName = event.getEventName();
List<PropagationPathVo> children = new ArrayList<>();
//获取专题数据
List<SubjectDataVo> specialDataList = getSubjectData(event, "0", Constants.FETCH_FIELDS_STATISTIC,1);
if (CollectionUtils.isNotEmpty(specialDataList)) {
//最早发布的时间
String publishDate = specialDataList.get(0).getPublishDate();
String earlyTime;
if (publishDate.length() > 10) {
earlyTime = DateUtil.formatStr(publishDate, "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd");
} else {
earlyTime = publishDate;
}
//获取最早发布的信息,若过多，取前一个
List<SubjectDataVo> earlyList = specialDataList.stream()
.filter(subjectDataVo -> {
String date = subjectDataVo.getPublishDate();
if (date.length() > 10) {
date = DateUtil.formatStr(date, "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd");
}
return earlyTime.equals(date);
})
.collect(Collectors.toList());
if (earlyList.size() > 1) {
earlyList = earlyList.subList(0, 1);
}
Map<SubjectDataVo, List<SubjectDataVo>> map = new HashMap<>();
//获取跟最早发布的信息的重复信息
earlyList.forEach(e -> {
List<SubjectDataVo> subjectDataVoList = esService.dataById(subjectId, e.getId());
if (CollectionUtils.isNotEmpty(subjectDataVoList)) {
map.put(e, subjectDataVoList);
}
});
//若最早发布的信息没有重复数据，则随机补充10个重复数最多的信息来源
// 本不该有的逻辑，奈何数据采集、处理 不给力
if (map.isEmpty()) {
for (SubjectDataVo subjectDataVo : earlyList) {
PropagationPathVo vo = new PropagationPathVo();
String topOrigin = subjectDataVo.getOrigin();
vo.setName(topOrigin);
//
vo.setChildren(pathByRepeat(topOrigin, event));
children.add(vo);
if (com.baomidou.mybatisplus.core.toolkit.CollectionUtils.isNotEmpty(specialDataList)) {
//用于来源去重
List<String> allOriginList = new ArrayList<>();
top = new PropagationPathVo();
top.setName(subjectName);
//获取发布时间最早的前10的资讯(来源不重复)
List<PropagationPathVo> secondList = new ArrayList<>();
List<SubjectDataVo> earlyList = topN(specialDataList,6);
earlyList.forEach(e->allOriginList.add(e.getOrigin()));
for (SubjectDataVo subjectDataVo : earlyList) {
String origin = subjectDataVo.getOrigin();
PropagationPathVo second = new PropagationPathVo();
second.setName(origin);
secondList.add(second);
List<String> thirdList = esService.groupByOrigin(subjectDataVo.getTitle(), subjectDataVo.getPublishDate());
thirdList.removeAll(allOriginList);
List<PropagationPathVo> lastList = new ArrayList<>();
if (thirdList.size() > 3) {
thirdList = thirdList.subList(0,3);
}
} else {//正常逻辑
Map<SubjectDataVo, LinkedHashMap<String, List<SubjectDataVo>>> dataMap = new HashMap<>();
for (Map.Entry<SubjectDataVo, List<SubjectDataVo>> entry : map.entrySet()) {
SubjectDataVo information = entry.getKey();
List<SubjectDataVo> value = entry.getValue();
//按来源分组
Map<String, List<SubjectDataVo>> originMap = value.stream().filter(e -> StringUtils.isNotEmpty(e.getOrigin()))
.collect(Collectors.groupingBy(SubjectDataVo::getOrigin));
//按统一来源信息数量 倒序 并截取前10
LinkedHashMap<String, List<SubjectDataVo>> orderMap = new LinkedHashMap<>();
originMap.entrySet().stream()
.sorted((o1, o2) -> o2.getValue().size() - o1.getValue().size()).limit(10)
.collect(Collectors.toList()).forEach(info -> orderMap.put(info.getKey(), info.getValue()));
dataMap.put(information, orderMap);
for (String s : thirdList) {
PropagationPathVo third = new PropagationPathVo();
third.setName(s);
lastList.add(third);
}
children = getPath(dataMap);
second.setChildren(lastList);
allOriginList.addAll(thirdList);
}
top.setChildren(secondList);
}
PropagationPathVo pathVo = null;
if (CollectionUtils.isNotEmpty(children)) {
pathVo = new PropagationPathVo();
pathVo.setName(subjectName);
if(null!=children.get(0)){
pathVo.setChildren(children.get(0).getChildren());
return top;
}
//获取发布时间最早的前N条资讯(来源不重复)
private List<SubjectDataVo> topN(List<SubjectDataVo> list,Integer num){
TreeSet<SubjectDataVo> subjectDataVos = new TreeSet<>(Comparator.comparing(SubjectDataVo::getOrigin));
for (SubjectDataVo subjectDataVo : list) {
subjectDataVos.add(subjectDataVo);
if (subjectDataVos.size() == num) {
break;
}
}
return pathVo;
return new ArrayList<>(subjectDataVos);
}
//
//
// @Override
// public Map<String, Object> statisticAnalysis(String subjectId) {
// Map<String, Object> map = new HashMap<>();
......
src/main/java/com/zzsn/event/service/impl/EsStatisticsServiceImpl.java
......@@ -42,6 +42,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* 舆情信息统计 es查询工具类
......@@ -449,6 +450,9 @@ public class EsStatisticsServiceImpl implements EsStatisticsService {
try {
SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
SearchHit[] hits = response.getHits().getHits();
if(hits.length==0){
return " There is nothing about this event in repeathold ";
}
SearchHit hit = hits[0];
title = hit.getSourceAsMap().get("title").toString();
} catch (Exception e) {
......@@ -533,6 +537,7 @@ public class EsStatisticsServiceImpl implements EsStatisticsService {
if (StringUtils.isNotEmpty(relationId)) {
param.setSearchLabelIds(relationId);
}
param.setSubjectId(subjectId);
if (StringUtils.isNotBlank(startTime)) {
param.setStartTime(startTime);
}
......@@ -552,6 +557,7 @@ public class EsStatisticsServiceImpl implements EsStatisticsService {
return dataSearchService.packageSearQuery(param);
}
// /**
// * 获取一级地域信息
// *
......@@ -621,4 +627,24 @@ public class EsStatisticsServiceImpl implements EsStatisticsService {
return countVO;
}
@Override
public long totalCount(List<String> eventIdList, String startTime, String endTime) {
Map<String, Object> map = new HashMap<>();
SearchRequest searchRequest = new SearchRequest(Constants.ES_DATA_FOR_SUBJECT);
String eventIds = eventIdList.stream().collect(Collectors.joining(","));
SearchSourceBuilder searchSourceBuilder = formatSourceBuilder(eventIds, null, startTime, endTime, null);
searchSourceBuilder.size(0);
searchSourceBuilder.trackTotalHits(true);
searchRequest.source(searchSourceBuilder);
try {
SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
long value = response.getHits().getTotalHits().value;
return value;
} catch (Exception e) {
e.printStackTrace();
}
return 0;
}
}
src/main/java/com/zzsn/event/service/impl/EventServiceImpl.java
package com.zzsn.event.service.impl;
import com.alibaba.fastjson.JSON;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
......@@ -13,11 +14,23 @@ import com.zzsn.event.mapper.EventMapper;
import com.zzsn.event.producer.ProduceInfo;
import com.zzsn.event.service.*;
import com.zzsn.event.util.*;
import com.zzsn.event.vo.AddEventParam;
import com.zzsn.event.vo.KeyWordsDTO;
import com.zzsn.event.vo.KeyWordsPage;
import com.zzsn.event.vo.SubjectKafkaVo;
import com.zzsn.event.vo.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.ParsedTopHits;
import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
......@@ -66,6 +79,11 @@ public class EventServiceImpl extends ServiceImpl<EventMapper, Event> implements
private RedisUtil redisUtil;
@Value("${files.storage}")
String filesStorage;
@Autowired
private RestHighLevelClient client;
@Autowired
private EsOpUtil esOpUtil;
@Value("${scoreRule.weekScore}")
Integer weekScore;
......@@ -98,9 +116,7 @@ public class EventServiceImpl extends ServiceImpl<EventMapper, Event> implements
event.setWechatHot(wechatHot);
event.setOtherHot(otherHot);
this.updateById(event);
}
}
......@@ -147,12 +163,14 @@ public class EventServiceImpl extends ServiceImpl<EventMapper, Event> implements
}
@Override
public IPage<Event> pageList(Event event, Integer pageNo, Integer pageSize) {
public IPage<Event> pageList(Event event, Integer pageNo, Integer pageSize, String order, String orderType) {
Integer offset = (pageNo - 1) * pageSize;
if (pageNo == 0) {
offset = 0;
}
List<Event> pageList = baseMapper.pageList(event, offset, pageSize);
List<Event> pageList = baseMapper.pageList(event, offset, pageSize, order, orderType);
Map<String, Object> map = getFistMap(pageList);
//获取总条数
Integer count = baseMapper.totalCount(event);
IPage<Event> pageData = new Page<>(pageNo, pageSize, count);
......@@ -160,6 +178,45 @@ public class EventServiceImpl extends ServiceImpl<EventMapper, Event> implements
return pageData;
}
private Map<String, Object> getFistMap(List<Event> pageList) {
Map<String, Object> resultlist = new HashMap<>();
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
//根据设备查询设备的相关信息
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.from(0);
searchSourceBuilder.size(0);
searchSourceBuilder.query(boolQueryBuilder);
searchSourceBuilder.sort("publishDate", SortOrder.ASC);
// 聚合搜索
TermsAggregationBuilder one = AggregationBuilders.terms("one").field("subjectId.keyword");
// TermsAggregationBuilder two = AggregationBuilders.terms("two").field("media");
// TermsAggregationBuilder three = AggregationBuilders.terms("three").field("pubTime.keyword");
//ES分组取每组第一条Java写法
TopHitsAggregationBuilder topHitsAggregationBuilder = AggregationBuilders.topHits("top_docs").size(1);
one.subAggregation(topHitsAggregationBuilder);
searchSourceBuilder.aggregation(one);
SearchRequest searchRequest = new SearchRequest(Constants.ES_DATA_FOR_SUBJECT);
searchRequest.source(searchSourceBuilder);
// 查询ES
SearchResponse searchResponse = null;
try {
searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
ParsedStringTerms oneTerms = searchResponse.getAggregations().get("one");
List<? extends Terms.Bucket> onebuckets = oneTerms.getBuckets();
for (Terms.Bucket threebucket : onebuckets) {
ParsedTopHits topDetail = threebucket.getAggregations().get("top_docs");
SearchHit[] hits = topDetail.getHits().getHits();
Map<String, Object> latestDocument = hits[0].getSourceAsMap();
resultlist.put(latestDocument.get("subjectId").toString(), latestDocument);
}
return resultlist;
}
@Override
public void extractHotWords(AddEventParam event) {
if (!StringUtils.isEmpty(event.getExtractHotWords()) && "1".equals(event.getExtractHotWords())) {
......@@ -256,6 +313,33 @@ public class EventServiceImpl extends ServiceImpl<EventMapper, Event> implements
return baseMapper.listByType(labelTypeId);
}
@Override
public DisplayInfo articleDetail(String articleId) {
Map<String, Object> map = esOpUtil.searchDoc(Constants.SUBJECT_INDEX, articleId);
DisplayInfo displayInfo = new DisplayInfo();
if (map != null) {
displayInfo = JSON.parseObject(JSON.toJSONString(map), DisplayInfo.class);
}
return displayInfo;
}
@Override
public Result<?> hotWords(String id, Integer number) {
Map<String, Object> map = esOpUtil.searchDoc(Constants.ES_DATA_FOR_SUBJECT, id);
String content = map.get("content").toString();
List<Map.Entry<String, Integer>> keywordsList = HanlpUtil.extractKeyWordsByText(content, number);
List<StatisticsKeyWordVo> rn = new ArrayList<>();
if (CollectionUtils.isNotEmpty(keywordsList)) {
for (Map.Entry<String, Integer> entry : keywordsList) {
StatisticsKeyWordVo statisticsKeyWordVo = new StatisticsKeyWordVo();
statisticsKeyWordVo.setName(entry.getKey());
statisticsKeyWordVo.setValue(entry.getValue());
rn.add(statisticsKeyWordVo);
}
}
return Result.OK(rn);
}
//生成文件夹路径
private String getFilePath() {
LocalDate currentDate = LocalDate.now();
......
src/main/java/com/zzsn/event/service/impl/SysBaseDomesticRegionServiceImpl.java 0 → 100644
package com.zzsn.event.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.event.entity.SysBaseDomesticRegion;
import com.zzsn.event.mapper.SysBaseDomesticRegionMapper;
import com.zzsn.event.service.ISysBaseDomesticRegionService;
import org.springframework.stereotype.Service;
/**
* @Description: 国内地域表
* @Author: jeecg-boot
* @Date: 2022-03-09
* @Version: V1.0
*/
@Service
public class SysBaseDomesticRegionServiceImpl extends ServiceImpl<SysBaseDomesticRegionMapper, SysBaseDomesticRegion> implements ISysBaseDomesticRegionService {
}
src/main/java/com/zzsn/event/task/AnalysisTask.java
package com.zzsn.event.task;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONWriter;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
......@@ -61,7 +62,7 @@ public class AnalysisTask {
* 按天发送kafka 获取进行中(未结束)的事件专题列表
*/
// @Scheduled(cron = "0 0 0 * * ?")
@Scheduled(cron = "0 * * * * ?")
// @Scheduled(cron = "0 * * * * ?")
public void subjectList() {
List<SubjectKafkaVo> data = new ArrayList<>();
Date today = new Date();
......@@ -79,22 +80,27 @@ public class AnalysisTask {
analysisTime = e.getTimeEnable();
}
Integer betweenTwoDate = DateUtil.betweenTwoDate(analysisTime, today);
if (increCount.compareTo(e.getIncreAnaRule()) > 0
|| totalCount.compareTo(e.getTotalAnaRule()) > 0
|| betweenTwoDate.compareTo(e.getTimeAnaRule()) > 0) {
e.setIndexName(Constants.SUBJECT_INDEX);
String query = queryString.replace("subject_id", subjectId);
e.setQuery(query);
String beforeQuery = beforeQueryString.replace("subject_id", subjectId).replace("start_time", startTime).replace("end_time", endTime);
e.setBeforeQuery(beforeQuery);
String repeatNumQuery = repeatNumQueryString.replace("subject_id", subjectId);
e.setRepeatNumQuery(repeatNumQuery);
e.setAnalysisTime(today);
data.add(e);
try {
if (increCount.compareTo(e.getIncreAnaRule()) > 0
|| totalCount.compareTo(e.getTotalAnaRule()) > 0
|| betweenTwoDate.compareTo(e.getTimeAnaRule()) > 0) {
e.setIndexName(Constants.SUBJECT_INDEX);
String query = queryString.replace("subject_id", subjectId);
e.setQuery(query);
String beforeQuery = beforeQueryString.replace("subject_id", subjectId).replace("start_time", startTime).replace("end_time", endTime);
e.setBeforeQuery(beforeQuery);
String repeatNumQuery = repeatNumQueryString.replace("subject_id", subjectId);
e.setRepeatNumQuery(repeatNumQuery);
e.setAnalysisTime(today);
data.add(e);
}
}catch (Exception exception){
log.error("事件{}，报错信息：{}",e,exception.getMessage());
}
});
if (CollectionUtils.isNotEmpty(data)) {
kafkaTemplate.send(Constants.EVENT_ANALYSIS_TOPIC, JSON.toJSONString(data));
kafkaTemplate.send(Constants.EVENT_VIEWPOINT_SEND_DATA, JSON.toJSONString(data));
}
log.info("进行中(未结束)的事件专题列表数据{}发送成功!",data);
}
......@@ -104,7 +110,7 @@ public class AnalysisTask {
* 按天发送 事件脉络 所需信息到kafka对应的topic
* 保证信息采集的及时性，审核人员审核的及时性
*/
@Scheduled(cron = "0 5 0 * * ?")
// @Scheduled(cron = "0 5 0 * * ?")
// @Scheduled(cron = "0 * * * * ?")
public void eventContext() {
Date today = new Date();
......@@ -141,7 +147,7 @@ public class AnalysisTask {
* 每天凌晨0点20分执行一次
* 发送 伪事件脉络 所需信息到kafka对应的topic
*/
@Scheduled(cron = "0 20 0 * * ?")
// @Scheduled(cron = "0 20 0 * * ?")
// @Scheduled(cron = "0 * * * * ?")
public void eventContext_fake() {
Date today = new Date();
......@@ -191,7 +197,7 @@ public class AnalysisTask {
* 定时生成传播路径
* 每天凌晨0点10分执行一次
*/
@Scheduled(cron = "0 0 0 * * ?")
// @Scheduled(cron = "0 0 0 * * ?")
// @Scheduled(cron = "0 * * * * ?")
public void propagationPath() {
Date today = new Date();
......@@ -209,6 +215,7 @@ public class AnalysisTask {
if (ObjectUtils.isNotEmpty(pathVo)) {
redisUtil.set(key, pathVo);
}
log.info("专题id为-{}-的专题-传播路径数据：{}-缓存成功!", subjectId, JSONObject.toJSON(pathVo));
}
} else {//已经结束的事件专题，缓存有效期一天
PropagationPathVo pathVo = analysisService.propagationPath(subjectId);
......
src/main/java/com/zzsn/event/util/FileUtil.java 0 → 100644
package com.zzsn.event.util;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.core.io.ClassPathResource;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.List;
/**
* @author 闫鑫
* @date 2022/6/21 12:01
*/
@Slf4j
public class FileUtil {
/**
* @Description: 读取resources 目录下的文件
* @param filePath 相对路径
* @Return: 文件实际地址
**/
public static String getFilePath(String filePath){
return Thread.currentThread().getContextClassLoader().getResource(filePath).getPath();
}
/**
* 读取classpath路径下的文本文件，返回list
* @param filePath
* @return
* @throws IOException
*/
public static List<String> readFileList(String filePath){
List<String> list = new ArrayList<>();
InputStream inputStream = null;
InputStreamReader inputStreamReader = null;
BufferedReader bufferedReader = null;
try {
// ClassPathResource类的构造方法接收路径名称，自动去classpath路径下找文件
ClassPathResource classPathResource = new ClassPathResource(filePath);
// 打成jar包后，只能获取输入流对象
inputStream = classPathResource.getInputStream();
inputStreamReader = new InputStreamReader(inputStream, "UTF-8");
bufferedReader = new BufferedReader(inputStreamReader);
String line = null;
while ((line = bufferedReader.readLine()) != null) {
if(oConvertUtils.isNotEmpty(line)){
list.add(line);
}
}
}catch (IOException e){
log.error("读取文件失败,filePath：{}，e：{}",filePath,e);
}finally {
// 统一在finally中关闭流，防止发生异常的情况下，文件流未能正常关闭
try {
if (inputStream != null) {
inputStream.close();
}
if (inputStreamReader != null) {
inputStreamReader.close();
}
if (bufferedReader != null) {
bufferedReader.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
return list;
}
public static InputStream DownloadFileFormServer(String strUrl) {
InputStream inputStream = null;
HttpURLConnection conn = null;
try {
URL url = new URL(strUrl);
conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
conn.setConnectTimeout(20 * 1000);
// 模板文件名
// 模板文件路径
inputStream = conn.getInputStream();
} catch (Exception e) {
} finally {
}
return inputStream;
}
/**
* 获取resource下的文件内容
*
* @param path
* @return
*/
public static String getResource(String path) throws IOException {
if (StringUtils.isBlank(path)) {
return null;
}
ClassPathResource classPathResource = new ClassPathResource(path);
StringBuilder stringBuilder = new StringBuilder();
InputStream inputStream =null;
try {
inputStream = classPathResource.getInputStream();
byte[] bytes = new byte[1024];
int lenth;
while ((lenth = inputStream.read(bytes)) != -1) {
stringBuilder.append(new String(bytes, 0, lenth));
}
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException("读取文件错误");
}finally {
inputStream.close();
}
return stringBuilder.toString();
}
public static String getFileNameFormPath(String fileName){
if(fileName == null || StringUtils.isEmpty(fileName)){
return null;
}
fileName = fileName.trim();
fileName = fileName.replace("\\", "/");
while(fileName.endsWith("/")){
fileName = fileName.substring(0, fileName.length() -1);
}
if(fileName.contains("/")){
fileName = fileName.substring(fileName.lastIndexOf("/") + 1);
}
return fileName;
}
/**
* 根据文件的字节大小计算显示大小
*
* @param size
* @return
*/
public static String readableFileSize(long size) {
if (size <= 0) return "0";
final String[] units = new String[]{"B", "KB", "MB", "GB", "TB"};
int digitGroups = (int) (Math.log10(size) / Math.log10(1024));
return new DecimalFormat("#,##0.#").format(size / Math.pow(1024, digitGroups)) + units[digitGroups];
}
}
src/main/java/com/zzsn/event/util/SortUtil.java 0 → 100644
package com.zzsn.event.util;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
/**
* @author kongliufeng
* @Description TODO: 自定义排序
* @create 2020-09-03 19:11
* @Version 1.0
*/
public class SortUtil {
public static final DateTimeFormatter dtf2 = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
public static List<Map.Entry<String, Integer>> sortMap(Map<String,Integer> map){
if(map!=null){
List<Map.Entry<String, Integer>> list = new ArrayList<>(map.entrySet());
Collections.sort(list,new Comparator<Map.Entry<String, Integer>>() {
@Override
public int compare(Map.Entry<String, Integer> o1,
Map.Entry<String, Integer> o2) {
return o2.getValue() - o1.getValue();
}
});
return list;
}
return new ArrayList<Map.Entry<String, Integer>>();
}
public static int dateLg(String s1,String s2){
if(s1.equals(s2)){
return 0;
}
int temp = 0;
if(s1.length()==10){
s1 = s1 +" 00:00:00";
}
if(s2.length()==10){
s2 = s2 +" 00:00:00";
}
try {
LocalDateTime localDateTime1 = LocalDateTime.parse(s1, dtf2);
LocalDateTime localDateTime2 = LocalDateTime.parse(s2, dtf2);
if(localDateTime1.isEqual(localDateTime2)){
temp = 0;
}
temp = localDateTime1.isBefore(localDateTime2)==true? 1:-1;
StringBuilder sb = new StringBuilder();
sb.append("s1对比s2").append(s1).append(":").append(s2).append("返回结果").append(temp);
return temp;
}catch (Exception e){
e.printStackTrace();
return -1;
}
}
public static void main(String[] args) {
String s1= "2020-10-22" ;
String s2= "2020-10-23";
System.out.println(s1.length());
System.out.println(dateLg(s1, s2));
}
}
src/main/java/com/zzsn/event/util/StopWordsUtil.java 0 → 100644
package com.zzsn.event.util;
import com.hankcs.hanlp.collection.trie.DoubleArrayTrie;
import java.util.Collection;
import java.util.Iterator;
import java.util.TreeMap;
/**
* @author kongliufeng
* @Description TODO:
* @create 2020-09-03 16:30
* @Version 1.0
*/
public class StopWordsUtil {
private static final String path = "hanlp/cloudstopwords.txt";
private static DoubleArrayTrie<String> tree;
static {
TreeMap<String, String> map = new TreeMap<>();
for (String word : FileUtil.readFileList(path)) {
map.put(word,word);
}
tree = new DoubleArrayTrie<>(map);
}
/**
* @Description 判断一个词是否为停用词
* @author kongliufeng
* @创建时间 2020/9/3 17:08
* @Version 1.0
*/
public static Boolean isStopWord(String word){
if(tree.containsKey(word)){
return true;
}
return false;
}
/**
* @Description 对已有的集合移除停用词
* @author kongliufeng
* @创建时间 2020/9/3 17:11
* @Version 1.0
*/
public static void removeStopWords(Collection<String> collection, String excludeWord){
if(collection!=null){
Iterator<String> iterator = collection.iterator();
while (iterator.hasNext()){
String next = iterator.next();
if(excludeWord!=null){
if(excludeWord.equals(next)){
iterator.remove();
}
}
if(tree.containsKey(next))
iterator.remove();
}
}
}
public static void removeStopWords(Collection<String> collection){
if(collection!=null){
Iterator<String> iterator = collection.iterator();
while (iterator.hasNext()){
if(tree.containsKey(iterator.next()))
iterator.remove();
}
}
}
}
src/main/java/com/zzsn/event/vo/DisplayInfo.java
......@@ -124,4 +124,5 @@ public class DisplayInfo {
* 是否是主条目,0为非主条目，1为主条目
*/
private Integer ismain;
private Integer similarNumber;
}
src/main/java/com/zzsn/event/vo/RepeatHold.java 0 → 100644
package com.zzsn.event.vo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@AllArgsConstructor
@NoArgsConstructor
public class RepeatHold {
private String articleId;
private String repeatMark;
}
src/main/java/com/zzsn/event/vo/SpecialInformation.java
......@@ -70,6 +70,7 @@ public class SpecialInformation {
//删除标记(1:删除；0:保留)
private Integer deleteFlag;
private String subjectId;
private List<String> subjectIdList;
//审核操作(0：未审核 1：审核通过 2：审核未通过 3：暂定 默认值为0)
private Integer checkStatus;
//阅读数
......
src/main/java/com/zzsn/event/vo/StatisticsKeyWordVo.java 0 → 100644
package com.zzsn.event.vo;
import lombok.Data;
@Data
public class StatisticsKeyWordVo {
//专题id
private String subjectId;
// 关键词id
private String kid;
//词频
private Integer value;
// name
private String name;
}
src/main/resources/application.yml
......@@ -115,8 +115,8 @@ mybatis-plus:
log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
map-underscore-to-camel-case: true
serviceProject:
# url: https://clb.ciglobal.cn/clb-api/datapull/
url: http://127.0.0.1:9988/datapull/
url: https://clb.ciglobal.cn/clb-api/datapull/
# url: http://127.0.0.1:9988/datapull/
#热词抽取地址
hotWords:
extractUrl: http://114.116.99.6:8055/task/dispose/extractKeyword
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论