Skip to content

E
event
提交 19ce910b 作者: 925993793@qq.com
操作

专题配置修改后处理方案实现；平台接口替换调试修改

上级 6ed7c675
正在显示 包含 661 行增加 和 207 行删除
src/main/java/com/zzsn/event/constant/Constants.java
......@@ -100,4 +100,6 @@ public class Constants {
}
return origin;
}
public static final String HISTORY_DATE_QUEUE = "HISTORY_DATE_QUEUE:";
}
src/main/java/com/zzsn/event/controller/FileController.java
......@@ -176,7 +176,12 @@ public class FileController {
clbFileOperationLog.setSubjectId(subjectId);
clbFileOperationLogService.updateById(clbFileOperationLog);
//读取文件内容
List<List<String>> lists = ExcelExportUtil.readExcel(new ByteArrayInputStream(fileData), 1, 9);
List<List<String>> lists;
if ("xls".equals(fileSuffix)) {
lists = ExcelExportUtil.readExcelXls(new ByteArrayInputStream(fileData), false,1, 9);
} else {
lists = ExcelExportUtil.readExcelXlsx(new ByteArrayInputStream(fileData), false,1, 9);
}
//存入es
informationService.importInfo(lists, subjectId, isTopping, isExamine, clbFileOperationLog, userVo);
} catch (Exception e) {
......@@ -262,8 +267,60 @@ public class FileController {
}
/**
* 专题资讯导入模板下载-研究中心
*
* @author lkg
* @date 2024/06/21
*/
@GetMapping("/downloadResearchTemplate")
public void downloadResearchTemplate(HttpServletResponse response) {
String filePath = "subjectDataImport/上传Excel模板.xls";
commonService.downloadTemplate(response, filePath);
}
/**
* 批量导入数据-研究中心
*
* @author lkg
* @date 2024/12/25
*/
@PostMapping("/importDataInfo")
public Result<?> importDataInfo(HttpServletRequest request){
MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
Map<String, MultipartFile> fileMap = multipartRequest.getFileMap();
if (fileMap.size() < 1) {
return Result.FAIL(500, "请上传excel文件");
}
MultipartFile multipartFile = fileMap.get(new ArrayList<>(fileMap.keySet()).get(0));
int index = multipartFile.getOriginalFilename().lastIndexOf(".");
String fileSuffix = multipartFile.getOriginalFilename().substring(index + 1);
if ("xls".equals(fileSuffix) || "xlsx".equals(fileSuffix)) {
CompletableFuture.runAsync(() -> {
String subjectId = request.getParameter("subjectId");
try {
byte[] fileData = multipartFile.getBytes();
//读取文件内容
List<List<String>> lists;
if ("xls".equals(fileSuffix)) {
lists = ExcelExportUtil.readExcelXls(new ByteArrayInputStream(fileData), true,1, 7);
} else {
lists = ExcelExportUtil.readExcelXlsx(new ByteArrayInputStream(fileData), true,1, 7);
}
informationService.importDataInfo(lists,subjectId);
} catch (IOException e) {
e.printStackTrace();
}
});
return Result.OK("正在进行处理");
} else {
return Result.FAIL(500, "不支持的文件类型");
}
}
/**
* 导出数据-word格式
* 导出数据-word格式-研究中心
*
* @param searchCondition 筛选条件
* @author lkg
......
src/main/java/com/zzsn/event/controller/SubjectManageController.java
......@@ -411,8 +411,8 @@ public class SubjectManageController {
*/
@GetMapping(value = "/bindKeyWordsList")
public Result<?> bindKeyWordsList(@RequestParam String subjectId) {
subjectKeywordsMapService.bindKeyWordsList(subjectId);
return Result.OK();
List<KeyWordsPage> bindKeyWordsList = subjectKeywordsMapService.bindKeyWordsList(subjectId);
return Result.OK(bindKeyWordsList);
}
/**
......@@ -860,8 +860,13 @@ public class SubjectManageController {
return Result.FAIL("请选择上传文件");
}
String originalFilename = file.getOriginalFilename();
if (!(originalFilename.endsWith(".xlsx") || originalFilename.endsWith(".xls"))) {
List<List<String>> dataList = ExcelExportUtil.readExcel(file.getInputStream(), 1, 2);
if (originalFilename.endsWith(".xlsx") || originalFilename.endsWith(".xls")) {
List<List<String>> dataList;
if (originalFilename.endsWith(".xls")) {
dataList = ExcelExportUtil.readExcelXls(file.getInputStream(), false,1, 2);
} else {
dataList = ExcelExportUtil.readExcelXlsx(file.getInputStream(), false,1, 2);
}
if (CollectionUtils.isNotEmpty(dataList)) {
boolean emptyFlag = false;
for (List<String> strings : dataList) {
......
src/main/java/com/zzsn/event/controller/SubjectSimpleController.java
package com.zzsn.event.controller;
import cn.hutool.http.HttpUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.zzsn.event.constant.Result;
import com.zzsn.event.entity.Subject;
import com.zzsn.event.service.SubjectService;
import com.zzsn.event.service.SubjectSimpleService;
import com.zzsn.event.vo.InfoDataSearchCondition;
import com.zzsn.event.vo.SearchWordVO;
import com.zzsn.event.vo.SubjectDetailVO;
import com.zzsn.event.vo.SubjectSimpleVO;
import com.zzsn.event.vo.*;
import com.zzsn.event.xxljob.service.IXxlJobInfoService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.*;
import java.util.List;
......@@ -56,6 +55,20 @@ public class SubjectSimpleController {
}
/**
* 专题配置校验
*
* @param subjectParamsCheckVO 参数
* @author lkg
* @date 2025/2/5
*/
@PostMapping("/configVerification")
public Result<?> configVerification(@RequestBody SubjectParamsCheckVO subjectParamsCheckVO){
Boolean verification = subjectSimpleService.configVerification(subjectParamsCheckVO);
return Result.OK(verification);
}
/**
* 编辑专题
*
* @param subjectSimpleVO 参数
......@@ -73,6 +86,19 @@ public class SubjectSimpleController {
}
/**
* 清空专题数据
*
* @param subjectId 专题id
* @author lkg
* @date 2025/2/6
*/
@GetMapping("/clearData")
public Result<?> clearData(String subjectId){
subjectSimpleService.clearSubjectData(subjectId);
return Result.OK("正在清空数据");
}
/**
* 专题详情(包含样例文章以及关键词)
*
* @param subjectId 专题id
......
src/main/java/com/zzsn/event/controller/common/InformationController.java
......@@ -13,6 +13,7 @@ import com.zzsn.event.enums.LabelTypeEnum;
import com.zzsn.event.es.EsService;
import com.zzsn.event.service.*;
import com.zzsn.event.util.HttpUtil;
import com.zzsn.event.util.PythonUtil;
import com.zzsn.event.util.user.AuthUtil;
import com.zzsn.event.util.user.UserUtil;
import com.zzsn.event.util.user.UserVo;
......@@ -51,9 +52,8 @@ public class InformationController {
private LabelEntityService labelEntityService;
@Autowired
private SysBaseLabelTypeService sysBaseLabelTypeService;
@Value("${python.judgeDuplicateUrl:}")
private String judgeDuplicateUrl;
@Autowired
private PythonUtil pythonUtil;
/**
......@@ -266,7 +266,7 @@ public class InformationController {
UserVo userVo = UserUtil.getLoginUser();
DisplayInfo displayInfo = JSON.parseObject(JSON.toJSONString(jsonObject.get("data")), DisplayInfo.class);
Integer category = (Integer) jsonObject.get("category");
boolean modelFlag = judgeDuplicate(displayInfo);
boolean modelFlag = pythonUtil.judgeDuplicate(displayInfo.getId(), displayInfo.getTitle(), displayInfo.getContent(), displayInfo.getSourceAddress(), displayInfo.getSubjectId());
if (modelFlag) {
return Result.FAIL(210, "判重存在重复资讯!");
} else {
......@@ -392,36 +392,4 @@ public class InformationController {
List<SubjectDataVo> recommendList = informationService.recommendList(subjectId, id, title, pageNo, pageSize);
return Result.OK(recommendList);
}
public boolean judgeDuplicate(DisplayInfo displayInfo) {
boolean repeat = false;
Map<String, String> headers = new HashMap<>();
headers.put("Content-Type", "application/json;charset=UTF-8");
headers.put("Accept", "application/json");
headers.put("Authorization", "!0gwY$5S@5V&A_+XEu)");
JSONObject jsonObjectParam = new JSONObject();
jsonObjectParam.put("id", displayInfo.getId());
jsonObjectParam.put("title", displayInfo.getTitle());
jsonObjectParam.put("content", displayInfo.getContent());
jsonObjectParam.put("sourceAddress", displayInfo.getSourceAddress());
JSONArray jsonArray = new JSONArray();
jsonArray.add(jsonObjectParam);
JSONObject paramObject = new JSONObject();
paramObject.put("data", jsonArray);
paramObject.put("hash_name", displayInfo.getSubjectId());
try {
String result = HttpUtil.doPostWithHeader(judgeDuplicateUrl, paramObject, 3000, headers);
if (StringUtils.isNotEmpty(result)) {
JSONObject jsonObject = JSONObject.parseObject(result);
if (StringUtils.isNotEmpty(jsonObject.getString("resultData")) && !jsonObject.getJSONArray("resultData").isEmpty()) {
JSONObject resultObject = (JSONObject) jsonObject.getJSONArray("resultData").get(0);
repeat = resultObject.getBoolean("duplicated");
}
}
} catch (Exception e) {
e.printStackTrace();
}
return repeat;
}
}
src/main/java/com/zzsn/event/controller/common/RelationBindController.java
......@@ -197,8 +197,6 @@ public class RelationBindController {
* @param category 类别(1-专题;2-事件)
* @param groupName 词组名称
* @param wordName 关键词
* @param pageNo 当前页
* @param pageSize 每页返回条数
* @author lkg
* @date 2024/12/17
*/
......@@ -206,9 +204,7 @@ public class RelationBindController {
public Result<?> bindKeyWordsList(@RequestParam String id,
@RequestParam Integer category,
@RequestParam(required = false) String groupName,
@RequestParam(required = false) String wordName,
@RequestParam(defaultValue = "1") Integer pageNo,
@RequestParam(defaultValue = "10") Integer pageSize) {
@RequestParam(required = false) String wordName) {
List<String> subjectIdList = new ArrayList<>();
if (!"0".equals(id)) {
List<String> typeIds = subjectTypeService.belowIdList(id, category);
......@@ -219,8 +215,8 @@ public class RelationBindController {
}
}
subjectIdList.add(id);
IPage<KeyWordsPage> page = keyWordsService.bindKeyWordsList(subjectIdList, groupName, wordName, pageNo, pageSize);
return Result.OK(page);
List<KeyWordsPage> bindKeyWordsList = keyWordsService.bindKeyWordsList(subjectIdList, groupName, wordName);
return Result.OK(bindKeyWordsList);
}
......
src/main/java/com/zzsn/event/controller/yjzx/EventManageController.java
......@@ -52,8 +52,6 @@ public class EventManageController {
private LabelEntityService labelEntityService;
@Autowired
private IXxlJobInfoService iXxlJobInfoService;
@Autowired
private ISubjectInfoSourceMapService subjectInfoSourceMapService;
@Autowired
private IKeyWordsService keyWordsService;
......
src/main/java/com/zzsn/event/entity/ClbLabelItem.java
......@@ -80,6 +80,10 @@ public class ClbLabelItem extends Model<ClbLabelItem> {
@TableField("path_ids")
private String pathIds;
/**所属标签编码*/
@TableField(exist = false)
private String labelCode;
@Override
protected Serializable pkVal() {
......
src/main/java/com/zzsn/event/entity/InfoSource.java
......@@ -50,61 +50,9 @@ public class InfoSource implements Serializable {
/**地区*/
@ApiModelProperty(value = "地区")
private String area;
/**是否需要登录*/
@ApiModelProperty(value = "是否需要登录")
private Integer ynLogin;
/**语种*/
@ApiModelProperty(value = "语种")
private String language;
/**是否公共*/
@ApiModelProperty(value = "是否公共")
private String ynPublic;
/**是否需要翻墙*/
@ApiModelProperty(value = "是否需要翻墙")
private Integer ynAbroad;
/**是否需要代理*/
@ApiModelProperty(value = "是否需要代理")
private Integer ynAgent;
/**动态爬取*/
@ApiModelProperty(value = "动态爬取")
private Integer ynBrowser;
/**调度时间间隔*/
@ApiModelProperty(value = "调度时间间隔")
private String period;
/**调度周期（cron自动生成）*/
@ApiModelProperty(value = "调度周期（cron自动生成）")
private String cron;
/**调度周期说明*/
@ApiModelProperty(value = "调度周期说明")
private String remarkCron;
/**信息源状态*/
@ApiModelProperty(value = "信息源状态")
private Integer status;
/**网站可信度*/
@ApiModelProperty(value = "网站可信度")
private Integer siteReliability;
/**信息源综合评分*/
@ApiModelProperty(value = "信息源综合评分")
private Integer score;
/**爬取深度*/
@ApiModelProperty(value = "爬取深度")
private Integer crawlDepth;
/**爬取方式*/
@ApiModelProperty(value = "爬取方式")
private Integer crawlType;
/**历史数据URL*/
@ApiModelProperty(value = "历史数据URL")
private String hisUriExp;
/**历史数据开始时间*/
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd")
@DateTimeFormat(pattern="yyyy-MM-dd")
@ApiModelProperty(value = "历史数据开始时间")
private Date hisDateStarttime;
/**历史数据结束时间*/
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd")
@DateTimeFormat(pattern="yyyy-MM-dd")
@ApiModelProperty(value = "历史数据结束时间")
private Date hisDateEndtime;
/**创建人*/
@ApiModelProperty(value = "创建人")
private String createBy;
......@@ -121,38 +69,7 @@ public class InfoSource implements Serializable {
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
@ApiModelProperty(value = "更新日期")
private Date updateTime;
/**所属部门*/
@ApiModelProperty(value = "所属部门")
private String sysOrgCode;
/**验证结果*/
@TableField(updateStrategy = FieldStrategy.IGNORED)
private String verification;
/**验证失败原因*/
private String verificationFailureCause;
/**所属单位*/
@TableField(updateStrategy = FieldStrategy.IGNORED)
private String company;
/**所属行业*/
@TableField(updateStrategy = FieldStrategy.IGNORED)
private String industry;
/**权威性*/
@TableField(updateStrategy = FieldStrategy.IGNORED)
private String authority;
/**可信度*/
@TableField(updateStrategy = FieldStrategy.IGNORED)
private String reliability;
/**原创度*/
@TableField(updateStrategy = FieldStrategy.IGNORED)
private String originality;
/**父网站*/
@TableField(updateStrategy = FieldStrategy.IGNORED)
private String parentSite;
/**是否保存快照（1：保存 0：不保存）*/
@TableField(updateStrategy = FieldStrategy.IGNORED)
private String ynSnapshot;
/**是否启动原创性校验（1：启动 0：不启动）*/
private String ynVerification;
/**原创来源（中文逗号隔开）*/
private String originalSource;
}
src/main/java/com/zzsn/event/entity/Subject.java
......@@ -103,7 +103,13 @@ public class Subject implements Serializable {
private String remark;
/**是否是平台创建的专题（0否--外部创建 1是--平台创建 ）*/
private String isPlatform;
/**第一次启用时间*/
@TableField(updateStrategy = FieldStrategy.IGNORED)
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
private Date firstOpenTime;
/**预估状态*/
private String estimateStatus;
/**专题分类id*/
@TableField(exist = false)
......
src/main/java/com/zzsn/event/es/EsService.java
......@@ -1507,8 +1507,7 @@ public class EsService {
}
if (aggregationBuilder != null) {
TermsAggregationBuilder groupLibraryType = AggregationBuilders.terms("groupLibraryType")
.field("classificationType")
.size(searchCondition.getIncludeValues().length);
.field("classificationType");
if (!Objects.isNull(searchCondition.getIncludeValues()) || !Objects.isNull(searchCondition.getExcludeValues())) {
if (!Objects.isNull(searchCondition.getIncludeValues())) {
groupLibraryType.size(searchCondition.getIncludeValues().length);
......@@ -1742,15 +1741,21 @@ public class EsService {
boolQuery.mustNot(QueryBuilders.matchQuery("type", "video"));
}
//删除状态查询
if (searchCondition.getDeleteFlag() != 0) {
boolQuery.must(QueryBuilders.termQuery("deleteFlag", "1"));
} else {
boolQuery.mustNot(QueryBuilders.termQuery("deleteFlag", "1"));
if (searchCondition.getDeleteFlag() != null) {
if (searchCondition.getDeleteFlag() != 0) {
boolQuery.must(QueryBuilders.termQuery("deleteFlag", "1"));
} else {
boolQuery.mustNot(QueryBuilders.termQuery("deleteFlag", "1"));
}
}
//专题库类别筛选
if (CollectionUtils.isNotEmpty(searchCondition.getClassificationType())) {
boolQuery.must(QueryBuilders.termsQuery("classificationType", searchCondition.getClassificationType()));
}
//是否原创
if (StringUtils.isNotEmpty(searchCondition.getOriginality())) {
boolQuery.must(QueryBuilders.matchQuery("originality", searchCondition.getOriginality()));
}
//得分范围筛选
Integer minScore = searchCondition.getMinScore();
if (minScore != null) {
......
src/main/java/com/zzsn/event/mapper/CommonMapper.java
package com.zzsn.event.mapper;
import com.zzsn.event.entity.ClbLabelItem;
import com.zzsn.event.util.tree.Node;
import com.zzsn.event.vo.*;
import org.apache.ibatis.annotations.Mapper;
......@@ -92,4 +93,13 @@ public interface CommonMapper {
List<String> getExcludeKeywords(@Param("databaseIds") List<String> databaseIds);
/**
* 信息源绑定的标签信息集合
*
* @param sidList 信息源id集合
* @author lkg
* @date 2025/2/5
*/
List<LabelItemMapVO> infoSourceLabelsBySidList(@Param("sidList") List<String> sidList);
}
src/main/java/com/zzsn/event/mapper/InfoSourceMapper.java
......@@ -10,6 +10,7 @@ import org.apache.ibatis.annotations.Param;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* @Description: 信息源表
......@@ -36,7 +37,7 @@ public interface InfoSourceMapper extends BaseMapper<InfoSource> {
* @author lkg
* @date 2024/5/7
*/
List<InfoSourceVo> queryInfoSource(@Param("sourceIdList") List<String> sourceIdList);
List<InfoSourceVo> queryInfoSource(@Param("sourceIdList") List<String> sourceIdList,@Param("subjectIds") List<String> subjectIds);
/**
* 专题绑定的信息源集合
......@@ -61,6 +62,15 @@ public interface InfoSourceMapper extends BaseMapper<InfoSource> {
Long bindSourceIdCount(@Param("infoSourceVo") InfoSourceCondition infoSourceCondition, @Param("subjectIds") List<String> subjectIds);
/**
*
*
* @param subjectIds 专题id集合
* @author lkg
* @date 2025/2/6
*/
List<InfoSourceVo> bindGroupSourceIdList(@Param("subjectIds") List<String> subjectIds);
/**
* 专题直接屏蔽的信息源列表(不包括专题屏蔽的信息源组下的信息源)
*
* @param infoSourceCondition 筛选条件
......@@ -109,4 +119,13 @@ public interface InfoSourceMapper extends BaseMapper<InfoSource> {
* @date 2025/1/4
*/
List<InfoSource> directSourceColumnList(@Param("webSiteName") String webSiteName);
/**
* 信息源下栏目id列表(校验专题配置时使用-研究中心)
*
* @param webSiteNames 信息源名称集合
* @author lkg
* @date 2025/1/4
*/
Set<String> directSourceColumnSet(@Param("webSiteNames") List<String> webSiteNames);
}
src/main/java/com/zzsn/event/mapper/KeyWordsMapper.java
......@@ -43,13 +43,10 @@ public interface KeyWordsMapper extends BaseMapper<KeyWords> {
* @param subjectIds 专题id集合
* @param groupName 词组名称
* @param wordName 关键词名称
* @param page 分页参数
* @author lkg
* @date 2024/5/7
*/
Page<KeyWordsPage> bindKeyWordsList(@Param("subjectIds") List<String> subjectIds,
@Param("groupName") String groupName, @Param("wordName") String wordName,
Page<KeyWordsPage> page);
List<KeyWordsPage> bindKeyWordsList(@Param("subjectIds") List<String> subjectIds,@Param("groupName") String groupName, @Param("wordName") String wordName);
/**
* 专题/事件绑定关键词组的数量
......
src/main/java/com/zzsn/event/mapper/SysDictItemMapper.java
......@@ -19,4 +19,6 @@ import java.util.List;
public interface SysDictItemMapper extends BaseMapper<SysDictItem> {
List<SysDictItem> listByDictCode(@Param("dictCode") String dictCode);
SysDictItem dictItemInfoByName(@Param("dictCode") String dictCode,@Param("itemName") String itemName);
}
src/main/java/com/zzsn/event/mapper/xml/CommonMapper.xml
......@@ -125,7 +125,8 @@
</select>
<select id="modelList" resultType="com.zzsn.event.vo.ModelVO">
select * from model
select *
from model
</select>
<select id="subjectModelBindLabels" resultType="com.zzsn.event.vo.LabelModelVo">
......@@ -169,4 +170,13 @@
</foreach>)
</if>
</select>
<select id="infoSourceLabelsBySidList" resultType="com.zzsn.event.vo.LabelItemMapVO">
select m.entity_code,m.label_code,m.label_item_code from clb_label_info_source_map m
inner join clb_label_item item on m.label_item_code = item.label_item_code
where m.entity_code in
<foreach collection="sidList" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</select>
</mapper>
src/main/java/com/zzsn/event/mapper/xml/InfoSourceMapper.xml
......@@ -24,7 +24,9 @@
</select>
<select id="queryInfoSource" resultType="com.zzsn.event.vo.InfoSourceVo">
select t.*, g.infoSourceNatureIds as natureIds, g.infoSourceNatureNames, k.infoSourceGroupNames from (
select t.*, g.infoSourceNatureIds as natureIds, g.infoSourceNatureNames,
k.infoSourceGroupNames,sm.is_exempt_keyword,sm.is_free_check
from (
select y.* from (
select a.*, c.type_name as infoSourceTypeName, c.id as infoSourceTypeId from info_source a
LEFT JOIN info_source_type_map b ON b.source_id = a.id
......@@ -47,6 +49,13 @@
SEPARATOR ',') as infoSourceGroupIds from info_source_group_map h
LEFT JOIN info_source_group j ON j.id = h.group_id
GROUP BY h.source_id) k ON t.id = k.source_id
LEFT JOIN subject_info_source_map sm ON sm.source_id = t.id
<if test="subjectIds != null and subjectIds.size() > 0">
AND sm.subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
</select>
<select id="bindSourceIdList" resultType="String">
......@@ -181,7 +190,37 @@
)
) q
</select>
<select id="bindGroupSourceIdList" resultType="com.zzsn.event.vo.InfoSourceVo">
SELECT
iso.id,
iso.web_site_name,
iso.site_name,
iso.site_uri,
iso.`status`,
iso.crawl_type,
iso.create_time,
sm.is_exempt_keyword,
sm.is_free_check
FROM
info_source_group_map m
INNER JOIN info_source iso ON m.source_id = iso.id
INNER JOIN subject_info_source_map sm ON sm.source_id = m.group_id
<if test="subjectIds != null and subjectIds.size() > 0">
AND sm.subject_id IN
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
WHERE
m.group_id IN ( SELECT source_id FROM subject_info_source_map WHERE type IN ( 2, 5 )
<if test="subjectIds != null and subjectIds.size() > 0">
and subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
GROUP BY source_id )
</select>
<select id="unBindSourcePageList" resultType="com.zzsn.event.vo.InfoSourceVo">
select s.id,s.info_source_code,s.web_site_name,s.site_name,s.site_uri,s.status,s.create_time from info_source s
inner join subject_info_source_map sm on s.id = sm.source_id
......@@ -238,4 +277,15 @@
from info_source iso
where iso.web_site_name = #{webSiteName}
</select>
<select id="directSourceColumnSet" resultType="String">
select iso.id from info_source iso
where 1=1
<if test="webSiteNames != null and webSiteNames.size() > 0">
and iso.web_site_name in
<foreach collection="webSiteNames" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
</select>
</mapper>
src/main/java/com/zzsn/event/mapper/xml/KeyWordsMapper.xml
......@@ -84,7 +84,6 @@
and x.key_word like CONCAT('%',#{wordName},'%')
</if>
order by x.create_time desc
limit #{offset},#{pageSize}
</select>
<select id="bindCount" resultType="Integer">
......
src/main/java/com/zzsn/event/mapper/xml/SubjectMapper.xml
......@@ -83,6 +83,7 @@
d.remark,
d.create_time,
d.status,
d.estimate_status,
d.create_by,
c.type_name as subjectTypeName
from subject d
......@@ -123,6 +124,7 @@
d.remark,
d.create_time,
d.status,
d.estimate_status,
d.create_by,
c.type_name as subjectTypeName
from subject d
......@@ -157,6 +159,7 @@
<result column="time_disable" property="timeDisable"/>
<result column="library" property="library"/>
<result column="face_public" property="facePublic"/>
<result column="first_open_time" property="firstOpenTime"/>
<collection property="sampleFileList" ofType="com.zzsn.event.entity.SubjectSampleFile">
<id column="id" property="id"/>
<result column="file_name" property="fileName"/>
......@@ -173,6 +176,7 @@
s.data_source,
s.face_public,
s.library,
s.first_open_time,
c.id as subjectTypeId,
c.type_name as subjectTypeName,
f.id,
......
src/main/java/com/zzsn/event/mapper/xml/SysDictItemMapper.xml
......@@ -6,4 +6,8 @@
select * from sys_dict_item item inner join sys_dict dict on item.dict_id = dict.id
where dict.dict_code = #{dictCode} and item.status = 1
</select>
<select id="dictItemInfoByName" resultType="com.zzsn.event.entity.SysDictItem">
select * from sys_dict_item item inner join sys_dict dict on item.dict_id = dict.id
where dict.dict_code = #{dictCode} and item.item_text = #{itemName}
</select>
</mapper>
\ No newline at end of file
src/main/java/com/zzsn/event/service/CommonService.java
package com.zzsn.event.service;
import com.alibaba.fastjson2.JSONObject;
import com.zzsn.event.entity.ClbLabelItem;
import com.zzsn.event.util.tree.Node;
import com.zzsn.event.vo.*;
import org.apache.ibatis.annotations.Param;
import javax.servlet.http.HttpServletResponse;
import java.util.List;
......@@ -99,4 +99,13 @@ public interface CommonService {
* @date 2025/1/4
*/
void downloadTemplate(HttpServletResponse response, String path);
/**
* 信息源绑定的标签信息集合
*
* @param sidList 信息源id集合
* @author lkg
* @date 2025/2/5
*/
List<LabelItemMapVO> infoSourceLabelsBySidList(List<String> sidList);
}
src/main/java/com/zzsn/event/service/IInfoSourceService.java
......@@ -8,6 +8,7 @@ import com.zzsn.event.entity.InfoSource;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* @Description: 信息源表
......@@ -95,4 +96,14 @@ public interface IInfoSourceService extends IService<InfoSource> {
* @date 2025/1/4
*/
List<InfoSource> directSourceColumnList(String webSiteName);
/**
* 信息源下栏目id列表(校验专题配置时使用-研究中心)
*
* @param webSiteNames 信息源名称集合
* @author lkg
* @date 2025/1/4
*/
Set<String> directSourceColumnSet(List<String> webSiteNames);
}
src/main/java/com/zzsn/event/service/IKeyWordsService.java
......@@ -69,12 +69,10 @@ public interface IKeyWordsService extends IService<KeyWords> {
* @param subjectIds 专题id集合
* @param groupName 词组名称
* @param wordName 关键词名称
* @param pageNo 当前页
* @param pageSize 返回条数
* @author lkg
* @date 2024/5/7
*/
IPage<KeyWordsPage> bindKeyWordsList(List<String> subjectIds, String groupName, String wordName, Integer pageNo, Integer pageSize);
List<KeyWordsPage> bindKeyWordsList(List<String> subjectIds, String groupName, String wordName);
/**
* 专题绑定关键词组的数量
......
src/main/java/com/zzsn/event/service/InformationService.java
......@@ -180,5 +180,17 @@ public interface InformationService {
*/
List<SubjectDataVo> recommendList(String subjectId, String id, String title, Integer pageNo, Integer pageSize);
/**
* 批量导入数据到发布库
* @author lkg
* @date 2025/2/5
*/
void importInfo(List<List<String>> lists, String subjectId, String isTopping, String isExamine, ClbFileOperationLog clbFileOperationLog, UserVo userVo);
/**
* 批量导入数据到发布库-研究中心
* @author lkg
* @date 2025/2/5
*/
void importDataInfo(List<List<String>> lists, String subjectId);
}
src/main/java/com/zzsn/event/service/SubjectSimpleService.java
package com.zzsn.event.service;
import com.zzsn.event.entity.Subject;
import com.zzsn.event.vo.InfoDataSearchCondition;
import com.zzsn.event.vo.SearchWordVO;
import com.zzsn.event.vo.SubjectDetailVO;
import com.zzsn.event.vo.SubjectSimpleVO;
import com.zzsn.event.vo.*;
import java.util.List;
......@@ -26,6 +23,15 @@ public interface SubjectSimpleService {
Subject createSubject(SubjectSimpleVO subjectSimpleVO);
/**
* 专题配置校验
*
* @param subjectParamsCheckVO 参数
* @author lkg
* @date 2025/2/5
*/
Boolean configVerification(SubjectParamsCheckVO subjectParamsCheckVO);
/**
* 编辑专题
*
* @param subjectSimpleVO 参数
......@@ -87,4 +93,13 @@ public interface SubjectSimpleService {
* @date 2025/1/14
*/
void batchRemove(InfoDataSearchCondition searchCondition);
/**
* 清空专题数据
*
* @param subjectId 专题id
* @author lkg
* @date 2025/2/6
*/
void clearSubjectData(String subjectId);
}
src/main/java/com/zzsn/event/service/SysDictItemService.java
......@@ -18,6 +18,8 @@ public interface SysDictItemService extends IService<SysDictItem> {
List<SysDictItem> listByDictCode(String dictCode);
SysDictItem dictItemInfoByName(String dictCode, String itemName);
/**
* 改变key值(字典值码转为字典值)
*
......
src/main/java/com/zzsn/event/service/impl/CommonServiceImpl.java
......@@ -340,6 +340,15 @@ public class CommonServiceImpl implements CommonService {
}
}
@Override
public List<LabelItemMapVO> infoSourceLabelsBySidList(List<String> sidList) {
List<LabelItemMapVO> clbLabelItems = new ArrayList<>();
if (CollectionUtils.isEmpty(sidList)) {
clbLabelItems = commonMapper.infoSourceLabelsBySidList(sidList);
}
return clbLabelItems;
}
//获取树状结构
private List<ModelVO> getTreeList(List<ModelVO> authTypeList){
List<ModelVO> topList = new ArrayList<>();
......
src/main/java/com/zzsn/event/service/impl/InfoSourceServiceImpl.java
......@@ -3,6 +3,7 @@ package com.zzsn.event.service.impl;
import cn.hutool.core.date.DateField;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.ObjectUtil;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
......@@ -20,6 +21,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
......@@ -49,8 +51,23 @@ public class InfoSourceServiceImpl extends ServiceImpl<InfoSourceMapper, InfoSou
List<InfoSourceVo> pageList = new ArrayList<>();
List<String> bindList = baseMapper.bindSourceIdList(infoSourceCondition,subjectIds,offset,pageSize);
if (CollectionUtils.isNotEmpty(bindList)) {
pageList = baseMapper.queryInfoSource(bindList);
List<InfoSourceVo> infoSourceVos = baseMapper.bindGroupSourceIdList(subjectIds);
Map<String, InfoSourceVo> collect1 = infoSourceVos.stream().collect(Collectors.toMap(InfoSourceVo::getId, Function.identity(), (exist, replace) -> exist));
pageList = baseMapper.queryInfoSource(bindList,subjectIds);
for (InfoSourceVo infoSourceVo1 : pageList) {
//数据配置
if (ObjectUtil.isEmpty(infoSourceVo1.getIsFreeCheck())){
InfoSourceVo infoSourceVo2 = collect1.get(infoSourceVo1.getId());
if (infoSourceVo2 != null) {
infoSourceVo1.setIsFreeCheck(infoSourceVo2.getIsFreeCheck());
}
}
if (ObjectUtil.isEmpty(infoSourceVo1.getIsExemptKeyword())){
InfoSourceVo infoSourceVo2 = collect1.get(infoSourceVo1.getId());
if (infoSourceVo2 != null) {
infoSourceVo1.setIsExemptKeyword(infoSourceVo2.getIsExemptKeyword());
}
}
//性质
if (StringUtils.isNotEmpty(infoSourceVo1.getNatureIds())) {
String natureIds = infoSourceVo1.getNatureIds();
......@@ -129,4 +146,9 @@ public class InfoSourceServiceImpl extends ServiceImpl<InfoSourceMapper, InfoSou
public List<InfoSource> directSourceColumnList(String webSiteName) {
return baseMapper.directSourceColumnList(webSiteName);
}
@Override
public Set<String> directSourceColumnSet(List<String> webSiteNames) {
return baseMapper.directSourceColumnSet(webSiteNames);
}
}
src/main/java/com/zzsn/event/service/impl/InformationServiceImpl.java
......@@ -2,6 +2,7 @@ package com.zzsn.event.service.impl;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.bean.copier.CopyOptions;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
......@@ -84,6 +85,8 @@ public class InformationServiceImpl implements InformationService {
private IClbFileOperationLogDetailsService clbFileOperationLogDetailsService;
@Autowired
private ICollectionMapService collectionMapService;
@Autowired
private SysDictItemService sysDictItemService;
@Override
......@@ -131,13 +134,13 @@ public class InformationServiceImpl implements InformationService {
DisplayInfo info = new DisplayInfo();
BeanUtils.copyProperties(specialInformation,info);
info.setPublishDate(EsDateUtil.esFieldDateMapping(info.getPublishDate()));
LambdaQueryWrapper<CollectionMap> query = Wrappers.lambdaQuery();
/*LambdaQueryWrapper<CollectionMap> query = Wrappers.lambdaQuery();
query.eq(CollectionMap::getUserId, userVo.getId());
query.eq(CollectionMap::getArticleId, info.getId());
int count = collectionMapService.count(query);
if (count > 0) {
info.setYnCollect(true);
}
}*/
//标签处理
List<LabelModelVo> modelVoList = modelMap.get(info.getSubjectId());
formatLabel(modelVoList, info);
......@@ -271,18 +274,16 @@ public class InformationServiceImpl implements InformationService {
String subjectId = manualAddVO.getSubjectId();
String subjectName = manualAddVO.getSubjectName();
String index = manualAddVO.getIndex();
Subjectdatabase subjectdatabase = (Subjectdatabase) esOpUtil.getInfoById(index, id, Subjectdatabase.class);
SpecialInformation subjectdatabase = (SpecialInformation) esOpUtil.getInfoById(index, id, SpecialInformation.class);
String dataId = subjectId + id;
subjectdatabase.setId(dataId);
LocalDateTime processDate = LocalDateTime.now();
subjectdatabase.setProcessDate(processDate);
subjectdatabase.setProcessDate(DateUtil.dateToString(new Date(), "yyyy-MM-dd'T'HH:mm:ss"));
subjectdatabase.setSubjectId(subjectId);
subjectdatabase.setSubjectName(subjectName);
subjectdatabase.setFlag("1");
subjectdatabase.setDeleteFlag(0L);
subjectdatabase.setCheckStatus(0L);
subjectdatabase.setTopNum(0L);
esOpUtil.docSaveByJson(Constants.SUBJECT_INDEX + "_" + processDate.getYear(), dataId, JSON.toJSONString(subjectdatabase));
subjectdatabase.setDeleteFlag(0);
subjectdatabase.setCheckStatus(0);
subjectdatabase.setTopNum(0);
esOpUtil.docSaveByJson(Constants.SUBJECT_INDEX + "_" + LocalDateTime.now().getYear(), dataId, JSON.toJSONString(subjectdatabase));
} catch (Exception e) {
e.printStackTrace();
}
......@@ -656,6 +657,7 @@ public class InformationServiceImpl implements InformationService {
specialInformation.setSubjectId(subjectId);
specialInformation.setFlag("1");
specialInformation.setMasterEntryId(specialInformation.getId());
specialInformation.setDataFrom(1);
//加上分类
setInfoSourceType(specialInformation);
//判断是否需要审核
......@@ -680,6 +682,63 @@ public class InformationServiceImpl implements InformationService {
clbFileOperationLogService.saveEntity(clbFileOperationLog);
}
@Override
public void importDataInfo(List<List<String>> lists, String subjectId) {
if (CollectionUtil.isNotEmpty(lists)) {
String index = EsIndexUtil.getIndexYear(Constants.SUBJECT_INDEX);
List<SpecialInformation> dataList = new ArrayList<>();
for (List<String> info : lists) {
SpecialInformation specialInformation = new SpecialInformation();
specialInformation.setId(codeGenerateUtil.geneIdNo(Constants.DATA_ADD_ID, 8));
if (StringUtils.isNotEmpty(info.get(0))) {
specialInformation.setTitle(info.get(0));
}
if (StringUtils.isNotEmpty(info.get(1))) {
specialInformation.setAuthor(info.get(1));
}
if (StringUtils.isNotEmpty(info.get(2))) {
specialInformation.setOrigin(info.get(2));
specialInformation.setSid(info.get(2));
}
if (StringUtils.isNotEmpty(info.get(3))) {
String library = info.get(3);
SysDictItem dictItem = sysDictItemService.dictItemInfoByName("Thematic_Library", library);
if (dictItem != null) {
specialInformation.setClassificationType(Integer.parseInt(dictItem.getItemValue()));
} else {
specialInformation.setClassificationType(0);
}
} else {
specialInformation.setClassificationType(0);
}
if (StringUtils.isNotEmpty(info.get(4))) {
specialInformation.setSourceAddress(info.get(4));
}
if (StringUtils.isNotEmpty(info.get(5))) {
specialInformation.setPublishDate(EsDateUtil.esFieldDateFormat(info.get(5)));
}
if (StringUtils.isNotEmpty(info.get(6))) {
specialInformation.setContent(info.get(6));
}
specialInformation.setCheckStatus(1);
specialInformation.setDeleteFlag(0);
specialInformation.setTopNum(0);
specialInformation.setDataFrom(1);
specialInformation.setSubjectId(subjectId);
String format = DateUtil.dateToString(new Date(), "yyyy-MM-dd'T'HH:mm:ss");
specialInformation.setCreateDate(format);
specialInformation.setProcessDate(format);
//加上分类
setInfoSourceType(specialInformation);
//匹配标签
List<Label> labels = getInfoSourceLabel(specialInformation.getOrigin(), specialInformation.getSourceAddress());
specialInformation.setLabels(labels);
dataList.add(specialInformation);
}
esOpUtil.docSaveBulk(index, dataList);
}
}
private void formatLabel(List<LabelModelVo> labelModelVos, DisplayInfo info) {
if (CollectionUtils.isNotEmpty(labelModelVos)) {
......@@ -768,7 +827,7 @@ public class InformationServiceImpl implements InformationService {
//信息加上分类
public void setInfoSourceType(SpecialInformation data) {
String url = Utility.domainURL(data.getSourceAddress());
String typeNum = null;
String typeNum;
if (StringUtils.isNotEmpty(url)) {
if (redisUtil.get(Constants.SITE_NAME_KEY + url) != null) {
typeNum = (String) redisUtil.get(Constants.SITE_NAME_KEY + url);
......@@ -780,4 +839,43 @@ public class InformationServiceImpl implements InformationService {
}
data.setInfoSourceType(typeNum);
}
private List<Label> getInfoSourceLabel(String origin,String sourceAddress){
List<String> sidList = new ArrayList<>();
if (StringUtils.isNotEmpty(origin)) {
LambdaQueryWrapper<InfoSource> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.eq(InfoSource::getSiteName, origin);
InfoSource infoSource = infoSourceService.getOne(queryWrapper);
if (infoSource != null) {
String sid = infoSource.getId();
sidList.add(sid);
} else {
if (StringUtils.isNotEmpty(sourceAddress)) {
String url = Utility.domainURL(sourceAddress);
if (StringUtils.isNotEmpty(url)) {
LambdaQueryWrapper<InfoSource> query = Wrappers.lambdaQuery();
query.like(InfoSource::getSiteUri, url);
List<InfoSource> sourceList = infoSourceService.list(query);
if (CollectionUtils.isNotEmpty(sourceList)) {
sidList = sourceList.stream().map(InfoSource::getId).collect(Collectors.toList());
}
}
}
}
}
List<Label> labels = new ArrayList<>();
List<LabelItemMapVO> clbLabelItems = commonService.infoSourceLabelsBySidList(sidList);
Label label = new Label();
if (CollectionUtils.isNotEmpty(clbLabelItems)) {
LabelItemMapVO clbLabelItem = clbLabelItems.get(0);
label.setSourceId(clbLabelItem.getEntityCode());
label.setLabelMark(clbLabelItem.getLabelCode());
label.setRelationId(clbLabelItem.getLabelCode() + "-" +clbLabelItem.getLabelItemCode());
} else {//默认其他
label.setLabelMark("LABEL-20250102-0006");
label.setRelationId("LABEL-20250102-0006-LV-20250102-0026");
}
labels.add(label);
return labels;
}
}
src/main/java/com/zzsn/event/service/impl/KeyWordsServiceImpl.java
......@@ -239,9 +239,8 @@ public class KeyWordsServiceImpl extends ServiceImpl<KeyWordsMapper, KeyWords> i
@Override
public IPage<KeyWordsPage> bindKeyWordsList(List<String> subjectIds, String groupName, String wordName, Integer pageNo, Integer pageSize) {
Page<KeyWordsPage> page = new Page<>(pageNo, pageSize);
return baseMapper.bindKeyWordsList(subjectIds, groupName, wordName, page);
public List<KeyWordsPage> bindKeyWordsList(List<String> subjectIds, String groupName, String wordName) {
return baseMapper.bindKeyWordsList(subjectIds, groupName, wordName);
}
@Override
......
src/main/java/com/zzsn/event/service/impl/ScoreModelServiceImpl.java
......@@ -12,7 +12,6 @@ import com.zzsn.event.mapper.ScoreModelMapper;
import com.zzsn.event.service.CommonService;
import com.zzsn.event.service.ISubjectInfoSourceMapService;
import com.zzsn.event.service.ScoreModelService;
import com.zzsn.event.util.HttpUtil;
import com.zzsn.event.vo.InfoSourceGroupPage;
import com.zzsn.event.vo.ScoreModelVo;
import com.zzsn.event.vo.SubjectScoreModel;
......@@ -21,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
......@@ -51,7 +49,7 @@ public class ScoreModelServiceImpl extends ServiceImpl<ScoreModelMapper, ScoreMo
ScoreModel scoreModelNew = new ScoreModel();
BeanUtils.copyProperties(scoreModel, scoreModelNew);
baseMapper.insert(scoreModelNew);
if (scoreModel.getChangeType() == 1) {
/*if (scoreModel.getChangeType() == 1) {
JSONObject jsonObject = commonService.siteInfo(scoreModel.getSubjectId(), scoreModel.getType(),category);
//调用python接口
try {
......@@ -59,7 +57,7 @@ public class ScoreModelServiceImpl extends ServiceImpl<ScoreModelMapper, ScoreMo
} catch (IOException e) {
log.error("传递专题配置信息异常!");
}
}
}*/
}
@Override
......
src/main/java/com/zzsn/event/service/impl/SubjectTypeServiceImpl.java
......@@ -127,7 +127,7 @@ public class SubjectTypeServiceImpl extends ServiceImpl<SubjectTypeMapper, Subje
queryWrapper.eq(SubjectType::getCategory, category).orderByAsc(SubjectType::getCreateTime);
List<SubjectType> allList = list(queryWrapper);
if (CollectionUtils.isNotEmpty(allList)) {
//绑定专题/事件的分类
//绑定有专题/事件的分类
List<SubjectTypeTreeVO> subjectTypeTreeVOS = baseMapper.typeBindCountTreeList(category);
if (CollectionUtils.isNotEmpty(subjectTypeTreeVOS)) {
List<String> collect = subjectTypeTreeVOS.stream().map(SubjectTypeTreeVO::getId).collect(Collectors.toList());
......
src/main/java/com/zzsn/event/service/impl/SysDictItemServiceImpl.java
......@@ -31,8 +31,13 @@ public class SysDictItemServiceImpl extends ServiceImpl<SysDictItemMapper, SysDi
return this.baseMapper.listByDictCode(dictCode);
}
@Override
public SysDictItem dictItemInfoByName(String dictCode, String itemName) {
return baseMapper.dictItemInfoByName(dictCode, itemName);
}
@Override
@Override
public void changeKey(List<CountVO> dataList, List<SysDictItem> dictItemList) {
if (CollectionUtils.isNotEmpty(dataList)) {
Map<String, List<SysDictItem>> map = dictItemList.stream().collect(Collectors.groupingBy(SysDictItem::getItemValue));
......
src/main/java/com/zzsn/event/util/EsOpUtil.java
......@@ -232,7 +232,7 @@ public class EsOpUtil<T> {
request.timeout(TimeValue.timeValueMinutes(10));
for (int i = 0; i < list.size(); i++) {
T b = list.get(i);
String id = null;
String id;
try {
Field field = b.getClass().getDeclaredField("id");
field.setAccessible(true);
......@@ -251,9 +251,7 @@ public class EsOpUtil<T> {
try {
BulkResponse bulk = client.bulk(request, RequestOptions.DEFAULT);
BulkItemResponse[] bulkItemResponses = bulk.getItems();
int length = bulkItemResponses.length;
for (int i = 0; i < length; ++i) {
BulkItemResponse response = bulkItemResponses[i];
for (BulkItemResponse response : bulkItemResponses) {
if (response.isFailed()) {
log.info("批量保存[{}]过程中,id为[{}]的保存失败,失败原因[{}]", response.getIndex(), response.getId(), response.getFailureMessage());
} else {
......@@ -346,6 +344,7 @@ public class EsOpUtil<T> {
IndexRequest request = new IndexRequest(index)
.id(id)
.source(jsonStr, XContentType.JSON);
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT);
return indexResponse.getId();
} catch (IOException e) {
......@@ -541,9 +540,9 @@ public class EsOpUtil<T> {
String index = m.get("index").toString();
String id = m.get("id").toString();
DeleteRequest deleteRequest = new DeleteRequest(index, id);
deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
bulkRequest.add(deleteRequest);
}
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
try {
client.bulk(bulkRequest, RequestOptions.DEFAULT);
} catch (Exception e) {
......@@ -657,7 +656,7 @@ public class EsOpUtil<T> {
* 调用client.updateByQuery方法，传入UpdateByQueryRequest对象和默认的RequestOptions，执行批量更新操作，并返回BulkByScrollResponse对象。
* Collections.emptyMap()是setScript脚本的参数，此方法没有设置其他参数，使用一个空的map
*/
public void updataMoreColumBatchByQuery(String index, BoolQueryBuilder boolQuery, Map<String, String> modifyColumValue) throws IOException, InterruptedException {
public void updateMoreColumBatchByQuery(String index, BoolQueryBuilder boolQuery, Map<String, String> modifyColumValue) throws IOException, InterruptedException {
UpdateByQueryRequest request = new UpdateByQueryRequest(index);
request.setQuery(boolQuery);
request.setScript(new Script(ScriptType.INLINE, "painless", getIdOrCode(modifyColumValue), Collections.emptyMap()));
......
src/main/java/com/zzsn/event/util/ExcelExportUtil.java
......@@ -4,6 +4,7 @@ import org.apache.poi.hssf.usermodel.*;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.*;
import java.io.IOException;
import java.io.InputStream;
import java.text.DecimalFormat;
import java.util.ArrayList;
......@@ -208,11 +209,52 @@ public class ExcelExportUtil {
* @param columnNum 有用数据的总列数
* @return java.util.List<java.util.List < java.lang.String>>
*/
public static List<List<String>> readExcel(InputStream inputStream, Integer firstRow, Integer columnNum) throws Exception {
public static List<List<String>> readExcelXls(InputStream inputStream,boolean singleSheet, Integer firstRow, Integer columnNum) throws IOException {
List<List<String>> dataList = new ArrayList<>();
//获取整个excel
HSSFWorkbook hb = new HSSFWorkbook(inputStream);
int sheets = hb.getNumberOfSheets();
if (singleSheet) {
sheets = 1;
}
for (int i = 0; i < sheets; i++) {
HSSFSheet sheet = hb.getSheetAt(i);
//第一行
int firstRowNum = sheet.getFirstRowNum();
//最后一行
int lastRowNum = sheet.getPhysicalNumberOfRows();
for (int j = firstRowNum + firstRow; j < lastRowNum; j++) {
//获取行
HSSFRow row = sheet.getRow(j);
if (row != null) {
List<String> list = new ArrayList<>();
for (int m = 0; m < columnNum; m++) {
String data = getValue(row.getCell(m)).trim();
list.add(data);
}
dataList.add(list);
}
}
}
return dataList;
}
/**
* 读取excel数据
*
* @param firstRow 第一行有用数据(0表示第一行)
* @param columnNum 有用数据的总列数
* @return java.util.List<java.util.List < java.lang.String>>
*/
public static List<List<String>> readExcelXlsx(InputStream inputStream,boolean singleSheet, Integer firstRow, Integer columnNum) throws IOException {
List<List<String>> dataList = new ArrayList<>();
//获取整个excel
XSSFWorkbook hb = new XSSFWorkbook(inputStream);
int sheets = hb.getNumberOfSheets();
if (singleSheet) {
sheets = 1;
}
for (int i = 0; i < sheets; i++) {
XSSFSheet sheet = hb.getSheetAt(i);
//第一行
......@@ -225,7 +267,7 @@ public class ExcelExportUtil {
if (row != null) {
List<String> list = new ArrayList<>();
for (int m = 0; m < columnNum; m++) {
String data = ExcelExportUtil.getValue(row.getCell(m)).trim();
String data = getValue(row.getCell(m)).trim();
list.add(data);
}
dataList.add(list);
......@@ -380,6 +422,23 @@ public class ExcelExportUtil {
return style;
}
public static String getValue(HSSFCell hssfCell) {
if (hssfCell == null || hssfCell.toString().trim().equals("")) {
return "";
}
CellType cellTypeEnum = hssfCell.getCellTypeEnum();
if (cellTypeEnum == CellType.NUMERIC) {
if (HSSFDateUtil.isCellDateFormatted(hssfCell)) {
return DateUtil.dateToString(hssfCell.getDateCellValue());
} else {
//防止数字变成科学计数法的形式
DecimalFormat df = new DecimalFormat("0");
return df.format(hssfCell.getNumericCellValue());
}
} else {
return hssfCell.getStringCellValue();
}
}
public static String getValue(XSSFCell xssfCell) {
if (xssfCell == null || xssfCell.toString().trim().equals("")) {
return "";
......
src/main/java/com/zzsn/event/util/PythonUtil.java
......@@ -23,6 +23,8 @@ public class PythonUtil {
@Value("${python.keyWordsExtractUrl}")
private String keywordExtractUrl;
@Value("${python.judgeDuplicateUrl:}")
private String judgeDuplicateUrl;
/**
* 提取关键词
......@@ -49,4 +51,46 @@ public class PythonUtil {
}
return wordsList;
}
/**
* 判重
*
* @param id 资讯id
* @param title 资讯标题
* @param content 资讯内容
* @param sourceAddress 原文链接
* @param subjectId 专题id
* @author lkg
* @date 2025/2/7
*/
public Boolean judgeDuplicate(String id, String title, String content, String sourceAddress, String subjectId) {
boolean repeat = false;
Map<String, String> headers = new HashMap<>();
headers.put("Content-Type", "application/json;charset=UTF-8");
headers.put("Accept", "application/json");
headers.put("Authorization", "!0gwY$5S@5V&A_+XEu)");
JSONObject jsonObjectParam = new JSONObject();
jsonObjectParam.put("id", id);
jsonObjectParam.put("title", title);
jsonObjectParam.put("content", content);
jsonObjectParam.put("sourceAddress", sourceAddress);
JSONArray jsonArray = new JSONArray();
jsonArray.add(jsonObjectParam);
JSONObject paramObject = new JSONObject();
paramObject.put("data", jsonArray);
paramObject.put("hash_name", subjectId);
try {
String result = HttpUtil.doPostWithHeader(judgeDuplicateUrl, paramObject, 3000, headers);
if (StringUtils.isNotEmpty(result)) {
JSONObject jsonObject = JSONObject.parseObject(result);
if (StringUtils.isNotEmpty(jsonObject.getString("resultData")) && !jsonObject.getJSONArray("resultData").isEmpty()) {
JSONObject resultObject = (JSONObject) jsonObject.getJSONArray("resultData").get(0);
repeat = resultObject.getBoolean("duplicated");
}
}
} catch (Exception e) {
e.printStackTrace();
}
return repeat;
}
}
src/main/java/com/zzsn/event/util/RedisUtil.java
......@@ -549,5 +549,8 @@ public class RedisUtil {
}
}
public void rpushMultipleValues(String key, Object... values) {
redisTemplate.opsForList().rightPushAll(key, values);
}
}
src/main/java/com/zzsn/event/vo/InfoDataSearchCondition.java
......@@ -38,6 +38,9 @@ public class InfoDataSearchCondition {
//专题库类型(对应字典编码为【Thematic_Library】的数据字典值)
private List<Integer> classificationType;
/**原创性(0-非原创;1-原创;2-疑似)*/
private String originality;
//审核操作(0：未审核 1：审核通过 2：审核未通过 3：暂定 默认值为0)
private Integer checkStatus;
......
src/main/java/com/zzsn/event/vo/InfoSourceVo.java
......@@ -259,4 +259,9 @@ public class InfoSourceVo {
private Long weekCount;
/**近一月采集量*/
private Long monthCount;
/**是否免过关键词 0否 1是*/
private Integer isExemptKeyword;
/**是否免审核 0否 1是*/
private Integer isFreeCheck;
}
src/main/java/com/zzsn/event/vo/KeyWordsPage.java
......@@ -90,4 +90,5 @@ public class KeyWordsPage {
/**专题绑定的关键词映射表主键id*/
private String subjectKeyWordId;
private String typeId;
private Integer bindingType;
}
src/main/java/com/zzsn/event/vo/LabelItemMapVO.java 0 → 100644
package com.zzsn.event.vo;
import lombok.Data;
/**
* 标签-实体关系
*
* @author lkg
* @date 2025/2/5
*/
@Data
public class LabelItemMapVO {
private String entityCode;
private String labelCode;
private String labelItemCode;
}
src/main/java/com/zzsn/event/vo/SubjectDetailVO.java
package com.zzsn.event.vo;
import com.baomidou.mybatisplus.annotation.FieldStrategy;
import com.baomidou.mybatisplus.annotation.TableField;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.zzsn.event.entity.SubjectSampleFile;
import lombok.Data;
......@@ -35,6 +37,10 @@ public class SubjectDetailVO {
private Integer dataSource;
/**划分专题库*/
private String library;
/**第一次启用时间*/
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
private Date firstOpenTime;
/**专题分类id*/
private String subjectTypeId;
......
src/main/java/com/zzsn/event/vo/SubjectPage.java
......@@ -155,5 +155,7 @@ public class SubjectPage {
private String taskCode;
/**分类(1-专题;2-事件)*/
private Integer category;
/**预估状态*/
private String estimateStatus;
}
src/main/java/com/zzsn/event/vo/SubjectParamsCheckVO.java 0 → 100644
package com.zzsn.event.vo;
import lombok.Data;
import java.util.List;
/**
* 专题参数校验
*
* @author lkg
* @date 2025/2/5
*/
@Data
public class SubjectParamsCheckVO {
/**专题新增参数*/
private SubjectSimpleVO subjectSimpleVO;
/**信息源标签*/
private SubjectSourceTagVO subjectSourceTagVO;
/**信息源名称列表*/
private List<String> infoSourceWebSiteNames;
/**信息源id列表-推荐信息源*/
private List<String> infoSourceIds;
}
src/main/java/com/zzsn/event/vo/SubjectSimpleVO.java
......@@ -34,6 +34,6 @@ public class SubjectSimpleVO {
private String library;
private Integer dataSource;
private Integer facePublic;
/**关键词*/
private List<SearchWordVO> keywords;
}
src/main/java/com/zzsn/event/vo/es/DisplayInfo.java
......@@ -7,6 +7,9 @@ import java.util.List;
@Data
public class DisplayInfo {
//es的索引名
private String dbIndex;
//说明：...Raw 表示原文，即原语言
//作者
private String author;
......@@ -104,6 +107,7 @@ public class DisplayInfo {
private List<String> databaseIds;
//是否风险
private Integer isRisk;
private Integer isFreeCheck;
//风险类型
private List<String> riskTypes;
//标签集合
......@@ -113,6 +117,8 @@ public class DisplayInfo {
//法规号
private String contentNo;
/**原创性*/
private String originality;
//来源 （政策发文机关）
private String organ;
......
src/main/java/com/zzsn/event/vo/es/SpecialInformation.java
......@@ -74,6 +74,7 @@ public class SpecialInformation {
//删除标记(1:删除；0:保留)
private Integer deleteFlag;
private String subjectId;
private String subjectName;
//审核操作(0：未审核 1：审核通过 2：审核未通过 3：暂定 默认值为0)
private Integer checkStatus;
//阅读数
......@@ -106,6 +107,7 @@ public class SpecialInformation {
private List<String> databaseIds;
//是否风险
private Integer isRisk;
private Integer isFreeCheck;
//风险类型
private List<String> riskTypes;
//资讯关联的附件id
......@@ -136,4 +138,7 @@ public class SpecialInformation {
private String originalSource;
/**原创性*/
private String originality;
/**数据来源(1-人工导入；空-正常采集)*/
private Integer dataFrom;
}
src/main/java/com/zzsn/event/vo/es/Subjectdatabase.java
......@@ -6,6 +6,7 @@ import lombok.Setter;
import lombok.experimental.FieldNameConstants;
import java.time.LocalDateTime;
import java.util.Date;
import java.util.List;
/**
......@@ -26,7 +27,7 @@ public class Subjectdatabase {
private List<AttachmentInfo> attachmentInfos;
private String author;
private String authorRaw;
private LocalDateTime checkDate;
private Date checkDate;
private Long checkStatus;
private Long classificationType;
private Long classificationTypePre;
......@@ -39,7 +40,7 @@ public class Subjectdatabase {
private String contentWithTag;
private String contentWithTagRaw;
private String country;
private LocalDateTime createDate;
private Date createDate;
private String customLabel;
private Long dataFrom;
private String dataHash;
......@@ -76,12 +77,12 @@ public class Subjectdatabase {
private Long orientation;
private String origin;
private String originRaw;
private LocalDateTime originalDate;
private Date originalDate;
private String originalId;
private String originalTime;
private LocalDateTime processDate;
private Date processDate;
private List<String> programaIds;
private LocalDateTime publishDate;
private Date publishDate;
private Long publishStatus;
private Long readNum;
private String repeatId;
......@@ -108,7 +109,7 @@ public class Subjectdatabase {
* 资讯唯一编码
*/
private String uniqueCode;
private LocalDateTime updateDate;
private Date updateDate;
private String videoImg;
private String videoPhoneUrl;
private Long videoTime;
......
src/main/resources/application-test.yml
......@@ -54,10 +54,17 @@ spring:
username: ciglobal
password: qwer@9988&zzsn
driver-class-name: com.mysql.cj.jdbc.Driver
multi-datasource2:
url: jdbc:mysql://1.95.78.131:3306/clb_system_label?characterEncoding=UTF-8&useUnicode=true&useSSL=false&tinyInt1isBit=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
username: ciglobal
password: qwer@9988&zzsn
driver-class-name: com.mysql.cj.jdbc.Driver
elasticsearch:
uris: ["1.95.69.2:9200"]
# uris: ["1.95.38.69:9700","1.95.3.121:9200","1.95.87.177:9700"]
username: elastic
password: elastic
# password: zzsn9988
connection-timeout: 300000
socket-timeout: 300000
cloud:
......@@ -111,7 +118,7 @@ spring:
mybatis-plus:
mapper-locations: classpath*:com/zzsn/event/**/xml/*.xml
configuration:
log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
map-underscore-to-camel-case: true
obs:
ak: VEHN7D0TJ9316H8AHCAV
......@@ -129,12 +136,23 @@ scoreRule:
translate:
url: http://114.116.116.241:8018/translate/getTranslateInfoWithTag
python:
#python打分接口
#python打分接口-弃用
scoreModelUrl: http://114.116.36.231:8080/score
#python开始执行接口
#python开始执行接口--弃用
subjectProcessorUrl: http://114.116.36.231:8085/subject/subject_processor
#判重
judgeDuplicateUrl: http://114.116.36.231:8080/subject/judg_duplicate
#热词抽取地址
hotWords:
extractUrl: http://114.116.99.6:8055/task/dispose/extractKeyword
\ No newline at end of file
judgeDuplicateUrl: http://1.95.13.40:8080/subject/judg_duplicate
#抽取关键词
keyWordsExtractUrl: http://1.95.91.200:7006/get_phrase/
jeecg:
shiro:
excludeUrls:
kafka:
topic:
subject:
run: subjectModelTest
clb:
subject:
default:
processing:
advanceMonth: 6
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论