提交 7116f7bf 作者: 925993793@qq.com

自定义专题-信息源绑定对接采集中心

上级 90b50cae
......@@ -4,6 +4,7 @@ import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
......@@ -27,11 +28,14 @@ import com.zzsn.event.xxljob.entity.XxlJobInfo;
import com.zzsn.event.xxljob.service.IXxlJobInfoService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.hwpf.HWPFDocument;
import org.apache.poi.hwpf.extractor.WordExtractor;
import org.apache.poi.ss.formula.functions.T;
import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.checkerframework.checker.units.qual.A;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
......@@ -114,6 +118,22 @@ public class SubjectManageController {
@Autowired
private CaiJiCenterHttpService caiJiCenterHttpService;
@Value("${infoSource.pageList:}")
private String INFO_SOURCE_PAGE_LIST;
@Value("${infoSource.columnList:}")
private String INFO_SOURCE_COLUMN_LIST;
@Value("${infoSource.add:}")
private String INFO_SOURCE_ADD;
@Value("${infoSource.waitInfoRemove:}")
private String WAIT_INFO_REMOVE;
@Value("${infoSource.columnDetail:}")
private String COLUMN_DETAIL;
@Value("${caiji.projectCode:zzsn_test}")
private String projectCode;
@Value("${caiji.projectName:克虏宝测试}")
private String projectName;
/**
* 专题列表-资讯转换时使用
*
......@@ -134,7 +154,7 @@ public class SubjectManageController {
* @date 2025/1/7
*/
@GetMapping("/dictItemList")
public Result<?> dictItemList(@RequestParam(defaultValue = "Thematic_Library")String dictCode){
public Result<?> dictItemList(@RequestParam(defaultValue = "Thematic_Library") String dictCode) {
List<SysDictItem> dictItemList = sysDictItemService.listByDictCode(dictCode);
return Result.OK(dictItemList);
}
......@@ -152,9 +172,9 @@ public class SubjectManageController {
public Result<?> researchCenterPageList(SubjectCondition subjectCondition,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
@RequestParam(required = false) String flagCode,HttpServletRequest request) {
@RequestParam(required = false) String flagCode, HttpServletRequest request) {
UserVo loginUser = UserUtil.getLoginUser();
Page<SubjectPage> pageList = subjectService.researchCenterPageList(subjectCondition, pageNo, pageSize,flagCode,request,loginUser.getUsername());
Page<SubjectPage> pageList = subjectService.researchCenterPageList(subjectCondition, pageNo, pageSize, flagCode, request, loginUser.getUsername());
return Result.OK(pageList);
}
......@@ -163,7 +183,7 @@ public class SubjectManageController {
*
* @param type 分类(1-个人;2-公开;3-个人+公开)
* @param subjectName 专题名称
* @param environment 环境 1-测试 2-正式
* @param environment 环境 1-测试 2-正式
* @param flagCode 数据权限code
* @param pageNo 当前页
* @param pageSize 每页返回条数
......@@ -179,7 +199,7 @@ public class SubjectManageController {
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
HttpServletRequest request) {
UserVo loginUser = UserUtil.getLoginUser();
Page<Node> pageList = subjectService.visiblePageList(loginUser.getUsername(), type, subjectName, pageNo, pageSize,environment,request,flagCode);
Page<Node> pageList = subjectService.visiblePageList(loginUser.getUsername(), type, subjectName, pageNo, pageSize, environment, request, flagCode);
return Result.OK(pageList);
}
......@@ -189,7 +209,7 @@ public class SubjectManageController {
*
* @param type 分类(1-个人;2-公开;3-个人+公开)
* @param subjectName 专题名称
* @param environment 环境 1-测试 2-正式
* @param environment 环境 1-测试 2-正式
* @param flagCode 数据权限code
* @param pageNo 当前页
* @param pageSize 每页返回条数
......@@ -198,14 +218,14 @@ public class SubjectManageController {
*/
@GetMapping("/visiblePageListNoSign")
public Result<?> visiblePageListNoSign(@RequestParam Integer type,
@RequestParam(required = false) String subjectName,
@RequestParam(required = false) String environment,
@RequestParam(required = false) String flagCode,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
HttpServletRequest request) {
@RequestParam(required = false) String subjectName,
@RequestParam(required = false) String environment,
@RequestParam(required = false) String flagCode,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
HttpServletRequest request) {
UserVo loginUser = UserUtil.getLoginUser();
Page<Node> pageList = subjectService.visiblePageList(loginUser.getUsername(), type, subjectName, pageNo, pageSize,environment,request,flagCode);
Page<Node> pageList = subjectService.visiblePageList(loginUser.getUsername(), type, subjectName, pageNo, pageSize, environment, request, flagCode);
return Result.OK(pageList);
}
......@@ -233,7 +253,7 @@ public class SubjectManageController {
* @date 2025/3/25
*/
@GetMapping(value = "/statisticInfo")
private Result<?> statisticInfo(@RequestParam List<String> subjectIds){
private Result<?> statisticInfo(@RequestParam List<String> subjectIds) {
List<SubjectStatisticInfo> statisticInfoList = subjectService.statisticInfo(subjectIds);
//异步更新专题统计信息表的数据
CompletableFuture.runAsync(() -> {
......@@ -409,7 +429,7 @@ public class SubjectManageController {
xxlJobInfoService.update(Wrappers.<XxlJobInfo>lambdaUpdate()
.eq(XxlJobInfo::getInfoSourceCode, byId.getSubjectCode())
.set(XxlJobInfo::getTriggerStatus, byId.getStatus()));
}else{
} else {
xxlJobInfoService.subjectInsert(byId);
}
......@@ -425,16 +445,17 @@ public class SubjectManageController {
cn.hutool.json.JSONObject entries = JSONUtil.parseObj(res);
//获取code状态码
Integer code = entries.getInt("code");
if (ObjectUtil.isEmpty(code) || code != 200) {
log.error("专题状态同步采集失败{}",res);
if (ObjectUtil.isEmpty(code) || code != 200) {
log.error("专题状态同步采集失败{}", res);
}
} catch (Exception e) {
log.error("专题状态同步采集失败{}",e.getMessage(),e);
log.error("专题状态同步采集失败{}", e.getMessage(), e);
}
return Result.OK();
}
/**
* 更新状态
*
......@@ -443,23 +464,24 @@ public class SubjectManageController {
*/
@PostMapping("/updateSubjectAllToCaiji")
public Result<?> updateSubjectAllToCaiji() {
List<Subject> list = subjectService.list(Wrappers.<Subject>lambdaQuery().eq(Subject::getSubjectType,1));
List<Subject> list = subjectService.list(Wrappers.<Subject>lambdaQuery().eq(Subject::getSubjectType, 1));
for (int i = 0; i < list.size(); i++) {
Subject subject = list.get(i);
log.info("专题状态同步采集{},index:{},total:{}",subject.getSubjectName(),i,list.size());
if (subject.getStatus()!=null && subject.getStatus() == 1) {
log.info("专题状态同步采集{},index:{},total:{}", subject.getSubjectName(), i, list.size());
if (subject.getStatus() != null && subject.getStatus() == 1) {
configurationMessageService.bindInfoSourceSend(subject.getId());
configurationMessageService.bindKeyWordsSend(subject.getId());
caiJiCenterHttpService.subjectStatusEdit(subject.getStatus(), subject.getId());
configurationMessageService.subjectEnterpriseSourceSync(subject.getId());
log.info("专题配置信息同步采集成功:{}",subject.getSubjectName());
}else{
log.info("专题配置信息同步采集成功:{}", subject.getSubjectName());
} else {
caiJiCenterHttpService.delSubject(subject.getId());
log.info("通知采集删除专题成功:{}",subject.getSubjectName());
log.info("通知采集删除专题成功:{}", subject.getSubjectName());
}
}
return Result.OK();
}
/**
* 更新状态
*
......@@ -471,14 +493,14 @@ public class SubjectManageController {
List<Event> list = eventService.list();
for (int i = 0; i < list.size(); i++) {
Event event = list.get(i);
log.info("事件状态同步采集{},index:{},total:{}",event.getEventName(),i,list.size());
if (event.getStatus()!=null && event.getStatus() == 1) {
log.info("事件状态同步采集{},index:{},total:{}", event.getEventName(), i, list.size());
if (event.getStatus() != null && event.getStatus() == 1) {
configurationMessageService.bindKeyWordsEventSend(event.getId());
caiJiCenterHttpService.subjectStatusEdit(event.getStatus(), event.getId());
log.info("事件配置信息同步采集成功:{}",event.getEventName());
}else{
log.info("事件配置信息同步采集成功:{}", event.getEventName());
} else {
caiJiCenterHttpService.delSubject(event.getId());
log.info("通知采集删除事件成功:{}",event.getEventName());
log.info("通知采集删除事件成功:{}", event.getEventName());
}
}
return Result.OK();
......@@ -505,11 +527,11 @@ public class SubjectManageController {
cn.hutool.json.JSONObject entries = JSONUtil.parseObj(res);
//获取code状态码
Integer code = entries.getInt("code");
if (ObjectUtil.isEmpty(code) || code != 200) {
log.error("专题删除同步采集失败{}",res);
if (ObjectUtil.isEmpty(code) || code != 200) {
log.error("专题删除同步采集失败{}", res);
}
} catch (Exception e) {
log.error("专题删除同步采集失败{}",e.getMessage(),e);
log.error("专题删除同步采集失败{}", e.getMessage(), e);
}
return Result.OK();
}
......@@ -539,11 +561,11 @@ public class SubjectManageController {
cn.hutool.json.JSONObject entries = JSONUtil.parseObj(res);
//获取code状态码
Integer code = entries.getInt("code");
if (ObjectUtil.isEmpty(code) || code != 200) {
log.error("专题删除同步采集失败{}",res);
if (ObjectUtil.isEmpty(code) || code != 200) {
log.error("专题删除同步采集失败{}", res);
}
} catch (Exception e) {
log.error("专题删除同步采集失败{}",e.getMessage(),e);
log.error("专题删除同步采集失败{}", e.getMessage(), e);
}
}
......@@ -618,8 +640,8 @@ public class SubjectManageController {
* @return
*/
@GetMapping(value = "/bindKeyWordsList")
public Result<?> bindKeyWordsList(@RequestParam(name = "id") String id,@RequestParam(name = "bindingType",required = false) String bindingType) {
List<KeyWordsPage> bindKeyWordsList = subjectKeywordsMapService.bindKeyWordsList(id,bindingType);
public Result<?> bindKeyWordsList(@RequestParam(name = "id") String id, @RequestParam(name = "bindingType", required = false) String bindingType) {
List<KeyWordsPage> bindKeyWordsList = subjectKeywordsMapService.bindKeyWordsList(id, bindingType);
return Result.OK(bindKeyWordsList);
}
......@@ -784,7 +806,6 @@ public class SubjectManageController {
}
/**
* 专题打分模型配置信息详情
*
......@@ -861,16 +882,18 @@ public class SubjectManageController {
List<InfoSourceLabelVO> bindList = clbLabelService.bindInfoSourceLabelInfo(subjectId);
return Result.OK(bindList);
}
/**
* 更新subject的数据范围
*
* @param subjectId 专题id
*/
@GetMapping("/subjectUpdateScope")
public Result<?> subjectUpdateScope(@RequestParam(name = "subjectId") String subjectId,
@RequestParam(name = "type") String type,
@RequestParam(name = "scop" , defaultValue = "") String scop
) {
Subject subject = subjectService.subjectUpdateScope(subjectId,type,scop);
@RequestParam(name = "type") String type,
@RequestParam(name = "scop", defaultValue = "") String scop
) {
Subject subject = subjectService.subjectUpdateScope(subjectId, type, scop);
return Result.OK(subject);
}
......@@ -922,7 +945,7 @@ public class SubjectManageController {
/**
* 专题绑定的定向信息源分页列表(专题绑定定向信息源时使用-研究中心)
* 专题绑定的定向信息源分页列表(专题绑定定向信息源时使用-研究中心)
*
* @param subjectId 专题id
* @param pageNo 当前页
......@@ -932,73 +955,195 @@ public class SubjectManageController {
*/
@GetMapping("/subjectBindDirectSourcePageList")
public Result<?> subjectBindDirectSourcePageList(@RequestParam String subjectId,
@RequestParam(required = false) String webSiteName,
@RequestParam(defaultValue = "1") Integer pageNo,
@RequestParam(defaultValue = "10") Integer pageSize) {
IPage<SubjectBindDirectSourceVO> page = infoSourceService.subjectBindDirectSourcePageList(subjectId, pageNo, pageSize);
IPage<SubjectBindDirectSourceVO> page = infoSourceService.subjectBindDirectSourcePageList(subjectId, webSiteName, pageNo, pageSize);
return Result.OK(page);
}
/**
* 专题已绑定的定向信息源栏目分页列表(专题绑定定向信息源时使用-研究中心)
*
* @param subjectId 专题id
* @param infoSourceId 信息源id
* @author lkg
* @date 2025/1/4
*/
@GetMapping("/subjectBindDirectSourceColumnPageList")
public Result<?> subjectBindDirectSourceColumnPageList(@RequestParam String subjectId,
@RequestParam String infoSourceId) {
//IPage<SubjectBindDirectSourceVO> page = infoSourceService.subjectBindDirectSourcePageList(subjectId, pageNo, pageSize);
String response = remoteGetColumnList(infoSourceId, null, null, 1, 1000);
JSONObject jsonObject = JSONObject.parseObject(response);
if (jsonObject.getInteger("code") != 200) {
return Result.FAIL(jsonObject.getString("message"));
} else {
List<InfoSource> bindSourceColumnList = infoSourceService.subjectBindSourceColumnList(subjectId, null);
JSONArray jsonArray = jsonObject.getJSONArray("result");
List<String> ids = new ArrayList<>();
for (Object column : jsonArray) {
JSONObject columnJson = JSONObject.parseObject(JSONObject.toJSONString(column));
ids.add(columnJson.getString("id"));
}
List<InfoSource> collect = bindSourceColumnList.stream().filter(infoSource -> ids.contains(infoSource.getId())).collect(Collectors.toList());
return Result.OK(collect);
}
}
/**
* 信息源分页列表(专题绑定定向信息源时使用-研究中心)
*
* @param searchWord 搜索词
* @param pageNo 当前页
* @param pageSize 返回条数
* @param infoName 信息源名称
* @param infoAddress 信息源地址
* @param pageNo 当前页
* @param pageSize 返回条数
* @author lkg
* @date 2025/1/4
*/
@GetMapping("/directInfoSourcePageList")
public Result<?> directSourcePageList(@RequestParam(required = false) String searchWord,
public Result<?> directSourcePageList(@RequestParam(required = false) String infoName,
@RequestParam(required = false) String infoAddress,
@RequestParam(defaultValue = "1") Integer pageNo,
@RequestParam(defaultValue = "10") Integer pageSize) {
IPage<SubjectBindDirectSourceVO> page = infoSourceService.directSourcePageList(searchWord, pageNo, pageSize);
return Result.OK(page);
//IPage<SubjectBindDirectSourceVO> page = infoSourceService.directSourcePageList(searchWord, pageNo, pageSize);
Map<String, String> variables = new HashMap<>();
variables.put("infoName", infoName);
variables.put("infoAddress", infoAddress);
variables.put("pageNo", pageNo.toString());
variables.put("pageSize", pageSize.toString());
String response = HttpUtil.doGet(INFO_SOURCE_PAGE_LIST, variables, null, "utf-8");
JSONObject jsonObject = JSONObject.parseObject(response);
if (jsonObject.getInteger("code") != 200) {
return Result.FAIL("采集中心接口异常:" + jsonObject.getString("message"));
} else {
IPage<Object> page = new Page<>(pageNo, pageSize);
page.setTotal(jsonObject.getLong("total"));
page.setRecords(jsonObject.getJSONArray("result"));
return Result.OK(page);
}
}
/**
* 信息源下栏目分页列表(专题绑定定向信息源时使用-研究中心)
*
* @param infoSourceId 信息源id
* @param columnName 栏目名称
* @param columnAddress 栏目地址
* @param pageNo 当前页
* @param pageSize 返回条数
* @author lkg
* @date 2025/1/4
*/
@GetMapping("/directSourceColumnPageList")
public Result<?> directSourceColumnPageList(@RequestParam String infoSourceId,
@RequestParam(required = false) String columnName,
@RequestParam(required = false) String columnAddress,
@RequestParam(defaultValue = "1") Integer pageNo,
@RequestParam(defaultValue = "10") Integer pageSize) {
//IPage<SubjectBindDirectSourceVO> page = infoSourceService.directSourcePageList(searchWord, pageNo, pageSize);
String response = remoteGetColumnList(infoSourceId, columnName, columnAddress, pageNo, pageSize);
JSONObject jsonObject = JSONObject.parseObject(response);
if (jsonObject.getInteger("code") != 200) {
return Result.FAIL(jsonObject.getString("message"));
} else {
IPage<Object> page = new Page<>(pageNo, pageSize);
page.setTotal(jsonObject.getLong("total"));
page.setRecords(jsonObject.getJSONArray("result"));
return Result.OK(page);
}
}
private String remoteGetColumnList(String infoSourceId, String columnName, String columnAddress, Integer pageNo, Integer pageSize) {
Map<String, String> variables = new HashMap<>();
variables.put("infoSourceId", infoSourceId);
variables.put("columnName", columnName);
variables.put("columnAddress", columnAddress);
variables.put("pageNo", pageNo.toString());
variables.put("pageSize", pageSize.toString());
return HttpUtil.doGet(INFO_SOURCE_COLUMN_LIST, variables, null, "utf-8");
}
/**
* 添加定向信息源(专题绑定定向信息源时使用-研究中心)
*
* @param subjectSourceVO 参数
* @param params 参数
* @author lkg
* @date 2025/1/4
*/
@PostMapping("/bindDirectInfoSource")
public Result<?> bindDirectInfoSource(@RequestBody SubjectSourceVO subjectSourceVO) {
List<String> sourceIdList = subjectSourceVO.getSourceIdList();
if (CollectionUtils.isEmpty(sourceIdList)) {
LambdaQueryWrapper<InfoSource> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.select(InfoSource::getId,InfoSource::getSiteName,InfoSource::getSiteUri)
.in(InfoSource::getWebSiteName, subjectSourceVO.getWebSiteNameList())
.groupBy(InfoSource::getId);
List<InfoSource> infoSources = infoSourceService.list(queryWrapper);
if (CollectionUtils.isNotEmpty(infoSources)) {
sourceIdList = infoSources.stream().map(InfoSource::getId).collect(Collectors.toList());
public Result<?> bindDirectInfoSource(@RequestBody JSONObject params) {
String subjectId = params.getString("subjectId");
Object bindInfoSourceList = params.get("bindInfoSourceList");
if (ObjectUtils.isEmpty(bindInfoSourceList)) {
return Result.FAIL("请选择信息源");
}
List<SubjectBindSourceVO> subjectBindSourceVOS = JSON.parseArray(JSON.toJSONString(bindInfoSourceList), SubjectBindSourceVO.class);
List<String> columnIds = new ArrayList<>();
List<String> infoSourceIds = new ArrayList<>();
for (SubjectBindSourceVO subjectBindSourceVO : subjectBindSourceVOS) {
infoSourceIds.add(subjectBindSourceVO.getWebSiteId());
List<String> infoColumnIds = subjectBindSourceVO.getColumnIds();
if (CollectionUtils.isEmpty(infoColumnIds)) {
String response = remoteGetColumnList(subjectBindSourceVO.getWebSiteId(), null, null, 1, 1000);
JSONObject jsonObject = JSONObject.parseObject(response);
if (jsonObject.getInteger("code") == 200) {
JSONArray jsonArray = jsonObject.getJSONArray("result");
for (Object column : jsonArray) {
JSONObject columnJson = JSONObject.parseObject(JSONObject.toJSONString(column));
columnIds.add(columnJson.getString("id"));
}
}
} else {
columnIds.addAll(infoColumnIds);
}
}
if (CollectionUtils.isNotEmpty(sourceIdList)) {
LambdaQueryWrapper<SubjectInfoSourceMap> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.eq(SubjectInfoSourceMap::getSubjectId, subjectSourceVO.getSubjectId())
.in(SubjectInfoSourceMap::getSourceId, sourceIdList)
.eq(SubjectInfoSourceMap::getType, 1);
List<SubjectInfoSourceMap> list = subjectInfoSourceMapService.list(queryWrapper);
if (CollectionUtils.isNotEmpty(list)) {
List<String> existSourceIdList = list.stream().map(SubjectInfoSourceMap::getSourceId).collect(Collectors.toList());
sourceIdList.removeAll(existSourceIdList);
//专题-信息源关系绑定
LambdaQueryWrapper<SubjectInfoSourceMiddleMap> query = Wrappers.lambdaQuery();
query.eq(SubjectInfoSourceMiddleMap::getSubjectId, subjectId)
.in(SubjectInfoSourceMiddleMap::getInfoSourceId, infoSourceIds);
List<SubjectInfoSourceMiddleMap> middleMapList = subjectInfoSourceMiddleMapService.list(query);
if (CollectionUtils.isNotEmpty(middleMapList)) {
List<String> existInfoSourceIdList = middleMapList.stream().map(SubjectInfoSourceMiddleMap::getInfoSourceId).collect(Collectors.toList());
infoSourceIds.removeAll(existInfoSourceIdList);
}
if (CollectionUtils.isNotEmpty(infoSourceIds)) {
List<SubjectInfoSourceMiddleMap> dataList = new ArrayList<>();
List<SubjectBindSourceVO> collect = subjectBindSourceVOS.stream().filter(subjectBindSourceVO -> infoSourceIds.contains(subjectBindSourceVO.getWebSiteId())).collect(Collectors.toList());
for (SubjectBindSourceVO subjectBindSourceVO : collect) {
SubjectInfoSourceMiddleMap subjectInfoSourceMiddleMap = new SubjectInfoSourceMiddleMap();
subjectInfoSourceMiddleMap.setSubjectId(subjectId);
subjectInfoSourceMiddleMap.setInfoSourceId(subjectBindSourceVO.getWebSiteId());
subjectInfoSourceMiddleMap.setInfoSourceName(subjectBindSourceVO.getWebSiteName());
subjectInfoSourceMiddleMap.setStatus(1);
dataList.add(subjectInfoSourceMiddleMap);
}
if (CollectionUtils.isNotEmpty(sourceIdList)) {
List<SubjectInfoSourceMap> dataList = new ArrayList<>();
for (String sourceId : sourceIdList) {
SubjectInfoSourceMap subjectInfoSourceMap = new SubjectInfoSourceMap();
subjectInfoSourceMap.setSubjectId(subjectSourceVO.getSubjectId());
subjectInfoSourceMap.setSourceId(sourceId);
subjectInfoSourceMap.setType(1);
dataList.add(subjectInfoSourceMap);
}
subjectInfoSourceMapService.saveBatch(dataList);
subjectInfoSourceMiddleMapService.saveBatch(dataList);
}
//专题栏目关系绑定
LambdaQueryWrapper<SubjectInfoSourceMap> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.eq(SubjectInfoSourceMap::getSubjectId, subjectId)
.in(SubjectInfoSourceMap::getSourceId, columnIds)
.eq(SubjectInfoSourceMap::getType, 1);
List<SubjectInfoSourceMap> list = subjectInfoSourceMapService.list(queryWrapper);
if (CollectionUtils.isNotEmpty(list)) {
List<String> existSourceIdList = list.stream().map(SubjectInfoSourceMap::getSourceId).collect(Collectors.toList());
columnIds.removeAll(existSourceIdList);
}
if (CollectionUtils.isNotEmpty(columnIds)) {
List<SubjectInfoSourceMap> dataList = new ArrayList<>();
for (String sourceId : columnIds) {
SubjectInfoSourceMap subjectInfoSourceMap = new SubjectInfoSourceMap();
subjectInfoSourceMap.setSubjectId(subjectId);
subjectInfoSourceMap.setSourceId(sourceId);
subjectInfoSourceMap.setType(1);
dataList.add(subjectInfoSourceMap);
}
subjectInfoSourceMapService.saveBatch(dataList);
}
//同步配置到采集
configurationMessageService.bindInfoSourceSend(subjectSourceVO.getSubjectId());
configurationMessageService.bindInfoSourceSend(subjectId);
return Result.OK();
}
......@@ -1012,7 +1157,7 @@ public class SubjectManageController {
*/
@PostMapping("/addInfoSource")
public Result<?> addInfoSource(@RequestBody SubjectInfoSourceMiddleMap subjectInfoSourceMiddleMap) {
//专题信息源关系数据
/*//专题信息源关系数据
Set<SubjectInfoSourceMap> mapDataSet = new HashSet<>();
//待配置信息源新增列表
Set<SubjectInfoSourceMiddleMap> configDataSet = new HashSet<>();
......@@ -1025,6 +1170,29 @@ public class SubjectManageController {
}
if (CollectionUtils.isNotEmpty(configDataSet)) {
subjectInfoSourceMiddleMapService.saveBatch(configDataSet);
}*/
try {
JSONObject params = new JSONObject();
JSONObject source = new JSONObject();
source.put("sourceName", subjectInfoSourceMiddleMap.getInfoSourceName());
source.put("address", subjectInfoSourceMiddleMap.getUrl());
params.put("configSource", source);
params.put("customerSign", projectCode);
params.put("customerName", projectName);
params.put("subjectSign", subjectInfoSourceMiddleMap.getSubjectId());
String response = HttpUtil.doPost(INFO_SOURCE_ADD, params, 60 * 1000);
JSONObject jsonObject = JSONObject.parseObject(response);
if (jsonObject.getInteger("code") == 204) {
return Result.FAIL(jsonObject.getString("message"));
} else if (jsonObject.getInteger("code") == 200) {
JSONObject result = jsonObject.getJSONObject("result");
subjectInfoSourceMiddleMap.setInfoSourceId(result.getString("id"));
subjectInfoSourceMiddleMap.setStatus(0);
subjectInfoSourceMiddleMapService.save(subjectInfoSourceMiddleMap);
return Result.OK();
}
} catch (IOException e) {
e.printStackTrace();
}
return Result.OK();
}
......@@ -1039,10 +1207,50 @@ public class SubjectManageController {
@PostMapping("/bindRecommendInfoSource")
public Result<?> bindRecommendInfoSource(@RequestBody List<SubjectInfoSourceMap> params) {
if (CollectionUtils.isNotEmpty(params)) {
params.forEach(item -> item.setType(1));
String subjectId = params.get(0).getSubjectId();
List<String> columnIds = new ArrayList<>();
params.forEach(item -> {
item.setType(1);
columnIds.add(item.getSourceId());
});
subjectInfoSourceMapService.saveBatch(params);
Map<String, String> variables = new HashMap<>();
variables.put("columnIds", String.join(",", columnIds));
String response = HttpUtil.doGet(COLUMN_DETAIL, variables, null, "utf-8");
JSONObject jsonObject = JSONObject.parseObject(response);
if (jsonObject.getInteger("code") != 200) {
return Result.FAIL("采集中心接口异常:" + jsonObject.getString("message"));
} else {
String result = jsonObject.getString("result");
List<JSONObject> jsonObjects = JSON.parseArray(result, JSONObject.class);
Set<String> infoSourceIds = jsonObjects.stream().collect(Collectors.groupingBy(item -> item.getString("infoSourceId"))).keySet();
LambdaQueryWrapper<SubjectInfoSourceMiddleMap> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.eq(SubjectInfoSourceMiddleMap::getSubjectId, subjectId).in(SubjectInfoSourceMiddleMap::getInfoSourceId, infoSourceIds);
List<SubjectInfoSourceMiddleMap> middleMapList = subjectInfoSourceMiddleMapService.list(queryWrapper);
if (CollectionUtils.isNotEmpty(middleMapList)) {
List<String> collect = infoSourceIds.stream().filter(item -> middleMapList.stream().noneMatch(e -> e.getInfoSourceId().equals(item))).collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(collect)) {
infoSourceIds = new HashSet<>(collect);
Set<String> finalInfoSourceIds = infoSourceIds;
jsonObjects = jsonObjects.stream().filter(item -> finalInfoSourceIds.contains(item.getString("infoSourceId"))).collect(Collectors.toList());
}
}
if (CollectionUtils.isNotEmpty(jsonObjects)) {
List<SubjectInfoSourceMiddleMap> dataList = new ArrayList<>();
for (JSONObject infoSource : jsonObjects) {
SubjectInfoSourceMiddleMap middleMap = new SubjectInfoSourceMiddleMap();
middleMap.setSubjectId(subjectId);
middleMap.setInfoSourceId(infoSource.getString("infoSourceId"));
middleMap.setInfoSourceName(infoSource.getString("infoName"));
middleMap.setUrl(infoSource.getString("infoAddress"));
middleMap.setStatus(1);
dataList.add(middleMap);
}
subjectInfoSourceMiddleMapService.saveBatch(dataList);
}
}
//同步配置到采集
configurationMessageService.bindInfoSourceSend(params.get(0).getSubjectId());
//configurationMessageService.bindInfoSourceSend(params.get(0).getSubjectId());
}
return Result.OK();
}
......@@ -1055,26 +1263,48 @@ public class SubjectManageController {
* @date 2025/1/9
*/
@PostMapping("/removeDirectInfoSource")
public Result<?> removeDirectInfoSource(@RequestBody List<SubjectBindDirectSourceVO> params){
public Result<?> removeDirectInfoSource(@RequestBody List<SubjectBindDirectSourceVO> params) {
for (SubjectBindDirectSourceVO param : params) {
String subjectId = param.getSubjectId();
String webSiteName = param.getWebSiteName();
Integer status = param.getStatus();
if (status == 1) {
List<String> sourceIdList;
String webSiteId = param.getWebSiteId();
List<String> columnIdList = new ArrayList<>();
List<InfoSource> children = param.getChildren();
String response = remoteGetColumnList(webSiteId, null, null, 1, 1000);
JSONObject jsonObject = JSONObject.parseObject(response);
if (jsonObject.getInteger("code") == 200) {
JSONArray jsonArray = jsonObject.getJSONArray("result");
for (Object column : jsonArray) {
JSONObject columnJson = JSONObject.parseObject(JSONObject.toJSONString(column));
columnIdList.add(columnJson.getString("id"));
}
}
//是否删除信息源下的所有栏目
boolean isAllFlag = true;
List<String> sourceIdList;
if (CollectionUtils.isNotEmpty(children)) {
sourceIdList = children.stream().map(InfoSource::getId).collect(Collectors.toList());
if (!CollectionUtils.isEqualCollection(sourceIdList, columnIdList)) {
isAllFlag = false;
}
} else {
List<String> webSiteNameList = new ArrayList<>();
webSiteNameList.add(webSiteName);
List<InfoSource> infoSources = infoSourceService.subjectBindSourceColumnList(subjectId,webSiteNameList);
sourceIdList = infoSources.stream().map(InfoSource::getId).collect(Collectors.toList());
sourceIdList = columnIdList;
}
if (CollectionUtils.isNotEmpty(sourceIdList)) {
subjectInfoSourceMapService.remove(new LambdaQueryWrapper<SubjectInfoSourceMap>()
.eq(SubjectInfoSourceMap::getSubjectId, subjectId)
.in(SubjectInfoSourceMap::getSourceId, sourceIdList));
}
subjectInfoSourceMapService.remove(new LambdaQueryWrapper<SubjectInfoSourceMap>()
.eq(SubjectInfoSourceMap::getSubjectId, subjectId)
.in(SubjectInfoSourceMap::getSourceId, sourceIdList));
if (isAllFlag) {
subjectInfoSourceMiddleMapService.remove(new LambdaQueryWrapper<SubjectInfoSourceMiddleMap>()
.eq(SubjectInfoSourceMiddleMap::getSubjectId, subjectId)
.eq(SubjectInfoSourceMiddleMap::getInfoSourceId, webSiteId));
}
} else if (status == 0) {
String webSiteName = param.getWebSiteName();
subjectInfoSourceMiddleMapService.remove(new LambdaQueryWrapper<SubjectInfoSourceMiddleMap>()
.eq(SubjectInfoSourceMiddleMap::getSubjectId, subjectId)
.eq(SubjectInfoSourceMiddleMap::getInfoSourceName, webSiteName));
......@@ -1085,6 +1315,50 @@ public class SubjectManageController {
return Result.OK();
}
/*
* 清空定向信息源(专题绑定定向信息源时使用-研究中心)
*
* @param subjectId
* @author lkg
* @date 2025/5/23
*/
@GetMapping("/clearDirectInfoSource")
public Result<?> clearDirectInfoSource(@RequestParam String subjectId) {
//删除所有定向信息源
subjectInfoSourceMapService.remove(new LambdaQueryWrapper<SubjectInfoSourceMap>()
.eq(SubjectInfoSourceMap::getSubjectId, subjectId)
.eq(SubjectInfoSourceMap::getType, 1));
//带配置信息源列表
List<SubjectInfoSourceMiddleMap> waitConfigSourceList = subjectInfoSourceMiddleMapService.list(new LambdaQueryWrapper<SubjectInfoSourceMiddleMap>()
.eq(SubjectInfoSourceMiddleMap::getSubjectId, subjectId).eq(SubjectInfoSourceMiddleMap::getStatus, 0));
if (CollectionUtils.isNotEmpty(waitConfigSourceList)) {
//调用采集中心接口,删除带配置逻辑
List<String> waitConfigSourceIds = waitConfigSourceList.stream().map(SubjectInfoSourceMiddleMap::getInfoSourceId).collect(Collectors.toList());
Map<String, String> variables = new HashMap<>();
variables.put("waitConfigIds", String.join(",", waitConfigSourceIds));
HttpUtil.doGet(WAIT_INFO_REMOVE, variables, null, "utf-8");
}
subjectInfoSourceMiddleMapService.remove(new LambdaQueryWrapper<SubjectInfoSourceMiddleMap>()
.eq(SubjectInfoSourceMiddleMap::getSubjectId, subjectId));
//同步配置到采集
configurationMessageService.bindInfoSourceSend(subjectId);
return Result.OK();
}
/**
* 专题绑定的定向信息源列表(专题配置校验时使用)
*
* @param subjectId 专题id
* @author lkg
* @date 2025/5/23
*/
@GetMapping("/subjectBindSourceColumnList")
public Result<?> subjectBindSourceColumnList(@RequestParam String subjectId) {
List<InfoSource> bindSourceColumnList = infoSourceService.subjectBindSourceColumnList(subjectId, null);
return Result.OK(bindSourceColumnList);
}
/**
* 信息源导入模板下载-研究中心
*
......@@ -1117,9 +1391,9 @@ public class SubjectManageController {
if (originalFilename.endsWith(".xlsx") || originalFilename.endsWith(".xls")) {
List<List<String>> dataList;
if (originalFilename.endsWith(".xls")) {
dataList = ExcelExportUtil.readExcelXls(file.getInputStream(), false,1, 2);
dataList = ExcelExportUtil.readExcelXls(file.getInputStream(), false, 1, 2);
} else {
dataList = ExcelExportUtil.readExcelXlsx(file.getInputStream(), false,1, 2);
dataList = ExcelExportUtil.readExcelXlsx(file.getInputStream(), false, 1, 2);
}
if (CollectionUtils.isNotEmpty(dataList)) {
boolean emptyFlag = false;
......@@ -1245,7 +1519,7 @@ public class SubjectManageController {
sourceIdList.add(countVO.getName());
}
LambdaQueryWrapper<InfoSource> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.select(InfoSource::getId,InfoSource::getSiteName,InfoSource::getSiteUri).in(InfoSource::getId,sourceIdList);
queryWrapper.select(InfoSource::getId, InfoSource::getSiteName, InfoSource::getSiteUri).in(InfoSource::getId, sourceIdList);
infoSources = infoSourceService.list(queryWrapper);
}
return Result.OK(infoSources);
......@@ -1394,7 +1668,7 @@ public class SubjectManageController {
String originalFilename = file.getOriginalFilename();
if (originalFilename.endsWith(".txt")) {
parseTxt(text, file);
} else if (originalFilename.endsWith(".docx")){
} else if (originalFilename.endsWith(".docx")) {
parseWordDocx(text, file);
} else if (originalFilename.endsWith(".doc")) {
parseWordDoc(text, file);
......@@ -1418,7 +1692,7 @@ public class SubjectManageController {
}
private List<StatisticsKeyWordVo> formatWordInfo(String text,List<String> wordsList){
private List<StatisticsKeyWordVo> formatWordInfo(String text, List<String> wordsList) {
List<StatisticsKeyWordVo> articleWordList = new ArrayList<>();
Map<String, Integer> hitWordsAndTimes = HanlpUtil.getHitWordsAndTimes(wordsList, text);
for (Map.Entry<String, Integer> entry : hitWordsAndTimes.entrySet()) {
......
......@@ -31,12 +31,18 @@ public class SubjectInfoSourceMiddleMap implements Serializable {
/**专题id*/
@TableField("subject_id")
private String subjectId;
/**信息源id或信息源组id*/
/**信息源id*/
@TableField("info_source_id")
private String infoSourceId;
/**信息源名称*/
@TableField("info_source_name")
private String infoSourceName;
/**网址*/
@TableField("url")
private String url;
/**状态(1-已有;0-待配置)*/
@TableField("status")
private Integer status;
/**创建时间*/
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
......
......@@ -89,7 +89,7 @@ public interface InfoSourceMapper extends BaseMapper<InfoSource> {
* @author lkg
* @date 2025/1/6
*/
Page<SubjectBindDirectSourceVO> subjectBindDirectInfoSourceList(@Param("subjectId") String subjectId, Page<Map<String, Object>> page);
Page<SubjectBindDirectSourceVO> subjectBindDirectInfoSourceList(@Param("subjectId") String subjectId ,@Param("webSiteName") String webSiteName, Page<Map<String, Object>> page);
/**
......
......@@ -335,7 +335,7 @@
<select id="subjectBindDirectInfoSourceList" resultType="com.zzsn.event.vo.SubjectBindDirectSourceVO">
select a.*
<!--select a.*
from (
select iso.web_site_name as webSiteName, 1 as `status`
from info_source iso
......@@ -348,7 +348,13 @@
where subject_id = #{subjectId}
group by info_source_name
) a
order by a.status desc, a.webSiteName asc
order by a.status desc, a.webSiteName asc-->
select info_source_id as webSiteId,info_source_name as webSiteName, `status`
from subject_info_source_middle_map
where subject_id = #{subjectId}
<if test="webSiteName!=null and webSiteName != ''">
and info_source_name like CONCAT('%',#{webSiteName},'%')
</if>
</select>
<select id="infoSourcePageList" resultType="com.zzsn.event.vo.SubjectBindDirectSourceVO">
......
......@@ -60,12 +60,13 @@ public interface IInfoSourceService extends IService<InfoSource> {
* 专题绑定的定向信息源分页列表(专题绑定定向信息源时使用-研究中心)
*
* @param subjectId 专题id
* @param webSiteName 信息源名称
* @param pageNo 当前页
* @param pageSize 返回条数
* @author lkg
* @date 2025/1/4
*/
IPage<SubjectBindDirectSourceVO> subjectBindDirectSourcePageList(String subjectId, Integer pageNo, Integer pageSize);
IPage<SubjectBindDirectSourceVO> subjectBindDirectSourcePageList(String subjectId,String webSiteName, Integer pageNo, Integer pageSize);
/**
* 信息源分页列表(专题绑定定向信息源时使用-研究中心)
......
......@@ -127,10 +127,9 @@ public class InfoSourceServiceImpl extends ServiceImpl<InfoSourceMapper, InfoSou
}
@Override
public IPage<SubjectBindDirectSourceVO> subjectBindDirectSourcePageList(String subjectId, Integer pageNo, Integer pageSize) {
public IPage<SubjectBindDirectSourceVO> subjectBindDirectSourcePageList(String subjectId,String webSiteName, Integer pageNo, Integer pageSize) {
Page<Map<String, Object>> page = new Page<>(pageNo, pageSize);
Page<SubjectBindDirectSourceVO> pageList = baseMapper.subjectBindDirectInfoSourceList(subjectId, page);
List<SubjectBindDirectSourceVO> records = pageList.getRecords();
/*List<SubjectBindDirectSourceVO> records = pageList.getRecords();
if (CollectionUtils.isNotEmpty(records)) {
List<String> webSiteNames = records.stream().map(SubjectBindDirectSourceVO::getWebSiteName).collect(Collectors.toList());
List<InfoSource> bindSourceColumnList = this.subjectBindSourceColumnList(subjectId, webSiteNames);
......@@ -142,8 +141,8 @@ public class InfoSourceServiceImpl extends ServiceImpl<InfoSourceMapper, InfoSou
subjectBindDirectSourceVO.setChildren(infoSources);
}
}
}
return pageList;
}*/
return baseMapper.subjectBindDirectInfoSourceList(subjectId,webSiteName, page);
}
@Override
......
package com.zzsn.event.task;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.zzsn.event.entity.InfoSource;
import com.zzsn.event.entity.SubjectInfoSourceMap;
import com.zzsn.event.entity.SubjectInfoSourceMiddleMap;
import com.zzsn.event.service.IInfoSourceService;
import com.zzsn.event.service.ISubjectInfoSourceMapService;
import com.zzsn.event.service.SubjectInfoSourceMiddleMapService;
import com.zzsn.event.util.HttpUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* 待配置信息源,状态监控更新任务(和采集中心交互)
*
* @author lkg
* @date 2025/5/23
*/
@Slf4j
@Component
public class WaitConfigInfoSourceTask {
@Value("${infoSource.columnListByWait:}")
private String columnListByWait;
@Autowired
private ISubjectInfoSourceMapService subjectInfoSourceMapService;
@Autowired
private SubjectInfoSourceMiddleMapService subjectInfoSourceMiddleMapService;
@Autowired
private IInfoSourceService infoSourceService;
@Value("${scheduling.yjzxEnable:false}")
Boolean yjzxEnable;
/**
* 每6小时,执行一次
*/
@Scheduled(cron = "0 0 0/6 * * ?")
public void syncConfig() {
if(!yjzxEnable){
return;
}
List<SubjectInfoSourceMiddleMap> waitConfigSourceList = subjectInfoSourceMiddleMapService.list(
new LambdaQueryWrapper<SubjectInfoSourceMiddleMap>()
.eq(SubjectInfoSourceMiddleMap::getStatus, 0)
);
if (CollectionUtils.isNotEmpty(waitConfigSourceList)) {
for (SubjectInfoSourceMiddleMap middleMap : waitConfigSourceList) {
String subjectId = middleMap.getSubjectId();
String waitForConfigId = middleMap.getInfoSourceId();
JSONObject jsonObject = getColumnListByWait(waitForConfigId);
if (jsonObject != null) {
middleMap.setStatus(1);
middleMap.setInfoSourceId(jsonObject.getString("infoSourceId"));
middleMap.setInfoSourceName(jsonObject.getString("infoSourceName"));
subjectInfoSourceMiddleMapService.updateById(middleMap);
List<JSONObject> columnList = JSON.parseArray(jsonObject.getString("columnList"), JSONObject.class);
List<SubjectInfoSourceMap> list = new ArrayList<>();
List<InfoSource> infoSourceList = new ArrayList<>();
for (JSONObject column : columnList) {
SubjectInfoSourceMap sms = new SubjectInfoSourceMap();
sms.setSubjectId(subjectId);
sms.setSourceId(column.getString("id"));
sms.setType(1);
list.add(sms);
//信息源
InfoSource infoSource = new InfoSource();
infoSource.setId(column.getString("id"));
infoSource.setSiteName(column.getString("columnName"));
infoSource.setSiteUri(column.getString("columnAddress"));
infoSource.setWebSiteName(jsonObject.getString("infoSourceName"));
infoSourceList.add(infoSource);
}
subjectInfoSourceMapService.saveBatch(list);
if (CollectionUtils.isNotEmpty(infoSourceList)) {
infoSourceService.saveOrUpdateBatch(infoSourceList);
}
log.info("待配置信息源-{},对应已配置信息源-{},同步更新完成",waitForConfigId, jsonObject.getString("infoSourceId"));
}
}
}
}
private JSONObject getColumnListByWait(String waitConfigId) {
JSONObject result = null;
Map<String, String> variables = new HashMap<>();
variables.put("waitConfigId", waitConfigId);
String response = HttpUtil.doGet(columnListByWait, variables, null, "utf-8");
JSONObject jsonObject = JSONObject.parseObject(response);
if (jsonObject.getInteger("code") == 200) {
result = jsonObject.getJSONObject("result");
}
return result;
}
}
......@@ -15,6 +15,8 @@ import java.util.List;
public class SubjectBindDirectSourceVO {
/**专题id*/
private String subjectId;
/**信息源id*/
private String webSiteId;
/**信息源名称*/
private String webSiteName;
/**状态(1-已配置;0-待配置)*/
......
package com.zzsn.event.vo;
import lombok.Data;
import java.util.List;
/**
*
*
* @author lkg
* @date 2024/4/28
*/
@Data
public class SubjectBindSourceVO {
private String webSiteId;
private String webSiteName;
private List<String> columnIds;
}
......@@ -60,11 +60,11 @@ spring:
password: qwer@9988&zzsn
driver-class-name: com.mysql.cj.jdbc.Driver
elasticsearch:
uris: ["1.95.69.2:9200"]
# uris: ["1.95.38.69:9700","1.95.3.121:9200","1.95.87.177:9700"]
# uris: ["1.95.69.2:9200"]
uris: ["1.95.38.69:9700","1.95.3.121:9200","1.95.87.177:9700"]
username: elastic
password: elastic
# password: zzsn9988
# password: elastic
password: zzsn9988
connection-timeout: 300000
socket-timeout: 300000
cloud:
......@@ -176,4 +176,11 @@ clb:
data-permit:
dataPermitGetQueryEntityTest: http://1.95.77.159:10089/permission/
dataPermitGetQueryEntityProd: http://1.95.14.24:8060/
\ No newline at end of file
dataPermitGetQueryEntityProd: http://1.95.14.24:8060/
infoSource:
pageList: http://1.95.79.85:8823/baseSourceInfo/api/infoSource/pageList
columnList: http://1.95.79.85:8823/baseSourceInfo/api/infoSource/columnList
add: http://1.95.79.85:8823/baseSourceInfo/api/infoSource/addInfoSourceTask
waitInfoRemove: http://1.95.79.85:8823/baseSourceInfo/api/infoSource/waitInfoRemove
columnListByWait: http://1.95.79.85:8823/baseSourceInfo/api/infoSource/columnListByWait
columnDetail: http://1.95.79.85:8823/baseSourceInfo/api/infoSource/columnDetail
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论