提交 a58dbbc6 作者: JQW

推荐资讯

上级 363efb25
...@@ -6,6 +6,8 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; ...@@ -6,6 +6,8 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.cloud.openfeign.EnableFeignClients;
import org.springframework.context.annotation.EnableMBeanExport; import org.springframework.context.annotation.EnableMBeanExport;
import org.springframework.jmx.support.RegistrationPolicy; import org.springframework.jmx.support.RegistrationPolicy;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.EnableAsync;
/** /**
* @author zhang ya nuo * @author zhang ya nuo
...@@ -16,6 +18,7 @@ import org.springframework.jmx.support.RegistrationPolicy; ...@@ -16,6 +18,7 @@ import org.springframework.jmx.support.RegistrationPolicy;
@EnableFeignClients(basePackages = {"org.jeecg","com.zzsn.clb"}) @EnableFeignClients(basePackages = {"org.jeecg","com.zzsn.clb"})
@EnableMBeanExport(registration = RegistrationPolicy.IGNORE_EXISTING) @EnableMBeanExport(registration = RegistrationPolicy.IGNORE_EXISTING)
@MapperScan(value={"com.zzsn.clb.**.mapper*"}) @MapperScan(value={"com.zzsn.clb.**.mapper*"})
@EnableAsync
public class ServiceWebApplication { public class ServiceWebApplication {
public static void main(String[] args) { public static void main(String[] args) {
SpringApplication.run(ServiceWebApplication.class, args); SpringApplication.run(ServiceWebApplication.class, args);
......
package com.zzsn.clb.config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.ThreadPoolExecutor;
@Configuration
public class ThreadPoolConfig {
// 线程名称前缀
@Value("${thread.prefix:jxyq-}")
private String threadPrefix;
// 核心线程数
@Value("${thread.core.size:16}")
private int coreSize;
// 最大线程数
@Value("${thread.max.size:16}")
private int maxSize;
// 队列长度
@Value("${thread.queue.size:30}")
private int queueSize;
// 通过bean注解注入
@Bean
public ThreadPoolTaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
//设置线程池参数信息
taskExecutor.setCorePoolSize(coreSize);
taskExecutor.setMaxPoolSize(maxSize);
taskExecutor.setQueueCapacity(queueSize);
taskExecutor.setThreadNamePrefix(threadPrefix);
taskExecutor.setWaitForTasksToCompleteOnShutdown(true);
taskExecutor.setAwaitTerminationSeconds(30);
//修改拒绝策略为使用当前线程执行
taskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
//初始化线程池
taskExecutor.initialize();
return taskExecutor;
}
}
\ No newline at end of file
...@@ -35,7 +35,7 @@ public interface IServiceProjectClient { ...@@ -35,7 +35,7 @@ public interface IServiceProjectClient {
* @return * @return
*/ */
@GetMapping(value = "/subject/display/list") @GetMapping(value = "/subject/display/list")
Result<?> getListBySubject(@RequestParam Map<String, Object> map, @RequestParam String userId,@RequestParam String video,@RequestParam Integer pageNo,@RequestParam Integer pageSize,@RequestParam String column,@RequestParam String order,@RequestParam String crawler,@RequestParam String isSubject,@RequestParam String subjectType,@RequestParam String labelIds); Result<JSONObject> getListBySubject(@RequestParam Map<String, Object> map, @RequestParam String userId,@RequestParam String video,@RequestParam Integer pageNo,@RequestParam Integer pageSize,@RequestParam String column,@RequestParam String order,@RequestParam String crawler,@RequestParam String isSubject,@RequestParam String subjectType,@RequestParam String labelIds);
/** /**
* 根据信息id查询数据 * 根据信息id查询数据
...@@ -43,7 +43,7 @@ public interface IServiceProjectClient { ...@@ -43,7 +43,7 @@ public interface IServiceProjectClient {
* @param id * @param id
* @return * @return
*/ */
@GetMapping(value = "/subject/display/queryById") @GetMapping(value = "/cloud/subject/display/queryById")
Result<JSONObject> getInfoById(@RequestParam(required = true) String id); Result<JSONObject> getInfoById(@RequestParam(required = true) String id);
...@@ -54,6 +54,6 @@ public interface IServiceProjectClient { ...@@ -54,6 +54,6 @@ public interface IServiceProjectClient {
* @param pageSize 每页显示条数 * @param pageSize 每页显示条数
* @return * @return
*/ */
@GetMapping(value = "/subject/subject/list") @GetMapping(value = "subject/subject/list")
Result<JSONObject> getSubjectList(@RequestParam(required = true) JSONObject jsonObject,@RequestParam(required = false) Integer pageNo,@RequestParam(required = false) Integer pageSize); Result<JSONObject> getSubjectList(@RequestParam(required = true) JSONObject jsonObject,@RequestParam(required = false) Integer pageNo,@RequestParam(required = false) Integer pageSize);
} }
...@@ -3,6 +3,7 @@ package com.zzsn.clb.member.mapper; ...@@ -3,6 +3,7 @@ package com.zzsn.clb.member.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zzsn.clb.member.entity.Subject; import com.zzsn.clb.member.entity.Subject;
import org.apache.ibatis.annotations.Param;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
...@@ -25,4 +26,11 @@ public interface SubjectMapper extends BaseMapper<Subject> { ...@@ -25,4 +26,11 @@ public interface SubjectMapper extends BaseMapper<Subject> {
* @return * @return
*/ */
List<Map<String,Object>> getNegativeLabelList(); List<Map<String,Object>> getNegativeLabelList();
/***
* 获取没有的专题及标签
* @param labelIds
* @return
*/
List<Map<String, Object>> getNoSubjectLabelList(@Param("labelIds") List<String> labelIds);
} }
...@@ -20,6 +20,30 @@ ...@@ -20,6 +20,30 @@
ORDER BY s.id, ORDER BY s.id,
le.sort ASC le.sort ASC
</select> </select>
<select id="getNoSubjectLabelList" resultType="java.util.Map">
SELECT
s.id subjectId,
le.id labelId
FROM
`subject` s
INNER JOIN subject_type_map stm ON s.id = stm.subject_id
INNER JOIN subject_model_map smm ON s.id = smm.subject_id
INNER JOIN python_model pm ON smm.model_id = pm.id
INNER JOIN sys_base_label_type lb ON lb.top_id = pm.label_id
INNER JOIN sys_base_label_type_map lm ON lb.id = lm.label_id
INNER JOIN label_entity le ON le.id = lm.relation_id
WHERE
le.`status` = 1
AND pm.type = 3
<if test="labelIds!=null and labelIds.size > 0">
<foreach collection="labelIds" item="item" open=" AND le.id NOT IN (" separator="," close=")">
#{item}
</foreach>
</if>
ORDER BY
s.id,
le.sort ASC
</select>
<select id="getNegativeLabelList" resultType="java.util.Map"> <select id="getNegativeLabelList" resultType="java.util.Map">
......
package com.zzsn.clb.member.service;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.clb.member.entity.MemRecommend;
import java.util.List;
import java.util.Map;
/**
* @Description: 会员-推荐资讯表
* @Author: jeecg-boot
* @Date: 2023-04-03
* @Version: V1.0
*/
public interface IMemRecommendAsynService {
/***
* 推荐得分写入
* @param memRecommend
* @param queryWrapper
*/
void insert(MemRecommend memRecommend, LambdaQueryWrapper<MemRecommend> queryWrapper);
}
package com.zzsn.clb.member.service; package com.zzsn.clb.member.service;
import com.baomidou.mybatisplus.extension.service.IService; import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.clb.member.entity.MemThumbsUp;
import com.zzsn.clb.member.entity.Subject; import com.zzsn.clb.member.entity.Subject;
import java.util.List; import java.util.List;
...@@ -12,5 +11,13 @@ public interface ISubjectTreeService extends IService<Subject> { ...@@ -12,5 +11,13 @@ public interface ISubjectTreeService extends IService<Subject> {
* 获取栏目树 * 获取栏目树
* @return * @return
*/ */
List<Map<String,Object>> getSubjectList(); List<Map<String, Object>> getSubjectList();
/***
* 获取没有的专题及标签
* @param labelIds
* @return
*/
List<Map<String, Object>> getNoSubjectLabelList(List<String> labelIds);
} }
package com.zzsn.clb.member.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.clb.member.entity.MemRecommend;
import com.zzsn.clb.member.mapper.MemRecommendMapper;
import com.zzsn.clb.member.service.IMemRecommendAsynService;
import com.zzsn.clb.member.service.IMemRecommendService;
import com.zzsn.clb.member.service.ISubjectTreeService;
import com.zzsn.common.Constants;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.search.join.ScoreMode;
import org.apache.shiro.SecurityUtils;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.Strings;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.NestedQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.jeecg.common.system.vo.LoginUser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* @Description: 会员-推荐资讯表
* @Author: jeecg-boot
* @Date: 2023-04-03
* @Version: V1.0
*/
@Component
public class MemRecommendAsynServiceImpl implements IMemRecommendAsynService {
@Autowired
private IMemRecommendService memRecommendService;
@Resource
ISubjectTreeService subjectTreeService;
@Async
public void insert(MemRecommend memRecommend, LambdaQueryWrapper<MemRecommend> queryWrapper) {
this.checkSubjectLabel(memRecommend);
LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
//如果是专题,则更新此专题下所有标签
if (memRecommend.getType() == 0) {
memRecommendService.updateScoreBySubjectId(memRecommend.getSubjectId(), user.getId(), memRecommend.getScore());
} else {
queryWrapper.eq(MemRecommend::getUserId, user.getId());
queryWrapper.eq(MemRecommend::getType, memRecommend.getType());
MemRecommend recommend = memRecommendService.getOne(queryWrapper);
//标签的肯定存在(关键词肯定存在),更新操作
if (recommend != null) {
recommend.setScore(memRecommend.getScore() + recommend.getScore());
memRecommendService.updateById(recommend);
} else if (StringUtils.isNotBlank(memRecommend.getKeyWord()) && memRecommend.getType() == 2) {
//关键词
recommend = new MemRecommend();
recommend.setUserId(user.getId());
recommend.setType(memRecommend.getType());
recommend.setKeyWord(memRecommend.getKeyWord());
recommend.setScore(memRecommend.getScore());
memRecommendService.save(recommend);
}
}
}
/***
* 检测是否有新的专题或者标签
* @param memRecommend
*/
public void checkSubjectLabel(MemRecommend memRecommend) {
if (StringUtils.isNotBlank(memRecommend.getSubjectId())) {
LoginUser user = (LoginUser) SecurityUtils.getSubject().getPrincipal();
LambdaQueryWrapper<MemRecommend> checkQueryWrapper = new LambdaQueryWrapper<>();
checkQueryWrapper.eq(MemRecommend::getUserId, user.getId());
checkQueryWrapper.eq(MemRecommend::getType, 1);
//获取当前用户已经存在的标签Id
List<String> labelIds = memRecommendService.list(checkQueryWrapper).stream().map(m -> m.getLabelId()).collect(Collectors.toList());
//获取当前用户不存在的标签id
List<Map<String, Object>> subjectList = subjectTreeService.getNoSubjectLabelList(labelIds);
List<MemRecommend> recList = new ArrayList<>();
//如果当前专题下没有添加标签或者有新的标签,则添加
subjectList.stream().filter(f -> f.get("subjectId").toString().equals(memRecommend.getSubjectId())).collect(Collectors.toList()).forEach(x -> {
MemRecommend model = new MemRecommend();
model.setSubjectId(x.get("subjectId").toString());
model.setLabelId(x.get("labelId").toString());
model.setUserId(user.getId());
model.setType(1);
model.setScore(0);
recList.add(model);
});
memRecommendService.saveBatch(recList);
}
}
}
package com.zzsn.clb.member.service.impl; package com.zzsn.clb.member.service.impl;
import com.baomidou.dynamic.datasource.annotation.DS; import com.baomidou.dynamic.datasource.annotation.DS;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.clb.member.entity.Subject; import com.zzsn.clb.member.entity.Subject;
import com.zzsn.clb.member.mapper.SubjectMapper; import com.zzsn.clb.member.mapper.SubjectMapper;
import com.zzsn.clb.member.service.ISubjectTreeService; import com.zzsn.clb.member.service.ISubjectTreeService;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import javax.annotation.Resource; import javax.annotation.Resource;
...@@ -17,13 +15,9 @@ import java.util.stream.Collectors; ...@@ -17,13 +15,9 @@ import java.util.stream.Collectors;
@DS("multi-clb-project") @DS("multi-clb-project")
public class SubjectTreeServiceImpl extends ServiceImpl<SubjectMapper, Subject> implements ISubjectTreeService { public class SubjectTreeServiceImpl extends ServiceImpl<SubjectMapper, Subject> implements ISubjectTreeService {
@Resource
SubjectMapper subjectMapper;
@Override @Override
public List<Map<String, Object>> getSubjectList() { public List<Map<String, Object>> getSubjectList() {
List<Map<String, Object>> list = subjectMapper.getSubjectList(); List<Map<String, Object>> list = this.getBaseMapper().getSubjectList();
List<String> idList = list.stream().distinct().map(m -> m.get("subjectId").toString()).collect(Collectors.toList()); List<String> idList = list.stream().distinct().map(m -> m.get("subjectId").toString()).collect(Collectors.toList());
List<Map<String, Object>> treeList = new ArrayList<>(); List<Map<String, Object>> treeList = new ArrayList<>();
String subjectId = ""; String subjectId = "";
...@@ -50,7 +44,7 @@ public class SubjectTreeServiceImpl extends ServiceImpl<SubjectMapper, Subject> ...@@ -50,7 +44,7 @@ public class SubjectTreeServiceImpl extends ServiceImpl<SubjectMapper, Subject>
map.put("name", "负面舆情"); map.put("name", "负面舆情");
map.put("child", chdlist); map.put("child", chdlist);
treeList.add(map); treeList.add(map);
list = subjectMapper.getNegativeLabelList(); list = this.getBaseMapper().getNegativeLabelList();
for (Map<String, Object> tmpMap : list) { for (Map<String, Object> tmpMap : list) {
Map<String, Object> childMap = new HashMap<>(); Map<String, Object> childMap = new HashMap<>();
childMap.put("key", tmpMap.get("labelId")); childMap.put("key", tmpMap.get("labelId"));
...@@ -59,4 +53,11 @@ public class SubjectTreeServiceImpl extends ServiceImpl<SubjectMapper, Subject> ...@@ -59,4 +53,11 @@ public class SubjectTreeServiceImpl extends ServiceImpl<SubjectMapper, Subject>
} }
return treeList; return treeList;
} }
@Override
public List<Map<String, Object>> getNoSubjectLabelList(List<String> labelIds) {
return this.getBaseMapper().getNoSubjectLabelList(labelIds);
}
} }
...@@ -41,6 +41,7 @@ import javax.annotation.Resource; ...@@ -41,6 +41,7 @@ import javax.annotation.Resource;
import java.io.IOException; import java.io.IOException;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
/*** /***
* 标签类的模型处理 * 标签类的模型处理
...@@ -64,13 +65,12 @@ public class ModelLabelTask { ...@@ -64,13 +65,12 @@ public class ModelLabelTask {
RedisUtil redisUtil; RedisUtil redisUtil;
@Scheduled(cron = "0 */10 * * * ?")// 每10分钟执行一次 //@Scheduled(cron = "0 */10 * * * ?")// 每10分钟执行一次
//@Bean
public void run() { public void run() {
if (redisUtil.get(redisCacheName) != null) { if (redisUtil.get(redisCacheName) != null) {
queryBgDate = redisUtil.get(redisCacheName).toString(); queryBgDate = redisUtil.get(redisCacheName).toString();
} else { } else {
queryBgDate = "2000-01-01T00:00:00"; queryBgDate = "2023-02-28T14:10:03";
} }
Date dateNow = new Date(); Date dateNow = new Date();
queryEndate = DateUtils.formatDate(dateNow, "yyyy-MM-dd") + "T" + DateUtils.formatDate(dateNow, "HH:mm:ss"); queryEndate = DateUtils.formatDate(dateNow, "yyyy-MM-dd") + "T" + DateUtils.formatDate(dateNow, "HH:mm:ss");
...@@ -108,7 +108,9 @@ public class ModelLabelTask { ...@@ -108,7 +108,9 @@ public class ModelLabelTask {
try { try {
JSONObject result = JSONObject.parseObject(HttpUtil.doPost(modelUrl, jo, 60000)); JSONObject result = JSONObject.parseObject(HttpUtil.doPost(modelUrl, jo, 60000));
if (result.getInteger("code") == 200 && result.getString("flag").equals("1")) { if (result.getInteger("code") == 200 && result.getString("flag").equals("1")) {
setEsDataList(dataList, map, result.getJSONArray("label_info"), subjectLabelList); List<Map<String, Object>> returnList = getEsDataList(map, result.getJSONArray("label_info"), subjectLabelList);
if (returnList != null)
dataList.addAll(returnList);
} }
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
...@@ -123,17 +125,17 @@ public class ModelLabelTask { ...@@ -123,17 +125,17 @@ public class ModelLabelTask {
/*** /***
* 对模型返回数据进行处理 * 对模型返回数据进行处理
* @param dataList
* @param mapData * @param mapData
* @param jsonArray * @param jsonArray
* @param subjectLabelList * @param subjectLabelList
*/ */
private void setEsDataList(List<Map<String, Object>> dataList, Map<String, Object> mapData, JSONArray jsonArray, List<Map<String, Object>> subjectLabelList) { private List<Map<String, Object>> getEsDataList(Map<String, Object> mapData, JSONArray jsonArray, List<Map<String, Object>> subjectLabelList) {
List<Map<String, Object>> oneDataList = new ArrayList<>();
//遍历数据模型返回的数据 //遍历数据模型返回的数据
for (Object x : jsonArray) { for (Object x : jsonArray) {
JSONObject jo = (JSONObject) x; JSONObject jo = (JSONObject) x;
if (jo.get("level1") == null || jo.get("level2") == null) if (jo.get("level1") == null || jo.get("level2") == null)
return; return null;
//获取标签分类数组 //获取标签分类数组
String[] labelTypeNames = {jo.getString("level1")}; String[] labelTypeNames = {jo.getString("level1")};
if (jo.getString("level1").contains("|")) { if (jo.getString("level1").contains("|")) {
...@@ -144,13 +146,52 @@ public class ModelLabelTask { ...@@ -144,13 +146,52 @@ public class ModelLabelTask {
if (jo.getString("level2").contains("|")) { if (jo.getString("level2").contains("|")) {
labelNames = jo.getString("level2").split("\\|"); labelNames = jo.getString("level2").split("\\|");
} }
//遍历标签分类 //遍历标签分类
for (String labelTypeName : labelTypeNames) { for (String labelTypeName : labelTypeNames) {
List<Map<String, Object>> returnList = this.setEsDataMap(mapData, labelTypeName, labelNames, subjectLabelList); List<Map<String, Object>> returnList = this.setEsDataMap(mapData, labelTypeName, labelNames, subjectLabelList);
if (returnList != null) if (returnList != null)
dataList.addAll(returnList); oneDataList.addAll(returnList);
} }
} }
return removeRepeatSubjectId(oneDataList);
}
/***
* 对同一个专题,不同标签分类的数据进行标签合并
* @param oneDataList
* @return
*/
private List<Map<String, Object>> removeRepeatSubjectId(List<Map<String, Object>> oneDataList) {
//对同一个专题,不同标签分类的数据进行标签合并
//筛出有重复的subjectId
Map<Object, Long> map = oneDataList.stream().collect(Collectors.groupingBy(m -> m.get("subjectId"), Collectors.counting()));
Stream<Object> stringStream = map.entrySet().stream().filter(entry -> entry.getValue() > 1).map(entry -> entry.getKey());
List<String> repeatIdList = new ArrayList<>();
stringStream.forEach(str -> {
repeatIdList.add(String.valueOf(str));
});
//对有重复subjectId的数据去重,重新装配标签
if (repeatIdList.size() > 0) {
log.debug("去重前数量:{}", oneDataList.size());
log.debug("重复subjectId:" + JSON.toJSONString(repeatIdList));
repeatIdList.forEach(x -> {
List<Map<String, Object>> repeatList = oneDataList.stream().filter(f -> f.get("subjectId").toString().equals(x)).collect(Collectors.toList());
log.debug("重复subjectId:{},数量:{}", x, repeatList.size());
List<Map<String, Object>> labelList = new ArrayList<>();
repeatList.stream().map(m -> ((List) m.get("labels"))).collect(Collectors.toList()).forEach(y -> {
labelList.addAll(y);
});
Map<String, Object> tmpSubject = repeatList.get(0);
tmpSubject.put("labels", labelList);
oneDataList.removeAll(repeatList);
oneDataList.add(tmpSubject);
log.debug("重复合并后ID:{}", tmpSubject.get("id"));
});
log.debug("现有数量:{}", oneDataList.size());
}
return oneDataList;
} }
/*** /***
...@@ -183,7 +224,7 @@ public class ModelLabelTask { ...@@ -183,7 +224,7 @@ public class ModelLabelTask {
//如果标签不存在,返回null //如果标签不存在,返回null
if (labelList == null) if (labelList == null)
return null; return null;
//装配入库信息,处理一条标签分类信息属于多个专题,如果这几类标签都属于一个专题,会出现重复情况,除非一级栏目是专题(负面舆情就有问题了) //装配入库信息,处理一条标签分类信息属于多个专题
List<String> subjectIdList = subjectLabelList.stream().filter(f -> f.get("labelTypeName").equals(labelTypeName)).map(m -> m.get("subjectId").toString()).distinct().collect(Collectors.toList()); List<String> subjectIdList = subjectLabelList.stream().filter(f -> f.get("labelTypeName").equals(labelTypeName)).map(m -> m.get("subjectId").toString()).distinct().collect(Collectors.toList());
return getMultSubjectData(subjectIdList, mapData, labelList); return getMultSubjectData(subjectIdList, mapData, labelList);
} }
...@@ -213,6 +254,10 @@ public class ModelLabelTask { ...@@ -213,6 +254,10 @@ public class ModelLabelTask {
return dataList; return dataList;
} }
/***
* 批量写入数据库
* @param addEsDataMapList
*/
public void addElasticsearchData(List<Map<String, Object>> addEsDataMapList) { public void addElasticsearchData(List<Map<String, Object>> addEsDataMapList) {
try { try {
...@@ -224,7 +269,7 @@ public class ModelLabelTask { ...@@ -224,7 +269,7 @@ public class ModelLabelTask {
requestData = new IndexRequest(Constants.ES_DATA_FOR_SUBJECT).id(addEsDataMap.get("id").toString()).source(addEsDataMap, XContentType.JSON); requestData = new IndexRequest(Constants.ES_DATA_FOR_SUBJECT).id(addEsDataMap.get("id").toString()).source(addEsDataMap, XContentType.JSON);
bulkRequest.add(requestData); bulkRequest.add(requestData);
} }
log.info("es同步数据数量:{}", bulkRequest.numberOfActions()); log.debug("es同步数据数量:{}", bulkRequest.numberOfActions());
//设置索引刷新规则 //设置索引刷新规则
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
//分批次提交,数量控制 //分批次提交,数量控制
......
...@@ -43,9 +43,9 @@ public class ImportExcelData { ...@@ -43,9 +43,9 @@ public class ImportExcelData {
SubjectMapper subjectMapper; SubjectMapper subjectMapper;
public void importData() { public void importData() {
String path = "C:\\Users\\EDY\\Documents\\WeChat Files\\wxid_7d7tl0zyfdxb21\\FileStorage\\File\\2023-02\\管理动态-下属机构工作.xlsx"; String path = "D:\\新数据22.xlsx";
try { try {
impData(1, 10, path, 6); impData(1, 10, path, 166);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
...@@ -178,7 +178,7 @@ public class ImportExcelData { ...@@ -178,7 +178,7 @@ public class ImportExcelData {
subjectData.setOrigin(list.get(7).trim()); subjectData.setOrigin(list.get(7).trim());
} }
if (StringUtils.isNotBlank(list.get(8))) { if (StringUtils.isNotBlank(list.get(8))) {
subjectData.setPublishDate(list.get(8)); subjectData.setPublishDate(list.get(8).substring(0,10));
} }
if (StringUtils.isNotBlank(list.get(9))) { if (StringUtils.isNotBlank(list.get(9))) {
......
...@@ -46,8 +46,36 @@ public class SubjectController { ...@@ -46,8 +46,36 @@ public class SubjectController {
@AutoLog(value = "专题数据") @AutoLog(value = "专题数据")
@ApiOperation(value = "专题数据", notes = "专题数据") @ApiOperation(value = "专题数据", notes = "专题数据")
@GetMapping(value = "/getSubjectCollect") @GetMapping(value = "/getSubjectCollect")
public Result<?> SubjectCollect(@RequestParam(required = true,defaultValue = "month") String type,@RequestParam(defaultValue = "6") Integer size,@RequestParam(defaultValue = "1") Integer subjectType) { public Result<?> SubjectCollect(@RequestParam(required = true, defaultValue = "month") String type, @RequestParam(defaultValue = "6") Integer size, @RequestParam(defaultValue = "1") Integer subjectType) {
return Result.OK(subjectService.getSubjectCollect(type,size,subjectType)); return Result.OK(subjectService.getSubjectCollect(type, size, subjectType));
}
/***
* 专题趋势
* @param queryBgDate 开始日期
* @param queryEndate 结束日期
* @return
*/
@AutoLog(value = "专题趋势")
@ApiOperation(value = "专题趋势", notes = "专题趋势")
@GetMapping(value = "/getDaySubjectTrend")
public Result<?> getDaySubjectTrend(@RequestParam(required = true, defaultValue = "") String queryBgDate, @RequestParam(required = true, defaultValue = "") String queryEndate) {
return Result.OK(subjectService.getDaySubjectTrend(queryBgDate, queryEndate));
}
/***
* 舆情走势
* @param queryBgDate 开始日期
* @param queryEndate 结束日期
* @param type 1-正面 2-负面
* @return
*/
@AutoLog(value = "舆情走势")
@ApiOperation(value = "舆情走势", notes = "舆情走势")
@GetMapping(value = "/getPubliOpinionTrend")
public Result<?> getPubliOpinionTrend(@RequestParam(required = true, defaultValue = "") String queryBgDate, @RequestParam(required = true, defaultValue = "") String queryEndate, @RequestParam(required = true, defaultValue = "1") Integer type) {
return Result.OK(subjectService.getPubliOpinionTrend(queryBgDate, queryEndate, type));
} }
...@@ -61,11 +89,11 @@ public class SubjectController { ...@@ -61,11 +89,11 @@ public class SubjectController {
@AutoLog(value = "根据专题id查询数据") @AutoLog(value = "根据专题id查询数据")
@ApiOperation(value = "根据专题id查询数据", notes = "根据专题id查询数据") @ApiOperation(value = "根据专题id查询数据", notes = "根据专题id查询数据")
@GetMapping(value = "/getListBySubject") @GetMapping(value = "/getListBySubject")
public Result<?> getListBySubject(Integer pageNo, Integer pageSize, @RequestParam(required = true) String subjectId,@RequestParam(required = false) String lableIds) { public Result<?> getListBySubject(Integer pageNo, Integer pageSize, @RequestParam(required = true) String subjectId, @RequestParam(required = false) String lableIds) {
//http://114.115.205.50:9988/subject/display/list?_t=1676630456&column=score&order=desc&field=id,,info&pageNo=1&pageSize=10&subjectId=1600743965735264258&checkStatusList=&isSubject=1 //http://114.115.205.50:9988/subject/display/list?_t=1676630456&column=score&order=desc&field=id,,info&pageNo=1&pageSize=10&subjectId=1600743965735264258&checkStatusList=&isSubject=1
Map<String, Object> map = new HashMap<>(); Map<String, Object> map = new HashMap<>();
map.put("subjectId", subjectId); map.put("subjectId", subjectId);
return serviceProjectClient.getListBySubject(map, null, null, pageNo, pageSize, "publishDate", "desc", null, "1", null,lableIds); return serviceProjectClient.getListBySubject(map, null, null, pageNo, pageSize, "publishDate", "desc", null, "1", null, lableIds);
} }
...@@ -86,15 +114,15 @@ public class SubjectController { ...@@ -86,15 +114,15 @@ public class SubjectController {
/** /**
* 根据信息id查询数据 * 根据信息id查询数据
* *
* @param id
* @return * @return
*/ */
@AutoLog(value = "根据信息id查询数据") @AutoLog(value = "根据信息id查询数据")
@ApiOperation(value = "根据信息id查询数据", notes = "根据信息id查询数据") @ApiOperation(value = "根据信息id查询数据", notes = "根据信息id查询数据")
// @GetMapping(value = "/import") @GetMapping(value = "/import")
public Result<?> importData() { public Result<?> importData() {
//http://114.115.236.206:9988/subject/display/queryById?_t=1676875032&id=21123000014507 //http://114.115.236.206:9988/subject/display/queryById?_t=1676875032&id=21123000014507
importExcelData.importData(); importExcelData.importData();
return Result.OK(); return Result.OK();
} }
} }
...@@ -2,7 +2,7 @@ package com.zzsn.common; ...@@ -2,7 +2,7 @@ package com.zzsn.common;
public class Constants { public class Constants {
//处理后的专题资讯信息存储索引 //处理后的专题资讯信息存储索引
public final static String ES_DATA_FOR_SUBJECT = "tmp_subjectdatabase"; public final static String ES_DATA_FOR_SUBJECT = "subjectdatabase";
//新采集库(22.04.23) //新采集库(22.04.23)
public final static String ES_BASE_DATA = "basedata"; public final static String ES_BASE_DATA = "basedata";
} }
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论