Skip to content

T
think-tank
提交 75e8e756 作者: ZhangJingKun
操作

修改导出一个月内的数据

上级 419a8c28
正在显示 包含 456 行增加 和 19 行删除
pom.xml
......@@ -104,7 +104,11 @@
<artifactId>spring-data-elasticsearch</artifactId>
<version>3.2.6.RELEASE</version>
</dependency>
<dependency>
<groupId>com.github.stuxuhai</groupId>
<artifactId>jpinyin</artifactId>
<version>1.1.8</version>
</dependency>
</dependencies>
......
src/main/java/com/zzsn/thinktank/controller/ThinktankBasicInfoController.java
......@@ -12,6 +12,7 @@ import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import org.springframework.web.util.WebUtils;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
......@@ -108,16 +109,29 @@ public class ThinktankBasicInfoController {
* @param
* @return
*/
@Transactional
//@Transactional
@PostMapping("/batchImport")
public Result<?> batchImport( HttpServletRequest request) {
MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
log.info("批量导入数据");
// log.info("批量导入数据:{}",file.getOriginalFilename());
// log.info("file:{}",file.getName());
MultipartHttpServletRequest multipartRequest = WebUtils.getNativeRequest(request, MultipartHttpServletRequest.class);
Map<String, MultipartFile> fileMap = multipartRequest.getFileMap();
if (fileMap.size() < 1) {
return Result.error("请上传文件!");
}
log.info(fileMap.size() + "");
// MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
// Map<String, MultipartFile> fileMap = multipartRequest.getFileMap();
// log.info("file size:{}",fileMap.size());
// if (fileMap.size() < 1) {
// return Result.error("请上传文件!");
// }
MultipartFile multipartFile = fileMap.get(new ArrayList<>(fileMap.keySet()).get(0));
Result result = thinktankBasicInfoService.batchImport(multipartFile);
// Result result = thinktankBasicInfoService.batchImport(file);
return result;
}
......@@ -136,6 +150,18 @@ public class ThinktankBasicInfoController {
return thinktankBasicInfoService.batchExport(thinktankBasicInfoListVo,request,response);
}
@GetMapping("/listBaseData")
public Result<?> getListBaseData(@RequestParam(name="id",required=true) String id,
@RequestParam(name="pageNo",required=true) Integer pageNo,
@RequestParam(name="pageSize",required=true) Integer pageSize){
log.info("智库成果分页查询:{}", id);
if(pageNo < 1)
pageNo = 1;
if(pageSize < 0)
pageSize = 10;
Integer office = pageSize * (pageNo - 1);
return thinktankBasicInfoService.getListBaseData(id,pageNo,pageSize,office);
}
}
src/main/java/com/zzsn/thinktank/entity/LabelEntity.java
package com.zzsn.thinktank.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.springframework.format.annotation.DateTimeFormat;
import java.util.Date;
@Data
@EqualsAndHashCode(callSuper = false)
@TableName("label_entity")
public class LabelEntity {
@TableId(type = IdType.ASSIGN_ID)
private String id;
private String name;
private String synonym;
private String explanation;
private long level;
private Integer level;
private String topId;
private String pathIds;
private long status;
private long sort;
private Integer status;
private Integer sort;
private String createBy;
private java.sql.Timestamp createTime;
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
private Date createTime;
private String updateBy;
private java.sql.Timestamp updateTime;
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
private Date updateTime;
private String sysOrgCode;
private String pid;
private String hasChild;
private String initial;
}
src/main/java/com/zzsn/thinktank/entity/SysBaseLabelTypeMap.java 0 → 100644
package com.zzsn.thinktank.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.experimental.Accessors;
import org.springframework.format.annotation.DateTimeFormat;
import java.io.Serializable;
/**
* @Description: 标签与类别映射表
* @Author: jeecg-boot
* @Date: 2022-05-13
* @Version: V1.0
*/
@Data
@TableName("sys_base_label_type_map")
@Accessors(chain = true)
@EqualsAndHashCode(callSuper = false)
public class SysBaseLabelTypeMap implements Serializable {
private static final long serialVersionUID = 1L;
/**主键*/
@TableId(type = IdType.ASSIGN_ID)
private String id;
/**关联id(企业或其他)*/
private String relationId;
/**标签id*/
private String labelId;
/**排序号(500强企业)*/
private Integer orderNo;
/**标签类别（如企业、自定义等）*/
private String labelType;
/**创建日期*/
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
private java.util.Date createTime;
}
src/main/java/com/zzsn/thinktank/mapper/SysBaseLabelTypeMapMapper.java 0 → 100644
package com.zzsn.thinktank.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zzsn.thinktank.entity.SysBaseLabelTypeMap;
import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface SysBaseLabelTypeMapMapper extends BaseMapper<SysBaseLabelTypeMap> {
}
src/main/java/com/zzsn/thinktank/mapper/ThinktankTagMapper.java
......@@ -36,4 +36,8 @@ public interface ThinktankTagMapper extends BaseMapper<LabelEntity> {
"and id = #{tagId} " +
"limit 1")
ThinktankTagTop getById(String tagName);
@Select("select max(sort) from label_entity where pid = #{pid}")
Integer queryMaxOrderNo(String pid);
}
src/main/java/com/zzsn/thinktank/service/SysBaseLabelTypeMapService.java 0 → 100644
package com.zzsn.thinktank.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.thinktank.entity.SysBaseLabelTypeMap;
public interface SysBaseLabelTypeMapService extends IService<SysBaseLabelTypeMap> {
}
src/main/java/com/zzsn/thinktank/service/ThinktankBasicInfoService.java
......@@ -34,5 +34,5 @@ public interface ThinktankBasicInfoService extends IService<ThinktankBasicInfo>
byte[] batchExport(ThinktankBasicInfoListVo thinktankBasicInfoListVo, HttpServletRequest request, HttpServletResponse response);
Result<?> getListBaseData(String id, Integer pageNo, Integer pageSize, Integer office);
}
src/main/java/com/zzsn/thinktank/service/ThinktankTagService.java
......@@ -2,6 +2,7 @@ package com.zzsn.thinktank.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.thinktank.entity.LabelEntity;
import com.zzsn.thinktank.entity.ThinktankBasicInfo;
import com.zzsn.thinktank.entity.ThinktankTagTop;
import com.zzsn.thinktank.vo.Result;
import com.zzsn.thinktank.vo.SysBaseLabelTypeVo;
......@@ -29,4 +30,10 @@ public interface ThinktankTagService extends IService<LabelEntity> {
void addTagTop(String tagId);
List<LabelEntity> getListByIds(String[] ids);
void addLabel(ThinktankBasicInfo thinktankBasicInfo);
void delLabel(List<String> ids);
void editLabel(ThinktankBasicInfo thinktankBasicInfo);
}
src/main/java/com/zzsn/thinktank/service/impl/SysBaseLabelTypeMapServiceImpl.java 0 → 100644
package com.zzsn.thinktank.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.thinktank.entity.SysBaseLabelTypeMap;
import com.zzsn.thinktank.mapper.SysBaseLabelTypeMapMapper;
import com.zzsn.thinktank.service.SysBaseLabelTypeMapService;
import org.springframework.stereotype.Service;
@Service
public class SysBaseLabelTypeMapServiceImpl extends ServiceImpl<SysBaseLabelTypeMapMapper, SysBaseLabelTypeMap> implements SysBaseLabelTypeMapService {
}
src/main/java/com/zzsn/thinktank/service/impl/ThinktankBasicInfoServiceImpl.java
......@@ -10,16 +10,15 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.thinktank.entity.*;
import com.zzsn.thinktank.mapper.ThinktankBasicInfoMapper;
import com.zzsn.thinktank.service.*;
import com.zzsn.thinktank.util.DateUtil;
import com.zzsn.thinktank.util.EsUtil;
import com.zzsn.thinktank.util.ExcelExportUtil;
import com.zzsn.thinktank.util.StringUtil;
import com.zzsn.thinktank.util.*;
import com.zzsn.thinktank.vo.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.usermodel.Workbook;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
......@@ -117,6 +116,8 @@ public class ThinktankBasicInfoServiceImpl extends ServiceImpl<ThinktankBasicInf
if(!b){
result.error500("保存失败！");
}
//同步新增智库数据到标签管理
thinktankTagService.addLabel(thinktankBasicInfo);
return result;
}
......@@ -129,6 +130,7 @@ public class ThinktankBasicInfoServiceImpl extends ServiceImpl<ThinktankBasicInf
if(!b){
result.error500("删除失败！");
}
thinktankTagService.delLabel(ids);
return result;
}
......@@ -162,6 +164,7 @@ public class ThinktankBasicInfoServiceImpl extends ServiceImpl<ThinktankBasicInf
if(!b){
result.error500("修改失败！");
}
thinktankTagService.editLabel(thinktankBasicInfo);
return result;
}
......@@ -541,6 +544,7 @@ public class ThinktankBasicInfoServiceImpl extends ServiceImpl<ThinktankBasicInf
if(b){
this.saveOrUpdate(thinktankBasicInfo);
thinktankTagService.addLabel(thinktankBasicInfo);
i++;
} else {
j++;
......@@ -644,6 +648,9 @@ public class ThinktankBasicInfoServiceImpl extends ServiceImpl<ThinktankBasicInf
//查询es
BoolQueryBuilder boolQuerylist = QueryBuilders.boolQuery();
boolQuerylist.must(QueryBuilders.termsQuery("sid",sidList));
//boolQuerylist.filter(QueryBuilders.rangeQuery("publishDate").gte("2024-05-18T00:00:01"));
Map<String, Long> mapGroup = null;
try {
mapGroup = esUtil.groupBy("basedata", "sid", boolQuerylist, 100);
......@@ -694,5 +701,42 @@ public class ThinktankBasicInfoServiceImpl extends ServiceImpl<ThinktankBasicInf
return list;
}
@Override
public Result<?> getListBaseData(String id, Integer pageNo, Integer pageSize, Integer office) {
List<InfoSourceGroupMap> infoSourceGroupMapList = infoSourceGroupMapService.getSourceGroupMap(Arrays.asList(id));
List<String> sidList = new ArrayList<>();
for (InfoSourceGroupMap infoSourceGroupMap : infoSourceGroupMapList) {
sidList.add(infoSourceGroupMap.getSourceId());
}
//设置分页参数
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// searchSourceBuilder.size(pageSize);
// searchSourceBuilder.from(office);
searchSourceBuilder.trackTotalHits(true);
//排序
searchSourceBuilder.sort("createDate", SortOrder.ASC);
//查询es
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.termsQuery("sid",sidList));
//查询数据
searchSourceBuilder.query(boolQuery);
Page<TransferData> page = null;
try {
page = esUtil.queryPage("basedata", searchSourceBuilder, TransferData.class, pageNo,pageSize);
} catch (IOException e) {
e.printStackTrace();
}
for (TransferData transferData : page.getRecords()) {
transferData.setCreateDate(EsDateUtil.esFieldDateMapping(transferData.getCreateDate()));
transferData.setPublishDate(EsDateUtil.esFieldDateMapping(transferData.getPublishDate()));
}
return Result.OK(page);
}
}
src/main/java/com/zzsn/thinktank/service/impl/ThinktankTagServiceImpl.java
......@@ -5,13 +5,13 @@ import cn.hutool.core.util.ObjectUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.thinktank.entity.LabelEntity;
import com.zzsn.thinktank.entity.SysBaseLabelType;
import com.zzsn.thinktank.entity.ThinktankTagTop;
import com.zzsn.thinktank.entity.*;
import com.zzsn.thinktank.mapper.ThinktankTagMapper;
import com.zzsn.thinktank.mapper.ThinktankTagTopMapper;
import com.zzsn.thinktank.service.SysBaseLabelTypeMapService;
import com.zzsn.thinktank.service.ThinktankBasicInfoService;
import com.zzsn.thinktank.service.ThinktankTagService;
import com.zzsn.thinktank.util.PinYinUtil;
import com.zzsn.thinktank.vo.Result;
import com.zzsn.thinktank.vo.SysBaseLabelTypeVo;
import com.zzsn.thinktank.vo.SysLabelVo;
......@@ -40,6 +40,9 @@ public class ThinktankTagServiceImpl extends ServiceImpl<ThinktankTagMapper, Lab
@Autowired
ThinktankBasicInfoService thinktankBasicInfoService;
@Autowired
SysBaseLabelTypeMapService sysBaseLabelTypeMapService;
@Value("${sys_base_label_type_id}")
......@@ -140,6 +143,51 @@ public class ThinktankTagServiceImpl extends ServiceImpl<ThinktankTagMapper, Lab
return list;
}
@Override
public void addLabel(ThinktankBasicInfo thinktankBasicInfo) {
LabelEntity labelEntity = new LabelEntity();
labelEntity.setId(thinktankBasicInfo.getId());
labelEntity.setName(thinktankBasicInfo.getChineseWhole());
labelEntity.setExplanation(thinktankBasicInfo.getChineseWhole());
labelEntity.setPathIds(thinktankBasicInfo.getId());
labelEntity.setStatus(1);
labelEntity.setPid("0");
Integer maxOrderNo = baseMapper.queryMaxOrderNo(labelEntity.getPid());
labelEntity.setSort(maxOrderNo);
labelEntity.setCreateTime(new Date());
labelEntity.setHasChild("0");
String initial = PinYinUtil.getInitialByWordMean(labelEntity.getName());
labelEntity.setInitial(initial);
this.save(labelEntity);
SysBaseLabelTypeMap typeMap = new SysBaseLabelTypeMap();
typeMap.setRelationId(thinktankBasicInfo.getId());
typeMap.setLabelId("1806202442359099393");
typeMap.setLabelType("custom_label");
typeMap.setCreateTime(new Date());
sysBaseLabelTypeMapService.save(typeMap);
}
@Override
public void delLabel(List<String> ids) {
this.removeByIds(ids);
LambdaQueryWrapper<SysBaseLabelTypeMap> lambdaQuery = Wrappers.lambdaQuery();
lambdaQuery.in(SysBaseLabelTypeMap::getRelationId, ids);
sysBaseLabelTypeMapService.remove(lambdaQuery);
}
@Override
public void editLabel(ThinktankBasicInfo thinktankBasicInfo) {
LabelEntity labelEntity = new LabelEntity();
labelEntity.setId(thinktankBasicInfo.getId());
labelEntity.setName(thinktankBasicInfo.getChineseWhole());
labelEntity.setExplanation(thinktankBasicInfo.getChineseWhole());
labelEntity.setUpdateTime(new Date());
String initial = PinYinUtil.getInitialByWordMean(labelEntity.getName());
labelEntity.setInitial(initial);
this.updateById(labelEntity);
}
/**
* 根据标签类别id查询所有的末端节点
......
src/main/java/com/zzsn/thinktank/util/EsDateUtil.java 0 → 100644
package com.zzsn.thinktank.util;
import java.text.SimpleDateFormat;
import java.util.Date;
/**
* Es对日期类型处理
* @author kongliufeng
* @create 2020-08-07 14:05
*/
public class EsDateUtil {
static SimpleDateFormat format;
/**
* yyyy-MM-dd HH:mm:ss ->yyyy-MM-ddTHH:mm:ss
* @param data
* @return
*/
public static String esFieldDateFormat(String data) {
if (data == null)
return data;
if (data.length() == 19) {//标准yyyy-MM-dd HH:mm:ss
return data.replace(" ", "T");
} else if (data.length() == 10) {//yyyy-MM-dd
return data;
}
return null;
}
/**
* yyyy-MM-dd HH:mm:ss ->yyyy-MM-ddTHH:mm:ss
* @param data
* @return
*/
public static String esFieldDateMapping(String data) {
if (data == null)
return data;
if (data.length() == 19) {//标准yyyy-MM-dd HH:mm:ss
return data.replace("T", " ");
} else if (data.length() == 10) {//yyyy-MM-dd
return data;
}
return null;
}
public static String dateToString(Date date) {
format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
return format.format(date);
}
}
src/main/java/com/zzsn/thinktank/util/PinYinUtil.java 0 → 100644
package com.zzsn.thinktank.util;
import com.github.stuxuhai.jpinyin.PinyinFormat;
/**
* 汉字拼音工具类
*
* @author lkg
* @date 2023/4/10
*/
public class PinYinUtil {
/**
* 获取首字母(大写)--根据词义准确抽取
*
* @param str 汉字
* @author lkg
* @date 2023/4/10
*/
public static String getInitialByWordMean(String str){
String initial = null;
try {
String shortPinyin = com.github.stuxuhai.jpinyin.PinyinHelper.getShortPinyin(str);
initial = String.valueOf(shortPinyin.charAt(0)).toUpperCase();
} catch (Exception e) {
e.printStackTrace();
}
return initial;
}
/**
* 抽取拼音--根据词义准确抽取
*
* @param str 词
* @param type 1-全拼;2-简拼(各字的首字母)
* @author lkg
* @date 2024/3/26
*/
public static String getPinYinByWordMean(String str,Integer type){
String pinyin = null;
try {
if (type == 1) {
pinyin = com.github.stuxuhai.jpinyin.PinyinHelper.convertToPinyinString(str, "", PinyinFormat.WITHOUT_TONE);
} else {
pinyin = com.github.stuxuhai.jpinyin.PinyinHelper.getShortPinyin(str);
}
} catch (Exception e) {
e.printStackTrace();
}
return pinyin;
}
}
src/main/java/com/zzsn/thinktank/vo/Label.java 0 → 100644
package com.zzsn.thinktank.vo;
import lombok.Data;
import java.io.Serializable;
/**
* 基础信息打上标签的实体对象
*/
@Data
public class Label implements Serializable {
//命中标识
private String hitRemarks;
//标签标识
private String labelMark;
//标签备注
private String labelRemarks;
//项目标签id
private String projectLabelId;
//关联标签id
private String relationId;
//关联标签名称
private String relationName;
//审核状态
private Integer status;
}
src/main/java/com/zzsn/thinktank/vo/TransferData.java 0 → 100644
package com.zzsn.thinktank.vo;
import com.alibaba.excel.annotation.ExcelProperty;
import lombok.Data;
import java.util.Date;
import java.util.List;
/**
* 数据
*/
@Data
public class TransferData {
//说明：...Raw 表示原文，即原语言
//作者
@ExcelProperty(value = "作者", index = 4)
private String author;
private String authorRaw;
//正文
@ExcelProperty(value = "正文", index = 3)
private String content;
private String contentRaw;
//带标签正文
private String contentWithTag;
private String contentWithTagRaw;
//入库时间
private String createDate;
private Date createDateTime;
//信息id
private String id;
//信息来源id(信息源或者关键词)
private String sid;
//语言
private String lang;
private String langRaw;
//来源（信息来源）
@ExcelProperty(value = "来源", index = 5)
private String origin;
private String originRaw;
//发布时间
@ExcelProperty(value = "发布时间", index = 6)
private String publishDate;
//原始时间（网页上，没有解析之前的时间）
private String originalTime;
//发布地址
@ExcelProperty(value = "网址", index = 7)
private String sourceAddress;
//摘要
@ExcelProperty(value = "摘要", index = 2)
private String summary;
private String summaryRaw;
//关键词
private String keyWords;
//标题
@ExcelProperty(value = "标题", index = 1)
private String title;
private String titleRaw;
//采集来源（如通用、定制、微信公众号等）
private String source;
//附加字段
private String type;
//标签信息
private List<Label> labels;
//视频下载链接
private String downLoadUrl;
//视频链接（原链接 网页版）
private String videoUrl;
//视频链接（原链接 手机版）
private String videoPhoneUrl;
//视频时长
private Long videoTime;
//视频第一帧图片
private String videoImg;
//重复id
private String repeatId;
//（1：主条目 0：非主条目）
private String flag;
//关联的主条目id
private String masterEntryId;
//原始id(去重服务生成的id,因为一条信息可以属于多个专题，原始id会发生改变，所以存储一个原始id,找寻对应关系)
private String originalId;
private String subjectId;
//快照地址
private String screenShotImg;
//数据所在的索引名称
private String index;
}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论