Skip to content

T
think-tank
提交 92411084 作者: yanxin
操作

导出增加“智库机构采集统计信息”

上级 89e1b47c
正在显示 包含 310 行增加 和 8 行删除
src/main/java/com/zzsn/thinktank/controller/ThinktankBasicInfoController.java
package com.zzsn.thinktank.controller;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.http.HttpUtil;
import cn.hutool.poi.excel.ExcelReader;
import cn.hutool.poi.excel.ExcelUtil;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.zzsn.thinktank.entity.ThinktankBasicInfo;
import com.zzsn.thinktank.service.LeaderCategoryService;
import com.zzsn.thinktank.service.ThinktankBasicInfoService;
......@@ -13,6 +17,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
......@@ -20,6 +25,7 @@ import javax.servlet.http.HttpServletResponse;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
/**
* @Version 1.0
......@@ -37,6 +43,8 @@ public class ThinktankBasicInfoController {
@Autowired
LeaderCategoryService leaderCategoryService;
@Value("${cj.url.infoSource.columnDetail:http://1.95.79.85:8823/baseSourceInfo/api/infoSource/columnDetail?columnIds={IDS}}")
private String CJ_IINFOSOURCE_COLUMNDETAIL;
/**
* 新增
......@@ -199,13 +207,65 @@ public class ThinktankBasicInfoController {
List<List<String>> rows = new ArrayList<>();
thinkTankBasicInfos.forEach(each -> rows.add(each.toList()));
ExcelExportUtil.exportExcelData(workbook, 0, Arrays.asList(arr), rows, "智库基本信息");
//统计信息
//智库栏目采集统计信息
String[] header = new String[]{"智库机构id", "智库机构编码", "中文全称", "关联信息源数量", "信息采集总量", "关联信息源编码",
"关联信息源名称", "栏目名称", "网址", "启用状态(1-启用;0-禁用)","信息源采集数量", "信息源近一月采集量"};
"关联信息源名称", "栏目名称", "网址", "启用状态(1-启用;0-禁用)","信息源采集数量", "信息源近一月采集量","采集状态"};
List<ThinkTankSourceVO> thinkTankSourceVOS = thinktankBasicInfoService.thinkTankCollectCount(thinkTankBasicInfos);
//补充信息源采集状态
List<String> allIds = thinkTankSourceVOS.stream().map(ThinkTankSourceVO::getSourceId).distinct().collect(Collectors.toList());
List<List<String>> groupIdList = CollectionUtil.split(allIds, 300);
Map<String, JSONObject> infoMap = new HashMap<>();
for (List<String> ids : groupIdList) {
String res = HttpUtil.get(CJ_IINFOSOURCE_COLUMNDETAIL.replace("{IDS}", StringUtils.join(ids, ",")));
JSONArray result = JSONObject.parseObject(res).getJSONArray("result");
for (int i = 0; i < result.size(); i++) {
JSONObject record = result.getJSONObject(i);
infoMap.put(record.getString("id"),record);
}
}
for (ThinkTankSourceVO thinkTankSourceVO : thinkTankSourceVOS) {
JSONObject record = infoMap.get(thinkTankSourceVO.getSourceId());
if(record==null){
continue;
}
//采集状态
switch (record.getString("abnormalCode")){
case "1":
thinkTankSourceVO.setAbnormalName("正常采集");
break;
case "101":
thinkTankSourceVO.setAbnormalName("访问异常");
break;
case "201":
thinkTankSourceVO.setAbnormalName("列表页解析异常");
break;
case "202":
thinkTankSourceVO.setAbnormalName("详情页配置异常");
break;
case "203":
thinkTankSourceVO.setAbnormalName("详情规则有差异");
break;
case "301":
thinkTankSourceVO.setAbnormalName("信息源漏采");
break;
case "503":
thinkTankSourceVO.setAbnormalName("账号异常");
break;
case "601":
thinkTankSourceVO.setAbnormalName("微信公众号异堂");
break;
default:
thinkTankSourceVO.setAbnormalName(record.getString("abnormalCode"));
}
}
List<List<String>> rowList = new ArrayList<>();
thinkTankSourceVOS.forEach(e->rowList.add(e.toList()));
ExcelExportUtil.exportExcelData(workbook, 1, Arrays.asList(header), rowList, "智库采集统计信息");
ExcelExportUtil.exportExcelData(workbook, 1, Arrays.asList(header), rowList, "智库栏目采集统计信息");
//智库机构采集统计信息
List<String> tIds = thinkTankBasicInfos.stream().map(ThinktankBasicInfo::getId).collect(Collectors.toList());
exportBindCount(workbook, 2, "智库机构采集统计信息",infoMap,tIds);
ByteArrayOutputStream outStream = new ByteArrayOutputStream();
workbook.write(outStream);
// 将字节流转换为InputStream
......@@ -260,4 +320,193 @@ public class ThinktankBasicInfoController {
}
return false;
}
/**
* 基于信息源组的采集状态统计导出
*
* @param workbook
* @param sheetNum
* @param sheetName
* @param infoMap
* @param tIds 智库id列表
*/
public void exportBindCount(XSSFWorkbook workbook, int sheetNum, String sheetName, Map<String, JSONObject> infoMap, List<String> tIds){
//查询所有的
List<CountVo> bindAll = thinktankBasicInfoService.bindInfoSource(tIds);
List<String> headers = new ArrayList<>();
headers.add("智库机构id");
headers.add("智库机构编码");
headers.add("中文全称");
headers.add("中文简称");
headers.add("英文全称");
headers.add("英文简称");
headers.add("国别");
headers.add("影响力及地位");
headers.add("是否展示");
headers.add("栏目数");
Map<String,Integer> columnNum = new HashMap<>();
headers.add("启用数");
Map<String,Integer> enableNum = new HashMap<>();
headers.add("未启用数");
Map<String,Integer> disableNum = new HashMap<>();
headers.add("正常采集");
Map<String,Integer> normalNum = new HashMap<>();
headers.add("访问异常");
Map<String,Integer> errorNum = new HashMap<>();
headers.add("列表页解析异常");
Map<String,Integer> listParseErrorNum = new HashMap<>();
headers.add("详情页配置异常");
Map<String,Integer> detailParseErrorNum = new HashMap<>();
headers.add("详情规则有差异");
Map<String,Integer> detailParseDiffNum = new HashMap<>();
headers.add("信息源漏采");
Map<String,Integer> missNum = new HashMap<>();
headers.add("账号异常");
Map<String,Integer> accountErrorNum = new HashMap<>();
headers.add("微信公众号异堂");
Map<String,Integer> wechatErrorNum = new HashMap<>();
headers.add("待配置");
Map<String,Integer> configNum = new HashMap<>();
headers.add("待验证");
Map<String,Integer> verifyNum = new HashMap<>();
headers.add("验证通过");
Map<String,Integer> verifyPassNum = new HashMap<>();
headers.add("验证不通过");
Map<String,Integer> verifyFailNum = new HashMap<>();
//查询智库列表
List<String> hasInfoIds = bindAll.stream().map(CountVo::getId).distinct().collect(Collectors.toList());
List<ThinktankBasicInfo> basicInfos = thinktankBasicInfoService.listByIds(hasInfoIds);
for (ThinktankBasicInfo basicInfo : basicInfos) {
String tid = basicInfo.getId();
//查询智库关联的信息源列表
List<String> sourceList = bindAll.stream().filter(item -> tid.equals(item.getId())).map(CountVo::getCode).collect(Collectors.toList());
for (String sourceId : sourceList) {
JSONObject record = infoMap.get(sourceId);
if(record==null){
continue;
}
countAdd(columnNum,tid);
//是否启用
if("1".equals(record.getString("enable"))){
countAdd(enableNum,tid);
}else{
countAdd(disableNum,tid);
}
//采集状态
switch (record.getString("abnormalCode")){
case "1":
countAdd(normalNum,tid);
break;
case "101":
countAdd(errorNum,tid);
break;
case "201":
countAdd(listParseErrorNum,tid);
break;
case "202":
countAdd(detailParseErrorNum,tid);
break;
case "203":
countAdd(detailParseDiffNum,tid);
break;
case "301":
countAdd(missNum,tid);
break;
case "503":
countAdd(accountErrorNum,tid);
break;
case "601":
countAdd(wechatErrorNum,tid);
break;
}
//状态
switch (record.getString("status")){
case "1":
countAdd(configNum,tid);
break;
case "2":
countAdd(verifyNum,tid);
break;
case "3":
countAdd(verifyPassNum,tid);
break;
case "4":
countAdd(verifyFailNum,tid);
break;
}
}
}
List<List<String>> rows = new ArrayList<>();
for (ThinktankBasicInfo basicInfo : basicInfos) {
List<String> dataItem = new ArrayList<>();
String id = basicInfo.getId();
//智库机构id
dataItem.add(basicInfo.getId());
//智库机构编码
dataItem.add(basicInfo.getCodeId());
//中文全称
dataItem.add(basicInfo.getChineseWhole());
//中文简称
dataItem.add(basicInfo.getChineseSimple());
//英文全称
dataItem.add(basicInfo.getEnglishWhole());
//英文简称
dataItem.add(basicInfo.getEnglishSimple());
//国别
dataItem.add(basicInfo.getBelongCountry());
//影响力及地位
dataItem.add(basicInfo.getInfluencePosition());
//是否展示
dataItem.add(String.valueOf(basicInfo.getYnDisplay()));
//栏目数
dataItem.add(getMapStr(columnNum,id));
//是否启用
dataItem.add(getMapStr(enableNum,id));
dataItem.add(getMapStr(disableNum,id));
//采集状态
dataItem.add(getMapStr(normalNum,id));
dataItem.add(getMapStr(errorNum,id));
dataItem.add(getMapStr(listParseErrorNum,id));
dataItem.add(getMapStr(detailParseErrorNum,id));
dataItem.add(getMapStr(detailParseDiffNum,id));
dataItem.add(getMapStr(missNum,id));
dataItem.add(getMapStr(accountErrorNum,id));
dataItem.add(getMapStr(wechatErrorNum,id));
//状态
dataItem.add(getMapStr(configNum,id));
dataItem.add(getMapStr(verifyNum,id));
dataItem.add(getMapStr(verifyPassNum,id));
dataItem.add(getMapStr(verifyFailNum,id));
rows.add(dataItem);
}
ExcelExportUtil.exportExcelData(workbook,sheetNum, headers, rows, sheetName);
}
private String getMapStr(Map<String, Integer> map, String id) {
if(map.containsKey(id)){
return map.get(id) + "";
}
return "0";
}
private String getSumMapStr(Map<String, Integer> map) {
if(map.isEmpty()){
return "0";
}
return map.values().stream().mapToInt(Integer::intValue).sum() + "";
}
private void countAdd(Map<String, Integer> map, String id) {
if (map.containsKey(id)) {
map.put(id, map.get(id) + 1);
} else {
map.put(id, 1);
}
}
}
src/main/java/com/zzsn/thinktank/entity/ThinktankBasicInfo.java
......@@ -102,6 +102,10 @@ public class ThinktankBasicInfo extends Model<ThinktankBasicInfo> {
//是否展示(1-是;0-否)
private Integer ynDisplay;
//信息源id
@TableField(exist = false)
private String sourceId;
public List<String> toList(){
List<String> list = new ArrayList<>();
list.add(getValue(id));
......
src/main/java/com/zzsn/thinktank/mapper/ThinktankBasicInfoMapper.java
......@@ -2,11 +2,13 @@ package com.zzsn.thinktank.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zzsn.thinktank.entity.ThinktankBasicInfo;
import com.zzsn.thinktank.vo.CountVo;
import com.zzsn.thinktank.vo.ThinktankBasicInfoExportVo;
import com.zzsn.thinktank.vo.ThinktankBasicInfoListVo;
import com.zzsn.thinktank.vo.ThinktankBasicInfoVo;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Update;
import java.util.List;
......@@ -38,4 +40,6 @@ public interface ThinktankBasicInfoMapper extends BaseMapper<ThinktankBasicInfo>
*/
List<ThinktankBasicInfo> exportListByCondition(@Param("ids") List<String> ids, @Param("codeId") String codeId, @Param("typeId") String typeId,
@Param("thinktankName") String thinktankName, @Param("tagId") String tagId, @Param("keyword") String keyword);
List<CountVo> bindInfoSource(@Param("ids")List<String> ids);
}
src/main/java/com/zzsn/thinktank/service/ThinktankBasicInfoService.java
......@@ -61,4 +61,11 @@ public interface ThinktankBasicInfoService extends IService<ThinktankBasicInfo>
* @date 2024/8/23
*/
List<ThinkTankSourceVO> thinkTankCollectCount(List<ThinktankBasicInfo> dataList);
/**
* 根据智库id查询绑定的信息源id
* @param ids
* @return
*/
List<CountVo> bindInfoSource(List<String> ids);
}
src/main/java/com/zzsn/thinktank/service/impl/ThinktankBasicInfoServiceImpl.java
......@@ -539,6 +539,11 @@ public class ThinktankBasicInfoServiceImpl extends ServiceImpl<ThinktankBasicInf
}
@Override
public List<CountVo> bindInfoSource(List<String> ids) {
return this.baseMapper.bindInfoSource(ids);
}
@Override
public List<ThinktankBasicInfo> exportListByCondition(String id, String codeId, String typeId, String thinktankName, String tagId, String keyword) {
List<String> ids = new ArrayList<>();
if (StringUtils.isNotEmpty(id)) {
......
src/main/java/com/zzsn/thinktank/vo/CountVo.java 0 → 100644
package com.zzsn.thinktank.vo;
import lombok.Data;
/**
* @Description: 统计
*/
@Data
public class CountVo {
private String id;
private String code;
private String name;
private Integer count;
}
src/main/java/com/zzsn/thinktank/vo/ThinkTankSourceVO.java
......@@ -43,6 +43,8 @@ public class ThinkTankSourceVO {
private Integer infoMonthlyCount;
/**智库下的资讯数量**/
private Integer groupCount;
/**信息源采集状态**/
private String abnormalName;
public List<String> toList(){
List<String> list = new ArrayList<>();
......@@ -74,6 +76,7 @@ public class ThinkTankSourceVO {
} else {
list.add(String.valueOf(infoMonthlyCount));
}
list.add(getValue(abnormalName));
return list;
}
......
src/main/resources/application.yml
......@@ -12,12 +12,12 @@ spring:
max-request-size: 1000MB
datasource:
url: jdbc:mysql://114.116.44.11:3306/clb_project?useUnicode=true&characterEncoding=utf-8&AllowPublicKeyRetrieval=True&serverTimezone=Asia/Shanghai&autoReconnect=true&rewriteBatchedStatements=true
url: jdbc:mysql://1.95.14.228:3306/clb_project?useUnicode=true&characterEncoding=utf-8&AllowPublicKeyRetrieval=True&serverTimezone=Asia/Shanghai&autoReconnect=true&rewriteBatchedStatements=true
username: ciglobal
password: _PSuWVQ4CLpX
redis:
database: 0
host: 114.116.90.53
host: 1.95.69.2
password: RPHZgkDQ4zGJ
port: 6380
mybatis-plus:
......@@ -28,12 +28,12 @@ mybatis-plus:
#生产es集群
es1:
endpoint1: 114.115.215.250
endpoint1: 1.95.38.69
endpoint1port: 9700
endpoint2: 1.95.3.121
endpoint2port: 9200
endpoint3: 114.115.215.96
endpoint3port: 9200
endpoint3: 1.95.87.177
endpoint3port: 9700
username: elastic
password: zzsn9988
......
src/main/resources/mapper/ThinktankBasicInfoMapper.xml
......@@ -141,6 +141,20 @@
</if>
order by create_time desc
</select>
<select id="bindInfoSource" resultType="com.zzsn.thinktank.vo.CountVo">
SELECT
group_id id,source_id code
FROM
thinktank_info_source_group_map
WHERE 1=1
<if test="ids != null and ids.size() > 0">
AND group_id in
<foreach item="item" index="index" collection="ids" open="(" separator="," close=")">
#{item}
</foreach>
</if>
</select>
</mapper>
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论