Skip to content

C
collection
提交 c17abf1b 作者: ZhangJingKun
操作

采集监控初始化提交

上级
正在显示 包含 2124 行增加 和 0 行删除
.idea/.gitignore 0 → 100644
# Default ignored files
/workspace.xml
\ No newline at end of file
.idea/compiler.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile default="true" name="Default" enabled="true" />
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="collection" />
</profile>
</annotationProcessing>
</component>
<component name="JavacSettings">
<option name="ADDITIONAL_OPTIONS_OVERRIDE">
<module name="collection" options="-parameters" />
</option>
</component>
</project>
\ No newline at end of file
.idea/encodings.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$" charset="UTF-8" />
</component>
</project>
\ No newline at end of file
.idea/misc.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
\ No newline at end of file
.idea/vcs.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
\ No newline at end of file
pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.3.5.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.zzsn.collection</groupId>
<artifactId>collection</artifactId>
<version>1.0-SNAPSHOT</version>
<name>collection</name>
<description>采集日志监控</description>
<properties>
<java.version>1.8</java.version>
<maven.test.skip>true</maven.test.skip>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<!--mybatis plus-->
<dependency>
<groupId>com.baomidou</groupId>
<artifactId>mybatis-plus-boot-starter</artifactId>
<version>3.4.1</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.21</version>
</dependency>
<!-- hutool工具类-->
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.3.8</version>
</dependency>
<!-- json-->
<dependency>
<groupId>com.alibaba.fastjson2</groupId>
<artifactId>fastjson2</artifactId>
<version>2.0.25</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.6</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<!--工具类依赖 -->
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.10</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.6</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
<resources>
<resource>
<directory>${project.basedir}/lib</directory>
<targetPath>BOOT-INF/lib/</targetPath>
<includes>
<include>**/*.jar</include>
</includes>
</resource>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
</build>
</project>
\ No newline at end of file
src/main/java/com/zzsn/collection/CollectionApplication.java 0 → 100644
package com.zzsn.collection;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/5/27 10:09
* @Content:
*/
@SpringBootApplication
public class CollectionApplication {
public static void main(String[] args) {
SpringApplication.run(CollectionApplication.class, args);
}
}
src/main/java/com/zzsn/collection/controller/CollectionController.java 0 → 100644
package com.zzsn.collection.controller;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/2/6 11:06
* @Content:
*/
@Controller
@RequestMapping("/api")
@Slf4j
public class CollectionController {
@GetMapping("/test")
public String test(Model model){
log.info("hello");
model.addAttribute("msg","hello word!");
return "hello";
}
}
src/main/java/com/zzsn/collection/controller/CollectionLogController.java 0 → 100644
package com.zzsn.collection.controller;
import com.zzsn.collection.entity.CollectionLog;
import com.zzsn.collection.service.CollectionLogService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.Map;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/5/27 10:29
* @Content:
*/
@Controller
@RequestMapping("/api/collectionlog")
@Slf4j
public class CollectionLogController {
@Autowired
CollectionLogService collectionLogService;
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
/*
https://blog.csdn.net/weixin_53949546/article/details/124944754
*/
/**
* 查询信息源的采集情况
* @param model
* @param infoSourceCode
* @return
*/
@GetMapping("/getJobLog")
public String getJobLog(Model model, @RequestParam(name="infoSourceCode",required=false) String infoSourceCode){
log.info("getJobLog:{}",infoSourceCode);
List<CollectionLog> list = collectionLogService.getJobLog(infoSourceCode);
model.addAttribute("list",list);
return "collection";
}
/**
* 采集情况统计
* @param model
* @return
*/
@GetMapping("/statistics")
public String statistics(Model model, @RequestParam(name="sign",required=false) String sign){
LocalDateTime date = LocalDateTime.now();
LocalDateTime localDateTime = date;
if("1".equals(sign)){
localDateTime = date.minusHours(1);
model.addAttribute("signValue","最近1小时统计结果");
} else {
localDateTime = date.minusHours(24);
model.addAttribute("signValue","最近24小时统计结果");
}
String createDate = dateTimeFormatter.format(localDateTime);
log.info("采集情况统计:{}---{}", sign, createDate);
List<Map<String, String>> list = collectionLogService.statistics(createDate);
model.addAttribute("list",list);
return "statisticsList";
}
}
src/main/java/com/zzsn/collection/entity/CollectionInfoSource.java 0 → 100644
package com.zzsn.collection.entity;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.Data;
import org.springframework.format.annotation.DateTimeFormat;
import java.util.Date;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/8/10 16:00
* @Content:
*/
@Data
@TableName("collection_info_source")
public class CollectionInfoSource {
//主键
private String id;
//信息源编码
private String infoSourceCode;
//栏目名称
private String siteName;
//资讯url
private String url;
//成功次数
private int successNum;
//失败次数
private int failNum;
//备注信息
private String remark;
//创建时间
@JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date createTime;
}
src/main/java/com/zzsn/collection/entity/CollectionLog.java 0 → 100644
package com.zzsn.collection.entity;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.Data;
import org.springframework.format.annotation.DateTimeFormat;
import java.util.Date;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/5/24 16:51
* @Content:
*/
@Data
@TableName("collection_log")
public class CollectionLog {
//主键
private String id;
//类别 1定时任务 2采集情况
private Integer type;
//信息源id
private String sid;
//信息源编码
private String infoSourceCode;
//栏目名称
private String siteName;
//程序标识
private String serverCode;
//标题
private String title;
//资讯唯一标识
private String uniqueCode;
//资讯url
private String url;
//备注信息
private String remark;
//创建时间
@JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date createTime;
//备用字段
private String coll1;
private String coll2;
private String coll3;
}
src/main/java/com/zzsn/collection/entity/SiteMsgTemple.java 0 → 100644
package com.zzsn.collection.entity;
import lombok.Data;
import java.io.Serializable;
import java.util.regex.Pattern;
@Data
public class SiteMsgTemple implements Serializable {
private static final long serialVersionUID = 1L;
/**主键*/
private String id;
/**信息源编码*/
private String infoSourceCode;
/**信息源名称*/
private String webSiteName;
/**栏目名称*/
private String siteName;
/**栏目地址*/
private String siteUri;
/**语种*/
private String language;
/**境外、公共、翻墙*/
private String checkedList;
/**历史数据URL*/
private String hisUriExp;
/**历史数据开始时间*/
// private java.util.Date hisDateStartTime;
private String hisDateStartTime;
/**历史数据结束时间*/
// private java.util.Date hisDateEndTime;
private String hisDateEndTime;
/**是否历史所有数据*/
private String ynHisDataAll;
/**网站级别*/
private String siteLevel;
/**状态*/
private Integer status;
/**列表页URL*/
private String listUrl;
/**表达式类型*/
private String listExpressionType;
/**
* 列表信息块位置
*/
private String infoBlockPosition;
/**
*抽取链接定位
*/
private String linkLocation;
/**匹配资讯的列表*/
private String informationItem;
/**匹配资讯的url*/
private String informationUrl;
/**匹配资讯标题*/
private String informationTitle;
/**匹配资讯发布时间*/
private String informationPublishDate;
/**匹配资讯来源*/
private String informationSource;
/**自定义实体*/
private Object extractInfo;
/**爬取深度*/
private Integer crawlDepth;
/**页码url*/
private String pageUrl;
/**匹配页码*/
private String matchPage;
/**开始页码*/
private Integer pageStart;
/**结束页码*/
private Integer pageEnd;
/**是否所有页*/
private String ynPageAll;
/**表达式类型*/
private String detailExpressionType;
/**详情页表URL*/
private String detailUrl;
/**匹配详情页标题*/
private String detailExpressionTitle;
/**匹配详情页时间*/
private String detailExpressionPublishDate;
/**匹配详情页来源*/
private String detailExpressionSource;
/**匹配详情页作者*/
private String detailExpressionAuthor;
/**匹配详情页摘要*/
private String detailExpressionSummary;
/**匹配详情页正文*/
private String detailExpressionContent;
/**自定义实体*/
private Object detailInfo;
/**是否下载附件*/
private String ynDownload;
/**数据表格页URL*/
private String formUrl;
/**数据表格标题*/
private String formTitle;
/**表达式类型*/
private Integer formType;
/**数据表格表达式*/
private String dataFormExpression;
/**自定义*/
private Object dataFormInfo;
/**页码URL*/
private String dataPageUrl;
/**页码规则*/
private String dataPageRule;
/**开始页码*/
private Integer dataPageStart;
/**结束页码*/
private Integer dataPageEnd;
/**是否所有页码*/
private String ynDataPageAll;
/**数据类型*/
private Integer dataType;
/**数据格式*/
private Integer dataFormat;
/**数据存储方式*/
private Integer dataStorageMode;
/**数据存储信息*/
private Object dataStorageInfo;
/**是否动态爬取*/
private Integer ynDynamicCrawl;
/**是否需要登陆*/
private Integer ynLogin;
/**登陆域名*/
private String domainName;
/**登陆链接*/
private String link;
/**登陆账号*/
private String account;
/**登陆密码*/
private String password;
/**userAgent*/
private String userAgent;
/**referer*/
private String referer;
/**cookies*/
private String cookies;
/**headers*/
private String headers;
/**其它参数*/
private String otherInfo;
/**爬虫类别*/
private Integer crawlType;
/**爬虫名称*/
private String crawlName;
/**爬虫地址*/
private String crawlAddress;
/**参数*/
private Object parameter;
/**cron表达式*/
private String cron;
// /**是否需要快照*/
// private String isScreenshot;
//++++++++++++++++++++++++++++++++++++++++++++++++++
private Pattern pattern;
/**是否保存快照（1：保存 0：不保存）*/
private String ynSnapshot;
/**是否为验证方法（1：是验证 0：不是）*/
private String verifyType;
}
src/main/java/com/zzsn/collection/mapper/CollectionInfoSourceMapper.java 0 → 100644
package com.zzsn.collection.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zzsn.collection.entity.CollectionInfoSource;
import org.apache.ibatis.annotations.Mapper;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/8/10 16:16
* @Content:
*/
@Mapper
public interface CollectionInfoSourceMapper extends BaseMapper<CollectionInfoSource> {
}
src/main/java/com/zzsn/collection/mapper/CollectionLogMapper.java 0 → 100644
package com.zzsn.collection.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zzsn.collection.entity.CollectionLog;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Select;
import java.util.List;
import java.util.Map;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/5/25 14:54
* @Content:
*/
@Mapper
public interface CollectionLogMapper extends BaseMapper<CollectionLog> {
// @Select("select * from collection_log where type = 1 and info_source_code = #{infoSourceCode} limit 10")
@Select("select * from collection_log where type = 1 limit 10")
List<CollectionLog> getJobLog(String infoSourceCode);
@Select("select a.name,a.code,b.countSource,c.countInfo from collection_enum a " +
"LEFT JOIN ( " +
"select server_code,count(*) as countSource from collection_log " +
"where type = 1 " +
"and create_time > #{createDate} " +
"group by server_code " +
") b " +
"on a.code = b.server_code " +
"LEFT JOIN ( " +
"select server_code,count(*) as countInfo from collection_log " +
"where type = 2 " +
"and create_time > #{createDate} " +
"group by server_code " +
") c " +
"on a.code = c.server_code " +
"where type = 'statistics' " +
"ORDER BY a.sort asc ")
List<Map<String, String>> statistics(String createDate);
}
src/main/java/com/zzsn/collection/service/CollectionInfoSourceService.java 0 → 100644
package com.zzsn.collection.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.collection.entity.CollectionInfoSource;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/8/10 16:14
* @Content:
*/
public interface CollectionInfoSourceService extends IService<CollectionInfoSource> {
}
src/main/java/com/zzsn/collection/service/CollectionLogService.java 0 → 100644
package com.zzsn.collection.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.collection.entity.CollectionLog;
import java.util.List;
import java.util.Map;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/5/24 16:41
* @Content:
*/
public interface CollectionLogService extends IService<CollectionLog> {
List<CollectionLog> getJobLog(String infoSourceCode);
List<Map<String, String>> statistics(String createDate);
}
src/main/java/com/zzsn/collection/service/impl/CollectionInfoSourceServiceImpl.java 0 → 100644
package com.zzsn.collection.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.collection.entity.CollectionInfoSource;
import com.zzsn.collection.mapper.CollectionInfoSourceMapper;
import com.zzsn.collection.service.CollectionInfoSourceService;
import org.springframework.stereotype.Component;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/8/10 16:15
* @Content:
*/
@Component
public class CollectionInfoSourceServiceImpl extends ServiceImpl<CollectionInfoSourceMapper, CollectionInfoSource> implements CollectionInfoSourceService {
}
src/main/java/com/zzsn/collection/service/impl/CollectionLogServiceImpl.java 0 → 100644
package com.zzsn.collection.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.collection.entity.CollectionLog;
import com.zzsn.collection.mapper.CollectionLogMapper;
import com.zzsn.collection.service.CollectionLogService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.util.Date;
import java.util.List;
import java.util.Map;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/5/24 16:19
* @Content:
*/
@Component
@Slf4j
public class CollectionLogServiceImpl extends ServiceImpl<CollectionLogMapper, CollectionLog> implements CollectionLogService {
@Override
public List<CollectionLog> getJobLog(String infoSourceCode) {
// LambdaQueryWrapper<CollectionLog> lambdaQuery = Wrappers.lambdaQuery();
// lambdaQuery.eq(CollectionLog::getType, 1);
// List<CollectionLog> list = baseMapper.selectList(lambdaQuery);
List<CollectionLog> list = baseMapper.getJobLog(infoSourceCode);
return list;
}
@Override
public List<Map<String, String>> statistics(String createDate) {
List<Map<String, String>> list = baseMapper.statistics(createDate);
return list;
}
}
src/main/java/com/zzsn/collection/service/impl/CommandLineRunnerImpl.java 0 → 100644
package com.zzsn.collection.service.impl;
import com.alibaba.fastjson.JSON;
import com.google.gson.Gson;
import com.zzsn.collection.entity.CollectionInfoSource;
import com.zzsn.collection.entity.SiteMsgTemple;
import com.zzsn.collection.service.CollectionInfoSourceService;
import com.zzsn.collection.util.FileUtil;
import com.zzsn.collection.util.HttpUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Component;
import com.alibaba.fastjson.JSONObject;
import java.io.File;
import java.util.Date;
import java.util.List;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/8/10 11:17
* @Content: 项目启动后立即执行
*/
@Component
@Slf4j
public class CommandLineRunnerImpl implements CommandLineRunner {
@Autowired
private RedisTemplate<String, String> redisTemplate;
@Autowired
private CollectionInfoSourceService collectionInfoSourceService;
@Value("${file.info-source-path}")
private String INFO_SOURCE_PATH;
//列表
private String verifyListUrlJN = "http://114.115.154.140:9010/crawlerVerify/listPage";//境内列表
private String verifyListUrlJW = "http://114.115.151.210:9010/crawlerVerify/listPage";//境外列表
//详情
private String verifyDetailUrlJN = "http://114.115.154.140:9010/crawlerVerify/detailPage";//境内列表
private String verifyDetailUrlJW = "http://114.115.151.210:9010/crawlerVerify/detailPage";//境外列表
@Override
public void run(String... args) throws Exception {
log.info("信息源可用验证任务启动！");
String filepath= INFO_SOURCE_PATH;
log.info("信息源验证文件："+filepath);
try {
File f = new File(filepath);
List<String> allLines = FileUtil.getFileLines(f, "utf-8");
log.info("信息源数量"+allLines.size());
for (String keysite : allLines) {
String value = redisTemplate.opsForValue().get("INFO_SOURCE_TO_REDIS::" + keysite);
SiteMsgTemple siteMsgTemple = new Gson().fromJson(value, SiteMsgTemple.class);
if(!yanzheng(siteMsgTemple)){
//第一次失败
if(!yanzheng(siteMsgTemple)){
//第二次失败
yanzheng(siteMsgTemple);//验证失败的重试3次
}
}
}
}catch (Exception e){
e.getMessage();
}
}
public Boolean yanzheng(SiteMsgTemple siteMsgTemple){
String infoSourceCode = siteMsgTemple.getInfoSourceCode();
String siteName = siteMsgTemple.getSiteName();
String siteUri = siteMsgTemple.getSiteUri();
log.info("验证信息源：｛｝---栏目名称：｛｝---栏目地址：｛｝", infoSourceCode, siteName, siteUri);
try {
String checkedList = siteMsgTemple.getCheckedList();//是否翻墙
boolean contains = checkedList.contains("3");
String result;//验证结果
Boolean mark = false;//列表页验证是否成功标识
if(contains){
//境外
result = HttpUtil.doPost(verifyListUrlJW, (JSONObject)JSONObject.toJSON(siteMsgTemple), 5*60*1000);
} else {
//境内
result = HttpUtil.doPost(verifyListUrlJN, (JSONObject)JSONObject.toJSON(siteMsgTemple), 5*60*1000);
}
if(result != null){
JSONObject jsonObject = JSON.parseObject(result);
JSONObject data = (JSONObject) jsonObject.get("data");
if(data.size() > 0){
Integer matchUrlNum = Integer.parseInt(data.getString("matchUrlNum"));
if(matchUrlNum > 0){
mark = true;
}
}
}
if(mark){
//列表页验证成功，验证详情页
String resultDetail;//验证结果
if(contains){
//境外
resultDetail = HttpUtil.doPost(verifyDetailUrlJW, (JSONObject)JSONObject.toJSON(siteMsgTemple), 5*60*1000);
} else {
//境内
resultDetail = HttpUtil.doPost(verifyDetailUrlJN, (JSONObject)JSONObject.toJSON(siteMsgTemple), 5*60*1000);
}
if(resultDetail != null){
JSONObject jsonObject = JSON.parseObject(resultDetail);
JSONObject data = (JSONObject) jsonObject.get("data");
if(data.size() > 0){
String content = data.getString("content");
String publishDate = data.getString("publishDate");
if(content != null && content.length() > 0 && publishDate != null && publishDate.length() > 0){
//验证通过
insertCollInfo(siteMsgTemple, true, "");
} else {
insertCollInfo(siteMsgTemple, false, "正文或发布时间为空，");
}
} else {
insertCollInfo(siteMsgTemple, false, "详情页验证失败，");
}
} else {
insertCollInfo(siteMsgTemple, false, "详情页验证失败，");
}
} else {
insertCollInfo(siteMsgTemple, false, "列表页验证失败，");
}
return null;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
*
* @param siteMsgTemple
* @param b 成功标志
* @param remark
*/
public void insertCollInfo(SiteMsgTemple siteMsgTemple, Boolean b, String remark){
CollectionInfoSource coll = collectionInfoSourceService.getById(siteMsgTemple.getId());
if (coll == null){
CollectionInfoSource newColl = new CollectionInfoSource();
newColl.setId(siteMsgTemple.getId());
newColl.setInfoSourceCode(siteMsgTemple.getInfoSourceCode());
newColl.setSiteName(siteMsgTemple.getSiteName());
newColl.setUrl(siteMsgTemple.getSiteUri());
if(b){
newColl.setSuccessNum(1);
} else {
newColl.setFailNum(1);
}
newColl.setRemark(remark);
newColl.setCreateTime(new Date());
collectionInfoSourceService.save(newColl);
} else {
if(b){
coll.setSuccessNum(coll.getSuccessNum() + 1);
} else {
coll.setFailNum(coll.getFailNum() + 1);
}
coll.setRemark(coll.getRemark() + remark);
collectionInfoSourceService.updateById(coll);
}
}
}
src/main/java/com/zzsn/collection/util/FileUtil.java 0 → 100644
package com.zzsn.collection.util;
import java.io.*;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
public class FileUtil {
public static List<String> getFileLines(File file, String encoding)
throws IOException
{
List<String> lines = new ArrayList<String>();
if (encoding == null)
{
encoding = "utf-8";
}
FileInputStream stream = new FileInputStream(file);
InputStreamReader reader = new InputStreamReader(stream, encoding);
BufferedReader bufferedReader = new BufferedReader(reader);
String line = null;
while ((line = bufferedReader.readLine()) != null)
{
lines.add(line);
}
bufferedReader.close();
reader.close();
stream.close();
return lines;
}
public static void outMsg(String fileName, String content) {
File files = new File(fileName);
FileWriter writer = null;
try {
if (!files.exists()) {
files.createNewFile();
}
// 打开一个写文件器，构造函数中的第二个参数true表示以追加形式写文件
writer = new FileWriter(files, true);
writer.write(content);
writer.write("\n");
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if(writer != null){
writer.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
public static String getFileBody(File file, String encoding)
{
StringBuffer buffer = new StringBuffer();
try {
List<String> lines = getFileLines(file, encoding);
for (String line : lines)
{
buffer.append(line);
buffer.append("\n");
}
} catch (IOException e) {
e.printStackTrace();
}
return buffer.toString();
}
public static String getFileBody(String path, String encoding)
{
File file = new File(path);
if (!file.exists())
{
return null;
}
return getFileBody(file, encoding);
}
public static void mergeFile(String dir, int depth,
String outPath, String encoding)
{
File dirFile = new File(dir);
List<File> filesForMerge = collectFiles(dirFile, depth, -1);
StringBuffer buffer = new StringBuffer();
for (File file : filesForMerge)
{
String temp = getFileBody(file, encoding);
buffer.append(temp);
buffer.append("\n");
}
try {
PrintWriter pw = new PrintWriter(outPath);
pw.write(buffer.toString());
pw.close();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static List<File> collectFiles(File dirFile, int depth, int sampleNum)
{
List<File> filesForMerge = new ArrayList<File>();
if (1 == depth)
{
File[] subFiles = dirFile.listFiles();
for (File file : subFiles)
{
if (file.isDirectory())
{
continue;
}
filesForMerge.add(file);
}
filesForMerge = sampleFiles(filesForMerge, sampleNum);
}
else
{
File[] subFiles = dirFile.listFiles();
for (File file : subFiles)
{
if (!file.isDirectory())
{
continue;
}
List<File> files = collectFiles(file, depth-1, sampleNum);
if (files.size() > 0)
{
filesForMerge.addAll(files);
}
}
}
return filesForMerge;
}
private static List<File> sampleFiles(List<File> origFiles, int sampleNum)
{
List<File> sampleFiles = new ArrayList<File>();
int totalSize = origFiles.size();
if (sampleNum >= 0 && totalSize > sampleNum)
{
if (sampleNum < totalSize / 10)
{
sampleNum = totalSize / 10;
}
Random random = new Random();
int count = 0;
while (count < sampleNum)
{
int pos = random.nextInt(totalSize);
File nextFile = origFiles.get(pos);
if (sampleFiles.contains(nextFile))
{
continue;
}
sampleFiles.add(nextFile);
count ++;
}
return sampleFiles;
}
return origFiles;
}
/**
* eg: cmread-books\dushi\16
* */
public static String getDifferPath(File subFile, File ancestorFile)
{
File parentFile = subFile.getParentFile();
String dirpath = "";
while (!parentFile.getPath().equalsIgnoreCase(ancestorFile.getPath()))
{
dirpath = String.format("%s\\%s", parentFile.getName(), dirpath);
parentFile = parentFile.getParentFile();
}
return dirpath;
}
public static void makeDir(String path)
{
File dirFile = new File(path);
if (!dirFile.exists())
{
dirFile.mkdirs();
}
}
/**
* find file/dir toFindFileName from ancestors of searchDirName
* */
public static File findDecestorFile(File searchDir, String toFindFileName)
{
if (searchDir.getName().equalsIgnoreCase(toFindFileName))
{
return searchDir;
}
if (searchDir.isDirectory())
{
File[] subFiles = searchDir.listFiles();
for (File subFile : subFiles)
{
File temp = findDecestorFile(subFile, toFindFileName);
if (temp != null)
{
return temp;
}
}
}
return null;
}
public static File findFile(String fileName, File parentFile, int compType)
{
File[] files = parentFile.listFiles();
switch (compType) {
case 0: //equal
for (File file : files)
{
if (file.getName().equalsIgnoreCase(fileName))
{
return file;
}
}
break;
case 1: //start with
for (File file : files)
{
if (file.getName().toLowerCase().startsWith(fileName.toLowerCase()))
{
return file;
}
}
break;
case 2: //contains
for (File file : files)
{
if (file.getName().toLowerCase().indexOf(fileName.toLowerCase()) != -1)
{
return file;
}
}
break;
default:
break;
}
return null;
}
private static final char BOM = '\uFEFF';
public static ArrayList<String> readTxtFile(String fileName,String encoding) {
ArrayList<String> rtn = new ArrayList<String>();
BufferedReader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(new FileInputStream(fileName), encoding));
String line = null;
boolean isFirstLine = true;
while ((line = reader.readLine()) != null) {
if (isFirstLine && line.length() > 0 && line.charAt(0) == BOM) {
line = line.substring(1);
isFirstLine = false;
}
rtn.add(line);
}
} catch (Exception e) {
String errMsg = String.format("Exception while reading file: %s", fileName, e.getMessage());
e.printStackTrace();
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e) {
String errMsg = String.format("Exception while closing file [%s]: %s", fileName, e.getMessage());
e.printStackTrace();
}
}
}
return rtn;
}
public static void copyFile(File source, File dest) throws IOException {
FileChannel inputChannel = null;
FileChannel outputChannel = null;
try {
inputChannel = new FileInputStream(source).getChannel();
outputChannel = new FileOutputStream(dest).getChannel();
outputChannel.transferFrom(inputChannel, 0, inputChannel.size());
} finally {
inputChannel.close();
outputChannel.close();
}
}
public static List<String> findFiles(String ph) {
List<String> filesList = new ArrayList<String>();
File file = new File(ph);
if (!file.exists()) {
return null;
}
File tempFile;
File[] files = file.listFiles();
for (int i = 0; i < files.length; i++) {
tempFile = files[i];
if (tempFile.isFile()) {
filesList.add(tempFile.getName().toString());
}
}
return filesList;
}
}
src/main/java/com/zzsn/collection/util/HttpUtil.java 0 → 100644
package com.zzsn.collection.util;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.*;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.ssl.TrustStrategy;
import org.apache.http.util.EntityUtils;
import org.springframework.util.CollectionUtils;
import javax.net.ssl.SSLContext;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.*;
/**
* @Description: Http工具类
* @Author: zhangshuo
* @Date: 2021-06-08
* @Version: V1.0
*/
@Slf4j
public class HttpUtil {
private static final CloseableHttpClient httpClient;
private static final String CHARSET = "utf-8";
// 采用静态代码块，初始化超时时间配置，再根据配置生成默认httpClient对象
static {
RequestConfig config = RequestConfig.custom().setConnectTimeout(10000).setSocketTimeout(10000).build();
httpClient = HttpClientBuilder.create().setDefaultRequestConfig(config).build();
}
/**
* HTTP Get 获取内容
*
* @param url 请求的url地址
* @param params 请求的参数
* @param charset 编码格式
* @return 页面内容
*/
public static String doGet(String url, Map<String, String> params, String charset) {
if (url == null || url.length() == 0) {
return null;
}
try {
if (params != null && !params.isEmpty()) {
List<NameValuePair> pairs = new ArrayList<NameValuePair>(params.size());
for (Map.Entry<String, String> entry : params.entrySet()) {
String value = entry.getValue();
if (value != null) {
pairs.add(new BasicNameValuePair(entry.getKey(), value));
}
}
// 将请求参数和url进行拼接
url += "?" + EntityUtils.toString(new UrlEncodedFormEntity(pairs, charset));
}
HttpGet httpGet = new HttpGet(url);
CloseableHttpResponse response = httpClient.execute(httpGet);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != 200) {
httpGet.abort();
throw new RuntimeException("HttpClient,error status code :" + statusCode);
}
HttpEntity entity = response.getEntity();
String result = null;
if (entity != null) {
result = EntityUtils.toString(entity, "utf-8");
}
EntityUtils.consume(entity);
response.close();
return result;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
/**
* HTTP Post 获取内容
*
* @param url 请求的url地址
* @return 页面内容
* @throws IOException
*/
public static String doPost(String url, JSONObject jsonObject, int ExTime)
throws IOException {
HttpClient httpclient = HttpClients.createDefault();
RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(ExTime).setConnectTimeout(ExTime).build();
HttpPost httpPost = new HttpPost(url);
httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");
httpPost.setHeader("Accept", "application/json");
httpPost.setConfig(requestConfig);
StringEntity se = new StringEntity(jsonObject.toJSONString(), "utf-8");
se.setContentType("application/json");
httpPost.setEntity(se);
HttpResponse response = httpclient.execute(httpPost);
String result = EntityUtils.toString(response.getEntity());
return result;
}
public static String doPost(String url, String jsonString, int ExTime)
throws IOException {
HttpClient httpclient = HttpClients.createDefault();
RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(ExTime).setConnectTimeout(ExTime).build();
HttpPost httpPost = new HttpPost(url);
httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");
httpPost.setHeader("Accept", "application/json");
httpPost.setConfig(requestConfig);
StringEntity se = new StringEntity(jsonString, "utf-8");
se.setContentType("application/json");
httpPost.setEntity(se);
HttpResponse response = httpclient.execute(httpPost);
String result = EntityUtils.toString(response.getEntity());
return result;
}
/**
* HTTP Post 获取内容
*
* @param url 请求的url地址
* @return 页面内容
* @throws IOException
*/
public static String doPostWithHeader(String url, JSONObject jsonObject, int ExTime ,Map<String,String> headers)
throws IOException {
HttpClient httpclient = HttpClients.createDefault();
RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(ExTime).setConnectTimeout(ExTime).build();
HttpPost httpPost = new HttpPost(url);
if (!CollectionUtils.isEmpty(headers)) {
for (Map.Entry<String, String> entry : headers.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
httpPost.setHeader(key, value);
}
}
httpPost.setConfig(requestConfig);
StringEntity se = new StringEntity(jsonObject.toJSONString(), "utf-8");
se.setContentType("application/json");
httpPost.setEntity(se);
HttpResponse response = httpclient.execute(httpPost);
String result = EntityUtils.toString(response.getEntity());
return result;
}
/**
* HTTP Post 获取内容
*
* @param url 请求的url地址
* @return 页面内容
* @throws IOException
*/
public static String doPost(String url, JSONArray data, int ExTime)
throws IOException {
HttpClient httpclient = HttpClients.createDefault();
RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(ExTime).setConnectTimeout(ExTime).build();
HttpPost httpPost = new HttpPost(url);
httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");
httpPost.setHeader("Accept", "application/json");
httpPost.setConfig(requestConfig);
StringEntity se = new StringEntity(data.toJSONString(), "utf-8");
se.setContentType("application/json");
httpPost.setEntity(se);
HttpResponse response = httpclient.execute(httpPost);
String result = EntityUtils.toString(response.getEntity());
return result;
}
/**
* HTTPS Get 获取内容(无SSL证书验证)
*
* @param url 请求的url地址
* @param params 请求的参数
* @param charset 编码格式
* @return 页面内容
*/
public static CloseableHttpResponse doGetSSL(String url, Map<String, String> params, String charset) {
if (url == null || url.length() == 0) {
return null;
}
try {
if (params != null && !params.isEmpty()) {
List<NameValuePair> pairs = new ArrayList<NameValuePair>(params.size());
for (Map.Entry<String, String> entry : params.entrySet()) {
String value = entry.getValue();
if (value != null) {
pairs.add(new BasicNameValuePair(entry.getKey(), value));
}
}
url += "?" + EntityUtils.toString(new UrlEncodedFormEntity(pairs, charset));
}
HttpGet httpGet = new HttpGet(url);
httpGet.setHeader("Content-Type", "application/x-www-form-urlencoded;charset=utf-8");
httpGet.setHeader(HttpHeaders.CONNECTION, "close");
// https 注意这里获取https内容，使用了忽略证书的方式，当然还有其他的方式来获取https内容
CloseableHttpClient httpsClient = createSSLClientDefault();
CloseableHttpResponse response = httpsClient.execute(httpGet);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != 200) {
httpGet.abort();
throw new RuntimeException("HttpClient,error status code :" + statusCode);
}
return response;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
/**
* 这里创建了忽略整数验证的CloseableHttpClient对象
*
* @return SSLClientDefault
*/
public static CloseableHttpClient createSSLClientDefault() {
try {
SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
// 信任所有
public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException {
return true;
}
}).build();
SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
return HttpClients.custom().setSSLSocketFactory(sslsf).build();
} catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) {
e.printStackTrace();
}
return HttpClients.createDefault();
}
/**
* ip代理请求
*
* @throws Exception
*/
public static CloseableHttpResponse getProxyHttpClient(String url) {
//获取代理ip信息
//TODO
String proxyHost = "";
int proxyPort = 0;
String userName = "";
String password = "";
try {
//设置代理IP和端口并设置链接、传输时间
HttpHost proxy = new HttpHost(proxyHost, proxyPort);
RequestConfig config = RequestConfig.custom().setProxy(proxy).setConnectTimeout(60000).setSocketTimeout(60000).build();
//设置账号密码
CredentialsProvider provider = new BasicCredentialsProvider();
provider.setCredentials(new AuthScope(proxy), new UsernamePasswordCredentials(userName, password));
CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(config)
.setDefaultCredentialsProvider(provider)
.build();
HttpGet httpGet = new HttpGet(url);
httpGet.setHeader("Content-Type", "application/x-www-form-urlencoded;charset=utf-8");
httpGet.setHeader(HttpHeaders.CONNECTION, "close");
CloseableHttpResponse response = httpClient.execute(httpGet);
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != 200) {
httpGet.abort();
throw new RuntimeException("HttpClient,error status code :" + statusCode);
}
return response;
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
public static String sendPost1(String url, String data, Map<String,String> header) {
String response = null;
try {
CloseableHttpClient httpclient = null;
CloseableHttpResponse httpresponse = null;
try {
httpclient = HttpClients.createDefault();
HttpPost method = new HttpPost(url);
StringEntity stringentity = new StringEntity(data, Charset.forName("UTF-8"));
stringentity.setContentEncoding("UTF-8");
for(Map.Entry<String, String> item : header.entrySet()){
method.setHeader(item.getKey(), item.getValue());
}
method.setEntity(stringentity);
httpresponse = httpclient.execute(method);
response = EntityUtils.toString(httpresponse.getEntity());
} finally {
if (httpclient != null) {
httpclient.close();
}
if (httpresponse != null) {
httpresponse.close();
}
}
} catch (Exception e) {
// throw new Exception("http link fail", e);
e.printStackTrace();
}
return response;
}
/**
*
* @param httpUrl 请求的url
* @param param form表单的参数（key,value形式）
* @return
*/
public static String doPostForm(String httpUrl, Map param,Integer expire) {
HttpURLConnection connection = null;
InputStream is = null;
OutputStream os = null;
BufferedReader br = null;
String result = null;
try {
URL url = new URL(httpUrl);
// 通过远程url连接对象打开连接
connection = (HttpURLConnection) url.openConnection();
// 设置连接请求方式
connection.setRequestMethod("POST");
// 设置连接主机服务器超时时间：15000毫秒
connection.setConnectTimeout(expire);
// 设置读取主机服务器返回数据超时时间：60000毫秒
connection.setReadTimeout(expire);
// 默认值为：false，当向远程服务器传送数据/写数据时，需要设置为true
connection.setDoOutput(true);
// 默认值为：true，当前向远程服务读取数据时，设置为true，该参数可有可无
connection.setDoInput(true);
// 设置传入参数的格式:请求参数应该是 name1=value1&name2=value2 的形式。
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
// 设置鉴权信息：Authorization: Bearer da3efcbf-0845-4fe3-8aba-ee040be542c0
//connection.setRequestProperty("Authorization", "Bearer da3efcbf-0845-4fe3-8aba-ee040be542c0");
// 通过连接对象获取一个输出流
os = connection.getOutputStream();
// 通过输出流对象将参数写出去/传输出去,它是通过字节数组写出的(form表单形式的参数实质也是key,value值的拼接，类似于get请求参数的拼接)
os.write(createLinkString(param).getBytes());
// 通过连接对象获取一个输入流，向远程读取
if (connection.getResponseCode() == 200) {
is = connection.getInputStream();
// 对输入流对象进行包装:charset根据工作项目组的要求来设置
br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
StringBuffer sbf = new StringBuffer();
String temp = null;
// 循环遍历一行一行读取数据
while ((temp = br.readLine()) != null) {
sbf.append(temp);
sbf.append("\r\n");
}
result = sbf.toString();
}
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
// 关闭资源
if (null != br) {
try {
br.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (null != os) {
try {
os.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (null != is) {
try {
is.close();
} catch (IOException e) {
e.printStackTrace();
}
}
// 断开与远程地址url的连接
connection.disconnect();
}
return result;
}
/**
* 把数组所有元素排序，并按照“参数=参数值”的模式用“&”字符拼接成字符串
* @param params 需要排序并参与字符拼接的参数组
* @return 拼接后字符串
*/
public static String createLinkString(Map<String, String> params) {
List<String> keys = new ArrayList<String>(params.keySet());
Collections.sort(keys);
StringBuilder prestr = new StringBuilder();
for (int i = 0; i < keys.size(); i++) {
String key = keys.get(i);
String value = params.get(key);
if (i == keys.size() - 1) {// 拼接时，不包括最后一个&字符
prestr.append(key).append("=").append(value);
} else {
prestr.append(key).append("=").append(value).append("&");
}
}
return prestr.toString();
}
public static String sendPost(String url, Map<String, Object> params,String charset,int ExTime) {
String content = "";
CloseableHttpClient httpClient = HttpClients.createDefault();
RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(ExTime).setConnectTimeout(ExTime).build();
HttpPost httpPost = new HttpPost(url);
JSONObject jsonObject = new JSONObject();
// 通过map集成entrySet方法获取entity
Set<Map.Entry<String, Object>> entrySet = params.entrySet();
// 循环遍历，获取迭代器
for (Map.Entry<String, Object> mapEntry : entrySet) {
if(mapEntry.getValue()!=null){
jsonObject.put(mapEntry.getKey(), mapEntry.getValue());
}
}
try {
if (null != params) {
//解决中文问题。
httpPost.addHeader("Content-type","application/json; charset=utf-8");
httpPost.setHeader("Accept", "application/json");
//System.out.println("urlEncodedFormEntity:" + urlEncodedFormEntity);
httpPost.setEntity(new StringEntity(jsonObject.toString(),"UTF-8"));
httpPost.setConfig(requestConfig);
}
System.out.println("execurting request:" + httpPost.getURI());
HttpResponse httpResponse = null;
httpResponse = httpClient.execute(httpPost);
HttpEntity httpEntity = httpResponse.getEntity();
if (httpEntity != null) {
content = EntityUtils.toString(httpEntity, charset);
}
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
// 关闭连接,释放资源
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return content;
}
}
src/main/resources/application.yml 0 → 100644
server:
port: 9393
servlet:
context-path: /collection
spring:
datasource:
url: jdbc:mysql://114.115.159.144:3306/caiji?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&useSSL=false&allowPublicKeyRetrieval=true
username: caiji
password: zzsn9988
redis:
database: 0
host: 114.116.90.53
password: clbzzsn
port: 6380
mybatis-plus:
mapper-locations: mapper/*.xml
configuration:
log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
map-underscore-to-camel-case: true
file:
info-source-path: E:\projects\code01\collection\src\main\resources\info_source.txt
src/main/resources/info_source.txt 0 → 100644
IN-20240810-0020
IN-20240807-0083
\ No newline at end of file
src/main/resources/logback-spring.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false">
<!--定义日志文件的存储地址 -->
<property name="LOG_HOME" value="../clbLogs/collection" />
<!--<property name="COLOR_PATTERN" value="%black(%contextName-) %red(%d{yyyy-MM-dd HH:mm:ss}) %green([%thread]) %highlight(%-5level) %boldMagenta( %replace(%caller{1}){'\t|Caller.{1}0|\r\n', ''})- %gray(%msg%xEx%n)" />-->
<!-- 控制台输出 -->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50}:%L - %msg%n</pattern>-->
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{50}:%L) - %msg%n</pattern>
</encoder>
</appender>
<!-- 按照每天生成日志文件 -->
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_HOME}/collection.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!--日志文件输出的文件名 -->
<FileNamePattern>${LOG_HOME}/collection-%d{yyyy-MM-dd}.%i.log</FileNamePattern>
<!--日志文件保留天数 -->
<MaxHistory>10</MaxHistory>
<maxFileSize>10MB</maxFileSize>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符 -->
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50}:%L - %msg%n</pattern>
</encoder>
</appender>
<!--myibatis log configure -->
<logger name="com.apache.ibatis" level="TRACE" />
<logger name="java.sql.Connection" level="DEBUG" />
<logger name="java.sql.Statement" level="DEBUG" />
<logger name="java.sql.PreparedStatement" level="DEBUG" />
<!-- 日志输出级别 -->
<root level="INFO">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
</root>
</configuration>
src/main/resources/templates/collection.html 0 → 100644
<!DOCTYPE html>
<html xmlns:th="http://www.thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>title</title>
</head>
<body>
<div th:each="collect,stat:${list}">
<span th:text="${collect.id}"/>
<span th:text="${collect.infoSourceCode}"/>
<span th:text="${collect.siteName}"/>
<span th:text="${collect.serverCode}"/>
</div>
</body>
</html>
\ No newline at end of file
src/main/resources/templates/hello.html 0 → 100644
<!DOCTYPE html>
<html lang="en" xmlns:th="http://www/thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
<div th:text="${msg}"></div>
</body>
</html>
src/main/resources/templates/index.html 0 → 100644
<!DOCTYPE html>
<html lang="en" xmlns:th="http://www/thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
<div th:text="hello"></div>
</body>
</html>
\ No newline at end of file
src/main/resources/templates/statisticsList.html 0 → 100644
<!DOCTYPE html>
<html lang="en" xmlns="http://www.w3.org/1999/xhtml" xmlns:th="http://www.thymeleaf.org">
<head>
<!-- 引入bootstrap样式 -->
<link rel="stylesheet" href="https://cdn.bootcss.com/bootstrap/3.3.7/css/bootstrap.min.css">
<script type="text/javascript" src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<script type="text/javascript" src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>
<meta charset="UTF-8">
<title>采集情况统计</title>
<style type="text/css">
td, th {
text-align: center;
}
</style>
<script>
$(function () {
//全选与全不选
$("#firstCb").click(function () {
var u = $("input[name='uid']")
for (var i = 0; i < u.length; i++) {
u[i].checked = this.checked;
}
});
//1. 获取数据行的奇数行的tr
$("tr:gt(0):odd").css("backgroundColor","#e8f4ff");
//2. 获取数据行的偶数行的tr
$("tr:gt(0):even").css("backgroundColor","#dbe8f2");
})
</script>
</head>
<body>
<div class="container">
<h3 style="text-align: center;color: black;font-family: 黑体">采集情况统计</h3>
<div style="float: left;margin-bottom: 20px;margin-top: 10px">
<form th:action="@{/api/collectionlog/statistics}" class="form-inline" method="get" id="form" autocomplete="off">
<div class="form-group">
<label for="sign">查询范围</label>
<select class="form-control" id="sign" name="sign" >
<option value="0">请选择</option>
<option value="1">最近1小时内统计</option>
<option value="0">最近24小时内统计</option>
</select>
</div>
<button type="submit" class="btn btn-primary">查询</button>
</form>
</div>
<table border="1" class="table table-bordered table-hover">
<tr style="background-color: #76aef0">
<th th:text="${signValue}" ></th>
<th></th>
<th></th>
<th></th>
</tr>
<tr style="background-color: #76aef0">
<th>采集名称</th>
<th>采集编码</th>
<th>信息源接收数量</th>
<th>资讯采集数量</th>
</tr>
<tr th:if="${list != null}" th:each="map,stat:${list}">
<td th:text="${map.get('name')}"></td>
<td th:text="${map.get('code')}"></td>
<td th:text="${map.get('countSource')}"></td>
<td th:text="${map.get('countInfo')}"></td>
</tr>
</table>
</div>
<script>
$(function () {
$("li").click(function () {
$(this).attr("class", "active")
})
})
</script>
</body>
</html>
\ No newline at end of file
target/classes/application.yml 0 → 100644
server:
port: 9393
servlet:
context-path: /collection
spring:
datasource:
url: jdbc:mysql://114.115.159.144:3306/caiji?useUnicode=true&characterEncoding=utf-8&serverTimezone=Asia/Shanghai&useSSL=false&allowPublicKeyRetrieval=true
username: caiji
password: zzsn9988
redis:
database: 0
host: 114.116.90.53
password: clbzzsn
port: 6380
mybatis-plus:
mapper-locations: mapper/*.xml
configuration:
log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
map-underscore-to-camel-case: true
file:
info-source-path: E:\projects\code01\collection\src\main\resources\info_source.txt
target/classes/info_source.txt 0 → 100644
IN-20240810-0020
IN-20240807-0083
\ No newline at end of file
target/classes/logback-spring.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false">
<!--定义日志文件的存储地址 -->
<property name="LOG_HOME" value="../clbLogs/collection" />
<!--<property name="COLOR_PATTERN" value="%black(%contextName-) %red(%d{yyyy-MM-dd HH:mm:ss}) %green([%thread]) %highlight(%-5level) %boldMagenta( %replace(%caller{1}){'\t|Caller.{1}0|\r\n', ''})- %gray(%msg%xEx%n)" />-->
<!-- 控制台输出 -->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50}:%L - %msg%n</pattern>-->
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{50}:%L) - %msg%n</pattern>
</encoder>
</appender>
<!-- 按照每天生成日志文件 -->
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_HOME}/collection.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!--日志文件输出的文件名 -->
<FileNamePattern>${LOG_HOME}/collection-%d{yyyy-MM-dd}.%i.log</FileNamePattern>
<!--日志文件保留天数 -->
<MaxHistory>10</MaxHistory>
<maxFileSize>10MB</maxFileSize>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符 -->
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50}:%L - %msg%n</pattern>
</encoder>
</appender>
<!--myibatis log configure -->
<logger name="com.apache.ibatis" level="TRACE" />
<logger name="java.sql.Connection" level="DEBUG" />
<logger name="java.sql.Statement" level="DEBUG" />
<logger name="java.sql.PreparedStatement" level="DEBUG" />
<!-- 日志输出级别 -->
<root level="INFO">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
</root>
</configuration>
target/classes/templates/collection.html 0 → 100644
<!DOCTYPE html>
<html xmlns:th="http://www.thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>title</title>
</head>
<body>
<div th:each="collect,stat:${list}">
<span th:text="${collect.id}"/>
<span th:text="${collect.infoSourceCode}"/>
<span th:text="${collect.siteName}"/>
<span th:text="${collect.serverCode}"/>
</div>
</body>
</html>
\ No newline at end of file
target/classes/templates/hello.html 0 → 100644
<!DOCTYPE html>
<html lang="en" xmlns:th="http://www/thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
<div th:text="${msg}"></div>
</body>
</html>
target/classes/templates/index.html 0 → 100644
<!DOCTYPE html>
<html lang="en" xmlns:th="http://www/thymeleaf.org">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
<div th:text="hello"></div>
</body>
</html>
\ No newline at end of file
target/classes/templates/statisticsList.html 0 → 100644
<!DOCTYPE html>
<html lang="en" xmlns="http://www.w3.org/1999/xhtml" xmlns:th="http://www.thymeleaf.org">
<head>
<!-- 引入bootstrap样式 -->
<link rel="stylesheet" href="https://cdn.bootcss.com/bootstrap/3.3.7/css/bootstrap.min.css">
<script type="text/javascript" src="https://cdn.bootcss.com/jquery/1.12.4/jquery.min.js"></script>
<script type="text/javascript" src="https://cdn.bootcss.com/bootstrap/3.3.7/js/bootstrap.min.js"></script>
<meta charset="UTF-8">
<title>采集情况统计</title>
<style type="text/css">
td, th {
text-align: center;
}
</style>
<script>
$(function () {
//全选与全不选
$("#firstCb").click(function () {
var u = $("input[name='uid']")
for (var i = 0; i < u.length; i++) {
u[i].checked = this.checked;
}
});
//1. 获取数据行的奇数行的tr
$("tr:gt(0):odd").css("backgroundColor","#e8f4ff");
//2. 获取数据行的偶数行的tr
$("tr:gt(0):even").css("backgroundColor","#dbe8f2");
})
</script>
</head>
<body>
<div class="container">
<h3 style="text-align: center;color: black;font-family: 黑体">采集情况统计</h3>
<div style="float: left;margin-bottom: 20px;margin-top: 10px">
<form th:action="@{/api/collectionlog/statistics}" class="form-inline" method="get" id="form" autocomplete="off">
<div class="form-group">
<label for="sign">查询范围</label>
<select class="form-control" id="sign" name="sign" >
<option value="0">请选择</option>
<option value="1">最近1小时内统计</option>
<option value="0">最近24小时内统计</option>
</select>
</div>
<button type="submit" class="btn btn-primary">查询</button>
</form>
</div>
<table border="1" class="table table-bordered table-hover">
<tr style="background-color: #76aef0">
<th th:text="${signValue}" ></th>
<th></th>
<th></th>
<th></th>
</tr>
<tr style="background-color: #76aef0">
<th>采集名称</th>
<th>采集编码</th>
<th>信息源接收数量</th>
<th>资讯采集数量</th>
</tr>
<tr th:if="${list != null}" th:each="map,stat:${list}">
<td th:text="${map.get('name')}"></td>
<td th:text="${map.get('code')}"></td>
<td th:text="${map.get('countSource')}"></td>
<td th:text="${map.get('countInfo')}"></td>
</tr>
</table>
</div>
<script>
$(function () {
$("li").click(function () {
$(this).attr("class", "active")
})
})
</script>
</body>
</html>
\ No newline at end of file
target/maven-archiver/pom.properties 0 → 100644
artifactId=collection
groupId=com.zzsn.collection
version=1.0-SNAPSHOT
target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst 0 → 100644
com\zzsn\collection\controller\CollectionController.class
com\zzsn\collection\controller\CollectionLogController.class
com\zzsn\collection\CollectionApplication.class
com\zzsn\collection\service\CollectionLogService.class
com\zzsn\collection\entity\CollectionLog.class
com\zzsn\collection\mapper\CollectionLogMapper.class
com\zzsn\collection\service\impl\CollectionLogServiceImpl.class
target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst 0 → 100644
E:\projects\code01\collection\src\main\java\com\zzsn\collection\CollectionApplication.java
E:\projects\code01\collection\src\main\java\com\zzsn\collection\controller\CollectionController.java
E:\projects\code01\collection\src\main\java\com\zzsn\collection\entity\CollectionLog.java
E:\projects\code01\collection\src\main\java\com\zzsn\collection\service\impl\CollectionLogServiceImpl.java
E:\projects\code01\collection\src\main\java\com\zzsn\collection\controller\CollectionLogController.java
E:\projects\code01\collection\src\main\java\com\zzsn\collection\service\CollectionLogService.java
E:\projects\code01\collection\src\main\java\com\zzsn\collection\mapper\CollectionLogMapper.java
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论