Skip to content

C
cs-mfa-gov
提交 10acbf00 作者: ZhangJingKun
操作

初始化提交

上级
正在显示 包含 721 行增加 和 0 行删除
.idea/.gitignore 0 → 100644
# Default ignored files
/workspace.xml
\ No newline at end of file
.idea/compiler.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile default="true" name="Default" enabled="true" />
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<module name="csmfagov" />
</profile>
</annotationProcessing>
</component>
<component name="JavacSettings">
<option name="ADDITIONAL_OPTIONS_OVERRIDE">
<module name="csmfagov" options="-parameters" />
</option>
</component>
</project>
\ No newline at end of file
.idea/encodings.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$" charset="UTF-8" />
</component>
</project>
\ No newline at end of file
.idea/misc.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>
\ No newline at end of file
csmfagov.iml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4" />
\ No newline at end of file
pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.3.5.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.zzsn</groupId>
<artifactId>cs-mfa-gov</artifactId>
<version>1.0-SNAPSHOT</version>
<name>cs-mfa-gov</name>
<description>中国领事服务网数据接口提供</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<!-- es-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
<version>2.2.6.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-elasticsearch</artifactId>
<version>4.0.5.RELEASE</version>
</dependency>
<!-- json-->
<dependency>
<groupId>com.alibaba.fastjson2</groupId>
<artifactId>fastjson2</artifactId>
<version>2.0.25</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.83</version>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-redis</artifactId>
<version>2.3.5.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
<resources>
<resource>
<directory>${project.basedir}/lib</directory>
<targetPath>BOOT-INF/lib/</targetPath>
<includes>
<include>**/*.jar</include>
</includes>
</resource>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
</build>
</project>
\ No newline at end of file
src/main/java/com/zzsn/csmfagov/CsmfagovApplication.java 0 → 100644
package com.zzsn.csmfagov;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableAsync;
@SpringBootApplication
@EnableAsync
public class CsmfagovApplication {
public static void main(String[] args) {
SpringApplication.run(CsmfagovApplication.class, args);
}
}
src/main/java/com/zzsn/csmfagov/controller/CsmfagovController.java 0 → 100644
package com.zzsn.csmfagov.controller;
import com.zzsn.csmfagov.server.CsmfagovServer;
import com.zzsn.csmfagov.vo.Result;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/2/6 11:06
* @Content:
*/
@RestController
@RequestMapping("/csmfagov/api")
@Slf4j
public class CsmfagovController {
@Autowired
CsmfagovServer csmfagovServer;
@GetMapping("/test")
public String test(){
log.info("hello!");
return "hello!";
}
@GetMapping("/getData")
public Result<?> getData(@RequestParam(name="groupId",required=false) String groupId){
log.info("groupId:{}", groupId);
//处理参数，groupId为空时默认给一个固定的
if(groupId == null || groupId.length() == 0){
groupId = "default";
}
return csmfagovServer.getData(groupId);
}
}
src/main/java/com/zzsn/csmfagov/server/CsmfagovServer.java 0 → 100644
package com.zzsn.csmfagov.server;
import com.zzsn.csmfagov.vo.Result;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/3/14 11:05
* @Content:
*/
public interface CsmfagovServer {
Result<?> getData(String groupId);
}
src/main/java/com/zzsn/csmfagov/server/impl/CsmfagovServerImpl.java 0 → 100644
package com.zzsn.csmfagov.server.impl;
import com.zzsn.csmfagov.server.CsmfagovServer;
import com.zzsn.csmfagov.util.EsDateUtil;
import com.zzsn.csmfagov.util.EsUtil;
import com.zzsn.csmfagov.vo.BaseData;
import com.zzsn.csmfagov.vo.Result;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
/**
* @Version 1.0
* @Author: ZhangJingKun
* @Date: 2024/3/14 11:06
* @Content:
*/
@Service
@Slf4j
public class CsmfagovServerImpl implements CsmfagovServer {
@Autowired
private RedisTemplate redisTemplate;
@Autowired
private EsUtil esUtil;
private static final String CSMFAGE_LAST_TIME = "csmfageLastTime";
@Override
public Result<?> getData(String groupId) {
Object object = redisTemplate.opsForValue().get(CSMFAGE_LAST_TIME + "_" + groupId);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder boolQueryBuilder = QueryBuilders
.boolQuery()
.must(QueryBuilders.termsQuery("sid", Arrays.asList("1534597839834664962", "1534597844930744322"))
);
if (object != null) {
String startTime = object.toString();
boolQueryBuilder.filter(QueryBuilders.rangeQuery("createDate").gte(EsDateUtil.esFieldDateFormat(startTime)));
}
searchSourceBuilder.query(boolQueryBuilder);
searchSourceBuilder.sort("createDate", SortOrder.DESC);
List<BaseData> list = esUtil.queryList("basedata", searchSourceBuilder, BaseData.class);
redisTemplate.opsForValue().set(CSMFAGE_LAST_TIME + "_" + groupId, LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd hh:mm:ss")));
if(list.size() == 0){
return Result.OK("没有新增数据！",list);
}
return Result.OK(list);
}
}
src/main/java/com/zzsn/csmfagov/util/EsDateUtil.java 0 → 100644
package com.zzsn.csmfagov.util;
/**
* Es对日期类型处理
* @author kongliufeng
* @create 2020-08-07 14:05
*/
public class EsDateUtil {
/**
* yyyy-MM-dd HH:mm:ss ->yyyy-MM-ddTHH:mm:ss
* @param data
* @return
*/
public static String esFieldDateFormat(String data) {
if (data == null)
return data;
if (data.length() == 19) {//标准yyyy-MM-dd HH:mm:ss
return data.replace(" ", "T");
} else if (data.length() == 10) {//yyyy-MM-dd
return data;
}
return null;
}
/**
* yyyy-MM-dd HH:mm:ss ->yyyy-MM-ddTHH:mm:ss
* @param data
* @return
*/
public static String esFieldDateMapping(String data) {
if (data == null)
return data;
if (data.length() == 19) {//标准yyyy-MM-dd HH:mm:ss
return data.replace("T", " ");
} else if (data.length() == 10) {//yyyy-MM-dd
return data;
}
return null;
}
}
src/main/java/com/zzsn/csmfagov/util/EsUtil.java 0 → 100644
package com.zzsn.csmfagov.util;
import com.alibaba.fastjson.JSON;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.*;
/**
* Description: es数据操作工具类
* Author: EDY
* Date: 2023/7/28
*/
@Component
@Slf4j
public class EsUtil {
/*es客户端*/
@Autowired
RestHighLevelClient client;
/**
* 获取低级客户端
*
* @return
*/
public RestClient getLowLevelClient() {
return client.getLowLevelClient();
}
/**
* 根据index,id索引文件
*
* @param index
* @param id
* @return
*/
public <T> T getInfoByid(String index, String id, Class<T> entry) {
T res = null;
try {
GetRequest searchRequest = new GetRequest(index, id);
GetResponse documentFields = client.get(searchRequest, RequestOptions.DEFAULT);
res = JSON.parseObject(documentFields.getSourceAsString(), entry);
return res;
} catch (IOException e) {
log.warn("查询doc异常，index=[{}],id=[{}], ex=[{}]", index, id, e.getMessage());
}
return res;
}
/**
* 根据id查询各类资讯详情
* */
public <T> T getInfoByid1(String index, String id, Class<T> entry){
try {
SearchRequest searchRequest = new SearchRequest(index);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("id", id);
searchSourceBuilder.query(termQueryBuilder);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
SearchHit[] hits = searchResponse.getHits().getHits();
T res = null;
if (hits.length>0){
res = JSON.parseObject(hits[0].getSourceAsString(), entry);
}
return res;
} catch (IOException e) {
log.info("查询异常{}",e.getMessage(),e);
return null;
}
}
/**
* 通用查询，
* */
public <T> List<T> queryList(String index, SearchSourceBuilder searchSourceBuilder, Class<T> entry) {
List<T> list = new ArrayList<>();
try {
SearchRequest searchRequest = new SearchRequest(index);
searchRequest.source(searchSourceBuilder);
searchSourceBuilder.size(1000);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
// 处理搜索结果
SearchHit[] hits = searchResponse.getHits().getHits();
if (hits.length>0){
Arrays.stream(hits).forEach(e -> {
T t = JSON.parseObject(e.getSourceAsString(), entry);
list.add(t);
});
}
return list;
} catch (IOException e) {
log.info("查询异常{}",e.getMessage(),e);
return list;
}
}
}
src/main/java/com/zzsn/csmfagov/vo/BaseData.java 0 → 100644
package com.zzsn.csmfagov.vo;
import lombok.Data;
import java.io.Serializable;
@Data
public class BaseData implements Serializable {
private Long id;
private Long sid;
private String title;
private String summary;
private String keywords;
private String author;
private String sourceSite;
private String sourceAddress;
private String type;
private String withTagFile;
private String publishDate;
private String createBy;
private String createDate;
private String content;
private String contentNoTag;
private String featureWords;
private String orientation;
private String lang;
private String words;
private String fileDownLoadPath;
//RELATE_PLACES 相关国家城市地点信息
private String relatePlaces;
//LOCALE 信息所属国别（国别，不是城市）
private String locale;
private Long hits;
private String origin;
private String fromWhere;
private String reliability;
private String influence;
private String contentImgCvtTag;
private String reprintCount; //重复数量
private String sourceType;
//分类id
private Long tid;
private Long btId; //情报分类主键ID
private String briefMark;//是否加入简报
private String readStatus;
private String sName;
private Integer isDelete;
private Long orgId;
private Long repeat;
private Double score;
}
src/main/java/com/zzsn/csmfagov/vo/Result.java 0 → 100644
package com.zzsn.csmfagov.vo;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Data;
import java.io.Serializable;
/**
* 接口返回数据格式
* @author scott
* @email jeecgos@163.com
* @date 2019年1月19日
*/
@Data
public class Result<T> implements Serializable {
private static final long serialVersionUID = 1L;
private static final Integer SC_OK_200 = 200;
private static final Integer SC_INTERNAL_SERVER_ERROR_500 = 500;
private static final Integer SC_JEECG_NO_AUTHZ=510;
/**
* 成功标志
*/
private boolean success = true;
/**
* 返回处理消息
*/
private String message = "操作成功！";
/**
* 返回代码
*/
private Integer code = 0;
/**
* 返回数据对象 data
*/
private T result;
/**
* 时间戳
*/
private long timestamp = System.currentTimeMillis();
/**
* python接口返回
*/
private String handleMsg;
/**
* python接口返回状态
*/
private boolean isHandleSuccess;
/**
* python接口返回状态
*/
private String logs;
/**
* 返回数据对象 data
*/
private T resultData;
public Result() {
}
public Result<T> success(String message) {
this.message = message;
this.code = SC_OK_200;
this.success = true;
return this;
}
@Deprecated
public static Result<Object> ok() {
Result<Object> r = new Result<Object>();
r.setSuccess(true);
r.setCode(SC_OK_200);
r.setMessage("成功");
return r;
}
@Deprecated
public static Result<Object> ok(String msg) {
Result<Object> r = new Result<Object>();
r.setSuccess(true);
r.setCode(SC_OK_200);
r.setMessage(msg);
return r;
}
@Deprecated
public static Result<Object> ok(Object data) {
Result<Object> r = new Result<Object>();
r.setSuccess(true);
r.setCode(SC_OK_200);
r.setResult(data);
return r;
}
public static<T> Result<T> OK() {
Result<T> r = new Result<T>();
r.setSuccess(true);
r.setCode(SC_OK_200);
r.setMessage("成功");
return r;
}
public static<T> Result<T> OK(T data) {
Result<T> r = new Result<T>();
r.setSuccess(true);
r.setCode(SC_OK_200);
r.setResult(data);
return r;
}
public static<T> Result<T> OK(String msg, T data) {
Result<T> r = new Result<T>();
r.setSuccess(true);
r.setCode(SC_OK_200);
r.setMessage(msg);
r.setResult(data);
return r;
}
public static Result<Object> error(String msg) {
return error(SC_INTERNAL_SERVER_ERROR_500, msg);
}
public static Result<Object> error(int code, String msg) {
Result<Object> r = new Result<Object>();
r.setCode(code);
r.setMessage(msg);
r.setSuccess(false);
return r;
}
public Result<T> error500(String message) {
this.message = message;
this.code = SC_INTERNAL_SERVER_ERROR_500;
this.success = false;
return this;
}
/**
* 无权限访问返回结果
*/
public static Result<Object> noauth(String msg) {
return error(SC_JEECG_NO_AUTHZ, msg);
}
@JsonIgnore
private String onlTable;
}
\ No newline at end of file
src/main/resources/application.yml 0 → 100644
server:
port: 9001
spring:
elasticsearch:
rest:
uris: ["114.115.215.250:9700"]
username: elastic
password: zzsn9988
redis:
database: 0
host: 114.116.90.53
password: clbzzsn
port: 6380
es1:
endpoint1: 114.115.215.250
endpoint1port: 9700
endpoint2: 114.116.19.92
endpoint2port: 9700
endpoint3: 114.116.54.108
endpoint3port: 9200
username: elastic
password: zzsn9988
src/main/resources/logback-spring.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false">
<!--定义日志文件的存储地址 -->
<property name="LOG_HOME" value="../clbLogs/csmfagov" />
<!--<property name="COLOR_PATTERN" value="%black(%contextName-) %red(%d{yyyy-MM-dd HH:mm:ss}) %green([%thread]) %highlight(%-5level) %boldMagenta( %replace(%caller{1}){'\t|Caller.{1}0|\r\n', ''})- %gray(%msg%xEx%n)" />-->
<!-- 控制台输出 -->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50}:%L - %msg%n</pattern>-->
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %highlight(%-5level) %cyan(%logger{50}:%L) - %msg%n</pattern>
</encoder>
</appender>
<!-- 按照每天生成日志文件 -->
<appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_HOME}/csmfagov.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<!--日志文件输出的文件名 -->
<FileNamePattern>${LOG_HOME}/csmfagov-%d{yyyy-MM-dd}.%i.log</FileNamePattern>
<!--日志文件保留天数 -->
<MaxHistory>10</MaxHistory>
<maxFileSize>10MB</maxFileSize>
</rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度%msg：日志消息，%n是换行符 -->
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50}:%L - %msg%n</pattern>
</encoder>
</appender>
<!--myibatis log configure -->
<logger name="com.apache.ibatis" level="TRACE" />
<logger name="java.sql.Connection" level="DEBUG" />
<logger name="java.sql.Statement" level="DEBUG" />
<logger name="java.sql.PreparedStatement" level="DEBUG" />
<!-- 日志输出级别 -->
<root level="INFO">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
</root>
</configuration>
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论