Skip to content

L
leader-base
提交 ebbca44a 作者: ZhangJingKun
操作

Merge remote-tracking branch 'origin/master'

上级 9476e40b a30a9c26
正在显示 包含 509 行增加 和 17 行删除
pom.xml
......@@ -94,6 +94,20 @@
<artifactId>fastjson</artifactId>
<version>1.2.83</version>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-redis</artifactId>
<version>2.3.5.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-pool2</artifactId>
<version>2.0</version>
</dependency>
</dependencies>
......
src/main/java/com/zzsn/leaderbase/LeaderBaseApplication.java
package com.zzsn.leaderbase;
import com.zzsn.leaderbase.config.TaskExecutor;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.scheduling.annotation.EnableAsync;
@SpringBootApplication
@MapperScan(value={"com.zzsn.leaderbase.mapper*"})
@EnableAsync
@MapperScan(value = {"com.zzsn.leaderbase.mapper*"})
public class LeaderBaseApplication {
public static void main(String[] args) {
SpringApplication.run(LeaderBaseApplication.class, args);
}
@Bean
public TaskExecutor schedulerRunner() {
return new TaskExecutor();
}
}
src/main/java/com/zzsn/leaderbase/config/Elasticsearch1Config.java 0 → 100644
package com.zzsn.leaderbase.config;
import lombok.Data;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
@ConfigurationProperties(prefix = "es1")
@Configuration
@Data
public class Elasticsearch1Config {
private String endpoint1;
private Integer endpoint1port;
private String endpoint2;
private Integer endpoint2port;
private String endpoint3;
private Integer endpoint3port;
private String username;
private String password;
@Bean
@Primary
public RestHighLevelClient elasticsearch1Client() {
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
RestClientBuilder builder = RestClient.builder(
new HttpHost(endpoint1, endpoint1port, "http"),
new HttpHost(endpoint2, endpoint2port, "http"),
new HttpHost(endpoint3, endpoint3port, "http"))
.setHttpClientConfigCallback(httpClientBuilder -> {
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
// 设置连接超时时间和套接字超时时间
RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
requestConfigBuilder.setConnectTimeout(300000); // 连接超时时间为300秒
requestConfigBuilder.setSocketTimeout(300000); // 套接字超时时间为300秒
httpClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build());
return httpClientBuilder;
});
return new RestHighLevelClient(builder);
}
// @Bean
// public RestHighLevelClient elasticsearch1Client() {
// final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
//
// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
//
// RestClientBuilder builder = RestClient.builder(HttpHost.create(endpoints))
// .setHttpClientConfigCallback(httpClientBuilder -> {
// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
// return httpClientBuilder;
// });
//
// return new RestHighLevelClient(builder);
// }
// @Bean
// public RestHighLevelClient elasticsearch1Client() {
// RestHighLevelClient client = new RestHighLevelClient(
// RestClient.builder(
// new HttpHost("",9200,"http"),
// new HttpHost("",9200,"http")
// )
// );
// return client;
//
// }
}
src/main/java/com/zzsn/leaderbase/config/RedisConfig.java 0 → 100644
package com.zzsn.leaderbase.config;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.cache.annotation.CachingConfigurerSupport;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
/**
* 开启缓存支持
* @author zyf
* @Return:
*/
@Slf4j
@EnableCaching
@Configuration
public class RedisConfig extends CachingConfigurerSupport {
/**
* RedisTemplate配置
* @param lettuceConnectionFactory
* @return
*/
@Bean
public RedisTemplate<String, Object> redisTemplate(LettuceConnectionFactory lettuceConnectionFactory) {
log.info(" --- redis config init --- ");
Jackson2JsonRedisSerializer<Object> jackson2JsonRedisSerializer =jacksonSerializer();
RedisTemplate<String, Object> redisTemplate = new RedisTemplate<String, Object>();
redisTemplate.setConnectionFactory(lettuceConnectionFactory);
RedisSerializer<?> stringSerializer = new StringRedisSerializer();
// key序列化
redisTemplate.setKeySerializer(stringSerializer);
// value序列化
redisTemplate.setValueSerializer(jackson2JsonRedisSerializer);
// Hash key序列化
redisTemplate.setHashKeySerializer(stringSerializer);
// Hash value序列化
redisTemplate.setHashValueSerializer(jackson2JsonRedisSerializer);
redisTemplate.afterPropertiesSet();
return redisTemplate;
}
private Jackson2JsonRedisSerializer jacksonSerializer() {
Jackson2JsonRedisSerializer jackson2JsonRedisSerializer = new Jackson2JsonRedisSerializer(Object.class);
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY);
objectMapper.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL);
jackson2JsonRedisSerializer.setObjectMapper(objectMapper);
return jackson2JsonRedisSerializer;
}
}
src/main/java/com/zzsn/leaderbase/config/TaskExecutor.java 0 → 100644
package com.zzsn.leaderbase.config;
import com.zzsn.leaderbase.service.DealLeaderDataService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@Slf4j
@Component
public class TaskExecutor implements CommandLineRunner {
ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(1);
private final Integer PERIOD=100;
@Autowired
private DealLeaderDataService dealLeaderDataService;
@Override
public void run(String... args)throws Exception {
scheduledExecutorService.scheduleAtFixedRate(()->{
try {
dealLeaderDataService.getData();
} catch (IOException e) {
e.printStackTrace();
}
},5,PERIOD, TimeUnit.SECONDS);
log.info("简易版定时任务启动成功！{}{}执行一次",PERIOD,TimeUnit.SECONDS);
}
}
src/main/java/com/zzsn/leaderbase/constant/CommonConstant.java
......@@ -49,4 +49,5 @@ public interface CommonConstant {
//处理后的专题资讯信息存储索引。
public final static String ES_DATA_FOR_SUBJECT = "subjectdatabase";
String LAST_TIME ="lastTime" ;
}
src/main/java/com/zzsn/leaderbase/service/DealLeaderDataService.java 0 → 100644
package com.zzsn.leaderbase.service;
import cn.hutool.core.collection.CollectionUtil;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.zzsn.leaderbase.constant.CommonConstant;
import com.zzsn.leaderbase.util.EsDateUtil;
import com.zzsn.leaderbase.util.EsUtil;
import com.zzsn.leaderbase.util.HttpUtil;
import com.zzsn.leaderbase.vo.InfoExtractionParam;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
@Slf4j
public class DealLeaderDataService {
@Autowired
private EsUtil esUtil;
@Value("${python.getInfoUrl}")
private String relationEntityUrl;
@Autowired
RedisTemplate redisTemplate;
public void getData() throws IOException {
log.info("获取数据");
Object object = redisTemplate.opsForValue().get(CommonConstant.LAST_TIME);
String startTime = null;
if (object != null) {
startTime = object.toString();
} else {
startTime = "2023-01-01 12:12:12";
}
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder boolQueryBuilder1 = QueryBuilders.boolQuery()
.must(QueryBuilders.termsQuery("subjectId", Arrays.asList("1750716233967157250", "1750716067187048450")));
boolQueryBuilder1.filter(QueryBuilders.rangeQuery("createDate").gte(EsDateUtil.esFieldDateFormat(startTime)));
searchSourceBuilder.query(boolQueryBuilder1);
searchSourceBuilder.sort("createDate", SortOrder.DESC);
Integer pagesize = 1;
for (int i = 1; true; i++) {
Page<InfoExtractionParam> subjectdatabase = esUtil.queryPage("subjectdatabase", searchSourceBuilder, InfoExtractionParam.class, i, pagesize);
List<InfoExtractionParam> records = subjectdatabase.getRecords();
log.info("页码：{},总页：{},总数量{}", i, subjectdatabase.getPages(), subjectdatabase.getTotal());
if (CollectionUtil.isNotEmpty(records)) {
JSONObject jsonObjectParam = new JSONObject();
Map<String, List<InfoExtractionParam>> map = new HashMap<>();
map.put("data_list", records);
jsonObjectParam.put("data", map);
String result = HttpUtil.doPost(relationEntityUrl, jsonObjectParam, 10000);
Object objectResult = parseResult(result);
JSONObject dataResult = null;
if (null != objectResult) {
dataResult = JSONObject.parseObject(objectResult.toString());
}
if (dataResult != null) {
log.info(String.valueOf(dataResult));
}
} else {
log.info("此轮数据处理完毕============================================");
break;
}
}
}
private Object parseResult(String result) {
JSONObject jsonObject = JSONObject.parseObject(result);
if (null != jsonObject && "200".equals(jsonObject.getString("code"))) {
return jsonObject.get("result");
}
log.error("python 服务结果异常，响应信息{}", result);
return null;
}
}
src/main/java/com/zzsn/leaderbase/util/EsDateUtil.java 0 → 100644
package com.zzsn.leaderbase.util;
/**
* Es对日期类型处理
* @author kongliufeng
* @create 2020-08-07 14:05
*/
public class EsDateUtil {
/**
* yyyy-MM-dd HH:mm:ss ->yyyy-MM-ddTHH:mm:ss
* @param data
* @return
*/
public static String esFieldDateFormat(String data) {
if (data == null)
return data;
if (data.length() == 19) {//标准yyyy-MM-dd HH:mm:ss
return data.replace(" ", "T");
} else if (data.length() == 10) {//yyyy-MM-dd
return data;
}
return null;
}
/**
* yyyy-MM-dd HH:mm:ss ->yyyy-MM-ddTHH:mm:ss
* @param data
* @return
*/
public static String esFieldDateMapping(String data) {
if (data == null)
return data;
if (data.length() == 19) {//标准yyyy-MM-dd HH:mm:ss
return data.replace("T", " ");
} else if (data.length() == 10) {//yyyy-MM-dd
return data;
}
return null;
}
}
src/main/java/com/zzsn/leaderbase/vo/InfoExtractionParam.java 0 → 100644
package com.zzsn.leaderbase.vo;
import lombok.Data;
@Data
public class InfoExtractionParam {
private String title;
private String content;
private String id;
private String publishDate;
private String origin;
}
src/main/java/com/zzsn/leaderbase/vo/SubjectInfoVo.java 0 → 100644
package com.zzsn.leaderbase.vo;
import lombok.Data;
import java.util.List;
@Data
public class SubjectInfoVo {
private String id;
//专题id
private String subjectId;
//文章链接
private String url;
//专题标题
private String title;
//专题摘要
private String summary;
//内容
private String content;
//作者
private String author;
//发布时间
private String publishDate;
//来源
private String origin;
//得分
private String score;
//企业名称
private String enterpriseName;
//倾向性
private String orientation;
//风险类型
private String riskType;
//企业性质
private String enterpriseNature;
//区域
private String area;
//行业
private String trade;
//信息类型
private String InfoType;
//专题库类型
private String libraryType;
//ids
private List<String> ids;
//置顶标识（0取消置顶 1置顶）
private String type;
private String sourceAddress;
//开始时间
private String startTime;
//结束时间
private String endTime;
//倾向性（前端交互参数约定 0：中性 1：负面 2：正面）
private Integer tendency;
//专题库类型(1：政策；2：领导讲话；3：专家观点；4：企业案例)
private Integer classificationType;
//审核操作(0：未审核 1：审核通过 2：审核未通过 3：暂定 默认值为0)
private List<Integer> checkStatusList;
private Integer checkStatus;
//删除标记(1:删除；0:保留)
private Integer deleteFlag = 0;
//国家
private String countryIds;
//城市
private String cityIds;
//企业
private String enterpriseIds;
//人物
private String characterIds;
//字段
private String fields;
//操作类型 add .update
private String action;
//日期最大值
private String maxValue;
//关键词检索范围 1标题 2正文 3 全部
private Integer searchScope;
//精确度("精确"| --)
private String searchAccuracy;
private String lang;
private String contentWithTag;
private String index;
}
src/main/resources/application.yml
......@@ -11,3 +11,21 @@ spring:
uris: ["114.116.90.53:9200"]
username: elastic
password: elastic
redis:
database: 0
host: 114.115.236.206
password: clbzzsn
port: 6379
python:
# getInfoUrl: http://114.115.130.239:1818/update_extraction/
getInfoUrl: http://192.168.1.116:1818/update_extraction/
es1:
endpoint1: 114.115.215.250
endpoint1port: 9700
endpoint2: 114.116.19.92
endpoint2port: 9700
endpoint3: 114.116.54.108
endpoint3port: 9200
username: elastic
password: zzsn9988
\ No newline at end of file
src/test/java/com/zzsn/leaderbase/LeaderBaseApplicationTests.java
......@@ -6,18 +6,18 @@ import org.springframework.boot.test.context.SpringBootTest;
import java.util.ArrayList;
import java.util.List;
@SpringBootTest
class LeaderBaseApplicationTests {
@Test
void contextLoads() {
}
@Test
void translate(){
}
}
//
//@SpringBootTest
//class LeaderBaseApplicationTests {
//
// @Test
// void contextLoads() {
// }
//
//
// @Test
// void translate(){
//
// }
//
//}
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论