Skip to content
项目
群组
代码片段
帮助
当前项目
正在载入...
登录 / 注册
切换导航面板
J
jxyq
概览
概览
详情
活动
周期分析
版本库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
统计图
问题
0
议题
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
CI / CD
CI / CD
流水线
作业
日程表
图表
维基
Wiki
代码片段
代码片段
成员
成员
折叠边栏
关闭边栏
活动
图像
聊天
创建新问题
作业
提交
问题看板
Open sidebar
贾庆卫
jxyq
Commits
a58dbbc6
提交
a58dbbc6
authored
4月 15, 2023
作者:
JQW
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
推荐资讯
上级
363efb25
隐藏空白字符变更
内嵌
并排
正在显示
13 个修改的文件
包含
331 行增加
和
33 行删除
+331
-33
ServiceWebApplication.java
...web/src/main/java/com/zzsn/clb/ServiceWebApplication.java
+3
-0
ThreadPoolConfig.java
...b/src/main/java/com/zzsn/clb/config/ThreadPoolConfig.java
+48
-0
IServiceProjectClient.java
...c/main/java/com/zzsn/clb/fegin/IServiceProjectClient.java
+3
-3
SubjectMapper.java
...c/main/java/com/zzsn/clb/member/mapper/SubjectMapper.java
+8
-0
SubjectMapper.xml
...ain/java/com/zzsn/clb/member/mapper/xml/SubjectMapper.xml
+24
-0
IMemRecommendAsynService.java
...com/zzsn/clb/member/service/IMemRecommendAsynService.java
+25
-0
ISubjectTreeService.java
...java/com/zzsn/clb/member/service/ISubjectTreeService.java
+9
-2
MemRecommendAsynServiceImpl.java
.../clb/member/service/impl/MemRecommendAsynServiceImpl.java
+109
-0
SubjectTreeServiceImpl.java
.../zzsn/clb/member/service/impl/SubjectTreeServiceImpl.java
+9
-8
ModelLabelTask.java
...e-web/src/main/java/com/zzsn/clb/task/ModelLabelTask.java
+55
-10
ImportExcelData.java
...-web/src/main/java/com/zzsn/clb/test/ImportExcelData.java
+3
-3
SubjectController.java
...n/java/com/zzsn/clb/web/controller/SubjectController.java
+34
-6
Constants.java
service-web/src/main/java/com/zzsn/common/Constants.java
+1
-1
没有找到文件。
service-web/src/main/java/com/zzsn/clb/ServiceWebApplication.java
浏览文件 @
a58dbbc6
...
@@ -6,6 +6,8 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
...
@@ -6,6 +6,8 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import
org.springframework.cloud.openfeign.EnableFeignClients
;
import
org.springframework.cloud.openfeign.EnableFeignClients
;
import
org.springframework.context.annotation.EnableMBeanExport
;
import
org.springframework.context.annotation.EnableMBeanExport
;
import
org.springframework.jmx.support.RegistrationPolicy
;
import
org.springframework.jmx.support.RegistrationPolicy
;
import
org.springframework.scheduling.annotation.Async
;
import
org.springframework.scheduling.annotation.EnableAsync
;
/**
/**
* @author zhang ya nuo
* @author zhang ya nuo
...
@@ -16,6 +18,7 @@ import org.springframework.jmx.support.RegistrationPolicy;
...
@@ -16,6 +18,7 @@ import org.springframework.jmx.support.RegistrationPolicy;
@EnableFeignClients
(
basePackages
=
{
"org.jeecg"
,
"com.zzsn.clb"
})
@EnableFeignClients
(
basePackages
=
{
"org.jeecg"
,
"com.zzsn.clb"
})
@EnableMBeanExport
(
registration
=
RegistrationPolicy
.
IGNORE_EXISTING
)
@EnableMBeanExport
(
registration
=
RegistrationPolicy
.
IGNORE_EXISTING
)
@MapperScan
(
value
={
"com.zzsn.clb.**.mapper*"
})
@MapperScan
(
value
={
"com.zzsn.clb.**.mapper*"
})
@EnableAsync
public
class
ServiceWebApplication
{
public
class
ServiceWebApplication
{
public
static
void
main
(
String
[]
args
)
{
public
static
void
main
(
String
[]
args
)
{
SpringApplication
.
run
(
ServiceWebApplication
.
class
,
args
);
SpringApplication
.
run
(
ServiceWebApplication
.
class
,
args
);
...
...
service-web/src/main/java/com/zzsn/clb/config/ThreadPoolConfig.java
0 → 100644
浏览文件 @
a58dbbc6
package
com
.
zzsn
.
clb
.
config
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.context.annotation.Bean
;
import
org.springframework.context.annotation.Configuration
;
import
org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor
;
import
java.util.concurrent.ThreadPoolExecutor
;
@Configuration
public
class
ThreadPoolConfig
{
// 线程名称前缀
@Value
(
"${thread.prefix:jxyq-}"
)
private
String
threadPrefix
;
// 核心线程数
@Value
(
"${thread.core.size:16}"
)
private
int
coreSize
;
// 最大线程数
@Value
(
"${thread.max.size:16}"
)
private
int
maxSize
;
// 队列长度
@Value
(
"${thread.queue.size:30}"
)
private
int
queueSize
;
// 通过bean注解注入
@Bean
public
ThreadPoolTaskExecutor
taskExecutor
()
{
ThreadPoolTaskExecutor
taskExecutor
=
new
ThreadPoolTaskExecutor
();
//设置线程池参数信息
taskExecutor
.
setCorePoolSize
(
coreSize
);
taskExecutor
.
setMaxPoolSize
(
maxSize
);
taskExecutor
.
setQueueCapacity
(
queueSize
);
taskExecutor
.
setThreadNamePrefix
(
threadPrefix
);
taskExecutor
.
setWaitForTasksToCompleteOnShutdown
(
true
);
taskExecutor
.
setAwaitTerminationSeconds
(
30
);
//修改拒绝策略为使用当前线程执行
taskExecutor
.
setRejectedExecutionHandler
(
new
ThreadPoolExecutor
.
CallerRunsPolicy
());
//初始化线程池
taskExecutor
.
initialize
();
return
taskExecutor
;
}
}
\ No newline at end of file
service-web/src/main/java/com/zzsn/clb/fegin/IServiceProjectClient.java
浏览文件 @
a58dbbc6
...
@@ -35,7 +35,7 @@ public interface IServiceProjectClient {
...
@@ -35,7 +35,7 @@ public interface IServiceProjectClient {
* @return
* @return
*/
*/
@GetMapping
(
value
=
"/subject/display/list"
)
@GetMapping
(
value
=
"/subject/display/list"
)
Result
<
?
>
getListBySubject
(
@RequestParam
Map
<
String
,
Object
>
map
,
@RequestParam
String
userId
,
@RequestParam
String
video
,
@RequestParam
Integer
pageNo
,
@RequestParam
Integer
pageSize
,
@RequestParam
String
column
,
@RequestParam
String
order
,
@RequestParam
String
crawler
,
@RequestParam
String
isSubject
,
@RequestParam
String
subjectType
,
@RequestParam
String
labelIds
);
Result
<
JSONObject
>
getListBySubject
(
@RequestParam
Map
<
String
,
Object
>
map
,
@RequestParam
String
userId
,
@RequestParam
String
video
,
@RequestParam
Integer
pageNo
,
@RequestParam
Integer
pageSize
,
@RequestParam
String
column
,
@RequestParam
String
order
,
@RequestParam
String
crawler
,
@RequestParam
String
isSubject
,
@RequestParam
String
subjectType
,
@RequestParam
String
labelIds
);
/**
/**
* 根据信息id查询数据
* 根据信息id查询数据
...
@@ -43,7 +43,7 @@ public interface IServiceProjectClient {
...
@@ -43,7 +43,7 @@ public interface IServiceProjectClient {
* @param id
* @param id
* @return
* @return
*/
*/
@GetMapping
(
value
=
"/subject/display/queryById"
)
@GetMapping
(
value
=
"/
cloud/
subject/display/queryById"
)
Result
<
JSONObject
>
getInfoById
(
@RequestParam
(
required
=
true
)
String
id
);
Result
<
JSONObject
>
getInfoById
(
@RequestParam
(
required
=
true
)
String
id
);
...
@@ -54,6 +54,6 @@ public interface IServiceProjectClient {
...
@@ -54,6 +54,6 @@ public interface IServiceProjectClient {
* @param pageSize 每页显示条数
* @param pageSize 每页显示条数
* @return
* @return
*/
*/
@GetMapping
(
value
=
"
/
subject/subject/list"
)
@GetMapping
(
value
=
"subject/subject/list"
)
Result
<
JSONObject
>
getSubjectList
(
@RequestParam
(
required
=
true
)
JSONObject
jsonObject
,
@RequestParam
(
required
=
false
)
Integer
pageNo
,
@RequestParam
(
required
=
false
)
Integer
pageSize
);
Result
<
JSONObject
>
getSubjectList
(
@RequestParam
(
required
=
true
)
JSONObject
jsonObject
,
@RequestParam
(
required
=
false
)
Integer
pageNo
,
@RequestParam
(
required
=
false
)
Integer
pageSize
);
}
}
service-web/src/main/java/com/zzsn/clb/member/mapper/SubjectMapper.java
浏览文件 @
a58dbbc6
...
@@ -3,6 +3,7 @@ package com.zzsn.clb.member.mapper;
...
@@ -3,6 +3,7 @@ package com.zzsn.clb.member.mapper;
import
com.baomidou.mybatisplus.core.mapper.BaseMapper
;
import
com.baomidou.mybatisplus.core.mapper.BaseMapper
;
import
com.zzsn.clb.member.entity.Subject
;
import
com.zzsn.clb.member.entity.Subject
;
import
org.apache.ibatis.annotations.Param
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
...
@@ -25,4 +26,11 @@ public interface SubjectMapper extends BaseMapper<Subject> {
...
@@ -25,4 +26,11 @@ public interface SubjectMapper extends BaseMapper<Subject> {
* @return
* @return
*/
*/
List
<
Map
<
String
,
Object
>>
getNegativeLabelList
();
List
<
Map
<
String
,
Object
>>
getNegativeLabelList
();
/***
* 获取没有的专题及标签
* @param labelIds
* @return
*/
List
<
Map
<
String
,
Object
>>
getNoSubjectLabelList
(
@Param
(
"labelIds"
)
List
<
String
>
labelIds
);
}
}
service-web/src/main/java/com/zzsn/clb/member/mapper/xml/SubjectMapper.xml
浏览文件 @
a58dbbc6
...
@@ -20,6 +20,30 @@
...
@@ -20,6 +20,30 @@
ORDER BY s.id,
ORDER BY s.id,
le.sort ASC
le.sort ASC
</select>
</select>
<select
id=
"getNoSubjectLabelList"
resultType=
"java.util.Map"
>
SELECT
s.id subjectId,
le.id labelId
FROM
`subject` s
INNER JOIN subject_type_map stm ON s.id = stm.subject_id
INNER JOIN subject_model_map smm ON s.id = smm.subject_id
INNER JOIN python_model pm ON smm.model_id = pm.id
INNER JOIN sys_base_label_type lb ON lb.top_id = pm.label_id
INNER JOIN sys_base_label_type_map lm ON lb.id = lm.label_id
INNER JOIN label_entity le ON le.id = lm.relation_id
WHERE
le.`status` = 1
AND pm.type = 3
<if
test=
"labelIds!=null and labelIds.size > 0"
>
<foreach
collection=
"labelIds"
item=
"item"
open=
" AND le.id NOT IN ("
separator=
","
close=
")"
>
#{item}
</foreach>
</if>
ORDER BY
s.id,
le.sort ASC
</select>
<select
id=
"getNegativeLabelList"
resultType=
"java.util.Map"
>
<select
id=
"getNegativeLabelList"
resultType=
"java.util.Map"
>
...
...
service-web/src/main/java/com/zzsn/clb/member/service/IMemRecommendAsynService.java
0 → 100644
浏览文件 @
a58dbbc6
package
com
.
zzsn
.
clb
.
member
.
service
;
import
com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper
;
import
com.baomidou.mybatisplus.extension.service.IService
;
import
com.zzsn.clb.member.entity.MemRecommend
;
import
java.util.List
;
import
java.util.Map
;
/**
* @Description: 会员-推荐资讯表
* @Author: jeecg-boot
* @Date: 2023-04-03
* @Version: V1.0
*/
public
interface
IMemRecommendAsynService
{
/***
* 推荐得分写入
* @param memRecommend
* @param queryWrapper
*/
void
insert
(
MemRecommend
memRecommend
,
LambdaQueryWrapper
<
MemRecommend
>
queryWrapper
);
}
service-web/src/main/java/com/zzsn/clb/member/service/ISubjectTreeService.java
浏览文件 @
a58dbbc6
package
com
.
zzsn
.
clb
.
member
.
service
;
package
com
.
zzsn
.
clb
.
member
.
service
;
import
com.baomidou.mybatisplus.extension.service.IService
;
import
com.baomidou.mybatisplus.extension.service.IService
;
import
com.zzsn.clb.member.entity.MemThumbsUp
;
import
com.zzsn.clb.member.entity.Subject
;
import
com.zzsn.clb.member.entity.Subject
;
import
java.util.List
;
import
java.util.List
;
...
@@ -12,5 +11,13 @@ public interface ISubjectTreeService extends IService<Subject> {
...
@@ -12,5 +11,13 @@ public interface ISubjectTreeService extends IService<Subject> {
* 获取栏目树
* 获取栏目树
* @return
* @return
*/
*/
List
<
Map
<
String
,
Object
>>
getSubjectList
();
List
<
Map
<
String
,
Object
>>
getSubjectList
();
/***
* 获取没有的专题及标签
* @param labelIds
* @return
*/
List
<
Map
<
String
,
Object
>>
getNoSubjectLabelList
(
List
<
String
>
labelIds
);
}
}
service-web/src/main/java/com/zzsn/clb/member/service/impl/MemRecommendAsynServiceImpl.java
0 → 100644
浏览文件 @
a58dbbc6
package
com
.
zzsn
.
clb
.
member
.
service
.
impl
;
import
com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper
;
import
com.baomidou.mybatisplus.extension.service.impl.ServiceImpl
;
import
com.zzsn.clb.member.entity.MemRecommend
;
import
com.zzsn.clb.member.mapper.MemRecommendMapper
;
import
com.zzsn.clb.member.service.IMemRecommendAsynService
;
import
com.zzsn.clb.member.service.IMemRecommendService
;
import
com.zzsn.clb.member.service.ISubjectTreeService
;
import
com.zzsn.common.Constants
;
import
org.apache.commons.lang.StringUtils
;
import
org.apache.lucene.search.join.ScoreMode
;
import
org.apache.shiro.SecurityUtils
;
import
org.elasticsearch.action.search.SearchRequest
;
import
org.elasticsearch.action.search.SearchResponse
;
import
org.elasticsearch.client.RequestOptions
;
import
org.elasticsearch.client.RestHighLevelClient
;
import
org.elasticsearch.common.Strings
;
import
org.elasticsearch.index.query.BoolQueryBuilder
;
import
org.elasticsearch.index.query.NestedQueryBuilder
;
import
org.elasticsearch.index.query.QueryBuilders
;
import
org.elasticsearch.search.SearchHit
;
import
org.elasticsearch.search.builder.SearchSourceBuilder
;
import
org.elasticsearch.search.sort.SortOrder
;
import
org.jeecg.common.system.vo.LoginUser
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.scheduling.annotation.Async
;
import
org.springframework.stereotype.Component
;
import
org.springframework.stereotype.Service
;
import
org.springframework.transaction.annotation.Transactional
;
import
javax.annotation.Resource
;
import
java.io.IOException
;
import
java.util.ArrayList
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.stream.Collectors
;
/**
* @Description: 会员-推荐资讯表
* @Author: jeecg-boot
* @Date: 2023-04-03
* @Version: V1.0
*/
@Component
public
class
MemRecommendAsynServiceImpl
implements
IMemRecommendAsynService
{
@Autowired
private
IMemRecommendService
memRecommendService
;
@Resource
ISubjectTreeService
subjectTreeService
;
@Async
public
void
insert
(
MemRecommend
memRecommend
,
LambdaQueryWrapper
<
MemRecommend
>
queryWrapper
)
{
this
.
checkSubjectLabel
(
memRecommend
);
LoginUser
user
=
(
LoginUser
)
SecurityUtils
.
getSubject
().
getPrincipal
();
//如果是专题,则更新此专题下所有标签
if
(
memRecommend
.
getType
()
==
0
)
{
memRecommendService
.
updateScoreBySubjectId
(
memRecommend
.
getSubjectId
(),
user
.
getId
(),
memRecommend
.
getScore
());
}
else
{
queryWrapper
.
eq
(
MemRecommend:
:
getUserId
,
user
.
getId
());
queryWrapper
.
eq
(
MemRecommend:
:
getType
,
memRecommend
.
getType
());
MemRecommend
recommend
=
memRecommendService
.
getOne
(
queryWrapper
);
//标签的肯定存在(关键词肯定存在),更新操作
if
(
recommend
!=
null
)
{
recommend
.
setScore
(
memRecommend
.
getScore
()
+
recommend
.
getScore
());
memRecommendService
.
updateById
(
recommend
);
}
else
if
(
StringUtils
.
isNotBlank
(
memRecommend
.
getKeyWord
())
&&
memRecommend
.
getType
()
==
2
)
{
//关键词
recommend
=
new
MemRecommend
();
recommend
.
setUserId
(
user
.
getId
());
recommend
.
setType
(
memRecommend
.
getType
());
recommend
.
setKeyWord
(
memRecommend
.
getKeyWord
());
recommend
.
setScore
(
memRecommend
.
getScore
());
memRecommendService
.
save
(
recommend
);
}
}
}
/***
* 检测是否有新的专题或者标签
* @param memRecommend
*/
public
void
checkSubjectLabel
(
MemRecommend
memRecommend
)
{
if
(
StringUtils
.
isNotBlank
(
memRecommend
.
getSubjectId
()))
{
LoginUser
user
=
(
LoginUser
)
SecurityUtils
.
getSubject
().
getPrincipal
();
LambdaQueryWrapper
<
MemRecommend
>
checkQueryWrapper
=
new
LambdaQueryWrapper
<>();
checkQueryWrapper
.
eq
(
MemRecommend:
:
getUserId
,
user
.
getId
());
checkQueryWrapper
.
eq
(
MemRecommend:
:
getType
,
1
);
//获取当前用户已经存在的标签Id
List
<
String
>
labelIds
=
memRecommendService
.
list
(
checkQueryWrapper
).
stream
().
map
(
m
->
m
.
getLabelId
()).
collect
(
Collectors
.
toList
());
//获取当前用户不存在的标签id
List
<
Map
<
String
,
Object
>>
subjectList
=
subjectTreeService
.
getNoSubjectLabelList
(
labelIds
);
List
<
MemRecommend
>
recList
=
new
ArrayList
<>();
//如果当前专题下没有添加标签或者有新的标签,则添加
subjectList
.
stream
().
filter
(
f
->
f
.
get
(
"subjectId"
).
toString
().
equals
(
memRecommend
.
getSubjectId
())).
collect
(
Collectors
.
toList
()).
forEach
(
x
->
{
MemRecommend
model
=
new
MemRecommend
();
model
.
setSubjectId
(
x
.
get
(
"subjectId"
).
toString
());
model
.
setLabelId
(
x
.
get
(
"labelId"
).
toString
());
model
.
setUserId
(
user
.
getId
());
model
.
setType
(
1
);
model
.
setScore
(
0
);
recList
.
add
(
model
);
});
memRecommendService
.
saveBatch
(
recList
);
}
}
}
service-web/src/main/java/com/zzsn/clb/member/service/impl/SubjectTreeServiceImpl.java
浏览文件 @
a58dbbc6
package
com
.
zzsn
.
clb
.
member
.
service
.
impl
;
package
com
.
zzsn
.
clb
.
member
.
service
.
impl
;
import
com.baomidou.dynamic.datasource.annotation.DS
;
import
com.baomidou.dynamic.datasource.annotation.DS
;
import
com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper
;
import
com.baomidou.mybatisplus.extension.service.impl.ServiceImpl
;
import
com.baomidou.mybatisplus.extension.service.impl.ServiceImpl
;
import
com.zzsn.clb.member.entity.Subject
;
import
com.zzsn.clb.member.entity.Subject
;
import
com.zzsn.clb.member.mapper.SubjectMapper
;
import
com.zzsn.clb.member.mapper.SubjectMapper
;
import
com.zzsn.clb.member.service.ISubjectTreeService
;
import
com.zzsn.clb.member.service.ISubjectTreeService
;
import
org.apache.commons.lang3.StringUtils
;
import
org.springframework.stereotype.Service
;
import
org.springframework.stereotype.Service
;
import
javax.annotation.Resource
;
import
javax.annotation.Resource
;
...
@@ -17,13 +15,9 @@ import java.util.stream.Collectors;
...
@@ -17,13 +15,9 @@ import java.util.stream.Collectors;
@DS
(
"multi-clb-project"
)
@DS
(
"multi-clb-project"
)
public
class
SubjectTreeServiceImpl
extends
ServiceImpl
<
SubjectMapper
,
Subject
>
implements
ISubjectTreeService
{
public
class
SubjectTreeServiceImpl
extends
ServiceImpl
<
SubjectMapper
,
Subject
>
implements
ISubjectTreeService
{
@Resource
SubjectMapper
subjectMapper
;
@Override
@Override
public
List
<
Map
<
String
,
Object
>>
getSubjectList
()
{
public
List
<
Map
<
String
,
Object
>>
getSubjectList
()
{
List
<
Map
<
String
,
Object
>>
list
=
subjectMapper
.
getSubjectList
();
List
<
Map
<
String
,
Object
>>
list
=
this
.
getBaseMapper
()
.
getSubjectList
();
List
<
String
>
idList
=
list
.
stream
().
distinct
().
map
(
m
->
m
.
get
(
"subjectId"
).
toString
()).
collect
(
Collectors
.
toList
());
List
<
String
>
idList
=
list
.
stream
().
distinct
().
map
(
m
->
m
.
get
(
"subjectId"
).
toString
()).
collect
(
Collectors
.
toList
());
List
<
Map
<
String
,
Object
>>
treeList
=
new
ArrayList
<>();
List
<
Map
<
String
,
Object
>>
treeList
=
new
ArrayList
<>();
String
subjectId
=
""
;
String
subjectId
=
""
;
...
@@ -50,7 +44,7 @@ public class SubjectTreeServiceImpl extends ServiceImpl<SubjectMapper, Subject>
...
@@ -50,7 +44,7 @@ public class SubjectTreeServiceImpl extends ServiceImpl<SubjectMapper, Subject>
map
.
put
(
"name"
,
"负面舆情"
);
map
.
put
(
"name"
,
"负面舆情"
);
map
.
put
(
"child"
,
chdlist
);
map
.
put
(
"child"
,
chdlist
);
treeList
.
add
(
map
);
treeList
.
add
(
map
);
list
=
subjectMapper
.
getNegativeLabelList
();
list
=
this
.
getBaseMapper
()
.
getNegativeLabelList
();
for
(
Map
<
String
,
Object
>
tmpMap
:
list
)
{
for
(
Map
<
String
,
Object
>
tmpMap
:
list
)
{
Map
<
String
,
Object
>
childMap
=
new
HashMap
<>();
Map
<
String
,
Object
>
childMap
=
new
HashMap
<>();
childMap
.
put
(
"key"
,
tmpMap
.
get
(
"labelId"
));
childMap
.
put
(
"key"
,
tmpMap
.
get
(
"labelId"
));
...
@@ -59,4 +53,11 @@ public class SubjectTreeServiceImpl extends ServiceImpl<SubjectMapper, Subject>
...
@@ -59,4 +53,11 @@ public class SubjectTreeServiceImpl extends ServiceImpl<SubjectMapper, Subject>
}
}
return
treeList
;
return
treeList
;
}
}
@Override
public
List
<
Map
<
String
,
Object
>>
getNoSubjectLabelList
(
List
<
String
>
labelIds
)
{
return
this
.
getBaseMapper
().
getNoSubjectLabelList
(
labelIds
);
}
}
}
service-web/src/main/java/com/zzsn/clb/task/ModelLabelTask.java
浏览文件 @
a58dbbc6
...
@@ -41,6 +41,7 @@ import javax.annotation.Resource;
...
@@ -41,6 +41,7 @@ import javax.annotation.Resource;
import
java.io.IOException
;
import
java.io.IOException
;
import
java.util.*
;
import
java.util.*
;
import
java.util.stream.Collectors
;
import
java.util.stream.Collectors
;
import
java.util.stream.Stream
;
/***
/***
* 标签类的模型处理
* 标签类的模型处理
...
@@ -64,13 +65,12 @@ public class ModelLabelTask {
...
@@ -64,13 +65,12 @@ public class ModelLabelTask {
RedisUtil
redisUtil
;
RedisUtil
redisUtil
;
@Scheduled
(
cron
=
"0 */10 * * * ?"
)
// 每10分钟执行一次
//@Scheduled(cron = "0 */10 * * * ?")// 每10分钟执行一次
//@Bean
public
void
run
()
{
public
void
run
()
{
if
(
redisUtil
.
get
(
redisCacheName
)
!=
null
)
{
if
(
redisUtil
.
get
(
redisCacheName
)
!=
null
)
{
queryBgDate
=
redisUtil
.
get
(
redisCacheName
).
toString
();
queryBgDate
=
redisUtil
.
get
(
redisCacheName
).
toString
();
}
else
{
}
else
{
queryBgDate
=
"20
00-01-01T00:00:00
"
;
queryBgDate
=
"20
23-02-28T14:10:03
"
;
}
}
Date
dateNow
=
new
Date
();
Date
dateNow
=
new
Date
();
queryEndate
=
DateUtils
.
formatDate
(
dateNow
,
"yyyy-MM-dd"
)
+
"T"
+
DateUtils
.
formatDate
(
dateNow
,
"HH:mm:ss"
);
queryEndate
=
DateUtils
.
formatDate
(
dateNow
,
"yyyy-MM-dd"
)
+
"T"
+
DateUtils
.
formatDate
(
dateNow
,
"HH:mm:ss"
);
...
@@ -108,7 +108,9 @@ public class ModelLabelTask {
...
@@ -108,7 +108,9 @@ public class ModelLabelTask {
try
{
try
{
JSONObject
result
=
JSONObject
.
parseObject
(
HttpUtil
.
doPost
(
modelUrl
,
jo
,
60000
));
JSONObject
result
=
JSONObject
.
parseObject
(
HttpUtil
.
doPost
(
modelUrl
,
jo
,
60000
));
if
(
result
.
getInteger
(
"code"
)
==
200
&&
result
.
getString
(
"flag"
).
equals
(
"1"
))
{
if
(
result
.
getInteger
(
"code"
)
==
200
&&
result
.
getString
(
"flag"
).
equals
(
"1"
))
{
setEsDataList
(
dataList
,
map
,
result
.
getJSONArray
(
"label_info"
),
subjectLabelList
);
List
<
Map
<
String
,
Object
>>
returnList
=
getEsDataList
(
map
,
result
.
getJSONArray
(
"label_info"
),
subjectLabelList
);
if
(
returnList
!=
null
)
dataList
.
addAll
(
returnList
);
}
}
}
catch
(
IOException
e
)
{
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
e
.
printStackTrace
();
...
@@ -123,17 +125,17 @@ public class ModelLabelTask {
...
@@ -123,17 +125,17 @@ public class ModelLabelTask {
/***
/***
* 对模型返回数据进行处理
* 对模型返回数据进行处理
* @param dataList
* @param mapData
* @param mapData
* @param jsonArray
* @param jsonArray
* @param subjectLabelList
* @param subjectLabelList
*/
*/
private
void
setEsDataList
(
List
<
Map
<
String
,
Object
>>
dataList
,
Map
<
String
,
Object
>
mapData
,
JSONArray
jsonArray
,
List
<
Map
<
String
,
Object
>>
subjectLabelList
)
{
private
List
<
Map
<
String
,
Object
>>
getEsDataList
(
Map
<
String
,
Object
>
mapData
,
JSONArray
jsonArray
,
List
<
Map
<
String
,
Object
>>
subjectLabelList
)
{
List
<
Map
<
String
,
Object
>>
oneDataList
=
new
ArrayList
<>();
//遍历数据模型返回的数据
//遍历数据模型返回的数据
for
(
Object
x
:
jsonArray
)
{
for
(
Object
x
:
jsonArray
)
{
JSONObject
jo
=
(
JSONObject
)
x
;
JSONObject
jo
=
(
JSONObject
)
x
;
if
(
jo
.
get
(
"level1"
)
==
null
||
jo
.
get
(
"level2"
)
==
null
)
if
(
jo
.
get
(
"level1"
)
==
null
||
jo
.
get
(
"level2"
)
==
null
)
return
;
return
null
;
//获取标签分类数组
//获取标签分类数组
String
[]
labelTypeNames
=
{
jo
.
getString
(
"level1"
)};
String
[]
labelTypeNames
=
{
jo
.
getString
(
"level1"
)};
if
(
jo
.
getString
(
"level1"
).
contains
(
"|"
))
{
if
(
jo
.
getString
(
"level1"
).
contains
(
"|"
))
{
...
@@ -144,13 +146,52 @@ public class ModelLabelTask {
...
@@ -144,13 +146,52 @@ public class ModelLabelTask {
if
(
jo
.
getString
(
"level2"
).
contains
(
"|"
))
{
if
(
jo
.
getString
(
"level2"
).
contains
(
"|"
))
{
labelNames
=
jo
.
getString
(
"level2"
).
split
(
"\\|"
);
labelNames
=
jo
.
getString
(
"level2"
).
split
(
"\\|"
);
}
}
//遍历标签分类
//遍历标签分类
for
(
String
labelTypeName
:
labelTypeNames
)
{
for
(
String
labelTypeName
:
labelTypeNames
)
{
List
<
Map
<
String
,
Object
>>
returnList
=
this
.
setEsDataMap
(
mapData
,
labelTypeName
,
labelNames
,
subjectLabelList
);
List
<
Map
<
String
,
Object
>>
returnList
=
this
.
setEsDataMap
(
mapData
,
labelTypeName
,
labelNames
,
subjectLabelList
);
if
(
returnList
!=
null
)
if
(
returnList
!=
null
)
d
ataList
.
addAll
(
returnList
);
oneD
ataList
.
addAll
(
returnList
);
}
}
}
}
return
removeRepeatSubjectId
(
oneDataList
);
}
/***
* 对同一个专题,不同标签分类的数据进行标签合并
* @param oneDataList
* @return
*/
private
List
<
Map
<
String
,
Object
>>
removeRepeatSubjectId
(
List
<
Map
<
String
,
Object
>>
oneDataList
)
{
//对同一个专题,不同标签分类的数据进行标签合并
//筛出有重复的subjectId
Map
<
Object
,
Long
>
map
=
oneDataList
.
stream
().
collect
(
Collectors
.
groupingBy
(
m
->
m
.
get
(
"subjectId"
),
Collectors
.
counting
()));
Stream
<
Object
>
stringStream
=
map
.
entrySet
().
stream
().
filter
(
entry
->
entry
.
getValue
()
>
1
).
map
(
entry
->
entry
.
getKey
());
List
<
String
>
repeatIdList
=
new
ArrayList
<>();
stringStream
.
forEach
(
str
->
{
repeatIdList
.
add
(
String
.
valueOf
(
str
));
});
//对有重复subjectId的数据去重,重新装配标签
if
(
repeatIdList
.
size
()
>
0
)
{
log
.
debug
(
"去重前数量:{}"
,
oneDataList
.
size
());
log
.
debug
(
"重复subjectId:"
+
JSON
.
toJSONString
(
repeatIdList
));
repeatIdList
.
forEach
(
x
->
{
List
<
Map
<
String
,
Object
>>
repeatList
=
oneDataList
.
stream
().
filter
(
f
->
f
.
get
(
"subjectId"
).
toString
().
equals
(
x
)).
collect
(
Collectors
.
toList
());
log
.
debug
(
"重复subjectId:{},数量:{}"
,
x
,
repeatList
.
size
());
List
<
Map
<
String
,
Object
>>
labelList
=
new
ArrayList
<>();
repeatList
.
stream
().
map
(
m
->
((
List
)
m
.
get
(
"labels"
))).
collect
(
Collectors
.
toList
()).
forEach
(
y
->
{
labelList
.
addAll
(
y
);
});
Map
<
String
,
Object
>
tmpSubject
=
repeatList
.
get
(
0
);
tmpSubject
.
put
(
"labels"
,
labelList
);
oneDataList
.
removeAll
(
repeatList
);
oneDataList
.
add
(
tmpSubject
);
log
.
debug
(
"重复合并后ID:{}"
,
tmpSubject
.
get
(
"id"
));
});
log
.
debug
(
"现有数量:{}"
,
oneDataList
.
size
());
}
return
oneDataList
;
}
}
/***
/***
...
@@ -183,7 +224,7 @@ public class ModelLabelTask {
...
@@ -183,7 +224,7 @@ public class ModelLabelTask {
//如果标签不存在,返回null
//如果标签不存在,返回null
if
(
labelList
==
null
)
if
(
labelList
==
null
)
return
null
;
return
null
;
//装配入库信息,处理一条标签分类信息属于多个专题
,如果这几类标签都属于一个专题,会出现重复情况,除非一级栏目是专题(负面舆情就有问题了)
//装配入库信息,处理一条标签分类信息属于多个专题
List
<
String
>
subjectIdList
=
subjectLabelList
.
stream
().
filter
(
f
->
f
.
get
(
"labelTypeName"
).
equals
(
labelTypeName
)).
map
(
m
->
m
.
get
(
"subjectId"
).
toString
()).
distinct
().
collect
(
Collectors
.
toList
());
List
<
String
>
subjectIdList
=
subjectLabelList
.
stream
().
filter
(
f
->
f
.
get
(
"labelTypeName"
).
equals
(
labelTypeName
)).
map
(
m
->
m
.
get
(
"subjectId"
).
toString
()).
distinct
().
collect
(
Collectors
.
toList
());
return
getMultSubjectData
(
subjectIdList
,
mapData
,
labelList
);
return
getMultSubjectData
(
subjectIdList
,
mapData
,
labelList
);
}
}
...
@@ -213,6 +254,10 @@ public class ModelLabelTask {
...
@@ -213,6 +254,10 @@ public class ModelLabelTask {
return
dataList
;
return
dataList
;
}
}
/***
* 批量写入数据库
* @param addEsDataMapList
*/
public
void
addElasticsearchData
(
List
<
Map
<
String
,
Object
>>
addEsDataMapList
)
{
public
void
addElasticsearchData
(
List
<
Map
<
String
,
Object
>>
addEsDataMapList
)
{
try
{
try
{
...
@@ -224,7 +269,7 @@ public class ModelLabelTask {
...
@@ -224,7 +269,7 @@ public class ModelLabelTask {
requestData
=
new
IndexRequest
(
Constants
.
ES_DATA_FOR_SUBJECT
).
id
(
addEsDataMap
.
get
(
"id"
).
toString
()).
source
(
addEsDataMap
,
XContentType
.
JSON
);
requestData
=
new
IndexRequest
(
Constants
.
ES_DATA_FOR_SUBJECT
).
id
(
addEsDataMap
.
get
(
"id"
).
toString
()).
source
(
addEsDataMap
,
XContentType
.
JSON
);
bulkRequest
.
add
(
requestData
);
bulkRequest
.
add
(
requestData
);
}
}
log
.
info
(
"es同步数据数量:{}"
,
bulkRequest
.
numberOfActions
());
log
.
debug
(
"es同步数据数量:{}"
,
bulkRequest
.
numberOfActions
());
//设置索引刷新规则
//设置索引刷新规则
bulkRequest
.
setRefreshPolicy
(
WriteRequest
.
RefreshPolicy
.
IMMEDIATE
);
bulkRequest
.
setRefreshPolicy
(
WriteRequest
.
RefreshPolicy
.
IMMEDIATE
);
//分批次提交,数量控制
//分批次提交,数量控制
...
...
service-web/src/main/java/com/zzsn/clb/test/ImportExcelData.java
浏览文件 @
a58dbbc6
...
@@ -43,9 +43,9 @@ public class ImportExcelData {
...
@@ -43,9 +43,9 @@ public class ImportExcelData {
SubjectMapper
subjectMapper
;
SubjectMapper
subjectMapper
;
public
void
importData
()
{
public
void
importData
()
{
String
path
=
"
C:\\Users\\EDY\\Documents\\WeChat Files\\wxid_7d7tl0zyfdxb21\\FileStorage\\File\\2023-02\\管理动态-下属机构工作
.xlsx"
;
String
path
=
"
D:\\新数据22
.xlsx"
;
try
{
try
{
impData
(
1
,
10
,
path
,
6
);
impData
(
1
,
10
,
path
,
16
6
);
}
catch
(
IOException
e
)
{
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
e
.
printStackTrace
();
}
}
...
@@ -178,7 +178,7 @@ public class ImportExcelData {
...
@@ -178,7 +178,7 @@ public class ImportExcelData {
subjectData
.
setOrigin
(
list
.
get
(
7
).
trim
());
subjectData
.
setOrigin
(
list
.
get
(
7
).
trim
());
}
}
if
(
StringUtils
.
isNotBlank
(
list
.
get
(
8
)))
{
if
(
StringUtils
.
isNotBlank
(
list
.
get
(
8
)))
{
subjectData
.
setPublishDate
(
list
.
get
(
8
));
subjectData
.
setPublishDate
(
list
.
get
(
8
)
.
substring
(
0
,
10
)
);
}
}
if
(
StringUtils
.
isNotBlank
(
list
.
get
(
9
)))
{
if
(
StringUtils
.
isNotBlank
(
list
.
get
(
9
)))
{
...
...
service-web/src/main/java/com/zzsn/clb/web/controller/SubjectController.java
浏览文件 @
a58dbbc6
...
@@ -46,8 +46,36 @@ public class SubjectController {
...
@@ -46,8 +46,36 @@ public class SubjectController {
@AutoLog
(
value
=
"专题数据"
)
@AutoLog
(
value
=
"专题数据"
)
@ApiOperation
(
value
=
"专题数据"
,
notes
=
"专题数据"
)
@ApiOperation
(
value
=
"专题数据"
,
notes
=
"专题数据"
)
@GetMapping
(
value
=
"/getSubjectCollect"
)
@GetMapping
(
value
=
"/getSubjectCollect"
)
public
Result
<?>
SubjectCollect
(
@RequestParam
(
required
=
true
,
defaultValue
=
"month"
)
String
type
,
@RequestParam
(
defaultValue
=
"6"
)
Integer
size
,
@RequestParam
(
defaultValue
=
"1"
)
Integer
subjectType
)
{
public
Result
<?>
SubjectCollect
(
@RequestParam
(
required
=
true
,
defaultValue
=
"month"
)
String
type
,
@RequestParam
(
defaultValue
=
"6"
)
Integer
size
,
@RequestParam
(
defaultValue
=
"1"
)
Integer
subjectType
)
{
return
Result
.
OK
(
subjectService
.
getSubjectCollect
(
type
,
size
,
subjectType
));
return
Result
.
OK
(
subjectService
.
getSubjectCollect
(
type
,
size
,
subjectType
));
}
/***
* 专题趋势
* @param queryBgDate 开始日期
* @param queryEndate 结束日期
* @return
*/
@AutoLog
(
value
=
"专题趋势"
)
@ApiOperation
(
value
=
"专题趋势"
,
notes
=
"专题趋势"
)
@GetMapping
(
value
=
"/getDaySubjectTrend"
)
public
Result
<?>
getDaySubjectTrend
(
@RequestParam
(
required
=
true
,
defaultValue
=
""
)
String
queryBgDate
,
@RequestParam
(
required
=
true
,
defaultValue
=
""
)
String
queryEndate
)
{
return
Result
.
OK
(
subjectService
.
getDaySubjectTrend
(
queryBgDate
,
queryEndate
));
}
/***
* 舆情走势
* @param queryBgDate 开始日期
* @param queryEndate 结束日期
* @param type 1-正面 2-负面
* @return
*/
@AutoLog
(
value
=
"舆情走势"
)
@ApiOperation
(
value
=
"舆情走势"
,
notes
=
"舆情走势"
)
@GetMapping
(
value
=
"/getPubliOpinionTrend"
)
public
Result
<?>
getPubliOpinionTrend
(
@RequestParam
(
required
=
true
,
defaultValue
=
""
)
String
queryBgDate
,
@RequestParam
(
required
=
true
,
defaultValue
=
""
)
String
queryEndate
,
@RequestParam
(
required
=
true
,
defaultValue
=
"1"
)
Integer
type
)
{
return
Result
.
OK
(
subjectService
.
getPubliOpinionTrend
(
queryBgDate
,
queryEndate
,
type
));
}
}
...
@@ -61,11 +89,11 @@ public class SubjectController {
...
@@ -61,11 +89,11 @@ public class SubjectController {
@AutoLog
(
value
=
"根据专题id查询数据"
)
@AutoLog
(
value
=
"根据专题id查询数据"
)
@ApiOperation
(
value
=
"根据专题id查询数据"
,
notes
=
"根据专题id查询数据"
)
@ApiOperation
(
value
=
"根据专题id查询数据"
,
notes
=
"根据专题id查询数据"
)
@GetMapping
(
value
=
"/getListBySubject"
)
@GetMapping
(
value
=
"/getListBySubject"
)
public
Result
<?>
getListBySubject
(
Integer
pageNo
,
Integer
pageSize
,
@RequestParam
(
required
=
true
)
String
subjectId
,
@RequestParam
(
required
=
false
)
String
lableIds
)
{
public
Result
<?>
getListBySubject
(
Integer
pageNo
,
Integer
pageSize
,
@RequestParam
(
required
=
true
)
String
subjectId
,
@RequestParam
(
required
=
false
)
String
lableIds
)
{
//http://114.115.205.50:9988/subject/display/list?_t=1676630456&column=score&order=desc&field=id,,info&pageNo=1&pageSize=10&subjectId=1600743965735264258&checkStatusList=&isSubject=1
//http://114.115.205.50:9988/subject/display/list?_t=1676630456&column=score&order=desc&field=id,,info&pageNo=1&pageSize=10&subjectId=1600743965735264258&checkStatusList=&isSubject=1
Map
<
String
,
Object
>
map
=
new
HashMap
<>();
Map
<
String
,
Object
>
map
=
new
HashMap
<>();
map
.
put
(
"subjectId"
,
subjectId
);
map
.
put
(
"subjectId"
,
subjectId
);
return
serviceProjectClient
.
getListBySubject
(
map
,
null
,
null
,
pageNo
,
pageSize
,
"publishDate"
,
"desc"
,
null
,
"1"
,
null
,
lableIds
);
return
serviceProjectClient
.
getListBySubject
(
map
,
null
,
null
,
pageNo
,
pageSize
,
"publishDate"
,
"desc"
,
null
,
"1"
,
null
,
lableIds
);
}
}
...
@@ -86,15 +114,15 @@ public class SubjectController {
...
@@ -86,15 +114,15 @@ public class SubjectController {
/**
/**
* 根据信息id查询数据
* 根据信息id查询数据
*
*
* @param id
* @return
* @return
*/
*/
@AutoLog
(
value
=
"根据信息id查询数据"
)
@AutoLog
(
value
=
"根据信息id查询数据"
)
@ApiOperation
(
value
=
"根据信息id查询数据"
,
notes
=
"根据信息id查询数据"
)
@ApiOperation
(
value
=
"根据信息id查询数据"
,
notes
=
"根据信息id查询数据"
)
//
@GetMapping(value = "/import")
@GetMapping
(
value
=
"/import"
)
public
Result
<?>
importData
()
{
public
Result
<?>
importData
()
{
//http://114.115.236.206:9988/subject/display/queryById?_t=1676875032&id=21123000014507
//http://114.115.236.206:9988/subject/display/queryById?_t=1676875032&id=21123000014507
importExcelData
.
importData
();
importExcelData
.
importData
();
return
Result
.
OK
();
return
Result
.
OK
();
}
}
}
}
service-web/src/main/java/com/zzsn/common/Constants.java
浏览文件 @
a58dbbc6
...
@@ -2,7 +2,7 @@ package com.zzsn.common;
...
@@ -2,7 +2,7 @@ package com.zzsn.common;
public
class
Constants
{
public
class
Constants
{
//处理后的专题资讯信息存储索引
//处理后的专题资讯信息存储索引
public
final
static
String
ES_DATA_FOR_SUBJECT
=
"
tmp_
subjectdatabase"
;
public
final
static
String
ES_DATA_FOR_SUBJECT
=
"subjectdatabase"
;
//新采集库(22.04.23)
//新采集库(22.04.23)
public
final
static
String
ES_BASE_DATA
=
"basedata"
;
public
final
static
String
ES_BASE_DATA
=
"basedata"
;
}
}
编写
预览
Markdown
格式
0%
重试
或
添加新文件
添加附件
取消
您添加了
0
人
到此讨论。请谨慎行事。
请先完成此评论的编辑!
取消
请
注册
或者
登录
后发表评论