网络事件定时任务

src/main/java/com/zzsn/event/task/NetWordEventTask.java

+package com.zzsn.event.task;
+
+import cn.hutool.core.date.DateUtil;
+import cn.hutool.http.HttpUtil;
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONObject;
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.core.toolkit.Wrappers;
+import com.zzsn.event.entity.EventNetwork;
+import com.zzsn.event.llm.LlmService;
+import com.zzsn.event.service.EventNetworkService;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.Collectors;
+
+/**
+ * 网络事件处理任务
+ *
+ * @author lkg
+ * @date 2025/7/3
+ */
+@Slf4j
+@Component
+public class NetWordEventTask {
+
+    @Autowired
+    private EventNetworkService eventNetworkService;
+    @Autowired
+    private LlmService llmService;
+
+    @Value("${scheduling.yjzxEnable:false}")
+    Boolean yjzxEnable;
+    @Value("${python.hot-crawler:}")
+    private String hotCrawlerPath;
+
+    final static String PROMPT = "根据提供的网络事件对象集合json字符串中事件标题-【title】字段，判断是否为政治经济领域相关的事件，并按以下格式返回符合条件的网络事件集合：[{\"title\": \"政治\"}]";
+
+    /**
+     * 定时拉取热榜数据 (1小时一次)
+     *
+     * @author lkg
+     * @date 2025/7/4
+     */
+    @Scheduled(cron = "0 0 0/1 * * ?")
+    public void execute() {
+        if(!yjzxEnable){
+            //研究中心需要此任务
+            return;
+        }
+        for (int i = 1; i < 3; i++) {
+            int finalI = i;
+            CompletableFuture.runAsync(() -> handler(finalI));
+        }
+    }
+
+
+    private void handler(Integer type) {
+        List<EventNetwork> networkList = getNetWordEventList(type);
+        if (CollectionUtils.isEmpty(networkList)) {
+            return;
+        }
+        String today = DateUtil.today();
+        LambdaQueryWrapper<EventNetwork> queryWrapper = Wrappers.lambdaQuery();
+        queryWrapper.eq(EventNetwork::getPublishDate, today).eq(EventNetwork::getType, type).eq(EventNetwork::getLatest, 1);
+        List<EventNetwork> oldList = eventNetworkService.list(queryWrapper);
+        //最终要处理的数据 过模型判断是否 是所需要的事件[政治经济领域相关]
+        List<EventNetwork> finalList;
+        if (CollectionUtils.isEmpty(oldList)) {
+            finalList = networkList;
+        } else {
+            finalList = networkList.stream().filter(network -> oldList.stream()
+                            .noneMatch(old -> old.getTitle().equals(network.getTitle())))
+                    .collect(Collectors.toList());
+            //变更状态 latest = 0，不是最新
+            List<EventNetwork> removedList = oldList.stream().filter(network -> networkList.stream()
+                            .noneMatch(fresh -> fresh.getTitle().equals(network.getTitle())))
+                    .collect(Collectors.toList());
+            if (CollectionUtils.isNotEmpty(removedList)) {
+                List<EventNetwork> collect = removedList.stream().filter(network -> network.getYnNeed() == 0).collect(Collectors.toList());
+                collect.forEach(network -> network.setLatest(0));
+                eventNetworkService.updateBatchById(collect);
+            }
+        }
+        if (CollectionUtils.isNotEmpty(finalList)) {
+            finalList.forEach(network -> network.setLatest(1));
+            String response = llmService.model(null, PROMPT, JSONObject.toJSONString(finalList));
+            if (StringUtils.isNotEmpty(response)) {
+                try {
+                    List<EventNetwork> hitList = JSON.parseArray(response, EventNetwork.class);
+                    finalList.forEach(network -> {
+                        List<String> titleLis = hitList.stream().map(EventNetwork::getTitle).collect(Collectors.toList());
+                        if (titleLis.contains(network.getTitle())) {
+                            network.setYnNeed(1);
+                        }
+                    });
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+            eventNetworkService.saveOrUpdateBatch(finalList);
+        }
+        log.info("{}-数据采集更新完成", type == 1 ? "百度热榜" : "新浪热榜");
+    }
+
+
+    /**
+     * 调用采集接口，获取网络事件
+     *
+     * @param type 来源类型(1-百度;2-新浪新闻)
+     * @author lkg
+     * @date 2025/7/4
+     */
+    private List<EventNetwork> getNetWordEventList(Integer type) {
+        try {
+            String option = HttpUtil.get(hotCrawlerPath.replace("OPTION", type.toString()));
+            if (StringUtils.isNotEmpty(option)) {
+                List<EventNetwork> eventNetworks = JSON.parseArray(option, EventNetwork.class);
+                eventNetworks.forEach(network -> network.setType(type));
+                return eventNetworks;
+            }
+        } catch (Exception e) {
+            log.info("{}-数据采集异常", type == 1 ? "百度热榜" : "新浪热榜");
+            e.printStackTrace();
+        }
+        return new ArrayList<>();
+    }
+}