Merge remote-tracking branch 'origin/master'

pom.xml

@@ -144,7 +144,22 @@
             <version>3.0.0</version>
         </dependency>
 
-
+        <dependency>
+            <groupId>org.zwobble.mammoth</groupId>
+            <artifactId>mammoth</artifactId>
+            <version>1.5.0</version>
+        </dependency>
+        <!--stream kafka-->
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-stream-kafka</artifactId>
+            <version>3.0.8.RELEASE</version>
+        </dependency>
+        <dependency>
+            <groupId>net.sourceforge.htmlcleaner</groupId>
+            <artifactId>htmlcleaner</artifactId>
+            <version>2.25</version>
+        </dependency>
     </dependencies>
 
     <build>

src/main/java/com/zzsn/knowbase/controller/KnowledgeController.java

@@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.zzsn.knowbase.entity.AiReportScienceFile;
 import com.zzsn.knowbase.entity.Knowledge;
 import com.zzsn.knowbase.service.IKnowledgeService;
+import com.zzsn.knowbase.util.DocUtil;
 import com.zzsn.knowbase.vo.AiReportScienceFileVo;
 import com.zzsn.knowbase.vo.KnowledgeParam;
 import com.zzsn.knowbase.vo.KnowledgeVO;
@@ -14,6 +15,9 @@ import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;
 
+import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+
 /**
  * @Description: 知识
  * @Author: chenshiqiang
@@ -47,9 +51,9 @@ public class KnowledgeController {
      * @return
      */
     @ApiOperation(value = "知识-添加", notes = "知识-添加")
-    @PostMapping(value = "/add")
-    public Result<?> add(@RequestBody Knowledge knowledge) {
-        knowledgeService.addKnowledge(knowledge);
+    @PostMapping(value = "/uploadKnowledge")
+    public Result<?> uploadKnowledge(HttpServletRequest request, @RequestBody Knowledge knowledge) {
+        knowledgeService.addKnowledge(request,knowledge);
         return Result.OK("添加成功！");
     }
 
@@ -77,5 +81,12 @@ public class KnowledgeController {
         knowledgeService.deleteKnowledge(id);
         return Result.OK("删除成功!");
     }
-
+    /**
+     * 文档解析
+     */
+    @GetMapping(value = "/parse")
+    public Result<?> parse(@RequestParam(name = "filePath")String filePath) throws IOException {
+        String s = DocUtil.docParseHtml(filePath);
+        return Result.OK(s);
+    }
 }

src/main/java/com/zzsn/knowbase/entity/KnowFile.java

+package com.zzsn.knowbase.entity;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/**
+ * todo
+ *
+ * @author ShiQiangChen
+ * @date 2024/1/5
+ */
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class KnowFile{
+    private String fileId;
+    private String filePath;
+    private String fileType;
+    private Integer fileSize;
+}

src/main/java/com/zzsn/knowbase/entity/Knowledge.java

@@ -103,10 +103,5 @@ public class Knowledge implements Serializable {
     private List<KnowFile> files;
 
 }
-class KnowFile{
-    private String fileId;
-    private String filePath;
-    private String fileType;
-    private Integer fileSize;
-}
+
 

src/main/java/com/zzsn/knowbase/kafka/cloudstream/KnowledgeChannel.java

+package com.zzsn.knowbase.kafka.cloudstream;
+
+import org.springframework.cloud.stream.annotation.Output;
+import org.springframework.messaging.MessageChannel;
+
+/**
+ * @author chenshiqiang
+ * @Description 知识信息通道
+ */
+public interface KnowledgeChannel {
+
+
+    /**
+     * 发送知识列表
+     *
+     * @author chenshiqiang
+     */
+    @Output
+    MessageChannel knowledgeContents();
+
+
+}

src/main/java/com/zzsn/knowbase/kafka/message/KnowledgeMessage.java

+package com.zzsn.knowbase.kafka.message;
+
+import com.zzsn.knowbase.entity.Content;
+import com.zzsn.knowbase.entity.KnowFile;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+/**
+ * 知识消息体
+ *
+ * @author ShiQiangChen
+ * @date 2024/1/5
+ */
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class KnowledgeMessage {
+
+    private String id;
+    private String title;
+    /**
+     * 来源
+     */
+    private String origin;
+    /**
+     * 发布时间
+     */
+    private String publishDate;
+    /**
+     * 审核状态
+     */
+    private Integer verifyStatus;
+    /**
+     * 审核人名字
+     */
+    private String verifierName;
+    /**
+     * 知识库id
+     */
+    private String knowledgeProjectId;
+    /**
+     * 类型
+     */
+    private String type;
+
+    private List<Content> contents;
+
+}

src/main/java/com/zzsn/knowbase/kafka/producer/ProduceInfo.java

+package com.zzsn.knowbase.kafka.producer;
+
+import com.alibaba.fastjson.JSON;
+import com.zzsn.knowbase.kafka.cloudstream.KnowledgeChannel;
+import com.zzsn.knowbase.kafka.message.KnowledgeMessage;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.cloud.stream.annotation.EnableBinding;
+import org.springframework.messaging.Message;
+import org.springframework.messaging.support.MessageBuilder;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.util.List;
+
+
+@Component
+@Slf4j
+@EnableBinding(KnowledgeChannel.class)
+public class ProduceInfo {
+
+    @Resource
+    private KnowledgeChannel channel;
+
+    /**
+     * 发送知识内容列表
+     *
+     * @param knowledgeMessage 知识内容列表
+     * @author chenshiqiang
+     */
+    public void sendKnowledgeContents(KnowledgeMessage knowledgeMessage) {
+        String msg = JSON.toJSONString(knowledgeMessage);
+        Message<String> message = MessageBuilder.withPayload(msg).build();
+        channel.knowledgeContents().send(message);
+    }
+}

src/main/java/com/zzsn/knowbase/service/IKnowledgeService.java

@@ -5,6 +5,7 @@ import com.zzsn.knowbase.entity.Knowledge;
 import com.zzsn.knowbase.vo.KnowledgeParam;
 import com.zzsn.knowbase.vo.KnowledgeVO;
 
+import javax.servlet.http.HttpServletRequest;
 import java.util.List;
 
 /**
@@ -18,7 +19,7 @@ public interface IKnowledgeService {
     /**
      * 新增
      */
-    void addKnowledge(Knowledge knowledge);
+    void addKnowledge(HttpServletRequest httpServletRequest,Knowledge knowledge);
 
     /**
      * 修改

src/main/java/com/zzsn/knowbase/service/impl/KnowledgeServiceImpl.java

@@ -5,12 +5,12 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.zzsn.knowbase.constant.Constants;
 import com.zzsn.knowbase.entity.Content;
+import com.zzsn.knowbase.entity.KnowFile;
 import com.zzsn.knowbase.entity.Knowledge;
+import com.zzsn.knowbase.kafka.message.KnowledgeMessage;
+import com.zzsn.knowbase.kafka.producer.ProduceInfo;
 import com.zzsn.knowbase.service.IKnowledgeService;
-import com.zzsn.knowbase.util.CodeGenerateUtil;
-import com.zzsn.knowbase.util.DateUtil;
-import com.zzsn.knowbase.util.EsDateUtil;
-import com.zzsn.knowbase.util.EsOpUtil;
+import com.zzsn.knowbase.util.*;
 import com.zzsn.knowbase.vo.KnowledgeParam;
 import com.zzsn.knowbase.vo.KnowledgeVO;
 import org.apache.commons.lang3.StringUtils;
@@ -25,13 +25,19 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.SortOrder;
+import org.htmlcleaner.CleanerProperties;
+import org.htmlcleaner.HtmlCleaner;
+import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import javax.servlet.http.HttpServletRequest;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * @Description: 知识
@@ -46,9 +52,12 @@ class KnowledgeServiceImpl implements IKnowledgeService {
     private EsOpUtil esOpUtil;
     @Autowired
     private CodeGenerateUtil codeGenerateUtil;
+    @Autowired
+    private ProduceInfo produceInfo;
 
     @Override
-    public void addKnowledge(Knowledge knowledge) {
+    public void addKnowledge(HttpServletRequest httpServletRequest,Knowledge knowledge) {
+
         if (null == knowledge.getId()) {
             knowledge.setId(codeGenerateUtil.geneIdNo(Constants.FINANCE, 8));
         }
@@ -57,12 +66,53 @@ class KnowledgeServiceImpl implements IKnowledgeService {
         }
         knowledge.setCreateTime(cn.hutool.core.date.DateUtil.formatDateTime(new Date()).replace(" ", "T"));
         knowledge.setDeleteFlag(0);
-        List<Content> contents = new ArrayList<>();
-        contents.add(Content.builder()
-                .contentId(codeGenerateUtil.geneIdNo(Constants.FINANCE, 8))
-                .content("content").build());
-        knowledge.setContents(contents);
+        List<KnowFile> knowFileList = new ArrayList<>();
+        knowFileList.add(KnowFile.builder()
+                .fileId(codeGenerateUtil.geneIdNo(Constants.FINANCE, 8))
+                .filePath("C:/bbb.docx")
+                .build());
+        knowledge.setFiles(knowFileList);
+        List<Content> contentList=new ArrayList<>();
+        List<String> contentStringList = new ArrayList<>();
+        String html = null;
+        try {
+             html = DocUtil.docParseHtml(knowledge.getFiles().get(0).getFilePath());
+            String htmlWithTable = html.replace("<p>", "");
+            contentStringList = Arrays.asList(htmlWithTable.split("</p>"));
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        if(!contentStringList.isEmpty()){
+            knowledge.setTitle(contentStringList.get(0));
+            for (String s : contentStringList) {
+                contentList.add(Content.builder()
+                        .contentId(codeGenerateUtil.geneIdNo(Constants.FINANCE, 8))
+                        .content(s)
+                        .build());
+            }
+        }
+        knowledge.setContents(contentList);
         esOpUtil.docSavaByEntity(Constants.ES_DATA_FOR_KNOWLEDGE, knowledge.getId(), knowledge);
+
+        KnowledgeMessage knowledgeMessage= new KnowledgeMessage();
+
+        BeanUtils.copyProperties( knowledge,knowledgeMessage);
+//        CleanerProperties props = new CleanerProperties();
+//        props.setPruneTags("table");
+//        String htmlWithoutTable = new HtmlCleaner(props).clean(html).getText().toString();
+//         htmlWithoutTable = htmlWithoutTable.replace("<p>", "");
+//        List<String>contentStringWithoutTableList = Arrays.asList(htmlWithoutTable.split("</p>"));
+        List<Content> messageContentList=contentList.stream()
+                .filter(item->item.getContent().length()>10)
+                .filter(item->!item.getContent().contains("<img"))
+                .filter(item->!item.getContent().contains("<table"))
+                .filter(item->!item.getContent().contains("<tr>"))
+                .filter(item->!item.getContent().contains("</tr>"))
+                .filter(item->!item.getContent().contains("<td>"))
+                .filter(item->!item.getContent().contains("</td>"))
+                .collect(Collectors.toList());
+        knowledgeMessage.setContents(messageContentList);
+        produceInfo.sendKnowledgeContents(knowledgeMessage);
     }
 
     @Override

src/main/java/com/zzsn/knowbase/util/DocUtil.java

+package com.zzsn.knowbase.util;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.zwobble.mammoth.DocumentConverter;
+import org.zwobble.mammoth.Result;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * 文档工具
+ *
+ * @author chenshiqiang
+ */
+@Slf4j
+@Component
+public class DocUtil {
+
+
+    /**
+     * 文档解析
+     */
+    public static  String docParseHtml(String filePath) throws IOException {
+        DocumentConverter converter = new DocumentConverter()
+                .disableDefaultStyleMap()
+//                .addStyleMap("p[style-name='Section Title'] => h1:fresh")
+//                .addStyleMap("p[style-name='Subsection Title'] => h2:fresh")
+                ;
+        Result<String> result = converter.convertToHtml(new File(filePath));
+        // The generated HTML
+        String html = result.getValue();
+        // Any warnings during conversion
+        Set<String> warnings = result.getWarnings();
+        return html;
+    }
+
+}

src/main/resources/application.yml

@@ -11,6 +11,13 @@ spring:
       uris: ["114.116.90.53:9200"]
       username: elastic
       password: elastic
+  cloud:
+    stream:
+      kafka:
+        binder:
+          brokers: 114.115.159.144:9092
+          zkNodes: 114.115.159.144:2181
+          requiredAcks: 1
 
 
   redis: