Skip to content

K
know-base
提交 bf3ec374 作者: yanxin
操作

文件导入整合minio

上级 310640c1
正在显示 包含 1687 行增加 和 383 行删除
pom.xml
......@@ -15,6 +15,7 @@
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
<minio.version>8.0.3</minio.version>
</properties>
<dependencies>
<dependency>
......@@ -211,9 +212,12 @@
<version>2.2.10</version>
</dependency>
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>${minio.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
......
src/main/java/com/zzsn/knowbase/config/MinioConfig.java 0 → 100644
package com.zzsn.knowbase.config;
import com.zzsn.knowbase.util.MinioUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* Minio文件上传配置文件
*/
@Slf4j
@Configuration
public class MinioConfig {
@Value(value = "${minio.minio_url}")
private String minioUrl;
@Value(value = "${minio.minio_name}")
private String minioName;
@Value(value = "${minio.minio_pass}")
private String minioPass;
@Value(value = "${minio.bucketName}")
private String bucketName;
@Bean
public void initMinio(){
if(!minioUrl.startsWith("http")){
minioUrl = "http://" + minioUrl;
}
if(!minioUrl.endsWith("/")){
minioUrl = minioUrl.concat("/");
}
MinioUtil.setMinioUrl(minioUrl);
MinioUtil.setMinioName(minioName);
MinioUtil.setMinioPass(minioPass);
MinioUtil.setBucketName(bucketName);
}
}
src/main/java/com/zzsn/knowbase/config/scheduler/Scheduler.java 0 → 100644
package com.zzsn.knowbase.config.scheduler;
import org.springframework.beans.factory.config.BeanDefinition;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Conditional;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Role;
import org.springframework.scheduling.annotation.ScheduledAnnotationBeanPostProcessor;
import org.springframework.scheduling.config.TaskManagementConfigUtils;
@Configuration
public class Scheduler {
@Conditional(SchedulerCondition.class)
@Bean(name = TaskManagementConfigUtils.SCHEDULED_ANNOTATION_PROCESSOR_BEAN_NAME)
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
public ScheduledAnnotationBeanPostProcessor scheduledAnnotationProcessor() {
return new ScheduledAnnotationBeanPostProcessor();
}
}
\ No newline at end of file
src/main/java/com/zzsn/knowbase/config/scheduler/SchedulerCondition.java 0 → 100644
package com.zzsn.knowbase.config.scheduler;
import org.springframework.context.annotation.Condition;
import org.springframework.context.annotation.ConditionContext;
import org.springframework.core.type.AnnotatedTypeMetadata;
public class SchedulerCondition implements Condition {
@Override
public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
return Boolean.valueOf(context.getEnvironment().getProperty("enable.scheduling"));
}
}
\ No newline at end of file
src/main/java/com/zzsn/knowbase/constant/Constants.java
......@@ -130,8 +130,12 @@ public class Constants {
public static final String FSP = System.getProperty("file.separator");
public static final String USER_HOME = System.getProperty("user.home");
/*
* 临时文件保存路径。
*/
public static final String TEMP_FILES_DIR = USER_HOME+FSP+"temp"+FSP;
public static final String APPLICATION_DATA_DIR = USER_HOME + FSP + "mrasdata" + FSP;
public static final String APPLICATION_DATA_DIR = USER_HOME+FSP+"mrasdata"+FSP;
//专题采集数据
public final static String SEND_TO_MACHINE_DATA = "SUBJECT_DATA:SYNtOMACHINE:";
......
src/main/java/com/zzsn/knowbase/entity/Enclosure.java 0 → 100644
package com.zzsn.knowbase.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.experimental.Accessors;
import org.jeecgframework.poi.excel.annotation.Excel;
import org.springframework.format.annotation.DateTimeFormat;
import java.io.Serializable;
/**
* @Description: 附件表
* @Author: jeecg-boot
* @Date: 2023-01-18
* @Version: V1.0
*/
@Data
@TableName("enclosure")
public class Enclosure implements Serializable {
/**id*/
@TableId(type = IdType.ASSIGN_ID)
@ApiModelProperty(value = "id")
private String id;
/**name*/
@Excel(name = "name", width = 15)
@ApiModelProperty(value = "name")
private String name;
/**extensionName*/
@Excel(name = "extensionName", width = 15)
@ApiModelProperty(value = "extensionName")
private String extensionName;
/**urlPath*/
@Excel(name = "urlPath", width = 15)
@ApiModelProperty(value = "urlPath")
private String urlPath;
/**contentMd5*/
@Excel(name = "contentMd5", width = 15)
@ApiModelProperty(value = "contentMd5")
private String contentMd5;
/**createTime*/
@Excel(name = "createTime", width = 20, format = "yyyy-MM-dd HH:mm:ss")
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
@ApiModelProperty(value = "createTime")
private java.util.Date createTime;
/**createBy*/
@Excel(name = "createBy", width = 15)
@ApiModelProperty(value = "createBy")
private String createBy;
@Excel(name = "fileSize", width = 15)
@ApiModelProperty(value = "fileSize")
private Long fileSize;
}
src/main/java/com/zzsn/knowbase/mapper/EnclosureMapper.java 0 → 100644
package com.zzsn.knowbase.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zzsn.knowbase.entity.Enclosure;
import org.apache.ibatis.annotations.Mapper;
/**
* @Description: 附件表
* @Author: jeecg-boot
* @Date: 2023-01-18
* @Version: V1.0
*/
@Mapper
public interface EnclosureMapper extends BaseMapper<Enclosure> {
}
src/main/java/com/zzsn/knowbase/service/IEnclosureService.java 0 → 100644
package com.zzsn.knowbase.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.knowbase.entity.Enclosure;
/**
* @Description: 附件表
* @Author: jeecg-boot
* @Date: 2023-01-18
* @Version: V1.0
*/
public interface IEnclosureService extends IService<Enclosure> {
Enclosure getByMd5(String contentMd5);
}
src/main/java/com/zzsn/knowbase/service/impl/EnclosureServiceImpl.java 0 → 100644
package com.zzsn.knowbase.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.knowbase.entity.Enclosure;
import com.zzsn.knowbase.mapper.EnclosureMapper;
import com.zzsn.knowbase.service.IEnclosureService;
import org.springframework.stereotype.Service;
/**
* @Description: 附件表
* @Author: jeecg-boot
* @Date: 2023-01-18
* @Version: V1.0
*/
@Service
public class EnclosureServiceImpl extends ServiceImpl<EnclosureMapper, Enclosure> implements IEnclosureService {
@Override
public Enclosure getByMd5(String contentMd5) {
LambdaQueryWrapper<Enclosure> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.eq(Enclosure::getContentMd5,contentMd5);
return this.getOne(queryWrapper);
}
}
src/main/java/com/zzsn/knowbase/service/impl/KnowledgeServiceImpl.java
......@@ -120,142 +120,160 @@ class KnowledgeServiceImpl implements IKnowledgeService {
knowledge.setDeleteFlag(0);
KnowledgeMessage knowledgeMessage = new KnowledgeMessage();
if (Integer.valueOf("0").equals(knowledge.getImportData())) {
List<KnowFile> knowFileList = new ArrayList<>();
knowFileList.add(knowFile);
knowledge.setFiles(knowFileList);
List<Content> contentList = new ArrayList<>();
List<String> contentStringList = new ArrayList<>();
String html = null;
//创建临时文件
File file = null;
try {
String path = filesStorage + knowledge.getFiles().get(0).getFilePath();
String filePath = null;
if (".pdf".equals(knowFile.getFileType())) {
knowledge.setImportData(3);
log.info("pdf");
try {
PDDocument document = PDDocument.load(new File(path));
StringBuilder allBuilder = new StringBuilder();
PDFTextStripper pdfTextStripper = new PDFTextStripper() {
private StringBuilder paragraphBuilder = new StringBuilder();
@Override
protected void startPage(PDPage page) throws IOException {
super.startPage(page);
paragraphBuilder.setLength(0);//make paragraphBuilder empty
}
File dirFile = new File(Constants.TEMP_FILES_DIR);
file = File.createTempFile(Constants.TEMP_FILES_DIR, knowFile.getFileType(), dirFile);
} catch (IOException e) {
log.info("创建临时文件失败");
return;
}
try (FileOutputStream fos = new FileOutputStream(file)){
String url = MinioUtil.getMinioUrl() + LoadToolUtil.urlEncoderRelative(knowFile.getFilePath());
try(InputStream inStream = FileUtil.DownloadFileFormServer(url)){
int bytesRead = 0;
int len = 8129;
byte[] buffer = new byte[len];
while ((bytesRead = inStream.read(buffer, 0, len)) != -1) {
fos.write(buffer, 0, bytesRead);
}
}
log.info("临时文件本地地址：{}",file.getPath());
List<KnowFile> knowFileList = new ArrayList<>();
knowFileList.add(knowFile);
knowledge.setFiles(knowFileList);
List<Content> contentList = new ArrayList<>();
List<String> contentStringList = new ArrayList<>();
String html = null;
try {
String path = filesStorage + knowledge.getFiles().get(0).getFilePath();
String filePath = null;
if (".pdf".equals(knowFile.getFileType())) {
knowledge.setImportData(3);
log.info("pdf");
try {
PDDocument document = PDDocument.load(new File(path));
StringBuilder allBuilder = new StringBuilder();
PDFTextStripper pdfTextStripper = new PDFTextStripper() {
private StringBuilder paragraphBuilder = new StringBuilder();
@Override
protected void startPage(PDPage page) throws IOException {
super.startPage(page);
paragraphBuilder.setLength(0);//make paragraphBuilder empty
}
@Override
protected void writeLineSeparator() throws IOException {
super.writeLineSeparator();
if (
paragraphBuilder.toString().endsWith(".") ||
paragraphBuilder.toString().endsWith("。") ||
paragraphBuilder.toString().endsWith("!") ||
paragraphBuilder.toString().endsWith("！")
) {
paragraphBuilder.append("\n");//mark paragraph
paragraphBuilder.append("😀");
} else {
paragraphBuilder.append("\n");//mark paragraph
@Override
protected void writeLineSeparator() throws IOException {
super.writeLineSeparator();
if (
paragraphBuilder.toString().endsWith(".") ||
paragraphBuilder.toString().endsWith("。") ||
paragraphBuilder.toString().endsWith("!") ||
paragraphBuilder.toString().endsWith("！")
) {
paragraphBuilder.append("\n");//mark paragraph
paragraphBuilder.append("😀");
} else {
paragraphBuilder.append("\n");//mark paragraph
}
}
}
@Override
protected void writeString(String string, List<TextPosition> textPositions) throws IOException {
super.writeString(string, textPositions);
paragraphBuilder.append(string);//add text content
}
@Override
protected void writeString(String string, List<TextPosition> textPositions) throws IOException {
super.writeString(string, textPositions);
paragraphBuilder.append(string);//add text content
}
@Override
protected void endPage(PDPage page) throws IOException {
super.endPage(page);
String paragraph = paragraphBuilder.toString().trim();//get paragraph
if (!paragraph.isEmpty()) {
allBuilder.append(paragraph);
@Override
protected void endPage(PDPage page) throws IOException {
super.endPage(page);
String paragraph = paragraphBuilder.toString().trim();//get paragraph
if (!paragraph.isEmpty()) {
allBuilder.append(paragraph);
}
}
}
};
pdfTextStripper.setSortByPosition(true);
pdfTextStripper.setStartPage(0);
pdfTextStripper.setEndPage(document.getNumberOfPages());
String text = pdfTextStripper.getText(document);
//knowledge.setContentAll(text);
//log.info("allBuilder:{}", allBuilder.toString().substring(1000));
contentStringList = Arrays.asList(allBuilder.toString().split("😀"));
} catch (Exception e) {
log.error("parsing pdf error :{}", e.getMessage());
}
} else if(".pptx".equals(knowFile.getFileType())){
knowledge.setImportData(2);
filePath = path;
log.info("pptx dealing filePath{}", filePath);
// 加载PPT文件
XMLSlideShow ppt = new XMLSlideShow(new FileInputStream(filePath));
// 遍历幻灯片
for (XSLFSlide slide : ppt.getSlides()) {
StringBuffer sb = new StringBuffer();
// 遍历形状
List<XSLFShape> shapes = slide.getShapes();
for (XSLFShape shape : shapes) {
// 检查形状是否包含文本
if (shape instanceof XSLFTextShape) {
XSLFTextShape textShape = (XSLFTextShape) shape;
String text = textShape.getText();
if (text != null) {
sb.append(text);
log.info("text :{}", text);
};
pdfTextStripper.setSortByPosition(true);
pdfTextStripper.setStartPage(0);
pdfTextStripper.setEndPage(document.getNumberOfPages());
String text = pdfTextStripper.getText(document);
//knowledge.setContentAll(text);
//log.info("allBuilder:{}", allBuilder.toString().substring(1000));
contentStringList = Arrays.asList(allBuilder.toString().split("😀"));
} catch (Exception e) {
log.error("parsing pdf error :{}", e.getMessage());
}
} else if(".pptx".equals(knowFile.getFileType())){
knowledge.setImportData(2);
filePath = path;
log.info("pptx dealing filePath{}", filePath);
// 加载PPT文件
XMLSlideShow ppt = new XMLSlideShow(new FileInputStream(filePath));
// 遍历幻灯片
for (XSLFSlide slide : ppt.getSlides()) {
StringBuffer sb = new StringBuffer();
// 遍历形状
List<XSLFShape> shapes = slide.getShapes();
for (XSLFShape shape : shapes) {
// 检查形状是否包含文本
if (shape instanceof XSLFTextShape) {
XSLFTextShape textShape = (XSLFTextShape) shape;
String text = textShape.getText();
if (text != null) {
sb.append(text);
log.info("text :{}", text);
}
}
}
contentStringList.add(sb.toString());
}
contentStringList.add(sb.toString());
}
ppt.close(); // 关闭PPT
} else if(".ppt".equals(knowFile.getFileType())){
knowledge.setImportData(2);
filePath = path;
log.info("pptx dealing filePath{}", filePath);
// 加载PPT文件
HSLFSlideShow ppt = new HSLFSlideShow(new FileInputStream(filePath));
// 遍历幻灯片
for (HSLFSlide slide : ppt.getSlides()) {
StringBuffer sb = new StringBuffer();
// 遍历形状
List<HSLFShape> shapes = slide.getShapes();
for (HSLFShape shape : shapes) {
// 检查形状是否包含文本
if (shape instanceof HSLFTextShape) {
HSLFTextShape textShape = (HSLFTextShape) shape;
String text = textShape.getText();
if (text != null) {
sb.append(text);
log.info("text :{}", text);
ppt.close(); // 关闭PPT
} else if(".ppt".equals(knowFile.getFileType())){
knowledge.setImportData(2);
filePath = path;
log.info("pptx dealing filePath{}", filePath);
// 加载PPT文件
HSLFSlideShow ppt = new HSLFSlideShow(new FileInputStream(filePath));
// 遍历幻灯片
for (HSLFSlide slide : ppt.getSlides()) {
StringBuffer sb = new StringBuffer();
// 遍历形状
List<HSLFShape> shapes = slide.getShapes();
for (HSLFShape shape : shapes) {
// 检查形状是否包含文本
if (shape instanceof HSLFTextShape) {
HSLFTextShape textShape = (HSLFTextShape) shape;
String text = textShape.getText();
if (text != null) {
sb.append(text);
log.info("text :{}", text);
}
}
}
contentStringList.add(sb.toString());
}
contentStringList.add(sb.toString());
ppt.close(); // 关闭PPT
} else {
filePath = path;
log.info("word dealing filePath{}", filePath);
html = DocUtil.convertDocStream2Html(new FileInputStream(file));
//log.info("html :{}", html);
String htmlWithTable = html.replace("</p>", "######</p>");
htmlWithTable = htmlWithTable.replace("</title>", "######</title>");
htmlWithTable = htmlWithTable.replace("</h1>", "######</h1>");
contentStringList = Arrays.asList(htmlWithTable.split("######"));
}
ppt.close(); // 关闭PPT
} else {
filePath = path;
File file = new File(filePath);
log.info("word dealing filePath{}", filePath);
html = DocUtil.convertDocStream2Html(new FileInputStream(file));
//log.info("html :{}", html);
String htmlWithTable = html.replace("</p>", "######</p>");
htmlWithTable = htmlWithTable.replace("</title>", "######</title>");
htmlWithTable = htmlWithTable.replace("</h1>", "######</h1>");
contentStringList = Arrays.asList(htmlWithTable.split("######"));
}
} catch (Exception e) {
log.error(e.getMessage());
}
} catch (Exception e) {
log.error(e.getMessage());
}
if (null == knowledge.getTitle()) {
if (null == knowledge.getTitle()) {
// for (String tem : contentStringList) {
// String text = Jsoup.parse(tem).text();
// log.info("info:{}", text);
......@@ -265,38 +283,45 @@ class KnowledgeServiceImpl implements IKnowledgeService {
// break;
// }
// }
if (knowFile.getFileName() != null && knowFile.getFileName().contains(".")) {
knowledge.setTitle(knowFile.getFileName().substring(0, knowFile.getFileName().lastIndexOf('.')));
if (knowFile.getFileName() != null && knowFile.getFileName().contains(".")) {
knowledge.setTitle(knowFile.getFileName().substring(0, knowFile.getFileName().lastIndexOf('.')));
} else {
knowledge.setTitle(knowFile.getFileName());
}
} else {
knowledge.setTitle(knowFile.getFileName());
}
}
List<String> messageContentList = contentStringList.stream()
.filter(item -> !item.contains("<img"))
.filter(item -> !item.contains("<table"))
.filter(item -> !item.contains("<tr>"))
.filter(item -> !item.contains("</tr>"))
.filter(item -> !item.contains("<td>"))
.filter(item -> !item.contains("</td>"))
.map(item -> Jsoup.parse(item).text())
.collect(Collectors.toList());
List<String> messageContentList = contentStringList.stream()
.filter(item -> !item.contains("<img"))
.filter(item -> !item.contains("<table"))
.filter(item -> !item.contains("<tr>"))
.filter(item -> !item.contains("</tr>"))
.filter(item -> !item.contains("<td>"))
.filter(item -> !item.contains("</td>"))
.map(item -> Jsoup.parse(item).text())
.collect(Collectors.toList());
messageContentList = messageContentList.
stream().filter(item -> item.length() > 10)
.collect(Collectors.toList());
if (!messageContentList.isEmpty()) {
for (String tem : messageContentList) {
contentList.add(Content.builder()
.contentId(codeGenerateUtil.geneIdNo(Constants.FINANCE, 8))
.content(tem)
.build());
messageContentList = messageContentList.
stream().filter(item -> item.length() > 10)
.collect(Collectors.toList());
if (!messageContentList.isEmpty()) {
for (String tem : messageContentList) {
contentList.add(Content.builder()
.contentId(codeGenerateUtil.geneIdNo(Constants.FINANCE, 8))
.content(tem)
.build());
}
}
knowledge.setContents(contentList);
knowledgeMessage.setContents(contentList);
} catch (Exception e) {
log.error("本地化并保存附件信息失败，系统异常：{}",e);
}finally{
if(file != null){
file.delete();
}
}
knowledge.setContents(contentList);
knowledgeMessage.setContents(contentList);
}
BeanUtils.copyProperties(knowledge, knowledgeMessage);
knowledgeMessage.setType(knowledge.getTypeId());
......
src/main/java/com/zzsn/knowbase/service/impl/LocalFileServiceImpl.java
......@@ -4,11 +4,9 @@ import com.alibaba.fastjson.JSON;
import com.zzsn.knowbase.constant.Constants;
import com.zzsn.knowbase.constant.DocumentConstants;
import com.zzsn.knowbase.constant.ErrorCodeEnum;
import com.zzsn.knowbase.entity.Document;
import com.zzsn.knowbase.entity.KbAuthorizedUser;
import com.zzsn.knowbase.entity.KnowFile;
import com.zzsn.knowbase.entity.Knowledge;
import com.zzsn.knowbase.entity.*;
import com.zzsn.knowbase.service.DocumentService;
import com.zzsn.knowbase.service.IEnclosureService;
import com.zzsn.knowbase.service.IKnowledgeService;
import com.zzsn.knowbase.service.ILocalFileService;
import com.zzsn.knowbase.util.*;
......@@ -67,6 +65,9 @@ public class LocalFileServiceImpl implements ILocalFileService {
@Autowired
private IKnowledgeService knowledgeService;
@Autowired
private IEnclosureService enclosureService;
@Value("${files.storage}")
String filesStorage;
@Value("${files.docservice.url.site}")
......@@ -77,45 +78,61 @@ public class LocalFileServiceImpl implements ILocalFileService {
@Override
public Result<KnowFile> upload(MultipartFile file) {
try {
String fileName = file.getOriginalFilename(); // 获取文件名称
String fileExtension = fileUtility.getFileExtension(fileName); // 获取文件扩展名
String fileType = fileUtility.getFileType(fileName); //获取文件类型
long fileSize = file.getSize(); // get file size
log.info("文件上传："+ fileName);
// check if the file size exceeds the maximum file size or is less than 0
if (fileUtility.getMaxFileSize() < fileSize || fileSize <= 0) {
Result result = Result.error("文件大小不正确！");
log.info("文件大小不正确!");
return result;
String contentMd5 = FileMd5Utils.getFileMD5String(file);
Enclosure enclosure = enclosureService.getByMd5(contentMd5);
if(enclosure == null){
enclosure = new Enclosure();
InputStream in = null;
try {
String fileName = file.getOriginalFilename(); // 获取文件名称
String fileType = fileUtility.getFileType(fileName); //获取文件类型
long fileSize = file.getSize(); // get file size
if (fileUtility.getMaxFileSize() < fileSize || fileSize <= 0) {
Result result = Result.error("文件大小超过范围！");
log.info("文件大小超过范围!");
return result;
}
if (!fileUtility.getFileExts().contains(fileType)) {
Result result = Result.error("不支持的文件类型！");
log.info("不支持的文件类型!");
return result;
}
enclosure.setExtensionName(fileType);
enclosure.setName(fileName);
enclosure.setContentMd5(contentMd5);
enclosure.setFileSize(file.getSize());
enclosure.setCreateTime(new Date());
enclosure.setCreateBy(SpringContextUtils.getUserInfo().getUsername());
log.info("文件上传到minio：{}", fileName);
in = file.getInputStream();
String fullFileName = MinioUtil.uploadByStream(in, MimeUtil.getMimeStrByName(fileName), null, MinioUtil.getRelativePath(fileName));
enclosure.setUrlPath(fullFileName);
enclosureService.save(enclosure);
} catch (Exception e) {
log.error("文件上传失败！{}", e.getMessage());
} finally {
if(in!=null){
try {
in.close();
} catch (IOException e) {
log.error(e.getMessage());
}
}
}
}
// check if file extension is supported by the editor
if (!fileUtility.getFileExts().contains(fileExtension)) {
Result result = Result.error("不支持的文件类型！");
log.info("不支持的文件类型!");
return result;
}
String fileId = codeGenerateUtil.geneIdNo(Constants.FINANCE, 8);
String filePath = getFilePath() + fileId + fileExtension;
//byte[] bytes = file.getBytes(); // get file in bytes
//Files.write(Paths.get(filePath), bytes);
file.transferTo(new File(filesStorage + filePath));
//封装es关联对象并返回
KnowFile knowFile = new KnowFile();
knowFile.setFileId(fileId);
knowFile.setFileName(fileName);
knowFile.setFilePath(filePath);
knowFile.setFileType(fileType);
knowFile.setFileSize(fileSize);
Result result = Result.OK(knowFile);
log.info("文件上传成功：" + fileName + "---" + filePath);
return result; // create user metadata and return it
knowFile.setFileId(enclosure.getId());
knowFile.setFileName(enclosure.getName());
knowFile.setFilePath(enclosure.getUrlPath());
knowFile.setFileType(enclosure.getExtensionName());
knowFile.setFileSize(enclosure.getFileSize());
log.info("文件上传成功name:{},path:{}",knowFile.getFileName(),knowFile.getFilePath());
return Result.OK(knowFile);
} catch (Exception e) {
e.printStackTrace();
log.error(e.getMessage());
}
// if the operation of file uploading is unsuccessful, an error occurs
Result result = Result.error("上传文件时出现问题！");
log.info("上传文件时出现问题！");
return result;
......
src/main/java/com/zzsn/knowbase/util/FileMd5Utils.java 0 → 100644
package com.zzsn.knowbase.util;
import org.springframework.web.multipart.MultipartFile;
import sun.misc.Cleaner;
import sun.nio.ch.DirectBuffer;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public class FileMd5Utils {
protected static char hexDigits[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e',
'f' };
protected static MessageDigest messageDigest = null;
static {
try {
messageDigest = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
System.err.println(FileMd5Utils.class.getName() + "初始化失败，MessageDigest不支持MD5Util.");
e.printStackTrace();
}
}
/**
* 计算文件的MD5
*
* @param fileName
* 文件的绝对路径
* @return
* @throws IOException
*/
public static String getFileMD5String(String fileName) throws IOException {
File f = new File(fileName);
return getFileMD5String(f);
}
/**
* 计算文件的MD5，重载方法
*
* @param file
* 文件对象
* @return
* @throws IOException
*/
public static String getFileMD5String(File file) throws IOException {
FileInputStream in = null;
FileChannel ch = null;
MappedByteBuffer byteBuffer = null;
try {
in = new FileInputStream(file);
ch = in.getChannel();
byteBuffer = ch.map(FileChannel.MapMode.READ_ONLY, 0, file.length());
messageDigest.update(byteBuffer);
} catch (Exception e) {
} finally {
if (byteBuffer != null) {
unmap(byteBuffer);
}
if (ch != null) {
ch.close();
}
if (in != null) {
in.close();
}
}
return bufferToHex(messageDigest.digest());
}
public static String getFileMD5String(MultipartFile file) throws IOException {
FileInputStream in = (FileInputStream) file.getInputStream();
FileChannel ch = in.getChannel();
MappedByteBuffer byteBuffer = ch.map(FileChannel.MapMode.READ_ONLY, 0, file.getSize());
messageDigest.update(byteBuffer);
return bufferToHex(messageDigest.digest());
}
/*
* 需要测试一下这个算法的长度是否正确。
*/
public static String getFileMD5String(FileInputStream in) throws IOException {
FileChannel ch = in.getChannel();
int len = in.read();
MappedByteBuffer byteBuffer = ch.map(FileChannel.MapMode.READ_ONLY, 0, len);
messageDigest.update(byteBuffer);
return bufferToHex(messageDigest.digest());
}
public static String getFileMD5String(InputStream in) throws IOException {
FileChannel ch = ((FileInputStream) in).getChannel();
int len = in.read();
MappedByteBuffer byteBuffer = ch.map(FileChannel.MapMode.READ_ONLY, 0, len);
messageDigest.update(byteBuffer);
return bufferToHex(messageDigest.digest());
}
public static String getMD5String(byte[] bytes) {
messageDigest.update(bytes);
return bufferToHex(messageDigest.digest());
}
public static String getMD5String(String s) {
return getMD5String(s.getBytes());
}
private static String bufferToHex(byte bytes[]) {
return bufferToHex(bytes, 0, bytes.length);
}
private static String bufferToHex(byte bytes[], int m, int n) {
StringBuffer stringbuffer = new StringBuffer(2 * n);
int k = m + n;
for (int l = m; l < k; l++) {
appendHexPair(bytes[l], stringbuffer);
}
return stringbuffer.toString();
}
private static void appendHexPair(byte bt, StringBuffer stringbuffer) {
char c0 = hexDigits[(bt & 0xf0) >> 4];
char c1 = hexDigits[bt & 0xf];
stringbuffer.append(c0);
stringbuffer.append(c1);
}
private static void unmap(MappedByteBuffer var0) {
if(var0 != null){
Cleaner var1 = ((DirectBuffer) var0).cleaner();
if (var1 != null) {
var1.clean();
}
}
}
}
src/main/java/com/zzsn/knowbase/util/FileUtil.java 0 → 100644
package com.zzsn.knowbase.util;
import org.apache.commons.lang3.StringUtils;
import javax.net.ssl.*;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class FileUtil {
private static ArrayList<String> readFileName = new ArrayList<String>();
public static InputStream DownloadFileFormServer(String strUrl) {
InputStream inputStream = null;
try {
URL url = new URL(strUrl);
if(strUrl.contains("https")){
//忽略证书
SSLContext sc = SSLContext.getInstance("SSL");
sc.init(null, new TrustManager[]{
new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] x509Certificates, String s) {
}
@Override
public void checkServerTrusted(X509Certificate[] x509Certificates, String s) {
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
}
}, new SecureRandom());
HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
conn.setRequestProperty("Accept-Encoding", "deflate");
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36");
// 设置证书忽略相关操作
conn.setSSLSocketFactory(sc.getSocketFactory());
conn.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String s, SSLSession sslSession) {
return true;
}
});
conn.setRequestMethod("GET");
conn.setConnectTimeout(20 * 1000);
conn.connect();
inputStream = conn.getInputStream();
}else{
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Accept-Encoding", "deflate");
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36");
conn.setRequestMethod("GET");
conn.setConnectTimeout(20 * 1000);
conn.connect();
inputStream = conn.getInputStream();
}
} catch (Exception e) {
} finally {
}
return inputStream;
}
public static int getFileSizeByUrl(String strUrl){
try {
URL url = new URL(strUrl);
if(strUrl.contains("https")){
//忽略证书
SSLContext sc = SSLContext.getInstance("SSL");
sc.init(null, new TrustManager[]{
new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] x509Certificates, String s) {
}
@Override
public void checkServerTrusted(X509Certificate[] x509Certificates, String s) {
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
}
}, new SecureRandom());
HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
conn.setRequestProperty("Accept-Encoding", "deflate");
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36");
// 设置证书忽略相关操作
conn.setSSLSocketFactory(sc.getSocketFactory());
conn.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String s, SSLSession sslSession) {
return true;
}
});
conn.setRequestMethod("GET");
conn.setConnectTimeout(20 * 1000);
conn.connect();
return conn.getContentLength();
}else{
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Accept-Encoding", "deflate");
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36");
conn.setRequestMethod("GET");
conn.setConnectTimeout(20 * 1000);
conn.connect();
return conn.getContentLength();
}
} catch (Exception e) {
}
return 0;
}
/*
* 清空文件列表中的记录
*/
public static void clearList() {
readFileName.clear();
}
public static ArrayList<String> getReadFileName() {
return readFileName;
}
public static void setReadFileName(ArrayList<String> readFileName) {
FileUtil.readFileName = readFileName;
}
/*
*
* 通过递归得到某一路径下所有的目录及其文件
*
*/
public static void getFiles(String filePath) {
File root = new File(filePath);
File[] files = root.listFiles();
for (File file : files) {
if (file.isDirectory()) {
/*
*
* 递归调用
*
*/
getFiles(file.getAbsolutePath());
} else {
String absolutePath = file.getAbsolutePath();
String regex = ".*[\\d]{4}年--[\\d]{1,2}月.*";
Matcher m = Pattern.compile(regex).matcher(absolutePath);
if (absolutePath != null && m.matches() && absolutePath.indexOf("--进出口--") == -1) {
readFileName.add(file.getAbsolutePath());
}
}
}
}
public static List<String> readFiletoList(String fileName) {
List<String> list = new ArrayList<String>();
BufferedReader br = null;
InputStream is = null;
InputStreamReader isr = null;
try {
is = new FileInputStream(fileName);
isr = new InputStreamReader(is, "gb2312");
br = new BufferedReader(isr);
// br = new BufferedReader(new FileReader(fileName),"gb2312");
String contentLine;
while ((contentLine = br.readLine()) != null) {
if (StringUtils.isNotEmpty(contentLine)) {
contentLine = contentLine.trim();
list.add(contentLine);
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (br != null) {
try {
br.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
if (isr != null) {
try {
isr.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
if (is != null) {
try {
is.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
return list;
}
/*
* 保存操作日志到文件
* 没有调用的地方
*/
public static void saveProjectInfo(String page,String fileName) {
BufferedWriter bw1 = null;
try {
bw1 = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName, true)));
bw1.write(page + "\r\n");
bw1.flush();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
if (bw1 != null) {
try {
bw1.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
}
src/main/java/com/zzsn/knowbase/util/LoadToolUtil.java 0 → 100644
package com.zzsn.knowbase.util;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author yanxin
*/
@Slf4j
public class LoadToolUtil {
/**
* 中文字符和中文引号
*/
private static Pattern decodePattern = Pattern.compile("[\u4e00-\u9fa5“”]+");
public static List<String> readFiletoList(String fileName){
List<String> list = new ArrayList<>();
BufferedReader br = null;
try{
br = new BufferedReader(new FileReader(fileName));
String contentLine ;
while ((contentLine = br.readLine()) != null) {
if(StringUtils.isNotEmpty(contentLine)){
contentLine = contentLine.trim();
list.add(contentLine);
}
}
}catch(Exception e){
e.printStackTrace();
}finally{
if(br != null){
try {
br.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
return list;
}
public static boolean judegExcelEdition(String fileName) {
String regex = "^.+\\.(?i)(xls)$";
return !fileName.matches(regex);
}
/**
* 找到文件的扩展名
*/
public static String getFileSuffix(String fileName) {
// TODO Auto-generated method stub
if(fileName == null || StringUtils.isEmpty(fileName) ){
return null;
}
String suffix = "";
String indexOf = ".";
if(fileName.lastIndexOf(indexOf) > -1){
suffix = fileName.substring(fileName.lastIndexOf("."));
}
return suffix;
}
/**
* 获取参数编码后的url
*/
public static String urlEncoder(String url) {
try {
Matcher m = decodePattern.matcher(url);
StringBuffer b = new StringBuffer();
while (m.find()) {
m.appendReplacement(b, URLEncoder.encode(m.group(0), "utf-8"));
}
m.appendTail(b);
url = b.toString();
url = url.replace("&amp;", "&");
} catch (Exception e) {
log.error("url编码转换失败e:{}",e.getMessage());
}
return url;
}
/**
* 获取相对路径，不带第一个/
*/
public static String urlEncoderRelative(String url) {
url = urlEncoder(url);
if(url.startsWith("/")){
url = url.substring(1);
}
return url;
}
}
src/main/java/com/zzsn/knowbase/util/MimeUtil.java 0 → 100644
package com.zzsn.knowbase.util;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.util.HashMap;
import java.util.Map;
/**
* mime类型匹配工具类
*/
@Slf4j
public class MimeUtil {
private static Map<String,String> map = new HashMap<>();
//根据文件名匹配mime类型
public static String getMimeStrByName(String fileName){
if(fileName.indexOf(".")==-1){
fileName = "."+fileName;
}else{
fileName = fileName.substring(fileName.lastIndexOf("."));
}
String mimeType = map.get(fileName);
if(StringUtils.isEmpty(mimeType)){
mimeType = "application/octet-stream";
}
return mimeType;
}
static {
map.put(".001","application/x-001");
map.put(".301","application/x-301");
map.put(".323","text/h323");
map.put(".906","application/x-906");
map.put(".907","drawing/907");
map.put(".a11","application/x-a11");
map.put(".acp","audio/x-mei-aac");
map.put(".ai","application/postscript");
map.put(".aif","audio/aiff");
map.put(".aifc","audio/aiff");
map.put(".aiff","audio/aiff");
map.put(".anv","application/x-anv");
map.put(".asa","text/asa");
map.put(".asf","video/x-ms-asf");
map.put(".asp","text/asp");
map.put(".asx","video/x-ms-asf");
map.put(".au","audio/basic");
map.put(".avi","video/avi");
map.put(".awf","application/vnd.adobe.workflow");
map.put(".biz","text/xml");
map.put(".bmp","application/x-bmp");
map.put(".bot","application/x-bot");
map.put(".c4t","application/x-c4t");
map.put(".c90","application/x-c90");
map.put(".cal","application/x-cals");
map.put(".cat","application/s-pki.seccat");
map.put(".cdf","application/x-netcdf");
map.put(".cdr","application/x-cdr");
map.put(".cel","application/x-cel");
map.put(".cer","application/x-x509-ca-cert");
map.put(".cg4","application/x-g4");
map.put(".cgm","application/x-cgm");
map.put(".cit","application/x-cit");
map.put(".class","java/*");
map.put(".cml","text/xml");
map.put(".cmp","application/x-cmp");
map.put(".cmx","application/x-cmx");
map.put(".cot","application/x-cot");
map.put(".crl","application/pkix-crl");
map.put(".crt","application/x-x509-ca-cert");
map.put(".csi","application/x-csi");
map.put(".css","text/css");
map.put(".cut","application/x-cut");
map.put(".dbf","application/x-dbf");
map.put(".dbm","application/x-dbm");
map.put(".dbx","application/x-dbx");
map.put(".dcd","text/xml");
map.put(".dcx","application/x-dcx");
map.put(".der","application/x-x509-ca-cert");
map.put(".dgn","application/x-dgn");
map.put(".dib","application/x-dib");
map.put(".dll","application/x-msdownload");
map.put(".doc","application/msword");
map.put(".dot","application/msword");
map.put(".drw","application/x-drw");
map.put(".dtd","text/xml");
map.put(".dwf","Model/vnd.dwf");
map.put(".dwg","application/x-dwg");
map.put(".dxb","application/x-dxb");
map.put(".dxf","application/x-dxf");
map.put(".edn","application/vnd.adobe.edn");
map.put(".emf","application/x-emf");
map.put(".eml","message/rfc822");
map.put(".ent","text/xml");
map.put(".epi","application/x-epi");
map.put(".eps","application/x-ps");
map.put(".etd","application/x-ebx");
map.put(".exe","application/x-msdownload");
map.put(".fax","image/fax");
map.put(".fdf","application/vnd.fdf");
map.put(".fif","application/fractals");
map.put(".fo","text/xml");
map.put(".frm","application/x-frm");
map.put(".g4","application/x-g4");
map.put(".gbr","application/x-gbr");
map.put(".gcd","application/x-gcd");
map.put(".gif","image/gif");
map.put(".gl2","application/x-gl2");
map.put(".gp4","application/x-gp4");
map.put(".hgl","application/x-hgl");
map.put(".hmr","application/x-hmr");
map.put(".hpg","application/x-hpgl");
map.put(".hpl","application/x-hpl");
map.put(".hqx","application/mac-binhex40");
map.put(".hrf","application/x-hrf");
map.put(".hta","application/hta");
map.put(".htc","text/x-component");
map.put(".htm","text/html");
map.put(".html","text/html");
map.put(".htt","text/webviewhtml");
map.put(".htx","text/html");
map.put(".icb","application/x-icb");
map.put(".ico","image/x-icon");
map.put(".iff","application/x-iff");
map.put(".ig4","application/x-g4");
map.put(".igs","application/x-igs");
map.put(".iii","application/x-iphone");
map.put(".img","application/x-img");
map.put(".ins","application/x-internet-signup");
map.put(".isp","application/x-internet-signup");
map.put(".IVF","video/x-ivf");
map.put(".java","java/*");
map.put(".jfif","image/jpeg");
map.put(".jpe","image/jpeg");
map.put(".jpeg","image/jpeg");
map.put(".jpg","image/jpeg");
map.put(".js","application/x-javascript");
map.put(".jsp","text/html");
map.put(".la1","audio/x-liquid-file");
map.put(".lar","application/x-laplayer-reg");
map.put(".latex","application/x-latex");
map.put(".lavs","audio/x-liquid-secure");
map.put(".lbm","application/x-lbm");
map.put(".lmsff","audio/x-la-lms");
map.put(".ls","application/x-javascript");
map.put(".ltr","application/x-ltr");
map.put(".m1v","video/x-mpeg");
map.put(".m2v","video/x-mpeg");
map.put(".m3u","audio/mpegurl");
map.put(".m4e","video/mpeg4");
map.put(".mac","application/x-mac");
map.put(".man","application/x-troff-man");
map.put(".math","text/xml");
map.put(".mdb","application/x-mdb");
map.put(".mfp","application/x-shockwave-flash");
map.put(".mht","message/rfc822");
map.put(".mhtml","message/rfc822");
map.put(".mi","application/x-mi");
map.put(".mid","audio/mid");
map.put(".midi","audio/mid");
map.put(".mil","application/x-mil");
map.put(".mml","text/xml");
map.put(".mnd","audio/x-musicnet-download");
map.put(".mns","audio/x-musicnet-stream");
map.put(".mocha","application/x-javascript");
map.put(".movie","video/x-sgi-movie");
map.put(".mp1","audio/mp1");
map.put(".mp2","audio/mp2");
map.put(".mp2v","video/mpeg");
map.put(".mp3","audio/mp3");
map.put(".mp4","video/mp4");
map.put(".mpa","video/x-mpg");
map.put(".mpd","application/-project");
map.put(".mpe","video/x-mpeg");
map.put(".mpeg","video/mpg");
map.put(".mpg","video/mpg");
map.put(".mpga","audio/rn-mpeg");
map.put(".mpp","application/-project");
map.put(".mps","video/x-mpeg");
map.put(".mpt","application/-project");
map.put(".mpv","video/mpg");
map.put(".mpv2","video/mpeg");
map.put(".mpw","application/s-project");
map.put(".mpx","application/-project");
map.put(".mtx","text/xml");
map.put(".mxp","application/x-mmxp");
map.put(".net","image/pnetvue");
map.put(".nrf","application/x-nrf");
map.put(".nws","message/rfc822");
map.put(".odc","text/x-ms-odc");
map.put(".out","application/x-out");
map.put(".p10","application/pkcs10");
map.put(".p12","application/x-pkcs12");
map.put(".p7b","application/x-pkcs7-certificates");
map.put(".p7c","application/pkcs7-mime");
map.put(".p7m","application/pkcs7-mime");
map.put(".p7r","application/x-pkcs7-certreqresp");
map.put(".p7s","application/pkcs7-signature");
map.put(".pc5","application/x-pc5");
map.put(".pci","application/x-pci");
map.put(".pcl","application/x-pcl");
map.put(".pcx","application/x-pcx");
map.put(".pdf","application/pdf");
map.put(".pdx","application/vnd.adobe.pdx");
map.put(".pfx","application/x-pkcs12");
map.put(".pgl","application/x-pgl");
map.put(".pic","application/x-pic");
map.put(".pko","application-pki.pko");
map.put(".pl","application/x-perl");
map.put(".plg","text/html");
map.put(".pls","audio/scpls");
map.put(".plt","application/x-plt");
map.put(".png","image/png");
map.put(".pot","applications-powerpoint");
map.put(".ppa","application/vs-powerpoint");
map.put(".ppm","application/x-ppm");
map.put(".pps","application-powerpoint");
map.put(".ppt","applications-powerpoint");
map.put(".pr","application/x-pr");
map.put(".prf","application/pics-rules");
map.put(".prn","application/x-prn");
map.put(".prt","application/x-prt");
map.put(".ps","application/postscript");
map.put(".ptn","application/x-ptn");
map.put(".pwz","application/powerpoint");
map.put(".r3t","text/vnd.rn-realtext3d");
map.put(".ra","audio/vnd.rn-realaudio");
map.put(".ram","audio/x-pn-realaudio");
map.put(".ras","application/x-ras");
map.put(".rat","application/rat-file");
map.put(".rdf","text/xml");
map.put(".rec","application/vnd.rn-recording");
map.put(".red","application/x-red");
map.put(".rgb","application/x-rgb");
map.put(".rjs","application/vnd.rn-realsystem-rjs");
map.put(".rjt","application/vnd.rn-realsystem-rjt");
map.put(".rlc","application/x-rlc");
map.put(".rle","application/x-rle");
map.put(".rm","application/vnd.rn-realmedia");
map.put(".rmf","application/vnd.adobe.rmf");
map.put(".rmi","audio/mid");
map.put(".rmj","application/vnd.rn-realsystem-rmj");
map.put(".rmm","audio/x-pn-realaudio");
map.put(".rmp","application/vnd.rn-rn_music_package");
map.put(".rms","application/vnd.rn-realmedia-secure");
map.put(".rmvb","application/vnd.rn-realmedia-vbr");
map.put(".rmx","application/vnd.rn-realsystem-rmx");
map.put(".rnx","application/vnd.rn-realplayer");
map.put(".rp","image/vnd.rn-realpix");
map.put(".rpm","audio/x-pn-realaudio-plugin");
map.put(".rsml","application/vnd.rn-rsml");
map.put(".rt","text/vnd.rn-realtext");
map.put(".rtf","application/x-rtf");
map.put(".rv","video/vnd.rn-realvideo");
map.put(".sam","application/x-sam");
map.put(".sat","application/x-sat");
map.put(".sdp","application/sdp");
map.put(".sdw","application/x-sdw");
map.put(".sit","application/x-stuffit");
map.put(".slb","application/x-slb");
map.put(".sld","application/x-sld");
map.put(".slk","drawing/x-slk");
map.put(".smi","application/smil");
map.put(".smil","application/smil");
map.put(".smk","application/x-smk");
map.put(".snd","audio/basic");
map.put(".sol","text/plain");
map.put(".sor","text/plain");
map.put(".spc","application/x-pkcs7-certificates");
map.put(".spl","application/futuresplash");
map.put(".spp","text/xml");
map.put(".ssm","application/streamingmedia");
map.put(".sst","application-pki.certstore");
map.put(".stl","application/-pki.stl");
map.put(".stm","text/html");
map.put(".sty","application/x-sty");
map.put(".svg","text/xml");
map.put(".swf","application/x-shockwave-flash");
map.put(".tdf","application/x-tdf");
map.put(".tg4","application/x-tg4");
map.put(".tga","application/x-tga");
map.put(".tif","image/tiff");
map.put(".tiff","image/tiff");
map.put(".tld","text/xml");
map.put(".top","drawing/x-top");
map.put(".torrent","application/x-bittorrent");
map.put(".tsd","text/xml");
map.put(".txt","text/plain");
map.put(".uin","application/x-icq");
map.put(".uls","text/iuls");
map.put(".vcf","text/x-vcard");
map.put(".vda","application/x-vda");
map.put(".vdx","application/vnd.visio");
map.put(".vml","text/xml");
map.put(".vpg","application/x-vpeg005");
map.put(".vsd","application/x-vsd");
map.put(".vss","application/vnd.visio");
map.put(".vst","application/x-vst");
map.put(".vsw","application/vnd.visio");
map.put(".vsx","application/vnd.visio");
map.put(".vtx","application/vnd.visio");
map.put(".vxml","text/xml");
map.put(".wav","audio/wav");
map.put(".wax","audio/x-ms-wax");
map.put(".wb1","application/x-wb1");
map.put(".wb2","application/x-wb2");
map.put(".wb3","application/x-wb3");
map.put(".wbmp","image/vnd.wap.wbmp");
map.put(".wiz","application/msword");
map.put(".wk3","application/x-wk3");
map.put(".wk4","application/x-wk4");
map.put(".wkq","application/x-wkq");
map.put(".wks","application/x-wks");
map.put(".wm","video/x-ms-wm");
map.put(".wma","audio/x-ms-wma");
map.put(".wmd","application/x-ms-wmd");
map.put(".wmf","application/x-wmf");
map.put(".wml","text/vnd.wap.wml");
map.put(".wmv","video/x-ms-wmv");
map.put(".wmx","video/x-ms-wmx");
map.put(".wmz","application/x-ms-wmz");
map.put(".wp6","application/x-wp6");
map.put(".wpd","application/x-wpd");
map.put(".wpg","application/x-wpg");
map.put(".wpl","application/-wpl");
map.put(".wq1","application/x-wq1");
map.put(".wr1","application/x-wr1");
map.put(".wri","application/x-wri");
map.put(".wrk","application/x-wrk");
map.put(".ws","application/x-ws");
map.put(".ws2","application/x-ws");
map.put(".wsc","text/scriptlet");
map.put(".wsdl","text/xml");
map.put(".wvx","video/x-ms-wvx");
map.put(".xdp","application/vnd.adobe.xdp");
map.put(".xdr","text/xml");
map.put(".xfd","application/vnd.adobe.xfd");
map.put(".xfdf","application/vnd.adobe.xfdf");
map.put(".xhtml","text/html");
map.put(".xls","application/x-xls");
map.put(".xlw","application/x-xlw");
map.put(".xml","text/xml");
map.put(".xpl","audio/scpls");
map.put(".xq","text/xml");
map.put(".xql","text/xml");
map.put(".xquery","text/xml");
map.put(".xsd","text/xml");
map.put(".xsl","text/xml");
map.put(".xslt","text/xml");
map.put(".xwd","application/x-xwd");
map.put(".x_b","application/x-x_b");
map.put(".x_t","application/x-x_t");
}
}
src/main/java/com/zzsn/knowbase/util/MinioUtil.java 0 → 100644
package com.zzsn.knowbase.util;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.lang.Snowflake;
import io.minio.*;
import io.minio.http.Method;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.web.multipart.MultipartFile;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.net.URLDecoder;
import java.util.Date;
/**
* minio文件上传工具类
*/
@Slf4j
public class MinioUtil {
private static String minioUrl;
private static String minioName;
private static String minioPass;
private static String bucketName;
public static void setMinioUrl(String minioUrl) {
MinioUtil.minioUrl = minioUrl;
}
public static void setMinioName(String minioName) {
MinioUtil.minioName = minioName;
}
public static void setMinioPass(String minioPass) {
MinioUtil.minioPass = minioPass;
}
public static void setBucketName(String bucketName) {
MinioUtil.bucketName = bucketName;
}
public static String getMinioUrl() {
return minioUrl;
}
public static String getBucketName() {
return bucketName;
}
private static MinioClient minioClient = null;
/**
* 上传文件
*
* @param file 文件
* @param bizPath 文件夹名称
* @param customBucket 自定义桶的名称
* @author lkg
* @date 2023/5/22
*/
public static String upload(MultipartFile file, String bizPath, String customBucket) {
String file_url = "";
//过滤上传文件夹名特殊字符，防止攻击
bizPath = StrAttackFilter.filter(bizPath);
String newBucket = bucketName;
if (StringUtils.isNotEmpty(customBucket)) {
newBucket = customBucket;
}
try {
initMinio(minioUrl, minioName, minioPass);
// 检查存储桶是否已经存在
if (minioClient.bucketExists(BucketExistsArgs.builder().bucket(newBucket).build())) {
log.info("Bucket already exists.");
} else {
// 创建一个名为ota的存储桶
minioClient.makeBucket(MakeBucketArgs.builder().bucket(newBucket).build());
log.info("create a new bucket.");
}
InputStream stream = file.getInputStream();
// 获取文件名
String orgName = file.getOriginalFilename();
if (StringUtils.isEmpty(orgName)) {
orgName = file.getName();
}
orgName = getFileName(orgName);
String objectName = bizPath + "/" + orgName.substring(0, orgName.lastIndexOf(".")) + "_" + System.currentTimeMillis() + orgName.substring(orgName.lastIndexOf("."));
// 使用putObject上传一个本地文件到存储桶中。
if (objectName.startsWith("/")) {
objectName = objectName.substring(1);
}
PutObjectArgs objectArgs = PutObjectArgs.builder().object(objectName)
.bucket(newBucket)
.contentType("application/octet-stream")
.stream(stream, stream.available(), -1).build();
minioClient.putObject(objectArgs);
stream.close();
file_url = newBucket + "/" + objectName;
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return file_url;
}
/**
* 文件上传
*
* @param file 文件
* @param bizPath 文件夹名称
* @return
*/
public static String upload(MultipartFile file, String bizPath) {
return upload(file, bizPath, null);
}
/**
* 获取文件流
*
* @param bucketName 桶名称
* @param objectName 文件路径：文件夹/文件 例如：test/test.docx、test/123/1234.docx
* @return
*/
public static InputStream getMinioFile(String bucketName, String objectName) {
InputStream inputStream = null;
try {
initMinio(minioUrl, minioName, minioPass);
GetObjectArgs objectArgs = GetObjectArgs.builder().object(objectName)
.bucket(bucketName).build();
inputStream = minioClient.getObject(objectArgs);
} catch (Exception e) {
log.info("文件获取失败" + e.getMessage());
}
return inputStream;
}
/**
* 删除文件
*
* @param bucketName 桶名称
* @param objectName 文件路径 格式： 文件夹/文件 例如：test/test.docx、test/123/1234.docx
* @throws Exception
*/
public static void removeObject(String bucketName, String objectName) {
try {
initMinio(minioUrl, minioName, minioPass);
RemoveObjectArgs objectArgs = RemoveObjectArgs.builder().object(objectName)
.bucket(bucketName).build();
minioClient.removeObject(objectArgs);
} catch (Exception e) {
log.info("文件删除失败" + e.getMessage());
}
}
/**
* 获取文件外链/预览链接(仅支持pdf文件和图片)
*
* @param bucketName 桶名称
* @param objectName 文件路径 格式：文件夹/文件 例如：test/test.docx、test/123/1234.docx
* @param expires 失效时长
* @return
*/
public static String getPreviewUrl(String bucketName, String objectName, Integer expires) {
initMinio(minioUrl, minioName, minioPass);
try {
GetPresignedObjectUrlArgs objectArgs;
if (expires != null) {
objectArgs = GetPresignedObjectUrlArgs.builder()
.object(objectName)
.bucket(bucketName)
.expiry(expires)
.method(Method.GET)
.build();
} else {
objectArgs = GetPresignedObjectUrlArgs.builder()
.object(objectName)
.bucket(bucketName)
.method(Method.GET)
.build();
}
String url = minioClient.getPresignedObjectUrl(objectArgs);
return URLDecoder.decode(url, "UTF-8");
} catch (Exception e) {
log.info("文件路径获取失败" + e.getMessage());
}
return null;
}
/**
* 初始化客户端
*
* @param minioUrl 地址
* @param minioName 账号
* @param minioPass 密码
*/
private static void initMinio(String minioUrl, String minioName, String minioPass) {
if (minioClient == null) {
try {
minioClient = MinioClient.builder()
.endpoint(minioUrl)
.credentials(minioName, minioPass)
.build();
} catch (Exception e) {
e.printStackTrace();
}
}
}
/**
* 上传文件到minio
*
* @param stream 文件流
* @param relativePath 文件路径
*/
public static String upload(InputStream stream, String relativePath) throws Exception {
initMinio(minioUrl, minioName, minioPass);
if (minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) {
log.info("Bucket already exists.");
} else {
// 创建一个名为ota的存储桶
minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).build());
log.info("create a new bucket.");
}
PutObjectArgs objectArgs = PutObjectArgs.builder().object(relativePath)
.bucket(bucketName)
.contentType("application/octet-stream")
.stream(stream, stream.available(), -1).build();
minioClient.putObject(objectArgs);
stream.close();
return "/" + bucketName + "/" + relativePath;
}
/**
* 上传文件
*
* @param file 文件
* @param contentType 类型，如果pdf文件或者图片，需要预览功能的话，必填.示例：pdf application/pdf；图片 image/png；image/jpeg
* @param bizPath 文件夹名称
* @param customBucket 自定义桶的名称
* @author lkg
* @date 2023/5/22
*/
public static String upload(MultipartFile file, String contentType, String bizPath, String customBucket) {
String fileUrl = null;
try {
if (StringUtils.isEmpty(bizPath)) {
bizPath = "report";
} else {
//过滤上传文件夹名特殊字符，防止攻击
bizPath = StrAttackFilter.filter(bizPath);
}
InputStream stream = file.getInputStream();
// 获取文件名
String orgName = file.getOriginalFilename();
if (StringUtils.isEmpty(orgName)) {
orgName = file.getName();
}
String objectName = bizPath + "/" + orgName;
// 使用putObject上传一个本地文件到存储桶中。
if (objectName.startsWith("/")) {
objectName = objectName.substring(1);
}
fileUrl = uploadByStream(stream,contentType, customBucket,objectName);
} catch (Exception e) {
e.printStackTrace();
}
return fileUrl;
}
/**
* 通过文件流上传文件
*
* @param stream 文件流
* @param contentType 类型，如果pdf文件或者图片，需要预览功能的话，必填.示例：pdf application/pdf；图片 image/png；image/jpeg
* @param customBucket 自定义桶名称
* @param relativePath 文件路径
* @author lkg
* @date 2023/5/22
*/
public static String uploadByStream(InputStream stream, String contentType, String customBucket, String relativePath) throws Exception {
initMinio(minioUrl, minioName, minioPass);
if (StringUtils.isEmpty(customBucket)) {
customBucket = bucketName;
}
if (!minioClient.bucketExists(BucketExistsArgs.builder().bucket(customBucket).build())) {
minioClient.makeBucket(MakeBucketArgs.builder().bucket(customBucket).build());
}
if (StringUtils.isEmpty(contentType)) {
contentType = "application/octet-stream";
}
int index = relativePath.lastIndexOf(".");
relativePath = relativePath.substring(0,index) + "_" + System.currentTimeMillis() + relativePath.substring(index);
if (relativePath.startsWith("/")) {
relativePath = relativePath.substring(1);
}
PutObjectArgs objectArgs = PutObjectArgs.builder().object(relativePath)
.bucket(customBucket)
.contentType(contentType)
.stream(stream, stream.available(), -1).build();
minioClient.putObject(objectArgs);
stream.close();
return "/" + customBucket + "/" + relativePath;
}
/**
* 通过字节上传文件
*
* @param bytes 字节
* @param contentType 类型，如果pdf文件或者图片，需要预览功能的话，必填.示例：pdf application/pdf；图片 image/png；image/jpeg
* @param customBucket 自定义桶名称
* @param relativePath 文件路径
* @author lkg
* @date 2023/5/22
*/
public static String uploadByByte(byte[] bytes, String contentType, String customBucket, String relativePath) throws Exception {
ByteArrayInputStream stream = new ByteArrayInputStream(bytes);
return uploadByStream(stream, contentType, customBucket, relativePath);
}
/**
* 获取永久预览链接
*
* @param filePath 文件路径(带桶) 桶/文件夹/文件名称 例如 jxyq/report/会议纪要.pdf
* @author lkg
* @date 2023/5/23
*/
public static String foreverPreviewUrl(String filePath){
int index = filePath.indexOf("/");
String bucketName = filePath.substring(0,index);
String filename = filePath.substring(index);
return getPreviewUrl(bucketName,filename,null);
}
/**
* 根据文件名生成文件存储路径(yyyy/yyyyMMdd/id)
* @param fileName
* @return
*/
public static String getRelativePath(String fileName) {
if(fileName == null || StringUtils.isEmpty(fileName) ){
return null;
}
String suffix = fileName;
if(fileName.lastIndexOf(".") > -1){
suffix = fileName.substring(fileName.lastIndexOf(".") + 1);
}
String date = DateUtil.formatDate(new Date()).replaceAll("-", "");
return date.substring(0,4)+"/"+date+"/"+ new Snowflake(1L,1L).nextIdStr()+"."+suffix;
}
/**
* 企业logo存储路径获取
* @param fileName
* @return
*/
public static String getEnterpriseLogoPath(String fileName) {
if(fileName == null || StringUtils.isEmpty(fileName) ){
return null;
}
String suffix = fileName;
if(fileName.lastIndexOf(".") > -1){
suffix = fileName.substring(fileName.lastIndexOf(".") + 1);
}
return "logo/enterprise/"+ new Snowflake(1L,1L).nextIdStr()+"."+suffix;
}
public static String getFileName(String fileName){
//判断是否带有盘符信息
// Check for Unix-style path
int unixSep = fileName.lastIndexOf('/');
// Check for Windows-style path
int winSep = fileName.lastIndexOf('\\');
// Cut off at latest possible point
int pos = (winSep > unixSep ? winSep : unixSep);
if (pos != -1) {
// Any sort of path separator found...
fileName = fileName.substring(pos + 1);
}
//替换上传文件名字的特殊字符
fileName = fileName.replace("=","").replace(",","").replace("&","").replace("#", "");
//替换上传文件名字中的空格
fileName=fileName.replaceAll("\\s","");
return fileName;
}
}
src/main/java/com/zzsn/knowbase/util/ObsUtil.java deleted 100644 → 0
//package com.zzsn.knowbase.util;
//
//
//import com.github.tobato.fastdfs.proto.storage.DownloadByteArray;
//import com.obs.services.ObsClient;
//import com.obs.services.model.*;
////import org.apache.commons.lang.StringUtils;
//import org.apache.commons.lang3.StringUtils;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.stereotype.Component;
//
//import javax.servlet.http.HttpServletRequest;
//import javax.servlet.http.HttpServletResponse;
//import java.io.*;
//import java.util.ArrayList;
//import java.util.List;
//
///**
// * Description: obs桶文件操作
// * Author: EDY
// * Date: 2023/10/9
// */
//@Component
//public class ObsUtil {
//
// @Autowired
// ObsClient obsClient;
// /**桶名称*/
// private String bucketName = "zzsn";
//
// /**判断桶是否存在*/
// public Boolean existsBucket(String bucket){
// return obsClient.headBucket(bucket);
// }
// public Boolean existsBucket(){
// return obsClient.headBucket(bucketName);
// }
//
// /**
// * 创建文件夹本质上来说是创建了一个大小为0且对象名以“/”结尾的对象。
// * 多级文件夹创建最后一级即可，比如src1/src2/src3/，创建src1/src2/src3/即可，无需创建src1/、src1/src2/。
// * keySuffixWithSlash为文件夹名称，以 / 结尾
// * */
// public boolean mkdir(String keySuffixWithSlash){
// PutObjectResult putObjectResult = obsClient.putObject(bucketName, keySuffixWithSlash, new ByteArrayInputStream(new byte[0]));
// if (putObjectResult.getStatusCode()==200) {
// return true;
// }else {
// return false;
// }
// }
//
// /**查询桶内文件夹下所有文件
// * folderPrefix 为文件夹名称，以 / 结尾
// * */
// public List<ObsObject> getPathFileList(String folderPrefix){
// List<ObsObject> res = new ArrayList<>();
// ListObjectsRequest request = new ListObjectsRequest(bucketName);
// request.setPrefix(folderPrefix);
// ObjectListing result = obsClient.listObjects(request);
// for (ObsObject obsObject : result.getObjects()) {
// res.add(obsObject);
// }
// return res;
// }
// /**
// * 获取文件夹下的文件数量
// * */
// public Integer getCount (String folderPrefix){
// ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName);
// listObjectsRequest.setPrefix(folderPrefix);
// listObjectsRequest.setMaxKeys(1000);
// int fileCount = 0;
//
// ObjectListing objectListing;
//
// do {
// objectListing = obsClient.listObjects(listObjectsRequest);
//
// List<S3Object> objectSummaries = objectListing.getObjectSummaries();
// fileCount += objectSummaries.size();
//
// String nextMarker = objectListing.getNextMarker();
// listObjectsRequest.setMarker(nextMarker);
// } while (objectListing.isTruncated());
//
// return fileCount;
// }
//
// /**删除桶内文件
// * objectKey为文件路径，起始为桶内某文件夹，或者直接为桶内文件
// * */
// public boolean delFile (String objectKey){
// DeleteObjectResult deleteObjectResult = obsClient.deleteObject(bucketName, objectKey);
// if (deleteObjectResult.getStatusCode()==200) {
// return true;
// }else {
// return false;
// }
// }
//
// /**文件上传
// * objectKey为文件路径
// * */
// public PutObjectResult uploadFile(String objectKey,byte[] bytes){
// PutObjectResult putObjectResult = obsClient.putObject(bucketName, objectKey, new ByteArrayInputStream(bytes));
// return putObjectResult;
// }
// /**文件上传
// * objectKey为文件路径
// * */
// public PutObjectResult uploadFile(String objectKey ,InputStream inputStream){
// PutObjectResult putObjectResult = obsClient.putObject(bucketName, objectKey, inputStream);
// return putObjectResult;
// }
// /**
// * 获取文件流
// *
// * */
// public InputStream getObjectStream(String objectKey){
// ObsObject obsObject = obsClient.getObject(bucketName, objectKey);
// return obsObject.getObjectContent();
// }
// /**
// * 获取文件流
// *
// * */
// public byte[] getObjectByte(String objectKey){
// ObsObject obsObject = obsClient.getObject(bucketName, objectKey);
// // 获取文件的输入流
// InputStream objectContent = obsObject.getObjectContent();
//
// // 将输入流转换为byte[]
// ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
// byte[] buffer = new byte[4096];
// int bytesRead;
// while (true) {
// try {
// if (!((bytesRead = objectContent.read(buffer)) != -1)) {
// break;
// }
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
// byteArrayOutputStream.write(buffer, 0, bytesRead);
// }
// byte[] fileBytes = byteArrayOutputStream.toByteArray();
// return fileBytes;
// }
//
// /*
// * 文件预览
// * @param fileName
// */
// public boolean previewImg(HttpServletRequest request, HttpServletResponse response) throws IOException {
// String filePath = request.getParameter("attachmentPath");
// String group = request.getParameter("group");
// if (StringUtils.isBlank(filePath)) {
// return false;
// }
// DownloadByteArray downloadByteArray = new DownloadByteArray();
// byte[] content = getObjectByte(filePath);
// if (content == null || content.length == 0) {
// return false;
// }
//
// response.addHeader("Pragma", "No-cache");
// response.addHeader("Cache-Control", "no-store,No-cache");
// response.setCharacterEncoding("UTF-8");
//// response.setContentType("application/json;charset=utf-8");
// String s = filePath.split("/")[filePath.split("/").length - 1];
// String mimeType = request.getServletContext().getMimeType(s);
// System.out.println("文件类型为" + mimeType);
// response.setContentType(request.getServletContext().getMimeType(s) + ";charset=utf-8");
// OutputStream out = response.getOutputStream();
// BufferedOutputStream bos = new BufferedOutputStream(out);
// try {
// bos.write(content, 0, content.length);
// bos.flush();
// } catch (Exception e) {
// e.printStackTrace();
// } finally {
// if (bos != null) {
// bos.close();
// }
// if (out != null) {
// out.close();
// }
// }
// return true;
// }
//
//}
src/main/java/com/zzsn/knowbase/util/StrAttackFilter.java 0 → 100644
package com.zzsn.knowbase.util;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
/**
* 文件上传字符串过滤特殊字符
*/
public class StrAttackFilter {
public static String filter(String str) throws PatternSyntaxException {
// 清除掉所有特殊字符
String regEx = "[`_《》~!@#$%^&*()+=|{}':;',\\[\\].<>?~！@#￥%……&*（）——+|{}【】‘；：”“’。，、？]";
Pattern p = Pattern.compile(regEx);
Matcher m = p.matcher(str);
return m.replaceAll("").trim();
}
// public static void main(String[] args) {
// String filter = filter("@#jeecg/《》【bo】￥%……&*（o）)))！@t<>,.,/?'\'~~`");
// System.out.println(filter);
// }
}
src/main/resources/application.yml
enable:
scheduling: false
server:
port: 9088
servlet:
......@@ -20,8 +22,10 @@ spring:
stream:
kafka:
binder:
brokers: 114.115.159.144:9092
zkNodes: 114.115.159.144:2181
# brokers: 114.115.159.144:9092
# zkNodes: 114.115.159.144:2181
brokers: 114.116.36.231:9092
zkNodes: 114.116.36.231:2181
requiredAcks: 1
redis:
database: 0
......@@ -61,8 +65,8 @@ document:
host: http://114.116.116.241:9088
# host: http://192.168.1.71:9088
files:
storage: /zzsn/nt/storage/know/
# storage: D:/storage/know/
# storage: /zzsn/nt/storage/know/
storage: D:/storage/know/
docservice:
url:
site: http://114.116.116.241:80/
......@@ -79,3 +83,9 @@ files:
history:
postfix: -hist
filesize-max: 52428800
minio:
minio_url: http://114.116.36.231:19000
minio_name: minioadmin
minio_pass: minioadmin
bucketName: know
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论