Skip to content

K
know-base
提交 bf3ec374 作者: yanxin
操作

文件导入整合minio

上级 310640c1
正在显示 包含 782 行增加 和 238 行删除
pom.xml
......@@ -15,6 +15,7 @@
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
<minio.version>8.0.3</minio.version>
</properties>
<dependencies>
<dependency>
......@@ -211,9 +212,12 @@
<version>2.2.10</version>
</dependency>
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>${minio.version}</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
......
src/main/java/com/zzsn/knowbase/config/MinioConfig.java 0 → 100644
package com.zzsn.knowbase.config;
import com.zzsn.knowbase.util.MinioUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* Minio文件上传配置文件
*/
@Slf4j
@Configuration
public class MinioConfig {
@Value(value = "${minio.minio_url}")
private String minioUrl;
@Value(value = "${minio.minio_name}")
private String minioName;
@Value(value = "${minio.minio_pass}")
private String minioPass;
@Value(value = "${minio.bucketName}")
private String bucketName;
@Bean
public void initMinio(){
if(!minioUrl.startsWith("http")){
minioUrl = "http://" + minioUrl;
}
if(!minioUrl.endsWith("/")){
minioUrl = minioUrl.concat("/");
}
MinioUtil.setMinioUrl(minioUrl);
MinioUtil.setMinioName(minioName);
MinioUtil.setMinioPass(minioPass);
MinioUtil.setBucketName(bucketName);
}
}
src/main/java/com/zzsn/knowbase/config/scheduler/Scheduler.java 0 → 100644
package com.zzsn.knowbase.config.scheduler;
import org.springframework.beans.factory.config.BeanDefinition;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Conditional;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Role;
import org.springframework.scheduling.annotation.ScheduledAnnotationBeanPostProcessor;
import org.springframework.scheduling.config.TaskManagementConfigUtils;
@Configuration
public class Scheduler {
@Conditional(SchedulerCondition.class)
@Bean(name = TaskManagementConfigUtils.SCHEDULED_ANNOTATION_PROCESSOR_BEAN_NAME)
@Role(BeanDefinition.ROLE_INFRASTRUCTURE)
public ScheduledAnnotationBeanPostProcessor scheduledAnnotationProcessor() {
return new ScheduledAnnotationBeanPostProcessor();
}
}
\ No newline at end of file
src/main/java/com/zzsn/knowbase/config/scheduler/SchedulerCondition.java 0 → 100644
package com.zzsn.knowbase.config.scheduler;
import org.springframework.context.annotation.Condition;
import org.springframework.context.annotation.ConditionContext;
import org.springframework.core.type.AnnotatedTypeMetadata;
public class SchedulerCondition implements Condition {
@Override
public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) {
return Boolean.valueOf(context.getEnvironment().getProperty("enable.scheduling"));
}
}
\ No newline at end of file
src/main/java/com/zzsn/knowbase/constant/Constants.java
......@@ -130,8 +130,12 @@ public class Constants {
public static final String FSP = System.getProperty("file.separator");
public static final String USER_HOME = System.getProperty("user.home");
/*
* 临时文件保存路径。
*/
public static final String TEMP_FILES_DIR = USER_HOME+FSP+"temp"+FSP;
public static final String APPLICATION_DATA_DIR = USER_HOME + FSP + "mrasdata" + FSP;
public static final String APPLICATION_DATA_DIR = USER_HOME+FSP+"mrasdata"+FSP;
//专题采集数据
public final static String SEND_TO_MACHINE_DATA = "SUBJECT_DATA:SYNtOMACHINE:";
......
src/main/java/com/zzsn/knowbase/entity/Enclosure.java 0 → 100644
package com.zzsn.knowbase.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.annotation.JsonFormat;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.experimental.Accessors;
import org.jeecgframework.poi.excel.annotation.Excel;
import org.springframework.format.annotation.DateTimeFormat;
import java.io.Serializable;
/**
* @Description: 附件表
* @Author: jeecg-boot
* @Date: 2023-01-18
* @Version: V1.0
*/
@Data
@TableName("enclosure")
public class Enclosure implements Serializable {
/**id*/
@TableId(type = IdType.ASSIGN_ID)
@ApiModelProperty(value = "id")
private String id;
/**name*/
@Excel(name = "name", width = 15)
@ApiModelProperty(value = "name")
private String name;
/**extensionName*/
@Excel(name = "extensionName", width = 15)
@ApiModelProperty(value = "extensionName")
private String extensionName;
/**urlPath*/
@Excel(name = "urlPath", width = 15)
@ApiModelProperty(value = "urlPath")
private String urlPath;
/**contentMd5*/
@Excel(name = "contentMd5", width = 15)
@ApiModelProperty(value = "contentMd5")
private String contentMd5;
/**createTime*/
@Excel(name = "createTime", width = 20, format = "yyyy-MM-dd HH:mm:ss")
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
@ApiModelProperty(value = "createTime")
private java.util.Date createTime;
/**createBy*/
@Excel(name = "createBy", width = 15)
@ApiModelProperty(value = "createBy")
private String createBy;
@Excel(name = "fileSize", width = 15)
@ApiModelProperty(value = "fileSize")
private Long fileSize;
}
src/main/java/com/zzsn/knowbase/mapper/EnclosureMapper.java 0 → 100644
package com.zzsn.knowbase.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zzsn.knowbase.entity.Enclosure;
import org.apache.ibatis.annotations.Mapper;
/**
* @Description: 附件表
* @Author: jeecg-boot
* @Date: 2023-01-18
* @Version: V1.0
*/
@Mapper
public interface EnclosureMapper extends BaseMapper<Enclosure> {
}
src/main/java/com/zzsn/knowbase/service/IEnclosureService.java 0 → 100644
package com.zzsn.knowbase.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.knowbase.entity.Enclosure;
/**
* @Description: 附件表
* @Author: jeecg-boot
* @Date: 2023-01-18
* @Version: V1.0
*/
public interface IEnclosureService extends IService<Enclosure> {
Enclosure getByMd5(String contentMd5);
}
src/main/java/com/zzsn/knowbase/service/impl/EnclosureServiceImpl.java 0 → 100644
package com.zzsn.knowbase.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.knowbase.entity.Enclosure;
import com.zzsn.knowbase.mapper.EnclosureMapper;
import com.zzsn.knowbase.service.IEnclosureService;
import org.springframework.stereotype.Service;
/**
* @Description: 附件表
* @Author: jeecg-boot
* @Date: 2023-01-18
* @Version: V1.0
*/
@Service
public class EnclosureServiceImpl extends ServiceImpl<EnclosureMapper, Enclosure> implements IEnclosureService {
@Override
public Enclosure getByMd5(String contentMd5) {
LambdaQueryWrapper<Enclosure> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.eq(Enclosure::getContentMd5,contentMd5);
return this.getOne(queryWrapper);
}
}
src/main/java/com/zzsn/knowbase/service/impl/LocalFileServiceImpl.java
......@@ -4,11 +4,9 @@ import com.alibaba.fastjson.JSON;
import com.zzsn.knowbase.constant.Constants;
import com.zzsn.knowbase.constant.DocumentConstants;
import com.zzsn.knowbase.constant.ErrorCodeEnum;
import com.zzsn.knowbase.entity.Document;
import com.zzsn.knowbase.entity.KbAuthorizedUser;
import com.zzsn.knowbase.entity.KnowFile;
import com.zzsn.knowbase.entity.Knowledge;
import com.zzsn.knowbase.entity.*;
import com.zzsn.knowbase.service.DocumentService;
import com.zzsn.knowbase.service.IEnclosureService;
import com.zzsn.knowbase.service.IKnowledgeService;
import com.zzsn.knowbase.service.ILocalFileService;
import com.zzsn.knowbase.util.*;
......@@ -67,6 +65,9 @@ public class LocalFileServiceImpl implements ILocalFileService {
@Autowired
private IKnowledgeService knowledgeService;
@Autowired
private IEnclosureService enclosureService;
@Value("${files.storage}")
String filesStorage;
@Value("${files.docservice.url.site}")
......@@ -77,45 +78,61 @@ public class LocalFileServiceImpl implements ILocalFileService {
@Override
public Result<KnowFile> upload(MultipartFile file) {
try {
String fileName = file.getOriginalFilename(); // 获取文件名称
String fileExtension = fileUtility.getFileExtension(fileName); // 获取文件扩展名
String fileType = fileUtility.getFileType(fileName); //获取文件类型
long fileSize = file.getSize(); // get file size
log.info("文件上传："+ fileName);
// check if the file size exceeds the maximum file size or is less than 0
if (fileUtility.getMaxFileSize() < fileSize || fileSize <= 0) {
Result result = Result.error("文件大小不正确！");
log.info("文件大小不正确!");
return result;
String contentMd5 = FileMd5Utils.getFileMD5String(file);
Enclosure enclosure = enclosureService.getByMd5(contentMd5);
if(enclosure == null){
enclosure = new Enclosure();
InputStream in = null;
try {
String fileName = file.getOriginalFilename(); // 获取文件名称
String fileType = fileUtility.getFileType(fileName); //获取文件类型
long fileSize = file.getSize(); // get file size
if (fileUtility.getMaxFileSize() < fileSize || fileSize <= 0) {
Result result = Result.error("文件大小超过范围！");
log.info("文件大小超过范围!");
return result;
}
if (!fileUtility.getFileExts().contains(fileType)) {
Result result = Result.error("不支持的文件类型！");
log.info("不支持的文件类型!");
return result;
}
enclosure.setExtensionName(fileType);
enclosure.setName(fileName);
enclosure.setContentMd5(contentMd5);
enclosure.setFileSize(file.getSize());
enclosure.setCreateTime(new Date());
enclosure.setCreateBy(SpringContextUtils.getUserInfo().getUsername());
log.info("文件上传到minio：{}", fileName);
in = file.getInputStream();
String fullFileName = MinioUtil.uploadByStream(in, MimeUtil.getMimeStrByName(fileName), null, MinioUtil.getRelativePath(fileName));
enclosure.setUrlPath(fullFileName);
enclosureService.save(enclosure);
} catch (Exception e) {
log.error("文件上传失败！{}", e.getMessage());
} finally {
if(in!=null){
try {
in.close();
} catch (IOException e) {
log.error(e.getMessage());
}
}
}
}
// check if file extension is supported by the editor
if (!fileUtility.getFileExts().contains(fileExtension)) {
Result result = Result.error("不支持的文件类型！");
log.info("不支持的文件类型!");
return result;
}
String fileId = codeGenerateUtil.geneIdNo(Constants.FINANCE, 8);
String filePath = getFilePath() + fileId + fileExtension;
//byte[] bytes = file.getBytes(); // get file in bytes
//Files.write(Paths.get(filePath), bytes);
file.transferTo(new File(filesStorage + filePath));
//封装es关联对象并返回
KnowFile knowFile = new KnowFile();
knowFile.setFileId(fileId);
knowFile.setFileName(fileName);
knowFile.setFilePath(filePath);
knowFile.setFileType(fileType);
knowFile.setFileSize(fileSize);
Result result = Result.OK(knowFile);
log.info("文件上传成功：" + fileName + "---" + filePath);
return result; // create user metadata and return it
knowFile.setFileId(enclosure.getId());
knowFile.setFileName(enclosure.getName());
knowFile.setFilePath(enclosure.getUrlPath());
knowFile.setFileType(enclosure.getExtensionName());
knowFile.setFileSize(enclosure.getFileSize());
log.info("文件上传成功name:{},path:{}",knowFile.getFileName(),knowFile.getFilePath());
return Result.OK(knowFile);
} catch (Exception e) {
e.printStackTrace();
log.error(e.getMessage());
}
// if the operation of file uploading is unsuccessful, an error occurs
Result result = Result.error("上传文件时出现问题！");
log.info("上传文件时出现问题！");
return result;
......
src/main/java/com/zzsn/knowbase/util/FileMd5Utils.java 0 → 100644
package com.zzsn.knowbase.util;
import org.springframework.web.multipart.MultipartFile;
import sun.misc.Cleaner;
import sun.nio.ch.DirectBuffer;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
public class FileMd5Utils {
protected static char hexDigits[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e',
'f' };
protected static MessageDigest messageDigest = null;
static {
try {
messageDigest = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
System.err.println(FileMd5Utils.class.getName() + "初始化失败，MessageDigest不支持MD5Util.");
e.printStackTrace();
}
}
/**
* 计算文件的MD5
*
* @param fileName
* 文件的绝对路径
* @return
* @throws IOException
*/
public static String getFileMD5String(String fileName) throws IOException {
File f = new File(fileName);
return getFileMD5String(f);
}
/**
* 计算文件的MD5，重载方法
*
* @param file
* 文件对象
* @return
* @throws IOException
*/
public static String getFileMD5String(File file) throws IOException {
FileInputStream in = null;
FileChannel ch = null;
MappedByteBuffer byteBuffer = null;
try {
in = new FileInputStream(file);
ch = in.getChannel();
byteBuffer = ch.map(FileChannel.MapMode.READ_ONLY, 0, file.length());
messageDigest.update(byteBuffer);
} catch (Exception e) {
} finally {
if (byteBuffer != null) {
unmap(byteBuffer);
}
if (ch != null) {
ch.close();
}
if (in != null) {
in.close();
}
}
return bufferToHex(messageDigest.digest());
}
public static String getFileMD5String(MultipartFile file) throws IOException {
FileInputStream in = (FileInputStream) file.getInputStream();
FileChannel ch = in.getChannel();
MappedByteBuffer byteBuffer = ch.map(FileChannel.MapMode.READ_ONLY, 0, file.getSize());
messageDigest.update(byteBuffer);
return bufferToHex(messageDigest.digest());
}
/*
* 需要测试一下这个算法的长度是否正确。
*/
public static String getFileMD5String(FileInputStream in) throws IOException {
FileChannel ch = in.getChannel();
int len = in.read();
MappedByteBuffer byteBuffer = ch.map(FileChannel.MapMode.READ_ONLY, 0, len);
messageDigest.update(byteBuffer);
return bufferToHex(messageDigest.digest());
}
public static String getFileMD5String(InputStream in) throws IOException {
FileChannel ch = ((FileInputStream) in).getChannel();
int len = in.read();
MappedByteBuffer byteBuffer = ch.map(FileChannel.MapMode.READ_ONLY, 0, len);
messageDigest.update(byteBuffer);
return bufferToHex(messageDigest.digest());
}
public static String getMD5String(byte[] bytes) {
messageDigest.update(bytes);
return bufferToHex(messageDigest.digest());
}
public static String getMD5String(String s) {
return getMD5String(s.getBytes());
}
private static String bufferToHex(byte bytes[]) {
return bufferToHex(bytes, 0, bytes.length);
}
private static String bufferToHex(byte bytes[], int m, int n) {
StringBuffer stringbuffer = new StringBuffer(2 * n);
int k = m + n;
for (int l = m; l < k; l++) {
appendHexPair(bytes[l], stringbuffer);
}
return stringbuffer.toString();
}
private static void appendHexPair(byte bt, StringBuffer stringbuffer) {
char c0 = hexDigits[(bt & 0xf0) >> 4];
char c1 = hexDigits[bt & 0xf];
stringbuffer.append(c0);
stringbuffer.append(c1);
}
private static void unmap(MappedByteBuffer var0) {
if(var0 != null){
Cleaner var1 = ((DirectBuffer) var0).cleaner();
if (var1 != null) {
var1.clean();
}
}
}
}
src/main/java/com/zzsn/knowbase/util/FileUtil.java 0 → 100644
package com.zzsn.knowbase.util;
import org.apache.commons.lang3.StringUtils;
import javax.net.ssl.*;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class FileUtil {
private static ArrayList<String> readFileName = new ArrayList<String>();
public static InputStream DownloadFileFormServer(String strUrl) {
InputStream inputStream = null;
try {
URL url = new URL(strUrl);
if(strUrl.contains("https")){
//忽略证书
SSLContext sc = SSLContext.getInstance("SSL");
sc.init(null, new TrustManager[]{
new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] x509Certificates, String s) {
}
@Override
public void checkServerTrusted(X509Certificate[] x509Certificates, String s) {
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
}
}, new SecureRandom());
HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
conn.setRequestProperty("Accept-Encoding", "deflate");
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36");
// 设置证书忽略相关操作
conn.setSSLSocketFactory(sc.getSocketFactory());
conn.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String s, SSLSession sslSession) {
return true;
}
});
conn.setRequestMethod("GET");
conn.setConnectTimeout(20 * 1000);
conn.connect();
inputStream = conn.getInputStream();
}else{
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Accept-Encoding", "deflate");
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36");
conn.setRequestMethod("GET");
conn.setConnectTimeout(20 * 1000);
conn.connect();
inputStream = conn.getInputStream();
}
} catch (Exception e) {
} finally {
}
return inputStream;
}
public static int getFileSizeByUrl(String strUrl){
try {
URL url = new URL(strUrl);
if(strUrl.contains("https")){
//忽略证书
SSLContext sc = SSLContext.getInstance("SSL");
sc.init(null, new TrustManager[]{
new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] x509Certificates, String s) {
}
@Override
public void checkServerTrusted(X509Certificate[] x509Certificates, String s) {
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
}
}, new SecureRandom());
HttpsURLConnection conn = (HttpsURLConnection) url.openConnection();
conn.setRequestProperty("Accept-Encoding", "deflate");
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36");
// 设置证书忽略相关操作
conn.setSSLSocketFactory(sc.getSocketFactory());
conn.setHostnameVerifier(new HostnameVerifier() {
@Override
public boolean verify(String s, SSLSession sslSession) {
return true;
}
});
conn.setRequestMethod("GET");
conn.setConnectTimeout(20 * 1000);
conn.connect();
return conn.getContentLength();
}else{
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestProperty("Accept-Encoding", "deflate");
conn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36");
conn.setRequestMethod("GET");
conn.setConnectTimeout(20 * 1000);
conn.connect();
return conn.getContentLength();
}
} catch (Exception e) {
}
return 0;
}
/*
* 清空文件列表中的记录
*/
public static void clearList() {
readFileName.clear();
}
public static ArrayList<String> getReadFileName() {
return readFileName;
}
public static void setReadFileName(ArrayList<String> readFileName) {
FileUtil.readFileName = readFileName;
}
/*
*
* 通过递归得到某一路径下所有的目录及其文件
*
*/
public static void getFiles(String filePath) {
File root = new File(filePath);
File[] files = root.listFiles();
for (File file : files) {
if (file.isDirectory()) {
/*
*
* 递归调用
*
*/
getFiles(file.getAbsolutePath());
} else {
String absolutePath = file.getAbsolutePath();
String regex = ".*[\\d]{4}年--[\\d]{1,2}月.*";
Matcher m = Pattern.compile(regex).matcher(absolutePath);
if (absolutePath != null && m.matches() && absolutePath.indexOf("--进出口--") == -1) {
readFileName.add(file.getAbsolutePath());
}
}
}
}
public static List<String> readFiletoList(String fileName) {
List<String> list = new ArrayList<String>();
BufferedReader br = null;
InputStream is = null;
InputStreamReader isr = null;
try {
is = new FileInputStream(fileName);
isr = new InputStreamReader(is, "gb2312");
br = new BufferedReader(isr);
// br = new BufferedReader(new FileReader(fileName),"gb2312");
String contentLine;
while ((contentLine = br.readLine()) != null) {
if (StringUtils.isNotEmpty(contentLine)) {
contentLine = contentLine.trim();
list.add(contentLine);
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (br != null) {
try {
br.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
if (isr != null) {
try {
isr.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
if (is != null) {
try {
is.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
return list;
}
/*
* 保存操作日志到文件
* 没有调用的地方
*/
public static void saveProjectInfo(String page,String fileName) {
BufferedWriter bw1 = null;
try {
bw1 = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileName, true)));
bw1.write(page + "\r\n");
bw1.flush();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
if (bw1 != null) {
try {
bw1.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
}
src/main/java/com/zzsn/knowbase/util/LoadToolUtil.java 0 → 100644
package com.zzsn.knowbase.util;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author yanxin
*/
@Slf4j
public class LoadToolUtil {
/**
* 中文字符和中文引号
*/
private static Pattern decodePattern = Pattern.compile("[\u4e00-\u9fa5“”]+");
public static List<String> readFiletoList(String fileName){
List<String> list = new ArrayList<>();
BufferedReader br = null;
try{
br = new BufferedReader(new FileReader(fileName));
String contentLine ;
while ((contentLine = br.readLine()) != null) {
if(StringUtils.isNotEmpty(contentLine)){
contentLine = contentLine.trim();
list.add(contentLine);
}
}
}catch(Exception e){
e.printStackTrace();
}finally{
if(br != null){
try {
br.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
return list;
}
public static boolean judegExcelEdition(String fileName) {
String regex = "^.+\\.(?i)(xls)$";
return !fileName.matches(regex);
}
/**
* 找到文件的扩展名
*/
public static String getFileSuffix(String fileName) {
// TODO Auto-generated method stub
if(fileName == null || StringUtils.isEmpty(fileName) ){
return null;
}
String suffix = "";
String indexOf = ".";
if(fileName.lastIndexOf(indexOf) > -1){
suffix = fileName.substring(fileName.lastIndexOf("."));
}
return suffix;
}
/**
* 获取参数编码后的url
*/
public static String urlEncoder(String url) {
try {
Matcher m = decodePattern.matcher(url);
StringBuffer b = new StringBuffer();
while (m.find()) {
m.appendReplacement(b, URLEncoder.encode(m.group(0), "utf-8"));
}
m.appendTail(b);
url = b.toString();
url = url.replace("&amp;", "&");
} catch (Exception e) {
log.error("url编码转换失败e:{}",e.getMessage());
}
return url;
}
/**
* 获取相对路径，不带第一个/
*/
public static String urlEncoderRelative(String url) {
url = urlEncoder(url);
if(url.startsWith("/")){
url = url.substring(1);
}
return url;
}
}
src/main/java/com/zzsn/knowbase/util/ObsUtil.java deleted 100644 → 0
//package com.zzsn.knowbase.util;
//
//
//import com.github.tobato.fastdfs.proto.storage.DownloadByteArray;
//import com.obs.services.ObsClient;
//import com.obs.services.model.*;
////import org.apache.commons.lang.StringUtils;
//import org.apache.commons.lang3.StringUtils;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.stereotype.Component;
//
//import javax.servlet.http.HttpServletRequest;
//import javax.servlet.http.HttpServletResponse;
//import java.io.*;
//import java.util.ArrayList;
//import java.util.List;
//
///**
// * Description: obs桶文件操作
// * Author: EDY
// * Date: 2023/10/9
// */
//@Component
//public class ObsUtil {
//
// @Autowired
// ObsClient obsClient;
// /**桶名称*/
// private String bucketName = "zzsn";
//
// /**判断桶是否存在*/
// public Boolean existsBucket(String bucket){
// return obsClient.headBucket(bucket);
// }
// public Boolean existsBucket(){
// return obsClient.headBucket(bucketName);
// }
//
// /**
// * 创建文件夹本质上来说是创建了一个大小为0且对象名以“/”结尾的对象。
// * 多级文件夹创建最后一级即可，比如src1/src2/src3/，创建src1/src2/src3/即可，无需创建src1/、src1/src2/。
// * keySuffixWithSlash为文件夹名称，以 / 结尾
// * */
// public boolean mkdir(String keySuffixWithSlash){
// PutObjectResult putObjectResult = obsClient.putObject(bucketName, keySuffixWithSlash, new ByteArrayInputStream(new byte[0]));
// if (putObjectResult.getStatusCode()==200) {
// return true;
// }else {
// return false;
// }
// }
//
// /**查询桶内文件夹下所有文件
// * folderPrefix 为文件夹名称，以 / 结尾
// * */
// public List<ObsObject> getPathFileList(String folderPrefix){
// List<ObsObject> res = new ArrayList<>();
// ListObjectsRequest request = new ListObjectsRequest(bucketName);
// request.setPrefix(folderPrefix);
// ObjectListing result = obsClient.listObjects(request);
// for (ObsObject obsObject : result.getObjects()) {
// res.add(obsObject);
// }
// return res;
// }
// /**
// * 获取文件夹下的文件数量
// * */
// public Integer getCount (String folderPrefix){
// ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName);
// listObjectsRequest.setPrefix(folderPrefix);
// listObjectsRequest.setMaxKeys(1000);
// int fileCount = 0;
//
// ObjectListing objectListing;
//
// do {
// objectListing = obsClient.listObjects(listObjectsRequest);
//
// List<S3Object> objectSummaries = objectListing.getObjectSummaries();
// fileCount += objectSummaries.size();
//
// String nextMarker = objectListing.getNextMarker();
// listObjectsRequest.setMarker(nextMarker);
// } while (objectListing.isTruncated());
//
// return fileCount;
// }
//
// /**删除桶内文件
// * objectKey为文件路径，起始为桶内某文件夹，或者直接为桶内文件
// * */
// public boolean delFile (String objectKey){
// DeleteObjectResult deleteObjectResult = obsClient.deleteObject(bucketName, objectKey);
// if (deleteObjectResult.getStatusCode()==200) {
// return true;
// }else {
// return false;
// }
// }
//
// /**文件上传
// * objectKey为文件路径
// * */
// public PutObjectResult uploadFile(String objectKey,byte[] bytes){
// PutObjectResult putObjectResult = obsClient.putObject(bucketName, objectKey, new ByteArrayInputStream(bytes));
// return putObjectResult;
// }
// /**文件上传
// * objectKey为文件路径
// * */
// public PutObjectResult uploadFile(String objectKey ,InputStream inputStream){
// PutObjectResult putObjectResult = obsClient.putObject(bucketName, objectKey, inputStream);
// return putObjectResult;
// }
// /**
// * 获取文件流
// *
// * */
// public InputStream getObjectStream(String objectKey){
// ObsObject obsObject = obsClient.getObject(bucketName, objectKey);
// return obsObject.getObjectContent();
// }
// /**
// * 获取文件流
// *
// * */
// public byte[] getObjectByte(String objectKey){
// ObsObject obsObject = obsClient.getObject(bucketName, objectKey);
// // 获取文件的输入流
// InputStream objectContent = obsObject.getObjectContent();
//
// // 将输入流转换为byte[]
// ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
// byte[] buffer = new byte[4096];
// int bytesRead;
// while (true) {
// try {
// if (!((bytesRead = objectContent.read(buffer)) != -1)) {
// break;
// }
// } catch (IOException e) {
// throw new RuntimeException(e);
// }
// byteArrayOutputStream.write(buffer, 0, bytesRead);
// }
// byte[] fileBytes = byteArrayOutputStream.toByteArray();
// return fileBytes;
// }
//
// /*
// * 文件预览
// * @param fileName
// */
// public boolean previewImg(HttpServletRequest request, HttpServletResponse response) throws IOException {
// String filePath = request.getParameter("attachmentPath");
// String group = request.getParameter("group");
// if (StringUtils.isBlank(filePath)) {
// return false;
// }
// DownloadByteArray downloadByteArray = new DownloadByteArray();
// byte[] content = getObjectByte(filePath);
// if (content == null || content.length == 0) {
// return false;
// }
//
// response.addHeader("Pragma", "No-cache");
// response.addHeader("Cache-Control", "no-store,No-cache");
// response.setCharacterEncoding("UTF-8");
//// response.setContentType("application/json;charset=utf-8");
// String s = filePath.split("/")[filePath.split("/").length - 1];
// String mimeType = request.getServletContext().getMimeType(s);
// System.out.println("文件类型为" + mimeType);
// response.setContentType(request.getServletContext().getMimeType(s) + ";charset=utf-8");
// OutputStream out = response.getOutputStream();
// BufferedOutputStream bos = new BufferedOutputStream(out);
// try {
// bos.write(content, 0, content.length);
// bos.flush();
// } catch (Exception e) {
// e.printStackTrace();
// } finally {
// if (bos != null) {
// bos.close();
// }
// if (out != null) {
// out.close();
// }
// }
// return true;
// }
//
//}
src/main/java/com/zzsn/knowbase/util/StrAttackFilter.java 0 → 100644
package com.zzsn.knowbase.util;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
/**
* 文件上传字符串过滤特殊字符
*/
public class StrAttackFilter {
public static String filter(String str) throws PatternSyntaxException {
// 清除掉所有特殊字符
String regEx = "[`_《》~!@#$%^&*()+=|{}':;',\\[\\].<>?~！@#￥%……&*（）——+|{}【】‘；：”“’。，、？]";
Pattern p = Pattern.compile(regEx);
Matcher m = p.matcher(str);
return m.replaceAll("").trim();
}
// public static void main(String[] args) {
// String filter = filter("@#jeecg/《》【bo】￥%……&*（o）)))！@t<>,.,/?'\'~~`");
// System.out.println(filter);
// }
}
src/main/resources/application.yml
enable:
scheduling: false
server:
port: 9088
servlet:
......@@ -20,8 +22,10 @@ spring:
stream:
kafka:
binder:
brokers: 114.115.159.144:9092
zkNodes: 114.115.159.144:2181
# brokers: 114.115.159.144:9092
# zkNodes: 114.115.159.144:2181
brokers: 114.116.36.231:9092
zkNodes: 114.116.36.231:2181
requiredAcks: 1
redis:
database: 0
......@@ -61,8 +65,8 @@ document:
host: http://114.116.116.241:9088
# host: http://192.168.1.71:9088
files:
storage: /zzsn/nt/storage/know/
# storage: D:/storage/know/
# storage: /zzsn/nt/storage/know/
storage: D:/storage/know/
docservice:
url:
site: http://114.116.116.241:80/
......@@ -79,3 +83,9 @@ files:
history:
postfix: -hist
filesize-max: 52428800
minio:
minio_url: http://114.116.36.231:19000
minio_name: minioadmin
minio_pass: minioadmin
bucketName: know
\ No newline at end of file
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论