引入cpdetector解决文件编码识别问题

This commit is contained in:
chenkailing
2020-12-26 19:13:50 +08:00
committed by kl
parent f2d929e6fa
commit 342c391a9b
23 changed files with 240 additions and 400 deletions

View File

@@ -5,7 +5,7 @@ import cn.keking.hutool.URLUtil;
import cn.keking.model.FileAttribute;
import cn.keking.model.FileType;
import cn.keking.model.ReturnResponse;
import cn.keking.service.FilePreviewCommonService;
import cn.keking.service.FileHandlerService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@@ -21,50 +21,48 @@ import java.util.UUID;
@Component
public class DownloadUtils {
private final Logger logger = LoggerFactory.getLogger(DownloadUtils.class);
private final static Logger logger = LoggerFactory.getLogger(DownloadUtils.class);
private final String fileDir = ConfigConstants.getFileDir();
private final FilePreviewCommonService filePreviewCommonService;
public DownloadUtils(FilePreviewCommonService filePreviewCommonService) {
this.filePreviewCommonService = filePreviewCommonService;
}
private static final String URL_PARAM_FTP_USERNAME = "ftp.username";
private static final String URL_PARAM_FTP_PASSWORD = "ftp.password";
private static final String URL_PARAM_FTP_CONTROL_ENCODING = "ftp.control.encoding";
private final FileHandlerService fileHandlerService;
public DownloadUtils(FileHandlerService fileHandlerService) {
this.fileHandlerService = fileHandlerService;
}
/**
* @param fileAttribute fileAttribute
* @param fileName 文件名
* @param fileName 文件名
* @return 本地文件绝对路径
*/
public ReturnResponse<String> downLoad(FileAttribute fileAttribute, String fileName) {
public ReturnResponse<String> downLoad(FileAttribute fileAttribute, String fileName) {
String urlStr = fileAttribute.getUrl();
String type = fileAttribute.getSuffix();
ReturnResponse<String> response = new ReturnResponse<>(0, "下载成功!!!", "");
UUID uuid = UUID.randomUUID();
if (null == fileName) {
fileName = uuid+ "."+type;
fileName = uuid + "." + type;
} else { // 文件后缀不一致时以type为准(针对simText【将类txt文件转为txt】)
fileName = fileName.replace(fileName.substring(fileName.lastIndexOf(".") + 1), type);
}
String realPath = fileDir + fileName;
File dirFile = new File(fileDir);
if (!dirFile.exists()) {
dirFile.mkdirs();
if (!dirFile.exists() && !dirFile.mkdirs()) {
logger.error("创建目录【{}】失败,可能是权限不够,请检查", fileDir);
}
try {
URL url = new URL(urlStr);
if (url.getProtocol() != null && (url.getProtocol().toLowerCase().startsWith("file")||url.getProtocol().toLowerCase().startsWith("http"))) {
if (url.getProtocol() != null && (url.getProtocol().toLowerCase().startsWith("file") || url.getProtocol().toLowerCase().startsWith("http"))) {
byte[] bytes = getBytesFromUrl(urlStr);
OutputStream os = new FileOutputStream(realPath);
saveBytesToOutStream(bytes, os);
} else if (url.getProtocol() != null && "ftp".equalsIgnoreCase(url.getProtocol())) {
String ftpUsername = filePreviewCommonService.getUrlParameterReg(fileAttribute.getUrl(), URL_PARAM_FTP_USERNAME);
String ftpPassword = filePreviewCommonService.getUrlParameterReg(fileAttribute.getUrl(), URL_PARAM_FTP_PASSWORD);
String ftpControlEncoding = filePreviewCommonService.getUrlParameterReg(fileAttribute.getUrl(), URL_PARAM_FTP_CONTROL_ENCODING);
String ftpUsername = fileHandlerService.getUrlParameterReg(fileAttribute.getUrl(), URL_PARAM_FTP_USERNAME);
String ftpPassword = fileHandlerService.getUrlParameterReg(fileAttribute.getUrl(), URL_PARAM_FTP_PASSWORD);
String ftpControlEncoding = fileHandlerService.getUrlParameterReg(fileAttribute.getUrl(), URL_PARAM_FTP_CONTROL_ENCODING);
FtpUtils.download(fileAttribute.getUrl(), realPath, ftpUsername, ftpPassword, ftpControlEncoding);
} else {
response.setCode(1);
@@ -73,8 +71,8 @@ public class DownloadUtils {
}
response.setContent(realPath);
response.setMsg(fileName);
if(FileType.simText.equals(fileAttribute.getType())){
convertTextPlainFileCharsetToUtf8(realPath);
if (FileType.simText.equals(fileAttribute.getType())) {
this.convertTextPlainFileCharsetToUtf8(realPath);
}
return response;
} catch (IOException e) {
@@ -92,17 +90,15 @@ public class DownloadUtils {
public byte[] getBytesFromUrl(String urlStr) throws IOException {
InputStream is = getInputStreamFromUrl(urlStr);
if (is != null) {
return getBytesFromStream(is);
} else {
if (is == null) {
urlStr = URLUtil.normalize(urlStr, true, true);
is = getInputStreamFromUrl(urlStr);
if (is == null) {
logger.error("文件下载异常url{}", urlStr);
throw new IOException("文件下载异常url" + urlStr);
}
return getBytesFromStream(is);
}
return getBytesFromStream(is);
}
public void saveBytesToOutStream(byte[] b, OutputStream os) throws IOException {
@@ -127,7 +123,7 @@ public class DownloadUtils {
private byte[] getBytesFromStream(InputStream is) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int len = 0;
int len;
while ((len = is.read(buffer)) != -1) {
baos.write(buffer, 0, len);
}
@@ -137,41 +133,37 @@ public class DownloadUtils {
return b;
}
/**
* 转换文本文件编码为utf8
* 探测源文件编码,探测到编码切不为utf8则进行转码
* @param filePath 文件路径
*/
private static void convertTextPlainFileCharsetToUtf8(String filePath) throws IOException {
File sourceFile = new File(filePath);
if(sourceFile.exists() && sourceFile.isFile() && sourceFile.canRead()) {
String encoding = null;
try {
FileCharsetDetector.Observer observer = FileCharsetDetector.guessFileEncoding(sourceFile);
// 为准确探测到编码,不适用猜测的编
encoding = observer.isFound()?observer.getEncoding():null;
// 为准确探测到编码,可以考虑使用GBK 大部分文件都是windows系统产生的
} catch (IOException e) {
// 编码探测失败,
e.printStackTrace();
}
if(encoding != null && !"UTF-8".equals(encoding)){
// 不为utf8,进行转码
File tmpUtf8File = new File(filePath+".utf8");
Writer writer = new OutputStreamWriter(new FileOutputStream(tmpUtf8File), StandardCharsets.UTF_8);
Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile),encoding));
char[] buf = new char[1024];
int read;
while ((read = reader.read(buf)) > 0){
writer.write(buf, 0, read);
/**
* 转换文本文件编码为utf8
* 探测源文件编码,探测到编码切不为utf8则进行转码
*
* @param filePath 文件路径
*/
private void convertTextPlainFileCharsetToUtf8(String filePath) throws IOException {
File sourceFile = new File(filePath);
if (sourceFile.exists() && sourceFile.isFile() && sourceFile.canRead()) {
String encoding = FileUtils.getFileEncode(filePath);
if (!FileUtils.DEFAULT_FILE_ENCODING.equals(encoding)) {
// 不为utf8,进行转
File tmpUtf8File = new File(filePath + ".utf8");
Writer writer = new OutputStreamWriter(new FileOutputStream(tmpUtf8File), StandardCharsets.UTF_8);
Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(sourceFile), encoding));
char[] buf = new char[1024];
int read;
while ((read = reader.read(buf)) > 0) {
writer.write(buf, 0, read);
}
reader.close();
writer.close();
// 删除源文件
if (!sourceFile.delete()) {
logger.error("源文件【{}】删除失败,请检查文件目录权限!", filePath);
}
// 重命名
if (tmpUtf8File.renameTo(sourceFile)) {
logger.error("临时文件【{}】重命名失败,请检查文件路径权限!", tmpUtf8File.getPath());
}
}
}
reader.close();
writer.close();
// 删除源文件
sourceFile.delete();
// 重命名
tmpUtf8File.renameTo(sourceFile);
}
}
}
}

View File

@@ -1,157 +0,0 @@
package cn.keking.utils;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.mozilla.intl.chardet.nsDetector;
import org.mozilla.intl.chardet.nsICharsetDetectionObserver;
/**
* 文本文件编码探测工具类
*
* @author HWliao
* @date 2017-12-24
*/
public class FileCharsetDetector {
/**
* 传入一个文件(File)对象,检查文件编码
*
* @param file File对象实例
* @return 文件编码若无则返回null
* @throws FileNotFoundException
* @throws IOException
*/
public static Observer guessFileEncoding(File file)
throws FileNotFoundException, IOException {
return guessFileEncoding(file, new nsDetector());
}
/**
* <pre>
* 获取文件的编码
* @param file
* File对象实例
* @param languageHint
* 语言提示区域代码 @see #nsPSMDetector ,取值如下:
* 1 : Japanese
* 2 : Chinese
* 3 : Simplified Chinese
* 4 : Traditional Chinese
* 5 : Korean
* 6 : Dont know(default)
* </pre>
*
* @return 文件编码egUTF-8,GBK,GB2312形式(不确定的时候,返回可能的字符编码序列)若无则返回null
* @throws FileNotFoundException
* @throws IOException
*/
public static Observer guessFileEncoding(File file, int languageHint)
throws FileNotFoundException, IOException {
return guessFileEncoding(file, new nsDetector(languageHint));
}
/**
* 获取文件的编码
*
* @param file
* @param det
* @return
* @throws FileNotFoundException
* @throws IOException
*/
private static Observer guessFileEncoding(File file, nsDetector det)
throws FileNotFoundException, IOException {
// new Observer
Observer observer = new Observer();
// set Observer
// The Notify() will be called when a matching charset is found.
det.Init(observer);
BufferedInputStream imp = new BufferedInputStream(new FileInputStream(
file));
byte[] buf = new byte[1024];
int len;
boolean done = false;
boolean isAscii = false;
while ((len = imp.read(buf, 0, buf.length)) != -1) {
// Check if the stream is only ascii.
isAscii = det.isAscii(buf, len);
if (isAscii) {
break;
}
// DoIt if non-ascii and not done yet.
done = det.DoIt(buf, len, false);
if (done) {
break;
}
}
imp.close();
det.DataEnd();
if (isAscii) {
observer.encoding = "ASCII";
observer.found = true;
}
if (!observer.isFound()) {
String[] prob = det.getProbableCharsets();
// // 这里将可能的字符集组合起来返回
// for (int i = 0; i < prob.length; i++) {
// if (i == 0) {
// encoding = prob[i];
// } else {
// encoding += "," + prob[i];
// }
// }
if (prob.length > 0) {
// 在没有发现情况下,去第一个可能的编码
observer.encoding = prob[0];
} else {
observer.encoding = null;
}
}
return observer;
}
/**
* @author liaohongwei
* @Description: 文件字符编码观察者, 但判断出字符编码时候调用
* @date 2016年6月20日 下午2:27:06
*/
public static class Observer implements nsICharsetDetectionObserver {
/**
* @Fields encoding : 字符编码
*/
private String encoding = null;
/**
* @Fields found : 是否找到字符集
*/
private boolean found = false;
@Override
public void Notify(String charset) {
this.encoding = charset;
this.found = true;
}
public String getEncoding() {
return encoding;
}
public boolean isFound() {
return found;
}
@Override
public String toString() {
return "Observer [encoding=" + encoding + ", found=" + found + "]";
}
}
}

View File

@@ -1,14 +1,18 @@
package cn.keking.utils;
import cpdetector.CharsetPrinter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.Objects;
public class DeleteFileUtil {
public class FileUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(DeleteFileUtil.class);
private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class);
public static final String DEFAULT_FILE_ENCODING = "UTF-8";
/**
* 删除单个文件
@@ -17,7 +21,7 @@ public class DeleteFileUtil {
* 要删除的文件的文件名
* @return 单个文件删除成功返回true否则返回false
*/
public static boolean deleteFile(String fileName) {
public static boolean deleteFileByName(String fileName) {
File file = new File(fileName);
// 如果文件路径所对应的文件存在并且是一个文件则直接删除
if (file.exists() && file.isFile()) {
@@ -34,6 +38,36 @@ public class DeleteFileUtil {
}
}
/**
* 判断文件编码格式
*
* @param filePath 绝对路径
* @return 编码格式
*/
public static String getFileEncode(String filePath) {
File file = new File(filePath);
CharsetPrinter cp = new CharsetPrinter();
try {
String encoding = cp.guessEncoding(file);
LOGGER.info("检测到文件【{}】编码: {}", filePath, encoding);
return encoding;
} catch (IOException e) {
LOGGER.warn("文件编码获取失败采用默认的编码格式UTF-8", e);
return DEFAULT_FILE_ENCODING;
}
}
/**
* 根据文件路径删除文件
*
* @param filePath 绝对路径
*/
public static void deleteFileByPath(String filePath) {
File file = new File(filePath);
if (file.exists() && !file.delete()) {
LOGGER.warn("压缩包源文件删除失败:{}", filePath);
}
}
/**
* 删除目录及目录下的文件
@@ -59,20 +93,20 @@ public class DeleteFileUtil {
for (int i = 0; i < Objects.requireNonNull(files).length; i++) {
// 删除子文件
if (files[i].isFile()) {
flag = DeleteFileUtil.deleteFile(files[i].getAbsolutePath());
flag = FileUtils.deleteFileByName(files[i].getAbsolutePath());
if (!flag) {
break;
}
} else if (files[i].isDirectory()) {
// 删除子目录
flag = DeleteFileUtil.deleteDirectory(files[i].getAbsolutePath());
flag = FileUtils.deleteDirectory(files[i].getAbsolutePath());
if (!flag) {
break;
}
}
}
dirFile.delete();
if (!flag) {
if (!dirFile.delete() || !flag) {
LOGGER.info("删除目录失败!");
return false;
}

View File

@@ -1,6 +1,6 @@
package cn.keking.utils;
import cn.keking.service.FilePreviewCommonService;
import cn.keking.service.FileHandlerService;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.rendering.ImageType;
import org.apache.pdfbox.rendering.PDFRenderer;
@@ -23,18 +23,18 @@ public class PdfUtils {
private final Logger logger = LoggerFactory.getLogger(PdfUtils.class);
private final FilePreviewCommonService filePreviewCommonService;
private final FileHandlerService fileHandlerService;
@Value("${server.tomcat.uri-encoding:UTF-8}")
private String uriEncoding;
public PdfUtils(FilePreviewCommonService filePreviewCommonService) {
this.filePreviewCommonService = filePreviewCommonService;
public PdfUtils(FileHandlerService fileHandlerService) {
this.fileHandlerService = fileHandlerService;
}
public List<String> pdf2jpg(String pdfFilePath, String pdfName, String baseUrl) {
List<String> imageUrls = new ArrayList<>();
Integer imageCount = filePreviewCommonService.getConvertedPdfImage(pdfFilePath);
Integer imageCount = fileHandlerService.getConvertedPdfImage(pdfFilePath);
String imageFileSuffix = ".jpg";
String pdfFolder = pdfName.substring(0, pdfName.length() - 4);
String urlPrefix = null;
@@ -70,7 +70,7 @@ public class PdfUtils {
imageUrls.add(urlPrefix + "/" + pageIndex + imageFileSuffix);
}
doc.close();
filePreviewCommonService.addConvertedPdfImage(pdfFilePath, pageCount);
fileHandlerService.addConvertedPdfImage(pdfFilePath, pageCount);
} catch (IOException e) {
logger.error("Convert pdf to jpg exception, pdfFilePath{}", pdfFilePath, e);
}

View File

@@ -31,7 +31,7 @@ public class ShedulerClean {
public void clean() {
logger.info("Cache clean start");
cacheService.cleanCache();
DeleteFileUtil.deleteDirectory(fileDir);
FileUtils.deleteDirectory(fileDir);
logger.info("Cache clean end");
}
}

View File

@@ -1,490 +0,0 @@
package cn.keking.utils;
import cn.keking.config.ConfigConstants;
import cn.keking.model.FileType;
import cn.keking.service.FilePreviewCommonService;
import cn.keking.web.filter.BaseUrlFilter;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.junrar.Archive;
import com.github.junrar.exception.RarException;
import com.github.junrar.rarfile.FileHeader;
import org.apache.commons.compress.archivers.sevenz.SevenZArchiveEntry;
import org.apache.commons.compress.archivers.sevenz.SevenZFile;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipFile;
import org.springframework.stereotype.Component;
import java.io.*;
import java.math.BigDecimal;
import java.text.CollationKey;
import java.text.Collator;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
*
* @author yudian-it
* @date 2017/11/27
*/
@Component
public class ZipReader {
static Pattern pattern = Pattern.compile("^\\d+");
private final FilePreviewCommonService filePreviewCommonService;
private final String fileDir = ConfigConstants.getFileDir();
private final ExecutorService executors = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
public ZipReader(FilePreviewCommonService filePreviewCommonService) {
this.filePreviewCommonService = filePreviewCommonService;
}
public String readZipFile(String filePath,String fileKey) {
String archiveSeparator = "/";
Map<String, FileNode> appender = new HashMap<>();
List<String> imgUrls = new LinkedList<>();
String baseUrl = BaseUrlFilter.getBaseUrl();
String archiveFileName = filePreviewCommonService.getFileNameFromPath(filePath);
try {
ZipFile zipFile = new ZipFile(filePath, filePreviewCommonService.getFileEncodeUTFGBK(filePath));
Enumeration<ZipArchiveEntry> entries = zipFile.getEntries();
// 排序
entries = sortZipEntries(entries);
List<Map<String, ZipArchiveEntry>> entriesToBeExtracted = new LinkedList<>();
while (entries.hasMoreElements()){
ZipArchiveEntry entry = entries.nextElement();
String fullName = entry.getName();
int level = fullName.split(archiveSeparator).length;
// 展示名
String originName = getLastFileName(fullName, archiveSeparator);
String childName = level + "_" + originName;
boolean directory = entry.isDirectory();
if (!directory) {
childName = archiveFileName + "_" + originName;
entriesToBeExtracted.add(Collections.singletonMap(childName, entry));
}
String parentName = getLast2FileName(fullName, archiveSeparator, archiveFileName);
parentName = (level-1) + "_" + parentName;
FileType type= filePreviewCommonService.typeFromUrl(childName);
if (type.equals(FileType.picture)){//添加图片文件到图片列表
imgUrls.add(baseUrl+childName);
}
FileNode node = new FileNode(originName, childName, parentName, new ArrayList<>(), directory, fileKey);
addNodes(appender, parentName, node);
appender.put(childName, node);
}
// 开启新的线程处理文件解压
executors.submit(new ZipExtractorWorker(entriesToBeExtracted, zipFile, filePath));
filePreviewCommonService.putImgCache(fileKey,imgUrls);
return new ObjectMapper().writeValueAsString(appender.get(""));
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
private Enumeration<ZipArchiveEntry> sortZipEntries(Enumeration<ZipArchiveEntry> entries) {
List<ZipArchiveEntry> sortedEntries = new LinkedList<>();
while(entries.hasMoreElements()){
sortedEntries.add(entries.nextElement());
}
sortedEntries.sort(Comparator.comparingInt(o -> o.getName().length()));
return Collections.enumeration(sortedEntries);
}
public String unRar(String filePath,String fileKey){
Map<String, FileNode> appender = new HashMap<>();
List<String> imgUrls = new ArrayList<>();
String baseUrl = BaseUrlFilter.getBaseUrl();
try {
Archive archive = new Archive(new FileInputStream(new File(filePath)));
List<FileHeader> headers = archive.getFileHeaders();
headers = sortedHeaders(headers);
String archiveFileName = filePreviewCommonService.getFileNameFromPath(filePath);
List<Map<String, FileHeader>> headersToBeExtracted =new ArrayList<>();
for (FileHeader header : headers) {
String fullName;
if (header.isUnicode()) {
fullName = header.getFileNameW();
}else {
fullName = header.getFileNameString();
}
// 展示名
String originName = getLastFileName(fullName, "\\");
String childName = originName;
boolean directory = header.isDirectory();
if (!directory) {
childName = archiveFileName + "_" + originName;
headersToBeExtracted.add(Collections.singletonMap(childName, header));
}
String parentName = getLast2FileName(fullName, "\\", archiveFileName);
FileType type = filePreviewCommonService.typeFromUrl(childName);
if (type.equals(FileType.picture)){//添加图片文件到图片列表
imgUrls.add(baseUrl+childName);
}
FileNode node = new FileNode(originName, childName, parentName, new ArrayList<>(), directory, fileKey);
addNodes(appender, parentName, node);
appender.put(childName, node);
}
executors.submit(new RarExtractorWorker(headersToBeExtracted, archive, filePath));
filePreviewCommonService.putImgCache(fileKey,imgUrls);
return new ObjectMapper().writeValueAsString(appender.get(""));
} catch (RarException | IOException e) {
e.printStackTrace();
}
return null;
}
public String read7zFile(String filePath,String fileKey) {
String archiveSeparator = "/";
Map<String, FileNode> appender = new HashMap<>();
List<String> imgUrls = new ArrayList<>();
String baseUrl= BaseUrlFilter.getBaseUrl();
String archiveFileName = filePreviewCommonService.getFileNameFromPath(filePath);
try {
SevenZFile zipFile = new SevenZFile(new File(filePath));
Iterable<SevenZArchiveEntry> entries = zipFile.getEntries();
// 排序
Enumeration<SevenZArchiveEntry> newEntries = sortSevenZEntries(entries);
List<Map<String, SevenZArchiveEntry>> entriesToBeExtracted = new ArrayList<>();
while (newEntries.hasMoreElements()){
SevenZArchiveEntry entry = newEntries.nextElement();
String fullName = entry.getName();
int level = fullName.split(archiveSeparator).length;
// 展示名
String originName = getLastFileName(fullName, archiveSeparator);
String childName = level + "_" + originName;
boolean directory = entry.isDirectory();
if (!directory) {
childName = archiveFileName + "_" + originName;
entriesToBeExtracted.add(Collections.singletonMap(childName, entry));
}
String parentName = getLast2FileName(fullName, archiveSeparator, archiveFileName);
parentName = (level-1) + "_" + parentName;
FileType type= filePreviewCommonService.typeFromUrl(childName);
if (type.equals(FileType.picture)){//添加图片文件到图片列表
imgUrls.add(baseUrl+childName);
}
FileNode node = new FileNode(originName, childName, parentName, new ArrayList<>(), directory, fileKey);
addNodes(appender, parentName, node);
appender.put(childName, node);
}
// 开启新的线程处理文件解压
executors.submit(new SevenZExtractorWorker(entriesToBeExtracted, filePath));
filePreviewCommonService.putImgCache(fileKey,imgUrls);
return new ObjectMapper().writeValueAsString(appender.get(""));
} catch (IOException e) {
e.printStackTrace();
return null;
}
}
private Enumeration<SevenZArchiveEntry> sortSevenZEntries(Iterable<SevenZArchiveEntry> entries) {
List<SevenZArchiveEntry> sortedEntries = new ArrayList<>();
for (SevenZArchiveEntry entry : entries) {
sortedEntries.add(entry);
}
return Collections.enumeration(sortedEntries);
}
private void addNodes(Map<String, FileNode> appender, String parentName, FileNode node) {
if (appender.containsKey(parentName)) {
appender.get(parentName).getChildList().add(node);
appender.get(parentName).getChildList().sort(sortComparator);
} else {
// 根节点
FileNode nodeRoot = new FileNode(parentName, parentName, "", new ArrayList<>(), true);
nodeRoot.getChildList().add(node);
appender.put("", nodeRoot);
appender.put(parentName, nodeRoot);
}
}
private List<FileHeader> sortedHeaders(List<FileHeader> headers) {
List<FileHeader> sortedHeaders = new ArrayList<>();
Map<Integer, FileHeader> mapHeaders = new TreeMap<>();
headers.forEach(header -> mapHeaders.put(new Integer(0).equals(header.getFileNameW().length()) ? header.getFileNameString().length() : header.getFileNameW().length(), header));
for (Map.Entry<Integer, FileHeader> entry : mapHeaders.entrySet()){
for (FileHeader header : headers) {
if (entry.getKey().equals(new Integer(0).equals(header.getFileNameW().length()) ? header.getFileNameString().length() : header.getFileNameW().length())) {
sortedHeaders.add(header);
}
}
}
return sortedHeaders;
}
private static String getLast2FileName(String fullName, String seperator, String rootName) {
if (fullName.endsWith(seperator)) {
fullName = fullName.substring(0, fullName.length()-1);
}
// 1.获取剩余部分
int endIndex = fullName.lastIndexOf(seperator);
String leftPath = fullName.substring(0, endIndex == -1 ? 0 : endIndex);
if (leftPath.length() > 1) {
// 2.获取倒数第二个
return getLastFileName(leftPath, seperator);
} else {
return rootName;
}
}
private static String getLastFileName(String fullName, String seperator) {
if (fullName.endsWith(seperator)) {
fullName = fullName.substring(0, fullName.length()-1);
}
String newName = fullName;
if (fullName.contains(seperator)) {
newName = fullName.substring(fullName.lastIndexOf(seperator) + 1);
}
return newName;
}
public static Comparator<FileNode> sortComparator = new Comparator<FileNode>() {
final Collator cmp = Collator.getInstance(Locale.US);
@Override
public int compare(FileNode o1, FileNode o2) {
// 判断两个对比对象是否是开头包含数字,如果包含数字则获取数字并按数字真正大小进行排序
BigDecimal num1,num2;
if (null != (num1 = isStartNumber(o1))
&& null != (num2 = isStartNumber(o2))) {
return num1.subtract(num2).intValue();
}
CollationKey c1 = cmp.getCollationKey(o1.getOriginName());
CollationKey c2 = cmp.getCollationKey(o2.getOriginName());
return cmp.compare(c1.getSourceString(), c2.getSourceString());
}
};
private static BigDecimal isStartNumber(FileNode src) {
Matcher matcher = pattern.matcher(src.getOriginName());
if (matcher.find()) {
return new BigDecimal(matcher.group());
}
return null;
}
public static class FileNode {
private String originName;
private String fileName;
private String parentFileName;
private boolean directory;
//用于图片预览时寻址
private String fileKey;
private List<FileNode> childList;
public FileNode(String originName, String fileName, String parentFileName, List<FileNode> childList, boolean directory) {
this.originName = originName;
this.fileName = fileName;
this.parentFileName = parentFileName;
this.childList = childList;
this.directory = directory;
}
public FileNode(String originName, String fileName, String parentFileName, List<FileNode> childList, boolean directory,String fileKey) {
this.originName = originName;
this.fileName = fileName;
this.parentFileName = parentFileName;
this.childList = childList;
this.directory = directory;
this.fileKey=fileKey;
}
public String getFileKey() {
return fileKey;
}
public void setFileKey(String fileKey) {
this.fileKey = fileKey;
}
public String getFileName() {
return fileName;
}
public void setFileName(String fileName) {
this.fileName = fileName;
}
public String getParentFileName() {
return parentFileName;
}
public void setParentFileName(String parentFileName) {
this.parentFileName = parentFileName;
}
public List<FileNode> getChildList() {
return childList;
}
public void setChildList(List<FileNode> childList) {
this.childList = childList;
}
@Override
public String toString() {
try {
return new ObjectMapper().writeValueAsString(this);
} catch (JsonProcessingException e) {
e.printStackTrace();
return "";
}
}
public String getOriginName() {
return originName;
}
public void setOriginName(String originName) {
this.originName = originName;
}
public boolean isDirectory() {
return directory;
}
public void setDirectory(boolean directory) {
this.directory = directory;
}
}
class ZipExtractorWorker implements Runnable {
private final List<Map<String, ZipArchiveEntry>> entriesToBeExtracted;
private final ZipFile zipFile;
private final String filePath;
public ZipExtractorWorker(List<Map<String, ZipArchiveEntry>> entriesToBeExtracted, ZipFile zipFile, String filePath) {
this.entriesToBeExtracted = entriesToBeExtracted;
this.zipFile = zipFile;
this.filePath = filePath;
}
@Override
public void run() {
for (Map<String, ZipArchiveEntry> entryMap : entriesToBeExtracted) {
String childName = entryMap.keySet().iterator().next();
ZipArchiveEntry entry = entryMap.values().iterator().next();
try {
extractZipFile(childName, zipFile.getInputStream(entry));
} catch (IOException e) {
e.printStackTrace();
}
}
try {
zipFile.close();
} catch (IOException e) {
e.printStackTrace();
}
if (new File(filePath).exists()) {
new File(filePath).delete();
}
}
private void extractZipFile(String childName, InputStream zipFile) {
String outPath = fileDir + childName;
try (OutputStream ot = new FileOutputStream(outPath)){
byte[] inByte = new byte[1024];
int len;
while ((-1 != (len = zipFile.read(inByte)))){
ot.write(inByte, 0, len);
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
class SevenZExtractorWorker implements Runnable {
private final List<Map<String, SevenZArchiveEntry>> entriesToBeExtracted;
private final String filePath;
public SevenZExtractorWorker(List<Map<String, SevenZArchiveEntry>> entriesToBeExtracted, String filePath) {
this.entriesToBeExtracted = entriesToBeExtracted;
this.filePath = filePath;
}
@Override
public void run() {
try {
SevenZFile sevenZFile = new SevenZFile(new File(filePath));
SevenZArchiveEntry entry = sevenZFile.getNextEntry();
while (entry != null) {
if (entry.isDirectory()) {
entry = sevenZFile.getNextEntry();
continue;
}
String childName = "default_file";
SevenZArchiveEntry entry1;
for (Map<String, SevenZArchiveEntry> entryMap : entriesToBeExtracted) {
childName = entryMap.keySet().iterator().next();
entry1 = entryMap.values().iterator().next();
if (entry.getName().equals(entry1.getName())) {
break;
}
}
FileOutputStream out = new FileOutputStream(fileDir + childName);
byte[] content = new byte[(int) entry.getSize()];
sevenZFile.read(content, 0, content.length);
out.write(content);
out.close();
entry = sevenZFile.getNextEntry();
}
sevenZFile.close();
} catch (IOException e) {
e.printStackTrace();
}
if (new File(filePath).exists()) {
new File(filePath).delete();
}
}
}
class RarExtractorWorker implements Runnable {
private final List<Map<String, FileHeader>> headersToBeExtracted;
private final Archive archive;
/**
* 用以删除源文件
*/
private final String filePath;
public RarExtractorWorker(List<Map<String, FileHeader>> headersToBeExtracted, Archive archive, String filePath) {
this.headersToBeExtracted = headersToBeExtracted;
this.archive = archive;
this.filePath = filePath;
}
@Override
public void run() {
for (Map<String, FileHeader> entryMap : headersToBeExtracted) {
String childName = entryMap.keySet().iterator().next();
extractRarFile(childName, entryMap.values().iterator().next(), archive);
}
try {
archive.close();
} catch (IOException e) {
e.printStackTrace();
}
if (new File(filePath).exists()) {
new File(filePath).delete();
}
}
private void extractRarFile(String childName, FileHeader header, Archive archive) {
String outPath = fileDir + childName;
try(OutputStream ot = new FileOutputStream(outPath)) {
archive.extractFile(header, ot);
} catch (IOException | RarException e) {
e.printStackTrace();
}
}
}
}