缓存及队列实现抽象，提供JDK和REDIS两种实现

README.en.md

@@ -57,15 +57,13 @@ Considering space issues, the pictures of other types of documents will not be s
 - Redisson 
 - Jodconverter
 > Dependencies
-- Redis 
+- Redis(Optional, Unnecessary by default)
 - OpenOffice or LibreOffice
 
 1. First step：`git pull https://github.com/kekingcn/file-online-preview.git`
 
 2. Second step：configure redis address and OpenOffice directory，such as
 ```
-#=============================================#Spring Redisson Configuration#===================================#
-spring.redisson.address = 192.168.1.204:6379
 ##The folder for files which are uploaded to the server(Because of running as jar)
 file.dir = C:\\Users\\yudian\\Desktop\\dev\\
 ## openoffice configuration

README.md

@@ -50,15 +50,13 @@ QQ群号：613025121
 - redisson 
 - jodconverter
 > 依赖外部环境
-- redis 
+- redis (可选，默认不用)
 - OpenOffice或者LibreOffice
 
 1. 第一步：pull项目https://github.com/kekingcn/file-online-preview.git
 
-2. 第二步：配置redis地址和OpenOffice目录，如
+2. 第二步：配置OpenOffice目录，如
 ```
-#=============================================#spring Redisson配置#===================================#
-spring.redisson.address = 192.168.1.204:6379
 ##资源映射路径(因为jar方式运行的原因)
 file.dir = C:\\Users\\yudian\\Desktop\\dev\\
 ## openoffice相关配置

jodconverter-web/pom.xml

@@ -149,6 +149,11 @@
             <artifactId>guava</artifactId>
             <version>19.0</version>
         </dependency>
+        <dependency>
+            <groupId>com.googlecode.concurrentlinkedhashmap</groupId>
+            <artifactId>concurrentlinkedhashmap-lru</artifactId>
+            <version>1.4.2</version>
+        </dependency>
     </dependencies>
     <build>
         <resources>

jodconverter-web/src/main/java/cn/keking/config/RedissonConfig.java

@@ -1,10 +1,9 @@
 package cn.keking.config;
 
 import io.netty.channel.nio.NioEventLoopGroup;
-import org.redisson.Redisson;
-import org.redisson.api.RedissonClient;
 import org.redisson.client.codec.Codec;
 import org.redisson.config.Config;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -14,6 +13,7 @@ import org.springframework.util.ClassUtils;
  * Created by kl on 2017/09/26.
  * redisson 客户端配置
  */
+@ConditionalOnExpression("'${cache.type:default}'.equals('redis')")
 @ConfigurationProperties(prefix = "spring.redisson")
 @Configuration
 public class RedissonConfig {
@@ -42,8 +42,8 @@ public class RedissonConfig {
 
     private String codec="org.redisson.codec.JsonJacksonCodec";
 
-    @Bean(destroyMethod = "shutdown")
+    @Bean
-    RedissonClient redisson() throws Exception {
+    Config config() throws Exception {
         Config config = new Config();
         config.useSingleServer().setAddress(address)
                 .setConnectionMinimumIdleSize(connectionMinimumIdleSize)
@@ -69,7 +69,7 @@ public class RedissonConfig {
         config.setThreads(thread);
         config.setEventLoopGroup(new NioEventLoopGroup());
         config.setUseLinuxNativeEpoll(false);
-        return Redisson.create(config);
+        return config;
     }
 
     public int getThread() {

jodconverter-web/src/main/java/cn/keking/service/FileConverQueueTask.java

@@ -2,9 +2,8 @@ package cn.keking.service;
 
 import cn.keking.model.FileAttribute;
 import cn.keking.model.FileType;
+import cn.keking.service.cache.CacheService;
 import cn.keking.utils.FileUtils;
-import org.redisson.api.RBlockingQueue;
-import org.redisson.api.RedissonClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -28,7 +27,7 @@ public class FileConverQueueTask {
     FilePreviewFactory previewFactory;
 
     @Autowired
-    RedissonClient redissonClient;
+    CacheService cacheService;
 
     @Autowired
     FileUtils fileUtils;
@@ -36,7 +35,7 @@ public class FileConverQueueTask {
     @PostConstruct
     public void startTask(){
         ExecutorService executorService = Executors.newFixedThreadPool(3);
-        executorService.submit(new ConverTask(previewFactory,redissonClient,fileUtils));
+        executorService.submit(new ConverTask(previewFactory,cacheService,fileUtils));
         logger.info("队列处理文件转换任务启动完成 ");
     }
 
@@ -44,13 +43,13 @@ public class FileConverQueueTask {
 
         FilePreviewFactory previewFactory;
 
-        RedissonClient redissonClient;
+        CacheService cacheService;
 
         FileUtils fileUtils;
 
-        public ConverTask(FilePreviewFactory previewFactory, RedissonClient redissonClient,FileUtils fileUtils) {
+        public ConverTask(FilePreviewFactory previewFactory, CacheService cacheService,FileUtils fileUtils) {
             this.previewFactory = previewFactory;
-            this.redissonClient = redissonClient;
+            this.cacheService = cacheService;
             this.fileUtils=fileUtils;
         }
 
@@ -58,8 +57,7 @@ public class FileConverQueueTask {
         public void run() {
             while (true) {
                 try {
-                    final RBlockingQueue<String> queue = redissonClient.getBlockingQueue(FileConverQueueTask.queueTaskName);
+                    String url = cacheService.takeQueueTask();
-                    String url = queue.take();
                     if(url!=null){
                         FileAttribute fileAttribute=fileUtils.getFileAttribute(url);
                         logger.info("正在处理转换任务，文件名称【{}】",fileAttribute.getName());

jodconverter-web/src/main/java/cn/keking/service/cache/CacheService.java

@@ -0,0 +1,29 @@
+package cn.keking.service.cache;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @auther: chenjh
+ * @time: 2019/4/2 16:45
+ * @description
+ */
+public interface CacheService {
+    final String REDIS_FILE_PREVIEW_PDF_KEY = "converted-preview-pdf-file";
+    final String REDIS_FILE_PREVIEW_IMGS_KEY = "converted-preview-imgs-file";//压缩包内图片文件集合
+
+    final Integer DEFAULT_PDF_CAPACITY = 500000;
+    final Integer DEFAULT_IMG_CAPACITY = 500000;
+
+    void initPDFCachePool(Integer capacity);
+    void initIMGCachePool(Integer capacity);
+    void putPDFCache(String key, String value);
+    void putImgCache(String key, List<String> value);
+    Map<String, String> getPDFCache();
+    String getPDFCache(String key);
+    Map<String, List<String>> getImgCache();
+    List<String> getImgCache(String key);
+
+    void addQueueTask(String url);
+    String takeQueueTask() throws InterruptedException;
+}

jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceJDKImpl.java

@@ -0,0 +1,102 @@
+package cn.keking.service.cache.impl;
+
+import cn.keking.service.cache.CacheService;
+import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
+import com.googlecode.concurrentlinkedhashmap.Weighers;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+
+/**
+ * @auther: chenjh
+ * @time: 2019/4/2 17:21
+ * @description
+ */
+@Service
+@ConditionalOnExpression("'${cache.type:default}'.equals('default')")
+public class CacheServiceJDKImpl implements CacheService {
+
+    private Map<String, String> pdfCache;
+
+    private Map<String, List<String>> imgCache;
+
+    private static final int QUEUE_SIZE = 500000;
+
+    private BlockingQueue blockingQueue = new ArrayBlockingQueue(QUEUE_SIZE);
+
+    @Override
+    public void initPDFCachePool(Integer capacity) {
+        pdfCache = new ConcurrentLinkedHashMap.Builder<String, String>()
+                .maximumWeightedCapacity(capacity).weigher(Weighers.singleton())
+                .build();
+    }
+
+    @Override
+    public void initIMGCachePool(Integer capacity) {
+        imgCache = new ConcurrentLinkedHashMap.Builder<String, List<String>>()
+                .maximumWeightedCapacity(capacity).weigher(Weighers.singleton())
+                .build();
+    }
+
+    @Override
+    public void putPDFCache(String key, String value) {
+        if (pdfCache == null) {
+            initPDFCachePool(CacheService.DEFAULT_PDF_CAPACITY);
+        }
+        pdfCache.put(key, value);
+    }
+
+    @Override
+    public void putImgCache(String key, List<String> value) {
+        if (imgCache == null) {
+            initIMGCachePool(CacheService.DEFAULT_IMG_CAPACITY);
+        }
+        imgCache.put(key, value);
+    }
+
+    @Override
+    public Map<String, String> getPDFCache() {
+        if (pdfCache == null) {
+            initPDFCachePool(CacheService.DEFAULT_PDF_CAPACITY);
+        }
+        return pdfCache;
+    }
+
+    @Override
+    public String getPDFCache(String key) {
+        if (pdfCache == null) {
+            initPDFCachePool(CacheService.DEFAULT_PDF_CAPACITY);
+        }
+        return pdfCache.get(key);
+    }
+
+    @Override
+    public Map<String, List<String>> getImgCache() {
+        if (imgCache == null) {
+            initPDFCachePool(CacheService.DEFAULT_IMG_CAPACITY);
+        }
+        return imgCache;
+    }
+
+    @Override
+    public List<String> getImgCache(String key) {
+        if (imgCache == null) {
+            initPDFCachePool(CacheService.DEFAULT_IMG_CAPACITY);
+        }
+        return imgCache.get(key);
+    }
+
+    @Override
+    public void addQueueTask(String url) {
+        blockingQueue.add(url);
+    }
+
+    @Override
+    public String takeQueueTask() throws InterruptedException {
+        return String.valueOf(blockingQueue.take());
+    }
+}

jodconverter-web/src/main/java/cn/keking/service/cache/impl/CacheServiceRedisImpl.java

@@ -0,0 +1,91 @@
+package cn.keking.service.cache.impl;
+
+import cn.keking.service.FileConverQueueTask;
+import cn.keking.service.cache.CacheService;
+import org.redisson.Redisson;
+import org.redisson.api.RBlockingQueue;
+import org.redisson.api.RMapCache;
+import org.redisson.api.RedissonClient;
+import org.redisson.config.Config;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * @auther: chenjh
+ * @time: 2019/4/2 18:02
+ * @description
+ */
+@ConditionalOnExpression("'${cache.type:default}'.equals('redis')")
+@Service
+public class CacheServiceRedisImpl implements CacheService {
+
+    private Config config;
+
+    @Autowired
+    public CacheServiceRedisImpl(Config config) {
+        this.config = config;
+        this.redissonClient = Redisson.create(config);
+    }
+
+    private RedissonClient redissonClient;
+
+    @Override
+    public void initPDFCachePool(Integer capacity) {
+
+    }
+
+    @Override
+    public void initIMGCachePool(Integer capacity) {
+
+    }
+
+    @Override
+    public void putPDFCache(String key, String value) {
+        RMapCache<String, String> convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_PDF_KEY);
+        convertedList.fastPut(key, value);
+    }
+
+    @Override
+    public void putImgCache(String key, List<String> value) {
+        RMapCache<String, List<String>> convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_IMGS_KEY);
+        convertedList.fastPut(key, value);
+    }
+
+    @Override
+    public Map<String, String> getPDFCache() {
+        return redissonClient.getMapCache(REDIS_FILE_PREVIEW_PDF_KEY);
+    }
+
+    @Override
+    public String getPDFCache(String key) {
+        RMapCache<String, String> convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_PDF_KEY);
+        return convertedList.get(key);
+    }
+
+    @Override
+    public Map<String, List<String>> getImgCache() {
+        return redissonClient.getMapCache(REDIS_FILE_PREVIEW_IMGS_KEY);
+    }
+
+    @Override
+    public List<String> getImgCache(String key) {
+        RMapCache<String, List<String>> convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_IMGS_KEY);
+        return convertedList.get(key);
+    }
+
+    @Override
+    public void addQueueTask(String url) {
+        RBlockingQueue<String> queue = redissonClient.getBlockingQueue(FileConverQueueTask.queueTaskName);
+        queue.addAsync(url);
+    }
+
+    @Override
+    public String takeQueueTask() throws InterruptedException {
+        RBlockingQueue<String> queue = redissonClient.getBlockingQueue(FileConverQueueTask.queueTaskName);
+        return queue.take();
+    }
+}

jodconverter-web/src/main/java/cn/keking/utils/FileUtils.java

@@ -2,9 +2,8 @@ package cn.keking.utils;
 
 import cn.keking.model.FileAttribute;
 import cn.keking.model.FileType;
+import cn.keking.service.cache.CacheService;
 import com.google.common.collect.Lists;
-import org.redisson.api.RMapCache;
-import org.redisson.api.RedissonClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -27,11 +26,9 @@ import java.util.Map;
 public class FileUtils {
     Logger log= LoggerFactory.getLogger(getClass());
 
-
-    final String REDIS_FILE_PREVIEW_PDF_KEY = "converted-preview-pdf-file";
-    final String REDIS_FILE_PREVIEW_IMGS_KEY = "converted-preview-imgs-file";//压缩包内图片文件集合
     @Autowired
-    RedissonClient redissonClient;
+    CacheService cacheService;
+
     @Value("${file.dir}")
     String fileDir;
 
@@ -48,8 +45,7 @@ public class FileUtils {
      * @return
      */
     public Map<String, String> listConvertedFiles() {
-        RMapCache<String, String> convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_PDF_KEY);
+        return cacheService.getPDFCache();
-        return convertedList;
     }
 
     /**
@@ -57,8 +53,7 @@ public class FileUtils {
      * @return
      */
     public String getConvertedFile(String key) {
-        RMapCache<String, String> convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_PDF_KEY);
+        return cacheService.getPDFCache(key);
-        return convertedList.get(key);
     }
 
     /**
@@ -170,8 +165,7 @@ public class FileUtils {
     }
 
     public void addConvertedFile(String fileName, String value){
-        RMapCache<String, String> convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_PDF_KEY);
+        cacheService.putPDFCache(fileName, value);
-        convertedList.fastPut(fileName, value);
     }
 
     /**
@@ -180,8 +174,7 @@ public class FileUtils {
      * @return
      */
     public List getRedisImgUrls(String fileKey){
-        RMapCache<String, List> convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_IMGS_KEY);
+        return cacheService.getImgCache(fileKey);
-        return convertedList.get(fileKey);
     }
 
     /**
@@ -190,8 +183,7 @@ public class FileUtils {
      * @param imgs
      */
     public void setRedisImgUrls(String fileKey,List imgs){
-        RMapCache<String, List> convertedList = redissonClient.getMapCache(REDIS_FILE_PREVIEW_IMGS_KEY);
+        cacheService.putImgCache(fileKey, imgs);
-         convertedList.fastPut(fileKey,imgs);
     }
     /**
      * 判断文件编码格式

jodconverter-web/src/main/java/cn/keking/web/controller/OnlinePreviewController.java

@@ -1,12 +1,10 @@
 package cn.keking.web.controller;
 
-import cn.keking.service.FileConverQueueTask;
 import cn.keking.service.FilePreview;
 import cn.keking.service.FilePreviewFactory;
 
+import cn.keking.service.cache.CacheService;
 import org.apache.commons.io.IOUtils;
-import org.redisson.api.RBlockingQueue;
-import org.redisson.api.RedissonClient;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.ui.Model;
@@ -26,7 +24,6 @@ import java.net.URLConnection;
 import java.net.URLDecoder;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Map;
 
 /**
  * @author yudian-it
@@ -38,7 +35,7 @@ public class OnlinePreviewController {
     FilePreviewFactory previewFactory;
 
     @Autowired
-    RedissonClient redissonClient;
+    CacheService cacheService;
 
     /**
      * @param url
@@ -126,8 +123,7 @@ public class OnlinePreviewController {
     @GetMapping("/addTask")
     @ResponseBody
     public String addQueueTask(String url) {
-        final RBlockingQueue<String> queue = redissonClient.getBlockingQueue(FileConverQueueTask.queueTaskName);
+        cacheService.addQueueTask(url);
-        queue.addAsync(url);
         return "success";
     }
 

jodconverter-web/src/main/resources/application-dev.properties

@@ -1,5 +1,3 @@
-#redis连接
-spring.redisson.address = 192.168.1.204:6379
 ##资源映射路径
 file.dir = C:\\Users\\yudian\\Desktop\\dev\\
 spring.resources.static-locations = classpath:/META-INF/resources/,classpath:/resources/,classpath:/static/,classpath:/public/,file:${file.dir}
@@ -12,4 +10,7 @@ spring.http.multipart.max-file-size=100MB
 ##文本类型
 simText = txt,html,xml,java,properties,mp3,mp4,sql
 #多媒体类型
-media=mp3,mp4,flv,rmvb
+media=mp3,mp4,flv,rmvb
+#缓存及队列实现类型，默认为JDK实现，可选redis(需要加spring.redisson.address等配置)
+#cache.type = redis
+#spring.redisson.address = 192.168.1.204:6379

jodconverter-web/src/main/resources/application-prod.properties

@@ -1,19 +1,11 @@
-#=============================================#spring Redisson<6F><6E><EFBFBD><EFBFBD>#===================================#
-spring.redisson.address = 10.19.140.7:6379
-spring.redisson.database = 0
-##<23><>Դӳ<D4B4><D3B3>·<EFBFBD><C2B7>(<28><>Ϊjar<61><72>ʽ<EFBFBD><CABD><EFBFBD>е<EFBFBD>ԭ<EFBFBD><D4AD>)
 file.dir = /data/file-preview/convertedFile/
 spring.resources.static-locations = classpath:/META-INF/resources/,classpath:/resources/,classpath:/static/,classpath:/public/,file:${file.dir}
-## openoffice<63><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
 office.home = /opt/openoffice4
-
-## <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
 server.tomcat.uri-encoding = utf-8
 converted.file.charset = utf-8
-## <20>ļ<EFBFBD><C4BC>ϴ<EFBFBD><CFB4><EFBFBD><EFBFBD>ֵ
 spring.http.multipart.max-file-size = 100MB
-
-## ֧<>ֵ<EFBFBD><D6B5><EFBFBD><EFBFBD>ı<EFBFBD><C4B1><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD>
 simText = txt,html,xml,java,properties,sql,js,md,json,conf,ini,vue,php,py,bat,gitignore,log,htm,css,cnf
-
+media=mp3,mp4,flv
-media=mp3,mp4,flv
+cache.type = redis
+spring.redisson.address = 10.19.140.7:6379
+spring.redisson.database = 0

jodconverter-web/src/main/resources/application-uat.properties

@@ -1,15 +1,12 @@
-#=============================================#spring Redisson<6F><6E><EFBFBD><EFBFBD>#===================================#
-spring.redisson.address = 192.168.1.204:6379
-spring.redisson.database = 3
-##<23><>Դӳ<D4B4><D3B3>·<EFBFBD><C2B7>(<28><>Ϊjar<61><72>ʽ<EFBFBD><CABD><EFBFBD>е<EFBFBD>ԭ<EFBFBD><D4AD>)
 file.dir = /data/filepreview/
 spring.resources.static-locations = classpath:/META-INF/resources/,classpath:/resources/,classpath:/static/,classpath:/public/,file:${file.dir}
-## openoffice<63><65><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
 openOfficePath = 123
 office.home = /opt/openoffice4
 server.tomcat.uri-encoding = utf-8
 converted.file.charset = utf-8
 spring.http.multipart.max-file-size = 100MB
-## ֧<>ֵ<EFBFBD><D6B5><EFBFBD><EFBFBD>ı<EFBFBD><C4B1><EFBFBD>ʽ<EFBFBD><CABD><EFBFBD>ļ<EFBFBD><C4BC><EFBFBD><EFBFBD><EFBFBD>
 simText = txt,html,xml,java,properties,sql,js,md,json,conf,ini,vue,php,py,bat,gitignore,log,htm,css,cnf
-media=mp3,mp4,flv
+media=mp3,mp4,flv
+cache.type = redis
+spring.redisson.address = 192.168.1.204:6379
+spring.redisson.database = 3