Merge pull request #2 from zongzibinbin/main

11

docs/mallchat.sql

@@ -25,6 +25,7 @@ INSERT INTO `item_config` VALUES (2, 2, 'https://cdn-icons-png.flaticon.com/128/
 INSERT INTO `item_config` VALUES (3, 2, 'https://cdn-icons-png.flaticon.com/512/6198/6198527.png ', '抹茶聊天前10名注册的用户才能获得的专属徽章', '2023-05-07 17:50:31.100', '2023-05-07 18:12:01.448');
 INSERT INTO `item_config` VALUES (4, 2, 'https://cdn-icons-png.flaticon.com/512/10232/10232583.png', '抹茶聊天前100名注册的用户才能获得的专属徽章', '2023-05-07 17:50:31.109', '2023-05-07 17:56:36.059');
 INSERT INTO `item_config` VALUES (5, 2, 'https://cdn-icons-png.flaticon.com/128/2909/2909937.png', '抹茶知识星球成员的专属徽章', '2023-05-07 17:50:31.109', '2023-05-07 17:56:36.059');
+INSERT INTO `item_config` VALUES (6, 2, 'https://s2.loli.net/2023/06/15/O9FwjH4ciAuMSnL.png', '抹茶项目contributor专属徽章', '2023-05-07 17:50:31.109', '2023-05-07 17:56:36.059');
 
 -- ----------------------------
 -- Table structure for message
@@ -181,4 +182,11 @@ CREATE TABLE `user_role` (
   KEY `idx_role_id` (`role_id`) USING BTREE,
   KEY `idx_create_time` (`create_time`) USING BTREE,
   KEY `idx_update_time` (`update_time`) USING BTREE
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='用户角色关系表';
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='用户角色关系表';
+
+DROP TABLE IF EXISTS `sensitive_word`;
+CREATE TABLE `sensitive_word` (
+  `word` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '敏感词'
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='敏感词库';
+INSERT INTO `sensitive_word` (`word`) VALUES ('TMD');
+INSERT INTO `sensitive_word` (`word`) VALUES ('tmd');

docs/version/2023-06-17.sql

@@ -0,0 +1,6 @@
+DROP TABLE IF EXISTS `sensitive_word`;
+CREATE TABLE `sensitive_word` (
+  `word` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '敏感词'
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='敏感词库';
+INSERT INTO `sensitive_word` (`word`) VALUES ('TMD');
+INSERT INTO `sensitive_word` (`word`) VALUES ('tmd');

mallchat-common/pom.xml

@@ -114,6 +114,14 @@
             <artifactId>redisson-spring-boot-starter</artifactId>
             <version>3.17.1</version>
         </dependency>
+
+        <!-- Used for unit testing -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>${junit.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/ac/ACTrie.java

@@ -0,0 +1,100 @@
+package com.abin.mallchat.common.common.algorithm.ac;
+
+import com.google.common.collect.Lists;
+
+import javax.annotation.concurrent.NotThreadSafe;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * aho-corasick算法（又称AC自动机算法）
+ * Created by berg on 2023/6/18.
+ */
+@NotThreadSafe
+public class ACTrie {
+
+    // 根节点
+    private ACTrieNode root;
+
+    public ACTrie(List<String> words) {
+        words = words.stream().distinct().collect(Collectors.toList()); // 去重
+        root = new ACTrieNode();
+        for (String word : words) {
+            addWord(word);
+        }
+        initFailover();
+    }
+
+    public void addWord(String word) {
+        ACTrieNode walkNode = root;
+        char[] chars = word.toCharArray();
+        for (int i = 0; i < word.length(); i++) {
+            walkNode.addChildrenIfAbsent(chars[i]);
+            walkNode = walkNode.childOf(chars[i]);
+            walkNode.setDepth(i + 1);
+        }
+        walkNode.setLeaf(true);
+    }
+
+    /**
+     * 初始化节点中的回退指针
+     */
+    private void initFailover() {
+        //第一层的fail指针指向root
+        Queue<ACTrieNode> queue = new LinkedList<>();
+        Map<Character, ACTrieNode> children = root.getChildren();
+        for (ACTrieNode node : children.values()) {
+            node.setFailover(root);
+            queue.offer(node);
+        }
+        //构建剩余层数节点的fail指针,利用层次遍历
+        while (!queue.isEmpty()) {
+            ACTrieNode parentNode = queue.poll();
+            for (Map.Entry<Character, ACTrieNode> entry : parentNode.getChildren().entrySet()) {
+                ACTrieNode childNode = entry.getValue();
+                ACTrieNode failover = parentNode.getFailover();
+                // 在树中找到以childNode为结尾的字符串的最长前缀匹配，failover指向了这个最长前缀匹配的父节点
+                while (failover != null && (!failover.hasChild(entry.getKey()))) {
+                    failover = failover.getFailover();
+                }
+                //回溯到了root节点
+                if (failover == null) {
+                    childNode.setFailover(root);
+                } else {
+                    // 更新当前节点的回退指针
+                    childNode.setFailover(failover.childOf(entry.getKey()));
+                }
+                queue.offer(childNode);
+            }
+        }
+    }
+
+    /**
+     * 查询句子中包含的敏感词的起始位置和结束位置
+     *
+     * @param text
+     */
+    public List<MatchResult> matches(String text) {
+        List<MatchResult> result = Lists.newArrayList();
+        ACTrieNode walkNode = root;
+        for (int i = 0; i < text.length(); i++) {
+            char c = text.charAt(i);
+            while (!walkNode.hasChild(c) && walkNode.getFailover() != null) {
+                walkNode = walkNode.getFailover();
+            }
+            //如果因为当前节点的孩子节点有这个字符，则将walkNode替换为下面的孩子节点
+            if (walkNode.hasChild(c)) {
+                walkNode = walkNode.childOf(c);
+                // 检索到了敏感词
+                if (walkNode.isLeaf()) {
+                    result.add(new MatchResult(i - walkNode.getDepth() + 1, i + 1));
+                    // 模式串回退到最长可匹配前缀位置并开启新一轮的匹配
+                    // 这种回退方式将一个不漏的匹配到所有的敏感词，匹配结果的区间可能会有重叠的部分
+                    walkNode = walkNode.getFailover();
+                }
+            }
+        }
+        return result;
+    }
+
+}

mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/ac/ACTrieNode.java

@@ -0,0 +1,46 @@
+package com.abin.mallchat.common.common.algorithm.ac;
+
+import com.google.common.collect.Maps;
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.Map;
+
+/**
+ * Created by berg on 2023/6/18.
+ */
+@Getter
+@Setter
+public class ACTrieNode {
+
+    // 子节点
+    private Map<Character, ACTrieNode> children = Maps.newHashMap();
+
+    // 匹配过程中，如果模式串不匹配，模式串指针会回退到failover继续进行匹配
+    private ACTrieNode failover = null;
+
+    private int depth;
+
+    private boolean isLeaf = false;
+
+    public void addChildrenIfAbsent(char c) {
+        children.computeIfAbsent(c, (key) -> new ACTrieNode());
+    }
+
+    public ACTrieNode childOf(char c) {
+        return children.get(c);
+    }
+
+    public boolean hasChild(char c) {
+        return children.containsKey(c);
+    }
+
+    @Override
+    public String toString() {
+        return "ACTrieNode{" +
+                "failover=" + failover +
+                ", depth=" + depth +
+                ", isLeaf=" + isLeaf +
+                '}';
+    }
+}

mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/ac/MatchResult.java

@@ -0,0 +1,24 @@
+package com.abin.mallchat.common.common.algorithm.ac;
+
+import lombok.*;
+
+/**
+ * Created by berg on 2023/6/18.
+ */
+@Getter
+@Setter
+@AllArgsConstructor
+public class MatchResult {
+
+    private int startIndex;
+
+    private int endIndex;
+
+    @Override
+    public String toString() {
+        return "MatchResult{" +
+                "startIndex=" + startIndex +
+                ", endIndex=" + endIndex +
+                '}';
+    }
+}

mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/package-info.java

@@ -0,0 +1,4 @@
+/**
+ * Created by berg on 2023/6/18.
+ */
+package com.abin.mallchat.common.common.algorithm;

mallchat-common/src/main/java/com/abin/mallchat/common/common/service/cache/AbstractRedisStringCache.java

@@ -35,6 +35,7 @@ public abstract class AbstractRedisStringCache<IN, OUT> implements BatchCache<IN
 
     @Override
     public Map<IN, OUT> getBatch(List<IN> req) {
+        req = req.stream().distinct().collect(Collectors.toList());
         List<String> keys = req.stream().map(this::getKey).collect(Collectors.toList());
         List<OUT> valueList = RedisUtils.mget(keys, outClass);
         List<IN> loadReqs = new ArrayList<>();

mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/SensitiveWordUtils.java

@@ -1,34 +1,41 @@
 package com.abin.mallchat.common.common.utils;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
-import java.util.ArrayList;
+import java.nio.file.Paths;
-import java.util.Collections;
+import java.util.*;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
 
 /**
- * 敏感词过滤
+ * 敏感词工具类
  *
  * @author zhaoyuhang
- * @since 2023/06/11
+ * @date 2023/06/19
  */
 public final class SensitiveWordUtils {
-    private static SensitiveWordList wordList;
+    private static Word root = new Word(' '); // 敏感词字典的根节点
     private final static char replace = '*'; // 替代字符
-    private final static char[] skip = new char[]{ // 遇到这些字符就会跳过
+    private final static String skipChars = " !*-+_=,，.@;:；：。、？?（）()【】[]《》<>“”\"‘’"; // 遇到这些字符就会跳过
-            ' ', '!', '*', '-', '+', '_', '=', ',', '，', '.', '@', ';', ':', '；', '：'
+    private final static Set<Character> skipSet = new HashSet<>(); // 遇到这些字符就会跳过
-    };
+
+    static {
+        for (char c : skipChars.toCharArray()) {
+            skipSet.add(c);
+        }
+    }
+
 
     /**
-     * 有敏感词
+     * 判断文本中是否存在敏感词
      *
      * @param text 文本
-     * @return boolean
+     * @return true: 存在敏感词, false: 不存在敏感词
      */
     public static boolean hasSensitiveWord(String text) {
         if (StringUtils.isBlank(text)) return false;
@@ -42,210 +49,154 @@ public final class SensitiveWordUtils {
      * @return 替换后的文本
      */
     public static String filter(String text) {
-        if (wordList == null || wordList.size() == 0 || StringUtils.isBlank(text)) return text;
+        StringBuilder result = new StringBuilder(text);
-        char[] __char__ = text.toCharArray(); // 把String转化成char数组，便于遍历
+        int index = 0;
-        int i, j;
+        while (index < result.length()) {
-        Word word;
+            char c = result.charAt(index);
-        boolean flag; // 是否需要替换
+            if (skip(c)) {
-        for (i = 0; i < __char__.length; i++) { // 遍历所有字符
+                index++;
-            char c = __char__[i];
+                continue;
-            word = wordList.binaryGet(c); // 使用二分查找来寻找字符，提高效率
+            }
-            if (word != null) { // word != null说明找到了
+            Word word = root;
-                flag = false;
+            int start = index;
-                j = i + 1;
+            boolean found = false;
-                while (j < __char__.length) { // 开始逐个比较后面的字符
+            for (int i = index; i < result.length(); i++) {
-                    if (skip(__char__[j])) { // 跳过空格之类的无关字符
+                c = result.charAt(i);
-                        j++;
+                if (skip(c)) {
-                        continue;
+                    continue;
-                    }
-                    if (word.next != null) { // 字符串尚未结束，不确定是否存在敏感词
-                        /*
-                        以下代码并没有使用二分查找，因为以同一个字符开头的敏感词较少
-                        例如，wordList中记录了所有敏感词的开头第一个字，它的数量通常会有上千个
-                        假如现在锁定了字符“T”开头的敏感词，而“T”开头的敏感词只有10个，这时使用二分查找的效率反而低于顺序查找
-                         */
-                        word = word.next.get(__char__[j]);
-                        if (word == null) {
-                            break;
-                        }
-                        j++;
-                    } else { // 字符串已结束，存在敏感词汇
-                        flag = true;
-                        break;
-                    }
                 }
-                if (word != null && word.next == null) {
+                if (c >= 'A' && c <= 'Z') {
-                    flag = true;
+                    c += 32;
                 }
-                if (flag) { // 如果flag==true，说明检测出敏感粗，需要替换
+                word = word.next.get(c);
-                    while (i < j) {
+                if (word == null) {
-                        // if(skip(__char__[i])){ // 跳过空格之类的无关字符，如果要把空格也替换成'*'，则删除这个if语句
+                    break;
-                        //     i++;
+                }
-                        //     continue;
+                if (word.end) {
-                        // }
+                    found = true;
-                        __char__[i] = replace;
+                    for (int j = start; j <= i; j++) {
-                        i++;
+                        result.setCharAt(j, replace);
                     }
-                    i--;
+                    index = i;
                 }
             }
+            if (!found) {
+                index++;
+            }
         }
-        return new String(__char__);
+        return result.toString();
     }
 
+
     /**
      * 加载敏感词列表
      *
      * @param words 敏感词数组
      */
     public static void loadWord(List<String> words) {
-        if (words == null) return;
+        if (!CollectionUtils.isEmpty(words)) {
-        words = words.stream().distinct().collect(Collectors.toList()); // 去重
+            Word newRoot = new Word(' ');
-        char[] chars;
+            words.forEach(word -> loadWord(word, newRoot));
-        SensitiveWordList now;
+            root = newRoot;
-        Word word;
-        wordList = new SensitiveWordList();
-        for (String __word__ : words) {
-            if (__word__ == null) continue;
-            chars = __word__.toCharArray();
-            now = wordList;
-            word = null;
-            for (char c : chars) {
-                if (word != null) {
-                    if (word.next == null) word.next = new SensitiveWordList();
-                    now = word.next;
-                }
-                word = now.get(c);
-                if (word == null) word = now.add(c);
-            }
         }
-        sort(wordList);
     }
 
     /**
-     * 加载敏感词txt文件，每个敏感词独占一行，不可出现空格，空行，逗号等非文字内容,必须使用UTF-8编码
+     * 加载敏感词
      *
-     * @param path txt文件的绝对地址
+     * @param word 词
+     */
+    public static void loadWord(String word, Word root) {
+        if (StringUtils.isBlank(word)) {
+            return;
+        }
+        Word current = root;
+        for (int i = 0; i < word.length(); i++) {
+            char c = word.charAt(i);
+            // 如果是大写字母, 转换为小写
+            if (c >= 'A' && c <= 'Z') {
+                c += 32;
+            }
+            if (skip(c)) {
+                continue;
+            }
+            Word next = current.next.get(c);
+            if (next == null) {
+                next = new Word(c);
+                current.next.put(c, next);
+            }
+            current = next;
+        }
+        current.end = true;
+    }
+
+
+    /**
+     * 从文本文件中加载敏感词列表
+     *
+     * @param path 文本文件的绝对路径
      */
     public static void loadWordFromFile(String path) {
-        String encoding = "UTF-8";
+        try (InputStream inputStream = Files.newInputStream(Paths.get(path))) {
-        File file = new File(path);
+            loadWord(inputStream);
-        try {
-            if (file.isFile() && file.exists()) {
-                InputStreamReader inputStreamReader = new InputStreamReader(
-                        Files.newInputStream(file.toPath()), encoding
-                );
-                BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
-                String line;
-                ArrayList<String> list = new ArrayList<>();
-                while ((line = bufferedReader.readLine()) != null) {
-                    list.add(line);
-                }
-                bufferedReader.close();
-                inputStreamReader.close();
-                loadWord(list);
-            }
         } catch (IOException e) {
             e.printStackTrace();
         }
     }
 
     /**
-     * 对敏感词多叉树递增排序
+     * 从流中加载敏感词列表
      *
-     * @param list 待排序List
+     * @param inputStream 文本文件输入流
+     * @throws IOException IO异常
      */
-    private static void sort(SensitiveWordList list) {
+    public static void loadWord(InputStream inputStream) throws IOException {
-        if (list == null) return;
+        try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
-        Collections.sort(list); // 递增排序
+            String line;
-        for (Word word : list) {
+            ArrayList<String> list = new ArrayList<>();
-            sort(word.next);
+            while ((line = reader.readLine()) != null) {
+                list.add(line);
+            }
+            loadWord(list);
         }
     }
 
     /**
-     * 判断是否跳过当前字符
+     * 判断是否需要跳过当前字符
      *
      * @param c 待检测字符
-     * @return true:需要跳过   false:不需要跳过
+     * @return true: 需要跳过, false: 不需要跳过
      */
     private static boolean skip(char c) {
-        for (char c1 : skip) {
+        return skipSet.contains(c);
-            if (c1 == c) return true;
-        }
-        return false;
     }
 
     /**
-     * 敏感词列表
+     * 敏感词类
-     *
-     * @author zhaoyuhang
-     * @since 2023/06/11
      */
-    public static class SensitiveWordList extends ArrayList<Word> {
+    private static class Word {
-        public Word get(char c) {
+        // 当前字符
-            for (Word w : this) {
+        private final char c;
-                if (w.c == c) return w;
-            }
-            return null;
-        }
 
-        /**
+        // 结束标识
-         * 二分查找，必须先升序排序
+        private boolean end;
-         *
-         * @param c 需要查找的字符
-         * @return Word对象：如果找到   null:如果没找到
-         */
-        public Word binaryGet(char c) {
-            int left, right, key;
-            Word word;
-            left = 0;
-            right = this.size() - 1;
-            while (left <= right) {
-                key = (left + right) / 2;
-                word = get(key);
-                if (word.c == c) {
-                    return word;
-                } else if (word.c > c) {
-                    right = key - 1;
-                } else {
-                    left = key + 1;
-                }
-            }
-            return null;
-        }
 
-        public Word add(char c) {
+        // 下一层级的敏感词字典
-            Word word = new Word(c);
+        private Map<Character, Word> next;
-            super.add(word);
-            return word;
-        }
-
-    }
-
-    /**
-     * 敏感词
-     *
-     * @author zhaoyuhang
-     * @since 2023/06/11
-     */
-    public static class Word implements Comparable<Word> {
-        public char c;
-        public SensitiveWordList next = null;
 
         public Word(char c) {
             this.c = c;
-        }
+            this.end = false;
-
+            this.next = new HashMap<>();
-        @Override
-        public int compareTo(Word word) {
-            return c - word.c;
-        }
-
-        public String toString() {
-            return c + "(" + (next == null ? null : next.size()) + ")";
         }
     }
+
+    public static void main(String[] args) {
+        String text = "白日,梦";
+        String filter = filter(text);
+        System.out.println(filter);
+
+
+    }
+
 }
-
-

mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/SensitiveWordUtils0.java

@@ -0,0 +1,70 @@
+package com.abin.mallchat.common.common.utils;
+
+import com.abin.mallchat.common.common.algorithm.ac.ACTrie;
+import com.abin.mallchat.common.common.algorithm.ac.MatchResult;
+import org.HdrHistogram.ConcurrentHistogram;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * 基于ac自动机实现的敏感词过滤工具类
+ * 可以用来替代{@link ConcurrentHistogram}
+ * 为了兼容提供了相同的api接口 {@code hasSensitiveWord}
+ *
+ * Created by berg on 2023/6/18.
+ */
+public class SensitiveWordUtils0 {
+
+    private final static char mask_char = '*'; // 替代字符
+
+    private static ACTrie ac_trie = null;
+
+    /**
+     * 有敏感词
+     *
+     * @param text 文本
+     * @return boolean
+     */
+    public static boolean hasSensitiveWord(String text) {
+        if (StringUtils.isBlank(text)) return false;
+        return !Objects.equals(filter(text), text);
+    }
+
+    /**
+     * 敏感词替换
+     *
+     * @param text 待替换文本
+     * @return 替换后的文本
+     */
+    public static String filter(String text) {
+        if (StringUtils.isBlank(text)) return text;
+        List<MatchResult> matchResults = ac_trie.matches(text);
+        StringBuffer result = new StringBuffer(text);
+        // matchResults是按照startIndex排序的，因此可以通过不断更新endIndex最大值的方式算出尚未被替代部分
+        int endIndex = 0;
+        for (MatchResult matchResult : matchResults) {
+            endIndex = Math.max(endIndex, matchResult.getEndIndex());
+            replaceBetween(result, matchResult.getStartIndex(), endIndex);
+        }
+        return result.toString();
+    }
+
+    private static void replaceBetween(StringBuffer buffer, int startIndex, int endIndex) {
+        for (int i = startIndex; i < endIndex; i++) {
+            buffer.setCharAt(i, mask_char);
+        }
+    }
+
+    /**
+     * 加载敏感词列表
+     *
+     * @param words 敏感词数组
+     */
+    public static void loadWord(List<String> words) {
+        if (words == null) return;
+        ac_trie = new ACTrie(words);
+    }
+
+}

mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/service/impl/SensitiveWordServiceImpl.java

@@ -4,8 +4,10 @@ import com.abin.mallchat.common.common.utils.SensitiveWordUtils;
 import com.abin.mallchat.common.sensitive.dao.SensitiveWordDao;
 import com.abin.mallchat.common.sensitive.domain.SensitiveWord;
 import com.abin.mallchat.common.sensitive.service.ISensitiveWordService;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.PostConstruct;
@@ -13,18 +15,25 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 @Service
+@Slf4j
 public class SensitiveWordServiceImpl implements ISensitiveWordService {
     @Autowired
     private SensitiveWordDao sensitiveWordDao;
+    @Autowired
+    private ThreadPoolTaskExecutor threadPoolTaskExecutor;
 
     @PostConstruct
     public void initSensitiveWord() {
-        List<SensitiveWord> list = sensitiveWordDao.list();
+        threadPoolTaskExecutor.execute(() -> {
-        if (!CollectionUtils.isEmpty(list)) {
+            log.info("[initSensitiveWord] start");
-            List<String> wordList = list.stream()
+            List<SensitiveWord> list = sensitiveWordDao.list();
-                    .map(SensitiveWord::getWord)
+            if (!CollectionUtils.isEmpty(list)) {
-                    .collect(Collectors.toList());
+                List<String> wordList = list.stream()
-            SensitiveWordUtils.loadWord(wordList);
+                        .map(SensitiveWord::getWord)
-        }
+                        .collect(Collectors.toList());
+                SensitiveWordUtils.loadWord(wordList);
+            }
+            log.info("[initSensitiveWord] end; loading sensitiveWords num:{}", list.size());
+        });
     }
 }

mallchat-common/src/main/java/com/abin/mallchat/common/user/domain/dto/ItemInfoDTO.java

@@ -19,9 +19,17 @@ import lombok.NoArgsConstructor;
 public class ItemInfoDTO {
     @ApiModelProperty(value = "徽章id")
     private Long itemId;
+    @ApiModelProperty(value = "是否需要刷新")
+    private Boolean needRefresh = Boolean.TRUE;
     @ApiModelProperty("徽章图像")
     private String img;
     @ApiModelProperty("徽章说明")
     private String describe;
 
+    public static ItemInfoDTO skip(Long itemId) {
+        ItemInfoDTO dto = new ItemInfoDTO();
+        dto.setItemId(itemId);
+        dto.setNeedRefresh(Boolean.FALSE);
+        return dto;
+    }
 }

mallchat-common/src/main/java/com/abin/mallchat/common/user/domain/dto/SummeryInfoDTO.java

@@ -21,6 +21,8 @@ import java.util.List;
 public class SummeryInfoDTO {
     @ApiModelProperty(value = "用户id")
     private Long uid;
+    @ApiModelProperty(value = "是否需要刷新")
+    private Boolean needRefresh = Boolean.TRUE;
     @ApiModelProperty(value = "用户昵称")
     private String name;
     @ApiModelProperty(value = "用户头像")
@@ -32,4 +34,10 @@ public class SummeryInfoDTO {
     @ApiModelProperty(value = "用户拥有的徽章id列表")
     List<Long> itemIds;
 
+    public static SummeryInfoDTO skip(Long uid) {
+        SummeryInfoDTO dto = new SummeryInfoDTO();
+        dto.setUid(uid);
+        dto.setNeedRefresh(Boolean.FALSE);
+        return dto;
+    }
 }

mallchat-common/src/test/java/com/abin/mallchat/common/common/algorithm/ac/ACTrieTest.java

@@ -0,0 +1,63 @@
+package com.abin.mallchat.common.common.algorithm.ac;
+
+import com.google.common.collect.Lists;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Created by berg on 2023/6/18.
+ */
+public class ACTrieTest {
+
+    private final static List<String> ALPHABET = Lists.newArrayList("abc", "bcd", "cde");
+
+    private static ACTrie trie(List<String> keywords) {
+        return new ACTrie(keywords);
+    }
+
+    @Test
+    public void test_TextIsLongerThanKeyword() {
+        final ACTrie trie = trie(ALPHABET);
+        final String text = " " + ALPHABET.get(0);
+        List<MatchResult> matchResults = trie.matches(text);
+        checkResult(matchResults.get(0), 1, 4, ALPHABET.get(0), text);
+    }
+
+    @Test
+    public void test_VariousKeywordsOneMatch() {
+        final ACTrie trie = trie(ALPHABET);
+        final String text = "bcd";
+        List<MatchResult> matchResults = trie.matches(text);
+        checkResult(matchResults.get(0), 0, 3, ALPHABET.get(1), text);
+    }
+
+    @Test
+    public void test_VariousKeywordsMultiMatch() {
+        final ACTrie trie = trie(ALPHABET);
+        final String text = "abcd";
+        List<MatchResult> matchResults = trie.matches(text);
+        assertEquals(2, matchResults.size());
+        checkResult(matchResults.get(0), 0, 3, ALPHABET.get(0), text);
+        checkResult(matchResults.get(1), 1, 4, ALPHABET.get(1), text);
+    }
+
+    @Test
+    public void test_VariousKeywordsMultiMatch2() {
+        final ACTrie trie = trie(ALPHABET);
+        final String text = "abcde";
+        List<MatchResult> matchResults = trie.matches(text);
+        assertEquals(3, matchResults.size());
+        checkResult(matchResults.get(0), 0, 3, ALPHABET.get(0), text);
+        checkResult(matchResults.get(1), 1, 4, ALPHABET.get(1), text);
+        checkResult(matchResults.get(2), 2, 5, ALPHABET.get(2), text);
+    }
+
+    private void checkResult(MatchResult matchResult, int expectedStart, int expectedEnd, String expectedKeyword, String text) {
+        assertEquals("Start of match should have been " + expectedStart, expectedStart, matchResult.getStartIndex());
+        assertEquals("End of match should have been " + expectedEnd, expectedEnd, matchResult.getEndIndex());
+        assertEquals(expectedKeyword, text.substring(expectedStart, expectedEnd));
+    }
+}

mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/controller/ChatController.java

@@ -15,6 +15,7 @@ import com.abin.mallchat.custom.user.service.impl.UserServiceImpl;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.*;
 
@@ -22,6 +23,7 @@ import javax.validation.Valid;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * <p>
@@ -57,6 +59,20 @@ public class ChatController {
         return ApiResult.success(memberPage);
     }
 
+    @GetMapping("/public/member/page/v1")
+    @ApiOperation("群成员列表/v1")
+    @FrequencyControl(time = 120, count = 20, target = FrequencyControl.Target.IP)
+    public ApiResult<CursorPageBaseResp<ChatMemberRespV1>> getMemberPage1(@Valid CursorPageBaseReq request) {
+        CursorPageBaseResp<ChatMemberResp> memberPage = chatService.getMemberPage(request);
+        filterBlackMember(memberPage);
+        List<ChatMemberRespV1> collect = memberPage.getList().stream().map(a -> {
+            ChatMemberRespV1 v1 = new ChatMemberRespV1();
+            BeanUtils.copyProperties(a, v1);
+            return v1;
+        }).collect(Collectors.toList());
+        return ApiResult.success(CursorPageBaseResp.init(memberPage, collect));
+    }
+
     @GetMapping("/member/list")
     @ApiOperation("房间内的所有群成员列表-@专用")
     public ApiResult<List<ChatMemberListResp>> getMemberList(@Valid ChatMessageMemberReq chatMessageMemberReq) {
@@ -81,25 +97,30 @@ public class ChatController {
     @Autowired
     private UserServiceImpl userService;
 
+    @GetMapping("/public/msg/page/v1")
+    @ApiOperation("消息列表/v1")
+    @FrequencyControl(time = 120, count = 20, target = FrequencyControl.Target.IP)
+    public ApiResult<CursorPageBaseResp<ChatMessageRespV1>> getMsgPage(@Valid ChatMessagePageReq request) {
+        CursorPageBaseResp<ChatMessageResp> msgPage = chatService.getMsgPage(request, RequestHolder.get().getUid());
+        filterBlackMsg(msgPage);
+        List<ChatMessageRespV1> collect = msgPage.getList().stream().map(a -> {
+            ChatMessageRespV1 v1 = new ChatMessageRespV1();
+            BeanUtils.copyProperties(a, v1);
+            return v1;
+        }).collect(Collectors.toList());
+        return ApiResult.success(CursorPageBaseResp.init(msgPage, collect));
+    }
+
     @GetMapping("/public/msg/page")
     @ApiOperation("消息列表")
     @FrequencyControl(time = 120, count = 20, target = FrequencyControl.Target.IP)
-    public ApiResult<CursorPageBaseResp<ChatMessageResp>> getMsgPage(@Valid ChatMessagePageReq request) {
+    public ApiResult<CursorPageBaseResp<ChatMessageResp>> getMsgPage1(@Valid ChatMessagePageReq request) {
 //        black(request);
         CursorPageBaseResp<ChatMessageResp> msgPage = chatService.getMsgPage(request, RequestHolder.get().getUid());
         filterBlackMsg(msgPage);
         return ApiResult.success(msgPage);
     }
 
-    private void black(CursorPageBaseReq baseReq) {
-        if (baseReq.getPageSize() > 50) {
-            log.info("limit request:{}", baseReq);
-            baseReq.setPageSize(10);
-            userService.blackIp(RequestHolder.get().getIp());
-        }
-
-    }
-
     private void filterBlackMsg(CursorPageBaseResp<ChatMessageResp> memberPage) {
         Set<String> blackMembers = getBlackUidSet();
         memberPage.getList().removeIf(a -> blackMembers.contains(a.getFromUser().getUid().toString()));

mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/domain/vo/response/ChatMemberRespV1.java

@@ -0,0 +1,30 @@
+package com.abin.mallchat.custom.chat.domain.vo.response;
+
+import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.Date;
+
+/**
+ * Description: 群成员列表的成员信息
+ * Author: <a href="https://github.com/zongzibinbin">abin</a>
+ * Date: 2023-03-23
+ */
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class ChatMemberRespV1 {
+    @ApiModelProperty("uid")
+    private Long uid;
+    /**
+     * @see com.abin.mallchat.common.user.domain.enums.ChatActiveStatusEnum
+     */
+    @ApiModelProperty("在线状态 1在线 2离线")
+    private Integer activeStatus;
+    @ApiModelProperty("最后一次上下线时间")
+    private Date lastOptTime;
+}

mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/domain/vo/response/ChatMessageRespV1.java

@@ -0,0 +1,60 @@
+package com.abin.mallchat.custom.chat.domain.vo.response;
+
+import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.Date;
+
+/**
+ * Description: 消息
+ * Author: <a href="https://github.com/zongzibinbin">abin</a>
+ * Date: 2023-03-23
+ */
+@Data
+@Builder
+@AllArgsConstructor
+@NoArgsConstructor
+public class ChatMessageRespV1 {
+
+    @ApiModelProperty("发送者信息")
+    private UserInfo fromUser;
+    @ApiModelProperty("消息详情")
+    private Message message;
+
+    @Data
+    public static class UserInfo {
+        @ApiModelProperty("用户id")
+        private Long uid;
+    }
+
+    @Data
+    public static class Message {
+        @ApiModelProperty("消息id")
+        private Long id;
+        @ApiModelProperty("消息发送时间")
+        private Date sendTime;
+        @ApiModelProperty("消息类型 1正常文本 2.撤回消息")
+        private Integer type;
+        @ApiModelProperty("消息内容不同的消息类型，内容体不同，见https://www.yuque.com/snab/mallcaht/rkb2uz5k1qqdmcmd")
+        private Object body;
+        @ApiModelProperty("消息标记")
+        private MessageMark messageMark;
+
+    }
+
+    @Data
+    public static class MessageMark {
+        @ApiModelProperty("点赞数")
+        private Integer likeCount;
+        @ApiModelProperty("该用户是否已经点赞 0否 1是")
+        private Integer userLike;
+        @ApiModelProperty("举报数")
+        private Integer dislikeCount;
+        @ApiModelProperty("该用户是否已经举报 0否 1是")
+        private Integer userDislike;
+    }
+
+}

mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/service/strategy/msg/TextMsgHandler.java

@@ -46,7 +46,7 @@ public class TextMsgHandler extends AbstractMsgHandler {
     private UserInfoCache userInfoCache;
     @Autowired
     private IRoleService iRoleService;
-    @Autowired
+    
     private static final PrioritizedUrlTitleDiscover URL_TITLE_DISCOVER = new PrioritizedUrlTitleDiscover();
 
     @Override

mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/service/impl/UserServiceImpl.java

@@ -143,7 +143,11 @@ public class UserServiceImpl implements UserService {
         List<Long> uidList = getNeedSyncUidList(req.getReqList());
         //加载用户信息
         Map<Long, SummeryInfoDTO> batch = userSummaryCache.getBatch(uidList);
-        return new ArrayList<>(batch.values());
+        return req.getReqList()
+                .stream()
+                .map(a -> batch.containsKey(a.getUid()) ? batch.get(a.getUid()) : SummeryInfoDTO.skip(a.getUid()))
+                .filter(Objects::nonNull)
+                .collect(Collectors.toList());
     }
 
     @Override
@@ -151,7 +155,7 @@ public class UserServiceImpl implements UserService {
         return req.getReqList().stream().map(a -> {
             ItemConfig itemConfig = itemCache.getById(a.getItemId());
             if (Objects.nonNull(a.getLastModifyTime()) && a.getLastModifyTime() >= itemConfig.getUpdateTime().getTime()) {
-                return null;
+                return ItemInfoDTO.skip(a.getItemId());
             }
             ItemInfoDTO dto = new ItemInfoDTO();
             dto.setItemId(itemConfig.getId());

mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/service/impl/WebSocketServiceImpl.java

@@ -90,7 +90,6 @@ public class WebSocketServiceImpl implements WebSocketService {
      */
     @SneakyThrows
     @Override
-    @FrequencyControl(time = 10, count = 2, spEl = "T(com.abin.mallchat.common.common.utils.RequestHolder).get().getIp()")
     @FrequencyControl(time = 100, count = 5, spEl = "T(com.abin.mallchat.common.common.utils.RequestHolder).get().getIp()")
     public void handleLoginReq(Channel channel) {
         //生成随机不重复的登录码

mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/websocket/NettyWebSocketServer.java

@@ -96,7 +96,6 @@ public class NettyWebSocketServer {
                 });
         // 启动服务器，监听端口，阻塞直到启动成功
         serverBootstrap.bind(WEB_SOCKET_PORT).sync();
-        System.out.println("启动成功");
     }
 
 }

pom.xml

@@ -34,7 +34,7 @@
         <mysql-connector.version>8.0.29</mysql-connector.version>
         <spring-data-commons.version>2.7.5</spring-data-commons.version>
         <jjwt.version>0.9.1</jjwt.version>
-        <aliyun-oss.version>2.5.0</aliyun-oss.version>
+        <aliyun-oss.version>3.16.0</aliyun-oss.version>
         <logstash-logback.version>7.2</logstash-logback.version>
         <minio.version>8.4.5</minio.version>
         <jaxb-api.version>2.3.1</jaxb-api.version>