diff --git a/docs/mallchat.sql b/docs/mallchat.sql index 96eb44d..91491cd 100644 --- a/docs/mallchat.sql +++ b/docs/mallchat.sql @@ -25,6 +25,7 @@ INSERT INTO `item_config` VALUES (2, 2, 'https://cdn-icons-png.flaticon.com/128/ INSERT INTO `item_config` VALUES (3, 2, 'https://cdn-icons-png.flaticon.com/512/6198/6198527.png ', '抹茶聊天前10名注册的用户才能获得的专属徽章', '2023-05-07 17:50:31.100', '2023-05-07 18:12:01.448'); INSERT INTO `item_config` VALUES (4, 2, 'https://cdn-icons-png.flaticon.com/512/10232/10232583.png', '抹茶聊天前100名注册的用户才能获得的专属徽章', '2023-05-07 17:50:31.109', '2023-05-07 17:56:36.059'); INSERT INTO `item_config` VALUES (5, 2, 'https://cdn-icons-png.flaticon.com/128/2909/2909937.png', '抹茶知识星球成员的专属徽章', '2023-05-07 17:50:31.109', '2023-05-07 17:56:36.059'); +INSERT INTO `item_config` VALUES (6, 2, 'https://s2.loli.net/2023/06/15/O9FwjH4ciAuMSnL.png', '抹茶项目contributor专属徽章', '2023-05-07 17:50:31.109', '2023-05-07 17:56:36.059'); -- ---------------------------- -- Table structure for message @@ -181,4 +182,11 @@ CREATE TABLE `user_role` ( KEY `idx_role_id` (`role_id`) USING BTREE, KEY `idx_create_time` (`create_time`) USING BTREE, KEY `idx_update_time` (`update_time`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='用户角色关系表'; \ No newline at end of file +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='用户角色关系表'; + +DROP TABLE IF EXISTS `sensitive_word`; +CREATE TABLE `sensitive_word` ( + `word` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '敏感词' +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='敏感词库'; +INSERT INTO `sensitive_word` (`word`) VALUES ('TMD'); +INSERT INTO `sensitive_word` (`word`) VALUES ('tmd'); \ No newline at end of file diff --git a/docs/version/2023-06-17.sql b/docs/version/2023-06-17.sql new file mode 100644 index 0000000..81218f3 --- /dev/null +++ b/docs/version/2023-06-17.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS `sensitive_word`; +CREATE TABLE `sensitive_word` ( + `word` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '敏感词' +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='敏感词库'; +INSERT INTO `sensitive_word` (`word`) VALUES ('TMD'); +INSERT INTO `sensitive_word` (`word`) VALUES ('tmd'); \ No newline at end of file diff --git a/mallchat-common/pom.xml b/mallchat-common/pom.xml index aefa9d6..f97a2e2 100644 --- a/mallchat-common/pom.xml +++ b/mallchat-common/pom.xml @@ -114,6 +114,14 @@ redisson-spring-boot-starter 3.17.1 + + + + junit + junit + ${junit.version} + test + diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/ac/ACTrie.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/ac/ACTrie.java new file mode 100644 index 0000000..523e4b2 --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/ac/ACTrie.java @@ -0,0 +1,100 @@ +package com.abin.mallchat.common.common.algorithm.ac; + +import com.google.common.collect.Lists; + +import javax.annotation.concurrent.NotThreadSafe; +import java.util.*; +import java.util.stream.Collectors; + +/** + * aho-corasick算法（又称AC自动机算法） + * Created by berg on 2023/6/18. + */ +@NotThreadSafe +public class ACTrie { + + // 根节点 + private ACTrieNode root; + + public ACTrie(List words) { + words = words.stream().distinct().collect(Collectors.toList()); // 去重 + root = new ACTrieNode(); + for (String word : words) { + addWord(word); + } + initFailover(); + } + + public void addWord(String word) { + ACTrieNode walkNode = root; + char[] chars = word.toCharArray(); + for (int i = 0; i < word.length(); i++) { + walkNode.addChildrenIfAbsent(chars[i]); + walkNode = walkNode.childOf(chars[i]); + walkNode.setDepth(i + 1); + } + walkNode.setLeaf(true); + } + + /** + * 初始化节点中的回退指针 + */ + private void initFailover() { + //第一层的fail指针指向root + Queue queue = new LinkedList<>(); + Map children = root.getChildren(); + for (ACTrieNode node : children.values()) { + node.setFailover(root); + queue.offer(node); + } + //构建剩余层数节点的fail指针,利用层次遍历 + while (!queue.isEmpty()) { + ACTrieNode parentNode = queue.poll(); + for (Map.Entry entry : parentNode.getChildren().entrySet()) { + ACTrieNode childNode = entry.getValue(); + ACTrieNode failover = parentNode.getFailover(); + // 在树中找到以childNode为结尾的字符串的最长前缀匹配，failover指向了这个最长前缀匹配的父节点 + while (failover != null && (!failover.hasChild(entry.getKey()))) { + failover = failover.getFailover(); + } + //回溯到了root节点 + if (failover == null) { + childNode.setFailover(root); + } else { + // 更新当前节点的回退指针 + childNode.setFailover(failover.childOf(entry.getKey())); + } + queue.offer(childNode); + } + } + } + + /** + * 查询句子中包含的敏感词的起始位置和结束位置 + * + * @param text + */ + public List matches(String text) { + List result = Lists.newArrayList(); + ACTrieNode walkNode = root; + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + while (!walkNode.hasChild(c) && walkNode.getFailover() != null) { + walkNode = walkNode.getFailover(); + } + //如果因为当前节点的孩子节点有这个字符，则将walkNode替换为下面的孩子节点 + if (walkNode.hasChild(c)) { + walkNode = walkNode.childOf(c); + // 检索到了敏感词 + if (walkNode.isLeaf()) { + result.add(new MatchResult(i - walkNode.getDepth() + 1, i + 1)); + // 模式串回退到最长可匹配前缀位置并开启新一轮的匹配 + // 这种回退方式将一个不漏的匹配到所有的敏感词，匹配结果的区间可能会有重叠的部分 + walkNode = walkNode.getFailover(); + } + } + } + return result; + } + +} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/ac/ACTrieNode.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/ac/ACTrieNode.java new file mode 100644 index 0000000..1068524 --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/ac/ACTrieNode.java @@ -0,0 +1,46 @@ +package com.abin.mallchat.common.common.algorithm.ac; + +import com.google.common.collect.Maps; +import lombok.Getter; +import lombok.Setter; + +import java.util.Map; + +/** + * Created by berg on 2023/6/18. + */ +@Getter +@Setter +public class ACTrieNode { + + // 子节点 + private Map children = Maps.newHashMap(); + + // 匹配过程中，如果模式串不匹配，模式串指针会回退到failover继续进行匹配 + private ACTrieNode failover = null; + + private int depth; + + private boolean isLeaf = false; + + public void addChildrenIfAbsent(char c) { + children.computeIfAbsent(c, (key) -> new ACTrieNode()); + } + + public ACTrieNode childOf(char c) { + return children.get(c); + } + + public boolean hasChild(char c) { + return children.containsKey(c); + } + + @Override + public String toString() { + return "ACTrieNode{" + + "failover=" + failover + + ", depth=" + depth + + ", isLeaf=" + isLeaf + + '}'; + } +} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/ac/MatchResult.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/ac/MatchResult.java new file mode 100644 index 0000000..36087e4 --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/ac/MatchResult.java @@ -0,0 +1,24 @@ +package com.abin.mallchat.common.common.algorithm.ac; + +import lombok.*; + +/** + * Created by berg on 2023/6/18. + */ +@Getter +@Setter +@AllArgsConstructor +public class MatchResult { + + private int startIndex; + + private int endIndex; + + @Override + public String toString() { + return "MatchResult{" + + "startIndex=" + startIndex + + ", endIndex=" + endIndex + + '}'; + } +} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/package-info.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/package-info.java new file mode 100644 index 0000000..3b8d300 --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/common/algorithm/package-info.java @@ -0,0 +1,4 @@ +/** + * Created by berg on 2023/6/18. + */ +package com.abin.mallchat.common.common.algorithm; \ No newline at end of file diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/service/cache/AbstractRedisStringCache.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/service/cache/AbstractRedisStringCache.java index 16b0f16..9f6b087 100644 --- a/mallchat-common/src/main/java/com/abin/mallchat/common/common/service/cache/AbstractRedisStringCache.java +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/common/service/cache/AbstractRedisStringCache.java @@ -35,6 +35,7 @@ public abstract class AbstractRedisStringCache implements BatchCache getBatch(List req) { + req = req.stream().distinct().collect(Collectors.toList()); List keys = req.stream().map(this::getKey).collect(Collectors.toList()); List valueList = RedisUtils.mget(keys, outClass); List loadReqs = new ArrayList<>(); diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/SensitiveWordUtils.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/SensitiveWordUtils.java new file mode 100644 index 0000000..63b4e24 --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/SensitiveWordUtils.java @@ -0,0 +1,182 @@ +package com.abin.mallchat.common.common.utils; + +import org.apache.commons.lang3.StringUtils; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.util.*; +import java.util.stream.Collectors; + +public final class SensitiveWordUtils { + private static Map wordMap; // 敏感词Map + private final static char replace = '*'; // 替代字符 + private final static char[] skip = new char[]{ // 遇到这些字符就会跳过 + ' ', '!', '*', '-', '+', '_', '=', ',', '，', '.', '@', ';', ':', '；', '：' + }; + + /** + * 判断文本中是否存在敏感词 + * + * @param text 文本 + * @return true: 存在敏感词, false: 不存在敏感词 + */ + public static boolean hasSensitiveWord(String text) { + if (StringUtils.isBlank(text)) return false; + return !Objects.equals(filter(text), text); + } + + /** + * 过滤敏感词并替换为指定字符 + * + * @param text 待替换文本 + * @return 替换后的文本 + */ + /** + * 敏感词替换 + * + * @param text 待替换文本 + * @return 替换后的文本 + */ + public static String filter(String text) { + if (wordMap == null || wordMap.isEmpty() || StringUtils.isBlank(text)) return text; + char[] chars = text.toCharArray(); // 将文本转换为字符数组 + int length = chars.length; // 文本长度 + StringBuilder result = new StringBuilder(length); // 存储替换后的结果 + int i = 0; // 当前遍历的字符索引 + while (i < length) { + char c = chars[i]; // 当前字符 + if (skip(c)) { // 如果是需要跳过的字符，则直接追加到结果中 + i++; + continue; + } + int startIndex = i; // 敏感词匹配的起始索引 + Map currentMap = wordMap; // 当前层级的敏感词字典 + int matchLength = 0; // 匹配到的敏感词长度 + for (int j = i; j < length; j++) { + char ch = chars[j]; // 当前遍历的字符 + if (skip(ch)) { // 如果是需要跳过的字符，则直接追加到结果中 + continue; + } + Word word = currentMap.get(ch); // 获取当前字符在当前层级的敏感词字典中对应的敏感词节点 + if (word == null) { // 如果未匹配到敏感词节点，则终止循环 + break; + } + if (word.end) { // 如果当前节点是敏感词的最后一个节点，则记录匹配长度 + matchLength = j - startIndex + 1; + } + currentMap = word.next; // 进入下一层级的敏感词字典 + if (word.next == null) { // 如果当前节点是敏感词的最后一个节点，则记录匹配长度 + matchLength = j - startIndex + 1; + } + } + if (matchLength > 0) { // 如果匹配到敏感词，则将对应的字符替换为指定替代字符 + for (int j = startIndex; j < startIndex + matchLength; j++) { + chars[j] = replace; + } + } + i += matchLength > 0 ? matchLength : 1; // 更新当前索引，跳过匹配到的敏感词 + } + result.append(chars); // 将匹配到的敏感词追加到结果中 + return result.toString(); + } + + + /** + * 加载敏感词列表 + * + * @param words 敏感词数组 + */ + public static void loadWord(List words) { + if (words == null) return; + words = words.stream().distinct().collect(Collectors.toList()); // 去重 + wordMap = new HashMap<>(); // 创建敏感词字典的根节点 + for (String word : words) { + if (word == null) continue; + char[] chars = word.toCharArray(); + Map currentMap = wordMap; // 当前层级的敏感词字典 + for (int i = 0; i < chars.length; i++) { + char c = chars[i]; + Word currentWord = currentMap.get(c); + if (currentWord == null) { + Word newWord = new Word(c); // 创建新的敏感词节点 + currentMap.put(c, newWord); // 将节点添加到当前层级的敏感词字典中 + if (i == chars.length - 1) { + newWord.end = true; // 添加结束标志 + } + currentMap = newWord.next = new HashMap<>(); // 进入下一层级 + } else { + currentMap = currentWord.next; // 存在该字符的节点，则进入下一层级 + } + } + } + } + + + /** + * 从文本文件中加载敏感词列表 + * + * @param path 文本文件的绝对路径 + */ + public static void loadWordFromFile(String path) { + String encoding = "UTF-8"; + File file = new File(path); + try { + if (file.isFile() && file.exists()) { + InputStreamReader inputStreamReader = new InputStreamReader( + Files.newInputStream(file.toPath()), encoding + ); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader); + String line; + ArrayList list = new ArrayList<>(); + while ((line = bufferedReader.readLine()) != null) { + list.add(line); + } + bufferedReader.close(); + inputStreamReader.close(); + loadWord(list); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * 判断是否需要跳过当前字符 + * + * @param c 待检测字符 + * @return true: 需要跳过, false: 不需要跳过 + */ + private static boolean skip(char c) { + for (char skipChar : skip) { + if (skipChar == c) return true; + } + return false; + } + + /** + * 敏感词类 + */ + private static class Word { + // 当前字符 + private char c; + + // 结束标识 + private boolean end; + + // 下一层级的敏感词字典 + private Map next; + + public Word(char c) { + this.c = c; + } + } + + public static void main(String[] args) { + List strings = Arrays.asList("白日梦", "白痴", "白痴是你","TMD"); + loadWord(strings); + System.out.println(filter("TMD,白痴是你吗")); + } +} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/SensitiveWordUtils0.java b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/SensitiveWordUtils0.java new file mode 100644 index 0000000..f67c694 --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/SensitiveWordUtils0.java @@ -0,0 +1,70 @@ +package com.abin.mallchat.common.common.utils; + +import com.abin.mallchat.common.common.algorithm.ac.ACTrie; +import com.abin.mallchat.common.common.algorithm.ac.MatchResult; +import org.HdrHistogram.ConcurrentHistogram; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Objects; + +/** + * 基于ac自动机实现的敏感词过滤工具类 + * 可以用来替代{@link ConcurrentHistogram} + * 为了兼容提供了相同的api接口 {@code hasSensitiveWord} + * + * Created by berg on 2023/6/18. + */ +public class SensitiveWordUtils0 { + + private final static char mask_char = '*'; // 替代字符 + + private static ACTrie ac_trie = null; + + /** + * 有敏感词 + * + * @param text 文本 + * @return boolean + */ + public static boolean hasSensitiveWord(String text) { + if (StringUtils.isBlank(text)) return false; + return !Objects.equals(filter(text), text); + } + + /** + * 敏感词替换 + * + * @param text 待替换文本 + * @return 替换后的文本 + */ + public static String filter(String text) { + if (StringUtils.isBlank(text)) return text; + List matchResults = ac_trie.matches(text); + StringBuffer result = new StringBuffer(text); + // matchResults是按照startIndex排序的，因此可以通过不断更新endIndex最大值的方式算出尚未被替代部分 + int endIndex = 0; + for (MatchResult matchResult : matchResults) { + endIndex = Math.max(endIndex, matchResult.getEndIndex()); + replaceBetween(result, matchResult.getStartIndex(), endIndex); + } + return result.toString(); + } + + private static void replaceBetween(StringBuffer buffer, int startIndex, int endIndex) { + for (int i = startIndex; i < endIndex; i++) { + buffer.setCharAt(i, mask_char); + } + } + + /** + * 加载敏感词列表 + * + * @param words 敏感词数组 + */ + public static void loadWord(List words) { + if (words == null) return; + ac_trie = new ACTrie(words); + } + +} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/dao/SensitiveWordDao.java b/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/dao/SensitiveWordDao.java new file mode 100644 index 0000000..72ccee5 --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/dao/SensitiveWordDao.java @@ -0,0 +1,17 @@ +package com.abin.mallchat.common.sensitive.dao; + +import com.abin.mallchat.common.sensitive.domain.SensitiveWord; +import com.abin.mallchat.common.sensitive.mapper.SensitiveWordMapper; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import org.springframework.stereotype.Service; + +/** + * 敏感词DAO + * + * @author zhaoyuhang + * @since 2023/06/11 + */ +@Service +public class SensitiveWordDao extends ServiceImpl { + +} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/domain/SensitiveWord.java b/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/domain/SensitiveWord.java new file mode 100644 index 0000000..ba9b2ba --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/domain/SensitiveWord.java @@ -0,0 +1,18 @@ +package com.abin.mallchat.common.sensitive.domain; + +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * 敏感词 + * + * @author zhaoyuhang + * @since 2023/06/11 + */ +@Data +@EqualsAndHashCode(callSuper = false) +@TableName("sensitive_word") +public class SensitiveWord { + private String word; +} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/mapper/SensitiveWordMapper.java b/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/mapper/SensitiveWordMapper.java new file mode 100644 index 0000000..5cfc22c --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/mapper/SensitiveWordMapper.java @@ -0,0 +1,14 @@ +package com.abin.mallchat.common.sensitive.mapper; + +import com.abin.mallchat.common.sensitive.domain.SensitiveWord; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * 敏感词Mapper + * + * @author zhaoyuhang + * @since 2023-05-21 + */ +public interface SensitiveWordMapper extends BaseMapper { + +} \ No newline at end of file diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/service/ISensitiveWordService.java b/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/service/ISensitiveWordService.java new file mode 100644 index 0000000..f62783a --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/service/ISensitiveWordService.java @@ -0,0 +1,5 @@ +package com.abin.mallchat.common.sensitive.service; + +public interface ISensitiveWordService { + +} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/service/impl/SensitiveWordServiceImpl.java b/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/service/impl/SensitiveWordServiceImpl.java new file mode 100644 index 0000000..19bab7a --- /dev/null +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/service/impl/SensitiveWordServiceImpl.java @@ -0,0 +1,39 @@ +package com.abin.mallchat.common.sensitive.service.impl; + +import com.abin.mallchat.common.common.utils.SensitiveWordUtils; +import com.abin.mallchat.common.sensitive.dao.SensitiveWordDao; +import com.abin.mallchat.common.sensitive.domain.SensitiveWord; +import com.abin.mallchat.common.sensitive.service.ISensitiveWordService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Service; + +import javax.annotation.PostConstruct; +import java.util.List; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class SensitiveWordServiceImpl implements ISensitiveWordService { + @Autowired + private SensitiveWordDao sensitiveWordDao; + @Autowired + private ThreadPoolTaskExecutor threadPoolTaskExecutor; + + @PostConstruct + public void initSensitiveWord() { + threadPoolTaskExecutor.execute(() -> { + log.info("[initSensitiveWord] start"); + List list = sensitiveWordDao.list(); + if (!CollectionUtils.isEmpty(list)) { + List wordList = list.stream() + .map(SensitiveWord::getWord) + .collect(Collectors.toList()); + SensitiveWordUtils.loadWord(wordList); + } + log.info("[initSensitiveWord] end; loading sensitiveWords num:{}", list.size()); + }); + } +} diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/user/domain/dto/ItemInfoDTO.java b/mallchat-common/src/main/java/com/abin/mallchat/common/user/domain/dto/ItemInfoDTO.java index 3fda5c5..253e13b 100644 --- a/mallchat-common/src/main/java/com/abin/mallchat/common/user/domain/dto/ItemInfoDTO.java +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/user/domain/dto/ItemInfoDTO.java @@ -19,9 +19,17 @@ import lombok.NoArgsConstructor; public class ItemInfoDTO { @ApiModelProperty(value = "徽章id") private Long itemId; + @ApiModelProperty(value = "是否需要刷新") + private Boolean needRefresh = Boolean.TRUE; @ApiModelProperty("徽章图像") private String img; @ApiModelProperty("徽章说明") private String describe; + public static ItemInfoDTO skip(Long itemId) { + ItemInfoDTO dto = new ItemInfoDTO(); + dto.setItemId(itemId); + dto.setNeedRefresh(Boolean.FALSE); + return dto; + } } diff --git a/mallchat-common/src/main/java/com/abin/mallchat/common/user/domain/dto/SummeryInfoDTO.java b/mallchat-common/src/main/java/com/abin/mallchat/common/user/domain/dto/SummeryInfoDTO.java index 70751d8..60cb132 100644 --- a/mallchat-common/src/main/java/com/abin/mallchat/common/user/domain/dto/SummeryInfoDTO.java +++ b/mallchat-common/src/main/java/com/abin/mallchat/common/user/domain/dto/SummeryInfoDTO.java @@ -21,6 +21,8 @@ import java.util.List; public class SummeryInfoDTO { @ApiModelProperty(value = "用户id") private Long uid; + @ApiModelProperty(value = "是否需要刷新") + private Boolean needRefresh = Boolean.TRUE; @ApiModelProperty(value = "用户昵称") private String name; @ApiModelProperty(value = "用户头像") @@ -32,4 +34,10 @@ public class SummeryInfoDTO { @ApiModelProperty(value = "用户拥有的徽章id列表") List itemIds; + public static SummeryInfoDTO skip(Long uid) { + SummeryInfoDTO dto = new SummeryInfoDTO(); + dto.setUid(uid); + dto.setNeedRefresh(Boolean.FALSE); + return dto; + } } diff --git a/mallchat-common/src/test/java/com/abin/mallchat/common/common/algorithm/ac/ACTrieTest.java b/mallchat-common/src/test/java/com/abin/mallchat/common/common/algorithm/ac/ACTrieTest.java new file mode 100644 index 0000000..2fb749f --- /dev/null +++ b/mallchat-common/src/test/java/com/abin/mallchat/common/common/algorithm/ac/ACTrieTest.java @@ -0,0 +1,63 @@ +package com.abin.mallchat.common.common.algorithm.ac; + +import com.google.common.collect.Lists; +import org.junit.Test; + +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Created by berg on 2023/6/18. + */ +public class ACTrieTest { + + private final static List ALPHABET = Lists.newArrayList("abc", "bcd", "cde"); + + private static ACTrie trie(List keywords) { + return new ACTrie(keywords); + } + + @Test + public void test_TextIsLongerThanKeyword() { + final ACTrie trie = trie(ALPHABET); + final String text = " " + ALPHABET.get(0); + List matchResults = trie.matches(text); + checkResult(matchResults.get(0), 1, 4, ALPHABET.get(0), text); + } + + @Test + public void test_VariousKeywordsOneMatch() { + final ACTrie trie = trie(ALPHABET); + final String text = "bcd"; + List matchResults = trie.matches(text); + checkResult(matchResults.get(0), 0, 3, ALPHABET.get(1), text); + } + + @Test + public void test_VariousKeywordsMultiMatch() { + final ACTrie trie = trie(ALPHABET); + final String text = "abcd"; + List matchResults = trie.matches(text); + assertEquals(2, matchResults.size()); + checkResult(matchResults.get(0), 0, 3, ALPHABET.get(0), text); + checkResult(matchResults.get(1), 1, 4, ALPHABET.get(1), text); + } + + @Test + public void test_VariousKeywordsMultiMatch2() { + final ACTrie trie = trie(ALPHABET); + final String text = "abcde"; + List matchResults = trie.matches(text); + assertEquals(3, matchResults.size()); + checkResult(matchResults.get(0), 0, 3, ALPHABET.get(0), text); + checkResult(matchResults.get(1), 1, 4, ALPHABET.get(1), text); + checkResult(matchResults.get(2), 2, 5, ALPHABET.get(2), text); + } + + private void checkResult(MatchResult matchResult, int expectedStart, int expectedEnd, String expectedKeyword, String text) { + assertEquals("Start of match should have been " + expectedStart, expectedStart, matchResult.getStartIndex()); + assertEquals("End of match should have been " + expectedEnd, expectedEnd, matchResult.getEndIndex()); + assertEquals(expectedKeyword, text.substring(expectedStart, expectedEnd)); + } +} diff --git a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/controller/ChatController.java b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/controller/ChatController.java index 65200ae..ac0154e 100644 --- a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/controller/ChatController.java +++ b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/controller/ChatController.java @@ -15,6 +15,7 @@ import com.abin.mallchat.custom.user.service.impl.UserServiceImpl; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.*; @@ -22,6 +23,7 @@ import javax.validation.Valid; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; /** *

@@ -57,6 +59,20 @@ public class ChatController { return ApiResult.success(memberPage); } + @GetMapping("/public/member/page/v1") + @ApiOperation("群成员列表/v1") + @FrequencyControl(time = 120, count = 20, target = FrequencyControl.Target.IP) + public ApiResult> getMemberPage1(@Valid CursorPageBaseReq request) { + CursorPageBaseResp memberPage = chatService.getMemberPage(request); + filterBlackMember(memberPage); + List collect = memberPage.getList().stream().map(a -> { + ChatMemberRespV1 v1 = new ChatMemberRespV1(); + BeanUtils.copyProperties(a, v1); + return v1; + }).collect(Collectors.toList()); + return ApiResult.success(CursorPageBaseResp.init(memberPage, collect)); + } + @GetMapping("/member/list") @ApiOperation("房间内的所有群成员列表-@专用") public ApiResult> getMemberList(@Valid ChatMessageMemberReq chatMessageMemberReq) { @@ -81,25 +97,30 @@ public class ChatController { @Autowired private UserServiceImpl userService; + @GetMapping("/public/msg/page/v1") + @ApiOperation("消息列表/v1") + @FrequencyControl(time = 120, count = 20, target = FrequencyControl.Target.IP) + public ApiResult> getMsgPage(@Valid ChatMessagePageReq request) { + CursorPageBaseResp msgPage = chatService.getMsgPage(request, RequestHolder.get().getUid()); + filterBlackMsg(msgPage); + List collect = msgPage.getList().stream().map(a -> { + ChatMessageRespV1 v1 = new ChatMessageRespV1(); + BeanUtils.copyProperties(a, v1); + return v1; + }).collect(Collectors.toList()); + return ApiResult.success(CursorPageBaseResp.init(msgPage, collect)); + } + @GetMapping("/public/msg/page") @ApiOperation("消息列表") @FrequencyControl(time = 120, count = 20, target = FrequencyControl.Target.IP) - public ApiResult> getMsgPage(@Valid ChatMessagePageReq request) { + public ApiResult> getMsgPage1(@Valid ChatMessagePageReq request) { // black(request); CursorPageBaseResp msgPage = chatService.getMsgPage(request, RequestHolder.get().getUid()); filterBlackMsg(msgPage); return ApiResult.success(msgPage); } - private void black(CursorPageBaseReq baseReq) { - if (baseReq.getPageSize() > 50) { - log.info("limit request:{}", baseReq); - baseReq.setPageSize(10); - userService.blackIp(RequestHolder.get().getIp()); - } - - } - private void filterBlackMsg(CursorPageBaseResp memberPage) { Set blackMembers = getBlackUidSet(); memberPage.getList().removeIf(a -> blackMembers.contains(a.getFromUser().getUid().toString())); diff --git a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/domain/vo/response/ChatMemberRespV1.java b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/domain/vo/response/ChatMemberRespV1.java new file mode 100644 index 0000000..bf7d24d --- /dev/null +++ b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/domain/vo/response/ChatMemberRespV1.java @@ -0,0 +1,30 @@ +package com.abin.mallchat.custom.chat.domain.vo.response; + +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Date; + +/** + * Description: 群成员列表的成员信息 + * Author: abin + * Date: 2023-03-23 + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ChatMemberRespV1 { + @ApiModelProperty("uid") + private Long uid; + /** + * @see com.abin.mallchat.common.user.domain.enums.ChatActiveStatusEnum + */ + @ApiModelProperty("在线状态 1在线 2离线") + private Integer activeStatus; + @ApiModelProperty("最后一次上下线时间") + private Date lastOptTime; +} diff --git a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/domain/vo/response/ChatMessageRespV1.java b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/domain/vo/response/ChatMessageRespV1.java new file mode 100644 index 0000000..c43507b --- /dev/null +++ b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/domain/vo/response/ChatMessageRespV1.java @@ -0,0 +1,60 @@ +package com.abin.mallchat.custom.chat.domain.vo.response; + +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Date; + +/** + * Description: 消息 + * Author: abin + * Date: 2023-03-23 + */ +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ChatMessageRespV1 { + + @ApiModelProperty("发送者信息") + private UserInfo fromUser; + @ApiModelProperty("消息详情") + private Message message; + + @Data + public static class UserInfo { + @ApiModelProperty("用户id") + private Long uid; + } + + @Data + public static class Message { + @ApiModelProperty("消息id") + private Long id; + @ApiModelProperty("消息发送时间") + private Date sendTime; + @ApiModelProperty("消息类型 1正常文本 2.撤回消息") + private Integer type; + @ApiModelProperty("消息内容不同的消息类型，内容体不同，见https://www.yuque.com/snab/mallcaht/rkb2uz5k1qqdmcmd") + private Object body; + @ApiModelProperty("消息标记") + private MessageMark messageMark; + + } + + @Data + public static class MessageMark { + @ApiModelProperty("点赞数") + private Integer likeCount; + @ApiModelProperty("该用户是否已经点赞 0否 1是") + private Integer userLike; + @ApiModelProperty("举报数") + private Integer dislikeCount; + @ApiModelProperty("该用户是否已经举报 0否 1是") + private Integer userDislike; + } + +} diff --git a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/service/strategy/msg/TextMsgHandler.java b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/service/strategy/msg/TextMsgHandler.java index 33762ca..9b91bc2 100644 --- a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/service/strategy/msg/TextMsgHandler.java +++ b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chat/service/strategy/msg/TextMsgHandler.java @@ -9,6 +9,7 @@ import com.abin.mallchat.common.chat.domain.enums.MessageTypeEnum; import com.abin.mallchat.common.chat.service.cache.MsgCache; import com.abin.mallchat.common.common.domain.enums.YesOrNoEnum; import com.abin.mallchat.common.common.utils.AssertUtil; +import com.abin.mallchat.common.common.utils.SensitiveWordUtils; import com.abin.mallchat.common.common.utils.discover.PrioritizedUrlTitleDiscover; import com.abin.mallchat.common.user.domain.entity.User; import com.abin.mallchat.common.user.domain.enums.RoleEnum; @@ -82,7 +83,7 @@ public class TextMsgHandler extends AbstractMsgHandler { MessageExtra extra = Optional.ofNullable(msg.getExtra()).orElse(new MessageExtra()); Message update = new Message(); update.setId(msg.getId()); - update.setContent(body.getContent()); + update.setContent(SensitiveWordUtils.filter(body.getContent())); update.setExtra(extra); //如果有回复消息 if (Objects.nonNull(body.getReplyMsgId())) { diff --git a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/service/impl/UserServiceImpl.java b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/service/impl/UserServiceImpl.java index f8c0106..dc63558 100644 --- a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/service/impl/UserServiceImpl.java +++ b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/service/impl/UserServiceImpl.java @@ -4,6 +4,7 @@ import cn.hutool.core.util.StrUtil; import com.abin.mallchat.common.common.event.UserBlackEvent; import com.abin.mallchat.common.common.event.UserRegisterEvent; import com.abin.mallchat.common.common.utils.AssertUtil; +import com.abin.mallchat.common.common.utils.SensitiveWordUtils; import com.abin.mallchat.common.user.dao.BlackDao; import com.abin.mallchat.common.user.dao.ItemConfigDao; import com.abin.mallchat.common.user.dao.UserBackpackDao; @@ -74,7 +75,9 @@ public class UserServiceImpl implements UserService { @Transactional public void modifyName(Long uid, ModifyNameReq req) { //判断名字是不是重复 - User oldUser = userDao.getByName(req.getName()); + String newName = req.getName(); + AssertUtil.isFalse(SensitiveWordUtils.hasSensitiveWord(newName), "名字中包含敏感词，请重新输入"); // 判断名字中有没有敏感词 + User oldUser = userDao.getByName(newName); AssertUtil.isEmpty(oldUser, "名字已经被抢占了，请换一个哦~~"); //判断改名卡够不够 UserBackpack firstValidItem = userBackpackDao.getFirstValidItem(uid, ItemEnum.MODIFY_NAME_CARD.getId()); @@ -140,7 +143,11 @@ public class UserServiceImpl implements UserService { List uidList = getNeedSyncUidList(req.getReqList()); //加载用户信息 Map batch = userSummaryCache.getBatch(uidList); - return new ArrayList<>(batch.values()); + return req.getReqList() + .stream() + .map(a -> batch.containsKey(a.getUid()) ? batch.get(a.getUid()) : SummeryInfoDTO.skip(a.getUid())) + .filter(Objects::nonNull) + .collect(Collectors.toList()); } @Override @@ -148,7 +155,7 @@ public class UserServiceImpl implements UserService { return req.getReqList().stream().map(a -> { ItemConfig itemConfig = itemCache.getById(a.getItemId()); if (Objects.nonNull(a.getLastModifyTime()) && a.getLastModifyTime() >= itemConfig.getUpdateTime().getTime()) { - return null; + return ItemInfoDTO.skip(a.getItemId()); } ItemInfoDTO dto = new ItemInfoDTO(); dto.setItemId(itemConfig.getId()); diff --git a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/service/impl/WebSocketServiceImpl.java b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/service/impl/WebSocketServiceImpl.java index e994bf0..7282473 100644 --- a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/service/impl/WebSocketServiceImpl.java +++ b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/service/impl/WebSocketServiceImpl.java @@ -90,7 +90,6 @@ public class WebSocketServiceImpl implements WebSocketService { */ @SneakyThrows @Override - @FrequencyControl(time = 10, count = 2, spEl = "T(com.abin.mallchat.common.common.utils.RequestHolder).get().getIp()") @FrequencyControl(time = 100, count = 5, spEl = "T(com.abin.mallchat.common.common.utils.RequestHolder).get().getIp()") public void handleLoginReq(Channel channel) { //生成随机不重复的登录码 diff --git a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/websocket/NettyWebSocketServer.java b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/websocket/NettyWebSocketServer.java index fb9d912..4ebf470 100644 --- a/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/websocket/NettyWebSocketServer.java +++ b/mallchat-custom-server/src/main/java/com/abin/mallchat/custom/user/websocket/NettyWebSocketServer.java @@ -15,6 +15,7 @@ import io.netty.handler.logging.LogLevel; import io.netty.handler.logging.LoggingHandler; import io.netty.handler.stream.ChunkedWriteHandler; import io.netty.handler.timeout.IdleStateHandler; +import io.netty.util.NettyRuntime; import io.netty.util.concurrent.Future; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Configuration; @@ -28,7 +29,7 @@ public class NettyWebSocketServer { public static final int WEB_SOCKET_PORT = 8090; // 创建线程池执行器 private EventLoopGroup bossGroup = new NioEventLoopGroup(1); - private EventLoopGroup workerGroup = new NioEventLoopGroup(8); + private EventLoopGroup workerGroup = new NioEventLoopGroup(NettyRuntime.availableProcessors()); /** * 启动 ws server @@ -95,7 +96,6 @@ public class NettyWebSocketServer { }); // 启动服务器，监听端口，阻塞直到启动成功 serverBootstrap.bind(WEB_SOCKET_PORT).sync(); - System.out.println("启动成功"); } }