From 8e6841ba9f0013f682e427b65e911a1ed32f4b84 Mon Sep 17 00:00:00 2001 From: zhongzb <972627721@qq.com> Date: Mon, 13 Nov 2023 22:31:03 +0800 Subject: [PATCH] =?UTF-8?q?feat:websocket=E5=8E=8B=E6=B5=8B=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/strategy/msg/TextMsgHandler.java | 2 +- .../common/algorithm/acpro/ACProTrie.java | 141 ++++++++++++++++++ .../sensitiveWord/ACFilter.java | 2 +- .../algorithm/sensitiveWord/ACProFilter.java | 35 +++++ .../sensitiveWord/DFAFilter.java | 2 +- .../sensitiveWord/IWordFactory.java | 2 +- .../sensitiveWord/SensitiveWordBs.java | 2 +- .../sensitiveWord/SensitiveWordFilter.java | 2 +- .../common/config/SensitiveWordConfig.java | 4 +- .../common/sensitive/MyWordFactory.java | 2 +- .../user/service/impl/UserServiceImpl.java | 2 +- .../abin/mallchat/common/SensitiveTest.java | 19 ++- 12 files changed, 201 insertions(+), 14 deletions(-) create mode 100644 mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/acpro/ACProTrie.java rename mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/{utils => algorithm}/sensitiveWord/ACFilter.java (96%) create mode 100644 mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/ACProFilter.java rename mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/{utils => algorithm}/sensitiveWord/DFAFilter.java (98%) rename mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/{utils => algorithm}/sensitiveWord/IWordFactory.java (79%) rename mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/{utils => algorithm}/sensitiveWord/SensitiveWordBs.java (96%) rename mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/{utils => algorithm}/sensitiveWord/SensitiveWordFilter.java (89%) diff --git a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/chat/service/strategy/msg/TextMsgHandler.java b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/chat/service/strategy/msg/TextMsgHandler.java index e7dbbaf..257dc1f 100644 --- a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/chat/service/strategy/msg/TextMsgHandler.java +++ b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/chat/service/strategy/msg/TextMsgHandler.java @@ -14,7 +14,7 @@ import com.abin.mallchat.common.common.domain.enums.YesOrNoEnum; import com.abin.mallchat.common.common.utils.AssertUtil; import com.abin.mallchat.common.common.utils.discover.PrioritizedUrlDiscover; import com.abin.mallchat.common.common.utils.discover.domain.UrlInfo; -import com.abin.mallchat.common.common.utils.sensitiveWord.SensitiveWordBs; +import com.abin.mallchat.common.common.algorithm.sensitiveWord.SensitiveWordBs; import com.abin.mallchat.common.user.domain.entity.User; import com.abin.mallchat.common.user.domain.enums.RoleEnum; import com.abin.mallchat.common.user.service.IRoleService; diff --git a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/acpro/ACProTrie.java b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/acpro/ACProTrie.java new file mode 100644 index 0000000..de85a3d --- /dev/null +++ b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/acpro/ACProTrie.java @@ -0,0 +1,141 @@ +package com.abin.mallchat.common.common.algorithm.acpro; + +import java.util.*; + +/** + *@author CtrlCver + *@date 2024/1/12 + *@description: AC自动机 + */ +public class ACProTrie { + + private final static char MASK = '*'; // 替代字符 + + private Word root; + + // 节点 + static class Word{ + // 判断是否是敏感词结尾 + boolean end=false; + // 失败回调节点/状态 + Word failOver=null; + // 记录字符偏移 + int depth=0; + // 下个自动机状态 + Map next=new HashMap<>(); + public boolean hasChild(char c) { + return next.containsKey(c); + } + } + //构建ACTrie + public void createACTrie(List list){ + Word currentNode = new Word(); + root=currentNode; + for(String key : list) + { + currentNode=root; + for(int j=0;j queue=new LinkedList<>(); + Map children=root.next; + for(Word node:children.values()) + { + node.failOver=root; + queue.offer(node); + } + while(!queue.isEmpty()) + { + Word parentNode=queue.poll(); + for(Map.Entry entry:parentNode.next.entrySet()) + { + Word childNode=entry.getValue(); + Word failOver=parentNode.failOver; + while(failOver!=null&&(!failOver.next.containsKey(entry.getKey()))){ + failOver=failOver.failOver; + } + if(failOver==null){ + childNode.failOver=root; + }else{ + childNode.failOver=failOver.next.get(entry.getKey()); + } + queue.offer(childNode); + } + } + } + // 匹配 + public String match(String matchWord) + { + Word walkNode=root; + char[] wordArray=matchWord.toCharArray(); + for(int i=0;i0) + { + length--; + wordArray[i+length]=MASK; + } + // 直接跳到最后的end节点failOver + i=i+length; + walkNode = sentinelB.failOver; + }else{ + int length=walkNode.depth; + while (length>0){ + length--; + wordArray[i-length]=MASK; + } + walkNode = walkNode.failOver; + } + } + } + } + return new String(wordArray); + } +} diff --git a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/ACFilter.java b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/ACFilter.java similarity index 96% rename from mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/ACFilter.java rename to mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/ACFilter.java index 19fc589..ee576ad 100644 --- a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/ACFilter.java +++ b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/ACFilter.java @@ -1,4 +1,4 @@ -package com.abin.mallchat.common.common.utils.sensitiveWord; +package com.abin.mallchat.common.common.algorithm.sensitiveWord; import com.abin.mallchat.common.common.algorithm.ac.ACTrie; import com.abin.mallchat.common.common.algorithm.ac.MatchResult; diff --git a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/ACProFilter.java b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/ACProFilter.java new file mode 100644 index 0000000..8791ed1 --- /dev/null +++ b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/ACProFilter.java @@ -0,0 +1,35 @@ +package com.abin.mallchat.common.common.algorithm.sensitiveWord; + +import com.abin.mallchat.common.common.algorithm.ac.ACTrie; +import com.abin.mallchat.common.common.algorithm.acpro.ACProTrie; +import io.micrometer.core.instrument.util.StringUtils; + +import java.util.List; +import java.util.Objects; + +/** + *@author CtrlCver + *@date 2024/1/12 + *@description: 基于ACFilter的优化增强版本 + */ +public class ACProFilter implements SensitiveWordFilter{ + private ACProTrie acProTrie; + + @Override + public boolean hasSensitiveWord(String text) { + if(StringUtils.isBlank(text)) return false; + return !Objects.equals(filter(text),text); + } + + @Override + public String filter(String text) { + return acProTrie.match(text); + } + + @Override + public void loadWord(List words) { + if (words == null) return; + acProTrie = new ACProTrie(); + acProTrie.createACTrie(words); + } +} diff --git a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/DFAFilter.java b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/DFAFilter.java similarity index 98% rename from mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/DFAFilter.java rename to mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/DFAFilter.java index 2a90a53..4e67b7f 100644 --- a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/DFAFilter.java +++ b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/DFAFilter.java @@ -1,4 +1,4 @@ -package com.abin.mallchat.common.common.utils.sensitiveWord; +package com.abin.mallchat.common.common.algorithm.sensitiveWord; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; diff --git a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/IWordFactory.java b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/IWordFactory.java similarity index 79% rename from mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/IWordFactory.java rename to mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/IWordFactory.java index 7aed4ea..4d9ad81 100644 --- a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/IWordFactory.java +++ b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/IWordFactory.java @@ -1,4 +1,4 @@ -package com.abin.mallchat.common.common.utils.sensitiveWord; +package com.abin.mallchat.common.common.algorithm.sensitiveWord; import java.util.List; diff --git a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/SensitiveWordBs.java b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/SensitiveWordBs.java similarity index 96% rename from mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/SensitiveWordBs.java rename to mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/SensitiveWordBs.java index b49faf4..a1c6e9e 100644 --- a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/SensitiveWordBs.java +++ b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/SensitiveWordBs.java @@ -1,4 +1,4 @@ -package com.abin.mallchat.common.common.utils.sensitiveWord; +package com.abin.mallchat.common.common.algorithm.sensitiveWord; import java.util.List; diff --git a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/SensitiveWordFilter.java b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/SensitiveWordFilter.java similarity index 89% rename from mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/SensitiveWordFilter.java rename to mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/SensitiveWordFilter.java index afe2533..cbe958b 100644 --- a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/SensitiveWordFilter.java +++ b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/algorithm/sensitiveWord/SensitiveWordFilter.java @@ -1,4 +1,4 @@ -package com.abin.mallchat.common.common.utils.sensitiveWord; +package com.abin.mallchat.common.common.algorithm.sensitiveWord; import java.util.List; diff --git a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/config/SensitiveWordConfig.java b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/config/SensitiveWordConfig.java index 262c880..6d0e082 100644 --- a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/config/SensitiveWordConfig.java +++ b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/common/config/SensitiveWordConfig.java @@ -1,7 +1,7 @@ package com.abin.mallchat.common.common.config; -import com.abin.mallchat.common.common.utils.sensitiveWord.DFAFilter; -import com.abin.mallchat.common.common.utils.sensitiveWord.SensitiveWordBs; +import com.abin.mallchat.common.common.algorithm.sensitiveWord.DFAFilter; +import com.abin.mallchat.common.common.algorithm.sensitiveWord.SensitiveWordBs; import com.abin.mallchat.common.sensitive.MyWordFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; diff --git a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/sensitive/MyWordFactory.java b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/sensitive/MyWordFactory.java index 8d9fef3..7a8b498 100644 --- a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/sensitive/MyWordFactory.java +++ b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/sensitive/MyWordFactory.java @@ -1,6 +1,6 @@ package com.abin.mallchat.common.sensitive; -import com.abin.mallchat.common.common.utils.sensitiveWord.IWordFactory; +import com.abin.mallchat.common.common.algorithm.sensitiveWord.IWordFactory; import com.abin.mallchat.common.sensitive.dao.SensitiveWordDao; import com.abin.mallchat.common.sensitive.domain.SensitiveWord; import org.springframework.beans.factory.annotation.Autowired; diff --git a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/user/service/impl/UserServiceImpl.java b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/user/service/impl/UserServiceImpl.java index a6b9d6c..dbeebf7 100644 --- a/mallchat-chat-server/src/main/java/com/abin/mallchat/common/user/service/impl/UserServiceImpl.java +++ b/mallchat-chat-server/src/main/java/com/abin/mallchat/common/user/service/impl/UserServiceImpl.java @@ -4,7 +4,7 @@ import cn.hutool.core.util.StrUtil; import com.abin.mallchat.common.common.event.UserBlackEvent; import com.abin.mallchat.common.common.event.UserRegisterEvent; import com.abin.mallchat.common.common.utils.AssertUtil; -import com.abin.mallchat.common.common.utils.sensitiveWord.SensitiveWordBs; +import com.abin.mallchat.common.common.algorithm.sensitiveWord.SensitiveWordBs; import com.abin.mallchat.common.user.dao.BlackDao; import com.abin.mallchat.common.user.dao.ItemConfigDao; import com.abin.mallchat.common.user.dao.UserBackpackDao; diff --git a/mallchat-chat-server/src/test/java/com/abin/mallchat/common/SensitiveTest.java b/mallchat-chat-server/src/test/java/com/abin/mallchat/common/SensitiveTest.java index 784b6e8..01818b3 100644 --- a/mallchat-chat-server/src/test/java/com/abin/mallchat/common/SensitiveTest.java +++ b/mallchat-chat-server/src/test/java/com/abin/mallchat/common/SensitiveTest.java @@ -1,11 +1,13 @@ package com.abin.mallchat.common; -import com.abin.mallchat.common.common.utils.sensitiveWord.ACFilter; -import com.abin.mallchat.common.common.utils.sensitiveWord.DFAFilter; +import com.abin.mallchat.common.common.algorithm.sensitiveWord.ACFilter; +import com.abin.mallchat.common.common.algorithm.sensitiveWord.ACProFilter; +import com.abin.mallchat.common.common.algorithm.sensitiveWord.DFAFilter; import org.junit.Test; -import java.util.Arrays; -import java.util.List; +import java.io.BufferedReader; +import java.io.FileReader; +import java.util.*; /** * Description: @@ -21,6 +23,7 @@ public class SensitiveTest { System.out.println(instance.hasSensitiveWord("adabcd")); } + @Test public void AC() { List sensitiveList = Arrays.asList("abcd", "abcbba", "adabca"); @@ -29,6 +32,14 @@ public class SensitiveTest { instance.hasSensitiveWord("adabcd"); } + @Test + public void ACPro() + { + List sensitiveList = Arrays.asList("白痴", "你是白痴", "白痴吗"); + ACProFilter acProFilter=new ACProFilter(); + acProFilter.loadWord(sensitiveList); + System.out.println(acProFilter.filter("你是白痴吗")); + } @Test public void DFAMulti() { List sensitiveList = Arrays.asList("白痴", "你是白痴", "白痴吗");