open-source/MallChat
1
0
Fork 0
mirror of https://github.com/zongzibinbin/MallChat.git synced 2026-03-13 21:53:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #105 from 1045078399/sensitive

Browse Source 
feat:敏感词封装
This commit is contained in:
zongzibinbin
2023-07-16 01:40:56 +08:00
committed by GitHub
parent 0f98094741 b6d9cad31c
commit ac34a522e7
12 changed files with 253 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
mallchat-common/src/main/java/com/abin/mallchat/common/common/config/SensitiveWordConfig.java Normal file
View File

@@ -0,0 +1,30 @@
package com.abin.mallchat.common.common.config;
import com.abin.mallchat.common.common.utils.sensitiveWord.DFAFilter;
import com.abin.mallchat.common.common.utils.sensitiveWord.SensitiveWordBs;
import com.abin.mallchat.common.sensitive.MyWordDeny;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class SensitiveWordConfig {
@Autowired
private MyWordDeny myWordDeny;
/**
* 初始化引导类
*
* @return 初始化引导类
* @since 1.0.0
*/
@Bean
public SensitiveWordBs sensitiveWordBs() {
return SensitiveWordBs.newInstance()
.filterStrategy(DFAFilter.getInstance())
.sensitiveWord(myWordDeny)
.init();
}
}

10
mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/SensitiveWordUtils0.java → mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/ACFilter.java
View File

@@ -1,4 +1,4 @@
package com.abin.mallchat.common.common.utils;
package com.abin.mallchat.common.common.utils.sensitiveWord;
import com.abin.mallchat.common.common.algorithm.ac.ACTrie;
import com.abin.mallchat.common.common.algorithm.ac.MatchResult;
@@ -15,7 +15,7 @@ import java.util.Objects;
*
* Created by berg on 2023/6/18.
*/
public class SensitiveWordUtils0 {
public class ACFilter implements SensitiveWordFilter {
private final static char mask_char = '*'; // 替代字符
@@ -27,7 +27,7 @@ public class SensitiveWordUtils0 {
* @param text 文本
* @return boolean
*/
public static boolean hasSensitiveWord(String text) {
public boolean hasSensitiveWord(String text) {
if (StringUtils.isBlank(text)) return false;
return !Objects.equals(filter(text), text);
}
@@ -38,7 +38,7 @@ public class SensitiveWordUtils0 {
* @param text 待替换文本
* @return 替换后的文本
*/
public static String filter(String text) {
public String filter(String text) {
if (StringUtils.isBlank(text)) return text;
List<MatchResult> matchResults = ac_trie.matches(text);
StringBuffer result = new StringBuffer(text);
@@ -62,7 +62,7 @@ public class SensitiveWordUtils0 {
*
* @param words 敏感词数组
*/
public static void loadWord(List<String> words) {
public void loadWord(List<String> words) {
if (words == null) return;
ac_trie = new ACTrie(words);
}

35
mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/SensitiveWordUtils.java → mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/DFAFilter.java
View File

@@ -1,4 +1,4 @@
package com.abin.mallchat.common.common.utils;
package com.abin.mallchat.common.common.utils.sensitiveWord;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils;
@@ -18,7 +18,10 @@ import java.util.*;
* @author zhaoyuhang
* @date 2023/06/19
*/
public final class SensitiveWordUtils {
public final class DFAFilter implements SensitiveWordFilter {
private DFAFilter() {
}
private static Word root = new Word(' '); // 敏感词字典的根节点
private final static char replace = '*'; // 替代字符
private final static String skipChars = " !*-+_=,，.@;:；：。、？?（）()【】[]《》<>“”\"‘’"; // 遇到这些字符就会跳过
@@ -30,6 +33,10 @@ public final class SensitiveWordUtils {
}
}
public static DFAFilter getInstance() {
return new DFAFilter();
}
/**
* 判断文本中是否存在敏感词
@@ -37,7 +44,7 @@ public final class SensitiveWordUtils {
* @param text 文本
* @return true: 存在敏感词, false: 不存在敏感词
*/
public static boolean hasSensitiveWord(String text) {
public boolean hasSensitiveWord(String text) {
if (StringUtils.isBlank(text)) return false;
return !Objects.equals(filter(text), text);
}
@@ -48,7 +55,7 @@ public final class SensitiveWordUtils {
* @param text 待替换文本
* @return 替换后的文本
*/
public static String filter(String text) {
public String filter(String text) {
StringBuilder result = new StringBuilder(text);
int index = 0;
while (index < result.length()) {
@@ -93,7 +100,7 @@ public final class SensitiveWordUtils {
*
* @param words 敏感词数组
*/
public static void loadWord(List<String> words) {
public void loadWord(List<String> words) {
if (!CollectionUtils.isEmpty(words)) {
Word newRoot = new Word(' ');
words.forEach(word -> loadWord(word, newRoot));
@@ -106,7 +113,7 @@ public final class SensitiveWordUtils {
*
* @param word 词
*/
public static void loadWord(String word, Word root) {
public void loadWord(String word, Word root) {
if (StringUtils.isBlank(word)) {
return;
}
@@ -136,7 +143,7 @@ public final class SensitiveWordUtils {
*
* @param path 文本文件的绝对路径
*/
public static void loadWordFromFile(String path) {
public void loadWordFromFile(String path) {
try (InputStream inputStream = Files.newInputStream(Paths.get(path))) {
loadWord(inputStream);
} catch (IOException e) {
@@ -150,7 +157,7 @@ public final class SensitiveWordUtils {
* @param inputStream 文本文件输入流
* @throws IOException IO异常
*/
public static void loadWord(InputStream inputStream) throws IOException {
public void loadWord(InputStream inputStream) throws IOException {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
String line;
ArrayList<String> list = new ArrayList<>();
@@ -167,7 +174,7 @@ public final class SensitiveWordUtils {
* @param c 待检测字符
* @return true: 需要跳过, false: 不需要跳过
*/
private static boolean skip(char c) {
private boolean skip(char c) {
return skipSet.contains(c);
}
@@ -186,17 +193,7 @@ public final class SensitiveWordUtils {
public Word(char c) {
this.c = c;
this.end = false;
this.next = new HashMap<>();
}
}
public static void main(String[] args) {
String text = "白日,梦";
String filter = filter(text);
System.out.println(filter);
}
}

18
mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/IWordDeny.java Normal file
View File

@@ -0,0 +1,18 @@
package com.abin.mallchat.common.common.utils.sensitiveWord;
import java.util.List;
/**
* 敏感词
*
* @author zhaoyuhang
* @date 2023/07/09
*/
public interface IWordDeny {
/**
* 获取结果
* @return 结果
* @since 0.0.13
*/
List<String> deny();
}

102
mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/SensitiveWordBs.java Normal file
View File

@@ -0,0 +1,102 @@
package com.abin.mallchat.common.common.utils.sensitiveWord;
import java.util.List;
/**
* 敏感词引导类
*
* @author zhaoyuhang
* @date 2023/07/08
*/
public class SensitiveWordBs {
/**
* 私有化构造器
*/
private SensitiveWordBs() {
}
/**
* 脱敏策略
*/
private SensitiveWordFilter sensitiveWordFilter = DFAFilter.getInstance();
/**
* 敏感词列表
*/
private IWordDeny wordDeny;
public static SensitiveWordBs newInstance() {
return new SensitiveWordBs();
}
/**
* 初始化
*
* 1. 根据配置，初始化对应的 map。比较消耗性能。
* @since 0.0.13
* @return this
*/
public SensitiveWordBs init() {
List<String> words = wordDeny.deny();
loadWord(words);
return this;
}
/**
* 过滤策略
*
* @param filter 过滤器
* @return 结果
* @since 0.7.0
*/
public SensitiveWordBs filterStrategy(SensitiveWordFilter filter) {
if (filter == null) {
throw new IllegalArgumentException("filter can not be null");
}
this.sensitiveWordFilter = filter;
return this;
}
public SensitiveWordBs sensitiveWord(IWordDeny wordDeny) {
if (wordDeny == null) {
throw new IllegalArgumentException("wordDeny can not be null");
}
this.wordDeny = wordDeny;
return this;
}
/**
* 有敏感词
*
* @param text 文本
* @return boolean
*/
public boolean hasSensitiveWord(String text) {
return sensitiveWordFilter.hasSensitiveWord(text);
}
/**
* 过滤
*
* @param text 文本
* @return {@link String}
*/
public String filter(String text) {
return sensitiveWordFilter.filter(text);
}
/**
* 加载敏感词列表
*
* @param words 敏感词数组
*/
private void loadWord(List<String> words) {
sensitiveWordFilter.loadWord(words);
}
}

37
mallchat-common/src/main/java/com/abin/mallchat/common/common/utils/sensitiveWord/SensitiveWordFilter.java Normal file
View File

@@ -0,0 +1,37 @@
package com.abin.mallchat.common.common.utils.sensitiveWord;
import java.util.List;
/**
* 敏感词过滤
*
* @author zhaoyuhang
* @date 2023/07/08
*/
public interface SensitiveWordFilter {
/**
* 有敏感词
*
* @param text 文本
* @return boolean
*/
boolean hasSensitiveWord(String text);
/**
* 过滤
*
* @param text 文本
* @return {@link String}
*/
String filter(String text);
/**
* 加载敏感词列表
*
* @param words 敏感词数组
*/
void loadWord(List<String> words);
}

24
mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/MyWordDeny.java Normal file
View File

@@ -0,0 +1,24 @@
package com.abin.mallchat.common.sensitive;
import com.abin.mallchat.common.common.utils.sensitiveWord.IWordDeny;
import com.abin.mallchat.common.sensitive.dao.SensitiveWordDao;
import com.abin.mallchat.common.sensitive.domain.SensitiveWord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.stream.Collectors;
@Component
public class MyWordDeny implements IWordDeny {
@Autowired
private SensitiveWordDao sensitiveWordDao;
@Override
public List<String> deny() {
return sensitiveWordDao.list()
.stream()
.map(SensitiveWord::getWord)
.collect(Collectors.toList());
}
}

5
mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/service/ISensitiveWordService.java
View File

@@ -1,5 +0,0 @@
package com.abin.mallchat.common.sensitive.service;
public interface ISensitiveWordService {
}

39
mallchat-common/src/main/java/com/abin/mallchat/common/sensitive/service/impl/SensitiveWordServiceImpl.java
View File

@@ -1,39 +0,0 @@
package com.abin.mallchat.common.sensitive.service.impl;
import com.abin.mallchat.common.common.utils.SensitiveWordUtils;
import com.abin.mallchat.common.sensitive.dao.SensitiveWordDao;
import com.abin.mallchat.common.sensitive.domain.SensitiveWord;
import com.abin.mallchat.common.sensitive.service.ISensitiveWordService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import javax.annotation.PostConstruct;
import java.util.List;
import java.util.stream.Collectors;
@Service
@Slf4j
public class SensitiveWordServiceImpl implements ISensitiveWordService {
@Autowired
private SensitiveWordDao sensitiveWordDao;
@Autowired
private ThreadPoolTaskExecutor threadPoolTaskExecutor;
@PostConstruct
public void initSensitiveWord() {
threadPoolTaskExecutor.execute(() -> {
log.info("[initSensitiveWord] start");
List<SensitiveWord> list = sensitiveWordDao.list();
if (!CollectionUtils.isEmpty(list)) {
List<String> wordList = list.stream()
.map(SensitiveWord::getWord)
.collect(Collectors.toList());
SensitiveWordUtils.loadWord(wordList);
}
log.info("[initSensitiveWord] end; loading sensitiveWords num:{}", list.size());
});
}
}