Merge branch 'main' into chat_context

# Conflicts:
#	mallchat-common/pom.xml
#	mallchat-custom-server/pom.xml
#	mallchat-custom-server/src/main/java/com/abin/mallchat/custom/chatai/handler/GPTChatAIHandler.java
This commit is contained in:
zhaoyuhang
2023-07-10 22:46:20 +08:00
81 changed files with 1870 additions and 455 deletions

1
MallChat Submodule

Submodule MallChat added at c47b48760d

View File

@@ -13,13 +13,22 @@
<a href="https://github.com/zongzibinbin/MallChat"><img src="https://img.shields.io/badge/github-项目地址-yellow.svg?style=plasticr"></a> <a href="https://github.com/zongzibinbin/MallChat"><img src="https://img.shields.io/badge/github-项目地址-yellow.svg?style=plasticr"></a>
<a href="https://gitee.com/zhongzhibinbin/MallChat"><img src="https://img.shields.io/badge/码云-项目地址-orange.svg?style=plasticr"></a> <a href="https://gitee.com/zhongzhibinbin/MallChat"><img src="https://img.shields.io/badge/码云-项目地址-orange.svg?style=plasticr"></a>
<a href="https://github.com/Evansy/MallChatWeb"><img src="https://img.shields.io/badge/前端-项目地址-blueviolet.svg?style=plasticr"></a> <a href="https://github.com/Evansy/MallChatWeb"><img src="https://img.shields.io/badge/前端-项目地址-blueviolet.svg?style=plasticr"></a>
<a href="https://github.com/zongzibinbin/MallChat/commits" target="_blank"><br>
[![MallChat](https://github.com/Evansy/MallChatWeb/actions/workflows/deploy.yml/badge.svg?branch=main)](https://github.com/Evansy/MallChatWeb/actions/workflows/deploy.yml) <a href="https://github.com/Evansy/MallChatWeb/actions/workflows/deploy.yml" target="_blank">
<img alt="Commit" src="https://github.com/Evansy/MallChatWeb/actions/workflows/deploy.yml/badge.svg?branch=main">
</a>
<a href="https://github.com/zongzibinbin/MallChat/commits" target="_blank"> <a href="https://github.com/zongzibinbin/MallChat/commits" target="_blank">
<img alt="Commit" src="https://img.shields.io/github/commit-activity/m/zongzibinbin/MallChat"></a> <img alt="Commit" src="https://img.shields.io/github/commit-activity/m/zongzibinbin/MallChat"></a>
<a href="https://github.com/zongzibinbin/MallChat/issues" target="_blank"> <a href="https://github.com/zongzibinbin/MallChat/issues" target="_blank">
<img alt="Issues" src="https://img.shields.io/github/issues/zongzibinbin/MallChat"></a> [![License](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/zongzibinbin/MallChat/blob/master/LICENSE) <img alt="Issues" src="https://img.shields.io/github/issues/zongzibinbin/MallChat">
[![GitHub stars](https://img.shields.io/github/stars/zongzibinbin/MallChat.svg?style=social)](https://github.com/zongzibinbin/MallChat/stargazers) </a>
<a href="https://github.com/zongzibinbin/MallChat/blob/master/LICENSE" target="_blank">
<img alt="License: Apache-2.0" src="https://img.shields.io/badge/License-Apache--2.0-blue.svg">
</a>
<a href="https://github.com/zongzibinbin/MallChat/stargazers" target="_blank">
<img alt="License" src="https://img.shields.io/github/stars/zongzibinbin/MallChat.svg?style=social">
</a>
</div> </div>
## 项目导航 ## 项目导航
@@ -130,7 +139,7 @@
<td><a href="https://github.com/Evansy/MallChatWeb/pull/17">虚拟列表</a></td> <td><a href="https://github.com/Evansy/MallChatWeb/pull/17">虚拟列表</a></td>
</tr> </tr>
<tr> <tr>
<td rowspan="3">后端</td> <td rowspan="5">后端</td>
<td rowspan="2"> <td rowspan="2">
<a href="https://github.com/1045078399"><img src="https://avatars.githubusercontent.com/u/82020261?v=4" style=" width: 80px; height: 80px;"></a> <a href="https://github.com/1045078399"><img src="https://avatars.githubusercontent.com/u/82020261?v=4" style=" width: 80px; height: 80px;"></a>
</td> </td>
@@ -145,6 +154,18 @@
</td> </td>
<td><a href="https://github.com/zongzibinbin/MallChat/pull/99">Ac自动机敏感词检测</a></td> <td><a href="https://github.com/zongzibinbin/MallChat/pull/99">Ac自动机敏感词检测</a></td>
</tr> </tr>
<tr>
<td rowspan="1">
<a href="https://github.com/linzhihan"><img src="https://avatars.githubusercontent.com/u/58815955?v=4" style=" width: 80px; height: 80px;"></a>
</td>
<td><a href="https://github.com/zongzibinbin/MallChat/pull/95">限流编程式</a></td>
</tr>
<tr>
<td rowspan="1">
<a href="https://github.com/zbzbzzz"><img src="https://avatars.githubusercontent.com/u/42697182?v=4" style=" width: 80px; height: 80px;"></a>
</td>
<td><a href="https://github.com/zongzibinbin/MallChat/pull/82">握手认证</a></td>
</tr>
</table> </table>

View File

@@ -191,3 +191,15 @@ CREATE TABLE `sensitive_word` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='敏感词库'; ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='敏感词库';
INSERT INTO `sensitive_word` (`word`) VALUES ('TMD'); INSERT INTO `sensitive_word` (`word`) VALUES ('TMD');
INSERT INTO `sensitive_word` (`word`) VALUES ('tmd'); INSERT INTO `sensitive_word` (`word`) VALUES ('tmd');
DROP TABLE IF EXISTS `user_emoji`;
CREATE TABLE `user_emoji` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
`uid` bigint(20) NOT NULL COMMENT '用户表ID',
`expression_url` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '表情地址',
`delete_status` int(1) NOT NULL DEFAULT '0' COMMENT '逻辑删除(0-正常,1-删除)',
`create_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '创建时间',
`update_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3) COMMENT '修改时间',
PRIMARY KEY (`id`) USING BTREE,
KEY `IDX_USER_EMOJIS_UID` (`uid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC COMMENT='用户表情包';

View File

@@ -0,0 +1,10 @@
CREATE TABLE `user_emoji` (
`id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'id',
`uid` bigint(20) NOT NULL COMMENT '用户表ID',
`expression_url` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '表情地址',
`delete_status` int(1) NOT NULL DEFAULT '0' COMMENT '逻辑删除(0-正常,1-删除)',
`create_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '创建时间',
`update_time` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3) COMMENT '修改时间',
PRIMARY KEY (`id`) USING BTREE,
KEY `IDX_USER_EMOJIS_UID` (`uid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC COMMENT='用户表情包';

View File

@@ -123,6 +123,16 @@
<groupId>com.alibaba</groupId> <groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId> <artifactId>fastjson</artifactId>
</dependency> </dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>5.3.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-test</artifactId>
</dependency>
</dependencies> </dependencies>
<build> <build>
<plugins> <plugins>

View File

@@ -0,0 +1,29 @@
package com.abin.mallchat.common.chat.domain.entity.msg;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import javax.validation.constraints.NotBlank;
import java.io.Serializable;
/**
* Description: 表情图片消息入参
* Author: <a href="https://github.com/zongzibinbin">abin</a>
* Date: 2023-06-04
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class EmojisMsgDTO implements Serializable {
private static final long serialVersionUID = 1L;
@ApiModelProperty("下载地址")
@NotBlank
private String url;
}

View File

@@ -1,5 +1,6 @@
package com.abin.mallchat.common.chat.domain.entity.msg; package com.abin.mallchat.common.chat.domain.entity.msg;
import com.abin.mallchat.common.common.utils.discover.domain.UrlInfo;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
@@ -23,7 +24,7 @@ import java.util.Map;
public class MessageExtra implements Serializable { public class MessageExtra implements Serializable {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
//url跳转链接 //url跳转链接
private Map<String, String> urlTitleMap; private Map<String, UrlInfo> urlContentMap;
//消息撤回详情 //消息撤回详情
private MsgRecall recall; private MsgRecall recall;
//艾特的uid //艾特的uid
@@ -36,4 +37,9 @@ public class MessageExtra implements Serializable {
private SoundMsgDTO soundMsgDTO; private SoundMsgDTO soundMsgDTO;
//文件消息 //文件消息
private VideoMsgDTO videoMsgDTO; private VideoMsgDTO videoMsgDTO;
/**
* 表情图片信息
*/
private EmojisMsgDTO emojisMsgDTO;
} }

View File

@@ -22,6 +22,7 @@ public enum MessageTypeEnum {
FILE(4, "文件"), FILE(4, "文件"),
SOUND(5, "语音"), SOUND(5, "语音"),
VIDEO(6, "视频"), VIDEO(6, "视频"),
EMOJI(7, "表情"),
; ;
private final Integer type; private final Integer type;

View File

@@ -2,9 +2,8 @@ package com.abin.mallchat.common.common.aspect;
import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.StrUtil;
import com.abin.mallchat.common.common.annotation.FrequencyControl; import com.abin.mallchat.common.common.annotation.FrequencyControl;
import com.abin.mallchat.common.common.exception.BusinessException; import com.abin.mallchat.common.common.domain.dto.FrequencyControlDTO;
import com.abin.mallchat.common.common.exception.CommonErrorEnum; import com.abin.mallchat.common.common.service.frequencycontrol.FrequencyControlUtil;
import com.abin.mallchat.common.common.utils.RedisUtils;
import com.abin.mallchat.common.common.utils.RequestHolder; import com.abin.mallchat.common.common.utils.RequestHolder;
import com.abin.mallchat.common.common.utils.SpElUtils; import com.abin.mallchat.common.common.utils.SpElUtils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
@@ -15,7 +14,12 @@ import org.aspectj.lang.reflect.MethodSignature;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.lang.reflect.Method; import java.lang.reflect.Method;
import java.util.*; import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import static com.abin.mallchat.common.common.service.frequencycontrol.FrequencyControlStrategyFactory.TOTAL_COUNT_WITH_IN_FIX_TIME_FREQUENCY_CONTROLLER;
/** /**
* Description: 频控实现 * Description: 频控实现
@@ -48,25 +52,25 @@ public class FrequencyControlAspect {
} }
keyMap.put(prefix + ":" + key, frequencyControl); keyMap.put(prefix + ":" + key, frequencyControl);
} }
//批量获取redis统计的值 // 将注解的参数转换为编程式调用需要的参数
ArrayList<String> keyList = new ArrayList<>(keyMap.keySet()); List<FrequencyControlDTO> frequencyControlDTOS = keyMap.entrySet().stream().map(entrySet -> buildFrequencyControlDTO(entrySet.getKey(), entrySet.getValue())).collect(Collectors.toList());
List<Integer> countList = RedisUtils.mget(keyList, Integer.class); // 调用编程式注解
for (int i = 0; i < keyList.size(); i++) { return FrequencyControlUtil.executeWithFrequencyControlList(TOTAL_COUNT_WITH_IN_FIX_TIME_FREQUENCY_CONTROLLER, frequencyControlDTOS, joinPoint::proceed);
String key = keyList.get(i); }
Integer count = countList.get(i);
FrequencyControl frequencyControl = keyMap.get(key); /**
if (Objects.nonNull(count) && count >= frequencyControl.count()) {//频率超过了 * 将注解参数转换为编程式调用所需要的参数
log.warn("frequencyControl limit key:{},count:{}", key, count); *
throw new BusinessException(CommonErrorEnum.FREQUENCY_LIMIT); * @param key 频率控制Key
} * @param frequencyControl 注解
} * @return 编程式调用所需要的参数-FrequencyControlDTO
try { */
return joinPoint.proceed(); private FrequencyControlDTO buildFrequencyControlDTO(String key, FrequencyControl frequencyControl) {
} finally { FrequencyControlDTO frequencyControlDTO = new FrequencyControlDTO();
//不管成功还是失败,都增加次数 frequencyControlDTO.setCount(frequencyControl.count());
keyMap.forEach((k, v) -> { frequencyControlDTO.setTime(frequencyControl.time());
RedisUtils.inc(k, v.time(), v.unit()); frequencyControlDTO.setUnit(frequencyControl.unit());
}); frequencyControlDTO.setKey(key);
} return frequencyControlDTO;
} }
} }

View File

@@ -0,0 +1,30 @@
package com.abin.mallchat.common.common.config;
import com.abin.mallchat.common.common.utils.sensitiveWord.DFAFilter;
import com.abin.mallchat.common.common.utils.sensitiveWord.SensitiveWordBs;
import com.abin.mallchat.common.sensitive.MyWordDeny;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class SensitiveWordConfig {
@Autowired
private MyWordDeny myWordDeny;
/**
* 初始化引导类
*
* @return 初始化引导类
* @since 1.0.0
*/
@Bean
public SensitiveWordBs sensitiveWordBs() {
return SensitiveWordBs.newInstance()
.filterStrategy(DFAFilter.getInstance())
.sensitiveWord(myWordDeny)
.init();
}
}

View File

@@ -0,0 +1,42 @@
package com.abin.mallchat.common.common.domain.dto;
import lombok.*;
import java.util.concurrent.TimeUnit;
@Data
@ToString
@Builder
@NoArgsConstructor
@AllArgsConstructor
/** 限流策略定义
* @author linzhihan
* @date 2023/07/03
*
*/
public class FrequencyControlDTO {
/**
* 代表频控的Key 如果target为Key的话 这里要传值用于构建redis的Key target为Ip或者UID的话会从上下文取值 Key字段无需传值
*/
private String key;
/**
* 频控时间范围,默认单位秒
*
* @return 时间范围
*/
private Integer time;
/**
* 频控时间单位,默认秒
*
* @return 单位
*/
private TimeUnit unit;
/**
* 单位时间内最大访问次数
*
* @return 次数
*/
private Integer count;
}

View File

@@ -0,0 +1,20 @@
package com.abin.mallchat.common.common.domain.vo.request;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import javax.validation.constraints.NotNull;
/**
* @author zhongzb create on 2021/05/31
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class IdReqVO {
@ApiModelProperty("id")
@NotNull
private long id;
}

View File

@@ -13,7 +13,7 @@ import lombok.Getter;
public enum CommonErrorEnum implements ErrorEnum { public enum CommonErrorEnum implements ErrorEnum {
SYSTEM_ERROR(-1, "系统出小差了,请稍后再试哦~~"), SYSTEM_ERROR(-1, "系统出小差了,请稍后再试哦~~"),
PARAM_VALID(-2, "参数校验失败"), PARAM_VALID(-2, "参数校验失败{0}"),
FREQUENCY_LIMIT(-3, "请求太频繁了,请稍后再试哦~~"), FREQUENCY_LIMIT(-3, "请求太频繁了,请稍后再试哦~~"),
LOCK_LIMIT(-4, "请求太频繁了,请稍后再试哦~~"), LOCK_LIMIT(-4, "请求太频繁了,请稍后再试哦~~"),
; ;

View File

@@ -0,0 +1,39 @@
package com.abin.mallchat.common.common.exception;
import lombok.Data;
/**
* 自定义限流异常
*
* @author linzhihan
* @date 2023/07/034
*/
@Data
public class FrequencyControlException extends RuntimeException {
private static final long serialVersionUID = 1L;
/**
*  错误码
*/
protected Integer errorCode;
/**
*  错误信息
*/
protected String errorMsg;
public FrequencyControlException() {
super();
}
public FrequencyControlException(String errorMsg) {
super(errorMsg);
this.errorMsg = errorMsg;
}
public FrequencyControlException(ErrorEnum error) {
super(error.getErrorMsg());
this.errorCode = error.getErrorCode();
this.errorMsg = error.getErrorMsg();
}
}

View File

@@ -42,7 +42,7 @@ public class GlobalExceptionHandler {
*/ */
@ExceptionHandler(value = NullPointerException.class) @ExceptionHandler(value = NullPointerException.class)
public ApiResult exceptionHandler(NullPointerException e) { public ApiResult exceptionHandler(NullPointerException e) {
log.error("null point exceptionThe reason is:{}", e.getMessage(), e); log.error("null point exceptionThe reason is: ", e);
return ApiResult.fail(CommonErrorEnum.SYSTEM_ERROR); return ApiResult.fail(CommonErrorEnum.SYSTEM_ERROR);
} }
@@ -73,4 +73,12 @@ public class GlobalExceptionHandler {
return ApiResult.fail(-1, String.format("不支持'%s'请求", e.getMethod())); return ApiResult.fail(-1, String.format("不支持'%s'请求", e.getMethod()));
} }
/**
* 限流异常
*/
@ExceptionHandler(value = FrequencyControlException.class)
public ApiResult frequencyControlExceptionHandler(FrequencyControlException e) {
log.info("frequencyControl exceptionThe reason is{}", e.getMessage(), e);
return ApiResult.fail(e.getErrorCode(), e.getMessage());
}
} }

View File

@@ -9,7 +9,6 @@ public class GlobalUncaughtExceptionHandler implements Thread.UncaughtException
@Override @Override
public void uncaughtException(Thread t, Throwable e) { public void uncaughtException(Thread t, Throwable e) {
log.error("Exception in thread {} ", t.getName(), e); log.error("Exception in thread {} ", t.getName(), e);
e.printStackTrace();
} }
} }

View File

@@ -0,0 +1,124 @@
package com.abin.mallchat.common.common.service.frequencycontrol;
import com.abin.mallchat.common.common.domain.dto.FrequencyControlDTO;
import com.abin.mallchat.common.common.exception.CommonErrorEnum;
import com.abin.mallchat.common.common.exception.FrequencyControlException;
import com.abin.mallchat.common.common.utils.AssertUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
import javax.annotation.PostConstruct;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* 抽象类频控服务 其他类如果要实现限流服务 直接注入使用通用限流类
* 后期会通过继承此类实现令牌桶等算法
*
* @author linzhihan
* @date 2023/07/03
* @see TotalCountWithInFixTimeFrequencyController 通用限流类
*/
@Slf4j
public abstract class AbstractFrequencyControlService<K extends FrequencyControlDTO> {
@PostConstruct
protected void registerMyselfToFactory() {
FrequencyControlStrategyFactory.registerFrequencyController(getStrategyName(), this);
}
/**
* @param frequencyControlMap 定义的注解频控 Map中的Key-对应redis的单个频控的Key Map中的Value-对应redis的单个频控的Key限制的Value
* @param supplier 函数式入参-代表每个频控方法执行的不同的业务逻辑
* @return 业务方法执行的返回值
* @throws Throwable
*/
private <T> T executeWithFrequencyControlMap(Map<String, K> frequencyControlMap, SupplierThrowWithoutParam<T> supplier) throws Throwable {
if (reachRateLimit(frequencyControlMap)) {
throw new FrequencyControlException(CommonErrorEnum.FREQUENCY_LIMIT);
}
try {
return supplier.get();
} finally {
//不管成功还是失败,都增加次数
addFrequencyControlStatisticsCount(frequencyControlMap);
}
}
/**
* 多限流策略的编程式调用方法 无参的调用方法
*
* @param frequencyControlList 频控列表 包含每一个频率控制的定义以及顺序
* @param supplier 函数式入参-代表每个频控方法执行的不同的业务逻辑
* @return 业务方法执行的返回值
* @throws Throwable 被限流或者限流策略定义错误
*/
@SuppressWarnings("unchecked")
public <T> T executeWithFrequencyControlList(List<K> frequencyControlList, SupplierThrowWithoutParam<T> supplier) throws Throwable {
boolean existsFrequencyControlHasNullKey = frequencyControlList.stream().anyMatch(frequencyControl -> ObjectUtils.isEmpty(frequencyControl.getKey()));
AssertUtil.isFalse(existsFrequencyControlHasNullKey, "限流策略的Key字段不允许出现空值");
Map<String, FrequencyControlDTO> frequencyControlDTOMap = frequencyControlList.stream().collect(Collectors.groupingBy(FrequencyControlDTO::getKey, Collectors.collectingAndThen(Collectors.toList(), list -> list.get(0))));
return executeWithFrequencyControlMap((Map<String, K>) frequencyControlDTOMap, supplier);
}
/**
* 单限流策略的调用方法-编程式调用
*
* @param frequencyControl 单个频控对象
* @param supplier 服务提供着
* @return 业务方法执行结果
* @throws Throwable
*/
public <T> T executeWithFrequencyControl(K frequencyControl, SupplierThrowWithoutParam<T> supplier) throws Throwable {
return executeWithFrequencyControlList(Collections.singletonList(frequencyControl), supplier);
}
@FunctionalInterface
public interface SupplierThrowWithoutParam<T> {
/**
* Gets a result.
*
* @return a result
*/
T get() throws Throwable;
}
@FunctionalInterface
public interface Executor {
/**
* Gets a result.
*
* @return a result
*/
void execute() throws Throwable;
}
/**
* 是否达到限流阈值 子类实现 每个子类都可以自定义自己的限流逻辑判断
*
* @param frequencyControlMap 定义的注解频控 Map中的Key-对应redis的单个频控的Key Map中的Value-对应redis的单个频控的Key限制的Value
* @return true-方法被限流 false-方法没有被限流
*/
protected abstract boolean reachRateLimit(Map<String, K> frequencyControlMap);
/**
* 增加限流统计次数 子类实现 每个子类都可以自定义自己的限流统计信息增加的逻辑
*
* @param frequencyControlMap 定义的注解频控 Map中的Key-对应redis的单个频控的Key Map中的Value-对应redis的单个频控的Key限制的Value
*/
protected abstract void addFrequencyControlStatisticsCount(Map<String, K> frequencyControlMap);
/**
* 获取策略名称
*
* @return 策略名称
*/
protected abstract String getStrategyName();
}

View File

@@ -0,0 +1,51 @@
package com.abin.mallchat.common.common.service.frequencycontrol;
import com.abin.mallchat.common.common.domain.dto.FrequencyControlDTO;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* 限流策略工厂
*
* @author linzhihan
* @date 2023/07/03
*/
public class FrequencyControlStrategyFactory {
/**
* 指定时间内总次数限流
*/
public static final String TOTAL_COUNT_WITH_IN_FIX_TIME_FREQUENCY_CONTROLLER = "TotalCountWithInFixTime";
/**
* 限流策略集合
*/
static Map<String, AbstractFrequencyControlService<?>> frequencyControlServiceStrategyMap = new ConcurrentHashMap<>(8);
/**
* 将策略类放入工厂
*
* @param strategyName 策略名称
* @param abstractFrequencyControlService 策略类
*/
public static <K extends FrequencyControlDTO> void registerFrequencyController(String strategyName, AbstractFrequencyControlService<K> abstractFrequencyControlService) {
frequencyControlServiceStrategyMap.put(strategyName, abstractFrequencyControlService);
}
/**
* 根据名称获取策略类
*
* @param strategyName 策略名称
* @return 对应的限流策略类
*/
@SuppressWarnings("unchecked")
public static <K extends FrequencyControlDTO> AbstractFrequencyControlService<K> getFrequencyControllerByName(String strategyName) {
return (AbstractFrequencyControlService<K>) frequencyControlServiceStrategyMap.get(strategyName);
}
/**
* 构造器私有
*/
private FrequencyControlStrategyFactory() {
}
}

View File

@@ -0,0 +1,63 @@
package com.abin.mallchat.common.common.service.frequencycontrol;
import com.abin.mallchat.common.common.domain.dto.FrequencyControlDTO;
import com.abin.mallchat.common.common.utils.AssertUtil;
import org.apache.commons.lang3.ObjectUtils;
import java.util.List;
/**
* 限流工具类 提供编程式的限流调用方法
*
* @author linzhihan
* @date 2023/07/03
*/
public class FrequencyControlUtil {
/**
* 单限流策略的调用方法-编程式调用
*
* @param strategyName 策略名称
* @param frequencyControl 单个频控对象
* @param supplier 服务提供着
* @return 业务方法执行结果
* @throws Throwable
*/
public static <T, K extends FrequencyControlDTO> T executeWithFrequencyControl(String strategyName, K frequencyControl, AbstractFrequencyControlService.SupplierThrowWithoutParam<T> supplier) throws Throwable {
AbstractFrequencyControlService<K> frequencyController = FrequencyControlStrategyFactory.getFrequencyControllerByName(strategyName);
return frequencyController.executeWithFrequencyControl(frequencyControl, supplier);
}
public static <K extends FrequencyControlDTO> void executeWithFrequencyControl(String strategyName, K frequencyControl, AbstractFrequencyControlService.Executor executor) throws Throwable {
AbstractFrequencyControlService<K> frequencyController = FrequencyControlStrategyFactory.getFrequencyControllerByName(strategyName);
frequencyController.executeWithFrequencyControl(frequencyControl, () -> {
executor.execute();
return null;
});
}
/**
* 多限流策略的编程式调用方法调用方法
*
* @param strategyName 策略名称
* @param frequencyControlList 频控列表 包含每一个频率控制的定义以及顺序
* @param supplier 函数式入参-代表每个频控方法执行的不同的业务逻辑
* @return 业务方法执行的返回值
* @throws Throwable 被限流或者限流策略定义错误
*/
public static <T, K extends FrequencyControlDTO> T executeWithFrequencyControlList(String strategyName, List<K> frequencyControlList, AbstractFrequencyControlService.SupplierThrowWithoutParam<T> supplier) throws Throwable {
boolean existsFrequencyControlHasNullKey = frequencyControlList.stream().anyMatch(frequencyControl -> ObjectUtils.isEmpty(frequencyControl.getKey()));
AssertUtil.isFalse(existsFrequencyControlHasNullKey, "限流策略的Key字段不允许出现空值");
AbstractFrequencyControlService<K> frequencyController = FrequencyControlStrategyFactory.getFrequencyControllerByName(strategyName);
return frequencyController.executeWithFrequencyControlList(frequencyControlList, supplier);
}
/**
* 构造器私有
*/
private FrequencyControlUtil() {
}
}

View File

@@ -0,0 +1,64 @@
package com.abin.mallchat.common.common.service.frequencycontrol;
import com.abin.mallchat.common.common.domain.dto.FrequencyControlDTO;
import com.abin.mallchat.common.common.utils.RedisUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import static com.abin.mallchat.common.common.service.frequencycontrol.FrequencyControlStrategyFactory.TOTAL_COUNT_WITH_IN_FIX_TIME_FREQUENCY_CONTROLLER;
/**
* 抽象类频控服务 -使用redis实现 固定时间内不超过固定次数的限流类
*
* @author linzhihan
* @date 2023/07/03
*/
@Slf4j
@Service
public class TotalCountWithInFixTimeFrequencyController extends AbstractFrequencyControlService<FrequencyControlDTO> {
/**
* 是否达到限流阈值 子类实现 每个子类都可以自定义自己的限流逻辑判断
*
* @param frequencyControlMap 定义的注解频控 Map中的Key-对应redis的单个频控的Key Map中的Value-对应redis的单个频控的Key限制的Value
* @return true-方法被限流 false-方法没有被限流
*/
@Override
protected boolean reachRateLimit(Map<String, FrequencyControlDTO> frequencyControlMap) {
//批量获取redis统计的值
List<String> frequencyKeys = new ArrayList<>(frequencyControlMap.keySet());
List<Integer> countList = RedisUtils.mget(frequencyKeys, Integer.class);
for (int i = 0; i < frequencyKeys.size(); i++) {
String key = frequencyKeys.get(i);
Integer count = countList.get(i);
int frequencyControlCount = frequencyControlMap.get(key).getCount();
if (Objects.nonNull(count) && count >= frequencyControlCount) {
//频率超过了
log.warn("frequencyControl limit key:{},count:{}", key, count);
return true;
}
}
return false;
}
/**
* 增加限流统计次数 子类实现 每个子类都可以自定义自己的限流统计信息增加的逻辑
*
* @param frequencyControlMap 定义的注解频控 Map中的Key-对应redis的单个频控的Key Map中的Value-对应redis的单个频控的Key限制的Value
*/
@Override
protected void addFrequencyControlStatisticsCount(Map<String, FrequencyControlDTO> frequencyControlMap) {
frequencyControlMap.forEach((k, v) -> RedisUtils.inc(k, v.getTime(), v.getUnit()));
}
@Override
protected String getStrategyName() {
return TOTAL_COUNT_WITH_IN_FIX_TIME_FREQUENCY_CONTROLLER;
}
}

View File

@@ -67,7 +67,7 @@ public class JwtUtils {
DecodedJWT jwt = verifier.verify(token); DecodedJWT jwt = verifier.verify(token);
return jwt.getClaims(); return jwt.getClaims();
} catch (Exception e) { } catch (Exception e) {
log.info("decode error,token:{}", token, e); log.error("decode error,token:{}", token, e);
} }
return null; return null;
} }

View File

@@ -3,14 +3,14 @@ package com.abin.mallchat.common.common.utils.discover;
import cn.hutool.core.util.ReUtil; import cn.hutool.core.util.ReUtil;
import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.StrUtil;
import com.abin.mallchat.common.common.utils.FutureUtils; import com.abin.mallchat.common.common.utils.FutureUtils;
import com.abin.mallchat.common.common.utils.discover.domain.UrlInfo;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.Nullable;
import org.jsoup.Connection; import org.jsoup.Connection;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.springframework.data.util.Pair; import org.springframework.data.util.Pair;
import javax.annotation.Nullable;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -20,46 +20,55 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
/** /**
* Description: urlTitle查询抽象类 * @author zhaoqichao
* Author: <a href="https://github.com/zongzibinbin">abin</a> * @date 2023/7/3 16:38
* Date: 2023-05-27
*/ */
@Slf4j @Slf4j
public abstract class AbstractUrlTitleDiscover implements UrlTitleDiscover { public abstract class AbstractUrlDiscover implements UrlDiscover {
//链接识别的正则 //链接识别的正则
private static final Pattern PATTERN = Pattern.compile("((http|https)://)?(www.)?([\\w_-]+(?:(?:\\.[\\w_-]+)+))([\\w.,@?^=%&:/~+#-]*[\\w@?^=%&/~+#-])?"); private static final Pattern PATTERN = Pattern.compile("((http|https)://)?(www.)?([\\w_-]+(?:(?:\\.[\\w_-]+)+))([\\w.,@?^=%&:/~+#-]*[\\w@?^=%&/~+#-])?");
@Nullable @Nullable
@Override @Override
public Map<String, String> getContentTitleMap(String content) { public Map<String, UrlInfo> getUrlContentMap(String content) {
if (StrUtil.isBlank(content)) { if (StrUtil.isBlank(content)) {
return new HashMap<>(); return new HashMap<>();
} }
List<String> matchList = ReUtil.findAll(PATTERN, content, 0); List<String> matchList = ReUtil.findAll(PATTERN, content, 0);
//并行请求 //并行请求
List<CompletableFuture<Pair<String, String>>> futures = matchList.stream().map(match -> CompletableFuture.supplyAsync(() -> { List<CompletableFuture<Pair<String, UrlInfo>>> futures = matchList.stream().map(match -> CompletableFuture.supplyAsync(() -> {
String title = getUrlTitle(match); UrlInfo urlInfo = getContent(match);
return StringUtils.isNotEmpty(title) ? Pair.of(match, title) : null; return Objects.isNull(urlInfo) ? null : Pair.of(match, urlInfo);
})).collect(Collectors.toList()); })).collect(Collectors.toList());
CompletableFuture<List<Pair<String, String>>> future = FutureUtils.sequenceNonNull(futures); CompletableFuture<List<Pair<String, UrlInfo>>> future = FutureUtils.sequenceNonNull(futures);
//结果组装 //结果组装
return future.join().stream().collect(Collectors.toMap(Pair::getFirst, Pair::getSecond, (a, b) -> a)); return future.join().stream().collect(Collectors.toMap(Pair::getFirst, Pair::getSecond, (a, b) -> a));
} }
@Nullable @Nullable
@Override @Override
public String getUrlTitle(String url) { public UrlInfo getContent(String url) {
Document document = getUrlDocument(assemble(url)); Document document = getUrlDocument(assemble(url));
if (Objects.isNull(document)) { if (Objects.isNull(document)) {
return null; return null;
} }
return getDocTitle(document);
return UrlInfo.builder()
.title(getTitle(document))
.description(getDescription(document))
.image(getImage(assemble(url),document)).build();
} }
private String assemble(String url) { private String assemble(String url) {
if (!StrUtil.startWith(url, "http")) { if (!StrUtil.startWith(url, "http")) {
return "http://" + url; return "http://" + url;
} }
return url; return url;
} }
@@ -69,8 +78,9 @@ public abstract class AbstractUrlTitleDiscover implements UrlTitleDiscover {
connect.timeout(2000); connect.timeout(2000);
return connect.get(); return connect.get();
} catch (Exception e) { } catch (Exception e) {
log.error("find title error:url:{}", matchUrl, e); log.error("find error:url:{}", matchUrl, e);
} }
return null; return null;
} }
} }

View File

@@ -0,0 +1,52 @@
package com.abin.mallchat.common.common.utils.discover;
import cn.hutool.core.util.StrUtil;
import io.jsonwebtoken.lang.Objects;
import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.Nullable;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
/**
* @author zhaoqichao
* @date 2023/7/3 16:54
*/
public class CommonUrlDiscover extends AbstractUrlDiscover {
@Nullable
@Override
public String getTitle(Document document) {
return document.title();
}
@Nullable
@Override
public String getDescription(Document document) {
String description = document.head().select("meta[name=description]").attr("content");
String keywords = document.head().select("meta[name=keywords]").attr("content");
String content = StrUtil.isNotBlank(description) ? description : keywords;
//只保留一句话的描述
return StrUtil.isNotBlank(content) ? content.substring(0, content.indexOf("")) : content;
}
@Nullable
@Override
public String getImage(String url, Document document) {
String image = document.select("link[type=image/x-icon]").attr("href");
//如果没有去匹配含有icon属性的logo
String href = StrUtil.isEmpty(image) ? document.select("link[rel$=icon]").attr("href") : image;
//如果icon中已经包含了url部分域名
if (StrUtil.isNotBlank(StrUtil.removeAny(StrUtil.removeAny(href, "/"), "favicon.ico")) &&
StrUtil.containsAny(StrUtil.removePrefix(url, "http://"), StrUtil.removeAny(StrUtil.removeAny(href, "/"), "favicon.ico"))) {
return "http://" + StrUtil.removePrefix(href, "/");
}
//如果url已经包含了logo
if (StrUtil.containsAny(url, "favicon")) {
return url;
}
//如果logo中有url
if (StrUtil.containsAny(href, "http") || StrUtil.containsAny(href, "https")) {
return href;
}
return StrUtil.format("{}/{}", url, StrUtil.removePrefix(href, "/"));
}
}

View File

@@ -1,15 +0,0 @@
package com.abin.mallchat.common.common.utils.discover;
import org.jsoup.nodes.Document;
/**
* Description: 通用的标题解析类
* Author: <a href="https://github.com/zongzibinbin">abin</a>
* Date: 2023-05-27
*/
public class CommonUrlTitleDiscover extends AbstractUrlTitleDiscover {
@Override
public String getDocTitle(Document document) {
return document.title();
}
}

View File

@@ -0,0 +1,60 @@
package com.abin.mallchat.common.common.utils.discover;
import cn.hutool.core.util.StrUtil;
import org.jetbrains.annotations.Nullable;
import org.jsoup.nodes.Document;
import java.util.ArrayList;
import java.util.List;
/**
* Description: 具有优先级的title查询器
* Author: <a href="https://github.com/zongzibinbin">abin</a>
* Date: 2023-05-27
*/
public class PrioritizedUrlDiscover extends AbstractUrlDiscover {
private final List<UrlDiscover> urlDiscovers = new ArrayList<>(2);
public PrioritizedUrlDiscover() {
urlDiscovers.add(new WxUrlDiscover());
urlDiscovers.add(new CommonUrlDiscover());
}
@Nullable
@Override
public String getTitle(Document document) {
for (UrlDiscover urlDiscover : urlDiscovers) {
String urlTitle = urlDiscover.getTitle(document);
if (StrUtil.isNotBlank(urlTitle)) {
return urlTitle;
}
}
return null;
}
@Nullable
@Override
public String getDescription(Document document) {
for (UrlDiscover urlDiscover : urlDiscovers) {
String urlDescription = urlDiscover.getDescription(document);
if (StrUtil.isNotBlank(urlDescription)) {
return urlDescription;
}
}
return null;
}
@Nullable
@Override
public String getImage(String url, Document document) {
for (UrlDiscover urlDiscover : urlDiscovers) {
String urlImage = urlDiscover.getImage(url,document);
if (StrUtil.isNotBlank(urlImage)) {
return urlImage;
}
}
return null;
}
}

View File

@@ -1,33 +0,0 @@
package com.abin.mallchat.common.common.utils.discover;
import cn.hutool.core.util.StrUtil;
import org.jsoup.nodes.Document;
import java.util.ArrayList;
import java.util.List;
/**
* Description: 具有优先级的title查询器
* Author: <a href="https://github.com/zongzibinbin">abin</a>
* Date: 2023-05-27
*/
public class PrioritizedUrlTitleDiscover extends AbstractUrlTitleDiscover {
private final List<UrlTitleDiscover> urlTitleDiscovers = new ArrayList<>(2);
public PrioritizedUrlTitleDiscover() {
urlTitleDiscovers.add(new CommonUrlTitleDiscover());
urlTitleDiscovers.add(new WxUrlTitleDiscover());
}
@Override
public String getDocTitle(Document document) {
for (UrlTitleDiscover urlTitleDiscover : urlTitleDiscovers) {
String urlTitle = urlTitleDiscover.getDocTitle(document);
if (StrUtil.isNotBlank(urlTitle)) {
return urlTitle;
}
}
return null;
}
}

View File

@@ -0,0 +1,46 @@
package com.abin.mallchat.common.common.utils.discover;
import cn.hutool.core.date.StopWatch;
import cn.hutool.core.util.StrUtil;
import com.abin.mallchat.common.common.utils.discover.domain.UrlInfo;
import org.jsoup.nodes.Document;
import javax.annotation.Nullable;
import java.util.Map;
/**
* @author zhaoqichao
* @date 2023/7/3 16:34
*/
public interface UrlDiscover {
@Nullable
Map<String,UrlInfo> getUrlContentMap(String content);
@Nullable
UrlInfo getContent(String url);
@Nullable
String getTitle(Document document);
@Nullable
String getDescription(Document document);
@Nullable
String getImage(String url, Document document);
public static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start();
String longStr = "其中包含一个URL www.baidu.com,一个带有端口号的URL http://www.jd.com:80, 一个带有路径的URL http://mallchat.cn, 还有美团技术文章https://mp.weixin.qq.com/s/hwTf4bDck9_tlFpgVDeIKg ";
// String longStr = "一个带有端口号的URL http://www.jd.com:80,";
// String longStr = "一个带有路径的URL http://mallchat.cn";
PrioritizedUrlDiscover discover = new PrioritizedUrlDiscover();
final Map<String, UrlInfo> map = discover.getUrlContentMap(longStr);
System.out.println(map);
stopWatch.stop();
long cost = stopWatch.getTotalTimeMillis();
System.out.println(cost);
}
}

View File

@@ -1,35 +0,0 @@
package com.abin.mallchat.common.common.utils.discover;
import cn.hutool.core.date.StopWatch;
import org.jsoup.nodes.Document;
import javax.annotation.Nullable;
import java.util.Map;
public interface UrlTitleDiscover {
@Nullable
Map<String, String> getContentTitleMap(String content);
@Nullable
String getUrlTitle(String url);
@Nullable
String getDocTitle(Document document);
public static void main(String[] args) {//用异步多任务查询并合并 974 //串行访问的速度1349 1291 1283 1559
StopWatch stopWatch = new StopWatch();
stopWatch.start();
String longStr = "这是一个很长的字符串再来 www.github.com其中包含一个URL www.baidu.com,, 一个带有端口号的URL http://www.jd.com:80, 一个带有路径的URL http://mallchat.cn, 还有美团技术文章https://mp.weixin.qq.com/s/hwTf4bDck9_tlFpgVDeIKg ";
PrioritizedUrlTitleDiscover discover = new PrioritizedUrlTitleDiscover();
Map<String, String> contentTitleMap = discover.getContentTitleMap(longStr);
System.out.println(contentTitleMap);
//
// Jsoup.connect("http:// www.github.com");
stopWatch.stop();
long cost = stopWatch.getTotalTimeMillis();
System.out.println(cost);
}//{http://mallchat.cn=MallChat, www.baidu.com=百度一下,你就知道, https://mp.weixin.qq.com/s/hwTf4bDck9_tlFpgVDeIKg=超大规模数据库集群保稳系列之二:数据库攻防演练建设实践, http://www.jd.com:80=京东(JD.COM)-正品低价、品质保障、配送及时、轻松购物!}
}

View File

@@ -0,0 +1,30 @@
package com.abin.mallchat.common.common.utils.discover;
import org.jetbrains.annotations.Nullable;
import org.jsoup.nodes.Document;
/**
* Description: 针对微信公众号文章的标题获取类
* Author: <a href="https://github.com/zongzibinbin">abin</a>
* Date: 2023-05-27
*/
public class WxUrlDiscover extends AbstractUrlDiscover {
@Nullable
@Override
public String getTitle(Document document) {
return document.getElementsByAttributeValue("property", "og:title").attr("content");
}
@Nullable
@Override
public String getDescription(Document document) {
return document.getElementsByAttributeValue("property", "og:description").attr("content");
}
@Nullable
@Override
public String getImage(String url, Document document) {
return document.getElementsByAttributeValue("property", "og:image").attr("content");
}
}

View File

@@ -1,15 +0,0 @@
package com.abin.mallchat.common.common.utils.discover;
import org.jsoup.nodes.Document;
/**
* Description: 针对微信公众号文章的标题获取类
* Author: <a href="https://github.com/zongzibinbin">abin</a>
* Date: 2023-05-27
*/
public class WxUrlTitleDiscover extends AbstractUrlTitleDiscover {
@Override
public String getDocTitle(Document document) {
return document.getElementsByAttributeValue("property", "og:title").attr("content");
}
}

View File

@@ -0,0 +1,32 @@
package com.abin.mallchat.common.common.utils.discover.domain;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* @author zhaoqichao
* @date 2023/7/3 16:12
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class UrlInfo {
/**
* 标题
**/
String title;
/**
* 描述
**/
String description;
/**
* 网站LOGO
**/
String image;
}

View File

@@ -136,8 +136,8 @@ public class MinIOTemplate {
String uid = Optional.ofNullable(req.getUid()).map(String::valueOf).orElse("000000"); String uid = Optional.ofNullable(req.getUid()).map(String::valueOf).orElse("000000");
cn.hutool.core.lang.UUID uuid = cn.hutool.core.lang.UUID.fastUUID(); cn.hutool.core.lang.UUID uuid = cn.hutool.core.lang.UUID.fastUUID();
String suffix = FileNameUtil.getSuffix(req.getFileName()); String suffix = FileNameUtil.getSuffix(req.getFileName());
String year = DateUtil.format(new Date(), DatePattern.NORM_YEAR_PATTERN); String yearAndMonth = DateUtil.format(new Date(), DatePattern.NORM_MONTH_PATTERN);
return req.getFilePath() + StrUtil.SLASH + year + StrUtil.SLASH + uid + StrUtil.SLASH + uuid + StrUtil.DOT + suffix; return req.getFilePath() + StrUtil.SLASH + yearAndMonth + StrUtil.SLASH + uid + StrUtil.SLASH + uuid + StrUtil.DOT + suffix;
} }
/** /**

View File

@@ -1,4 +1,4 @@
package com.abin.mallchat.common.common.utils; package com.abin.mallchat.common.common.utils.sensitiveWord;
import com.abin.mallchat.common.common.algorithm.ac.ACTrie; import com.abin.mallchat.common.common.algorithm.ac.ACTrie;
import com.abin.mallchat.common.common.algorithm.ac.MatchResult; import com.abin.mallchat.common.common.algorithm.ac.MatchResult;
@@ -15,7 +15,7 @@ import java.util.Objects;
* *
* Created by berg on 2023/6/18. * Created by berg on 2023/6/18.
*/ */
public class SensitiveWordUtils0 { public class ACFilter implements SensitiveWordFilter {
private final static char mask_char = '*'; // 替代字符 private final static char mask_char = '*'; // 替代字符
@@ -27,7 +27,7 @@ public class SensitiveWordUtils0 {
* @param text 文本 * @param text 文本
* @return boolean * @return boolean
*/ */
public static boolean hasSensitiveWord(String text) { public boolean hasSensitiveWord(String text) {
if (StringUtils.isBlank(text)) return false; if (StringUtils.isBlank(text)) return false;
return !Objects.equals(filter(text), text); return !Objects.equals(filter(text), text);
} }
@@ -38,7 +38,7 @@ public class SensitiveWordUtils0 {
* @param text 待替换文本 * @param text 待替换文本
* @return 替换后的文本 * @return 替换后的文本
*/ */
public static String filter(String text) { public String filter(String text) {
if (StringUtils.isBlank(text)) return text; if (StringUtils.isBlank(text)) return text;
List<MatchResult> matchResults = ac_trie.matches(text); List<MatchResult> matchResults = ac_trie.matches(text);
StringBuffer result = new StringBuffer(text); StringBuffer result = new StringBuffer(text);
@@ -62,7 +62,7 @@ public class SensitiveWordUtils0 {
* *
* @param words 敏感词数组 * @param words 敏感词数组
*/ */
public static void loadWord(List<String> words) { public void loadWord(List<String> words) {
if (words == null) return; if (words == null) return;
ac_trie = new ACTrie(words); ac_trie = new ACTrie(words);
} }

View File

@@ -1,4 +1,4 @@
package com.abin.mallchat.common.common.utils; package com.abin.mallchat.common.common.utils.sensitiveWord;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
@@ -18,7 +18,10 @@ import java.util.*;
* @author zhaoyuhang * @author zhaoyuhang
* @date 2023/06/19 * @date 2023/06/19
*/ */
public final class SensitiveWordUtils { public final class DFAFilter implements SensitiveWordFilter {
private DFAFilter() {
}
private static Word root = new Word(' '); // 敏感词字典的根节点 private static Word root = new Word(' '); // 敏感词字典的根节点
private final static char replace = '*'; // 替代字符 private final static char replace = '*'; // 替代字符
private final static String skipChars = " !*-+_=,.@;:;:。、??()【】[]《》<>“”\""; // 遇到这些字符就会跳过 private final static String skipChars = " !*-+_=,.@;:;:。、??()【】[]《》<>“”\""; // 遇到这些字符就会跳过
@@ -30,6 +33,10 @@ public final class SensitiveWordUtils {
} }
} }
public static DFAFilter getInstance() {
return new DFAFilter();
}
/** /**
* 判断文本中是否存在敏感词 * 判断文本中是否存在敏感词
@@ -37,7 +44,7 @@ public final class SensitiveWordUtils {
* @param text 文本 * @param text 文本
* @return true: 存在敏感词, false: 不存在敏感词 * @return true: 存在敏感词, false: 不存在敏感词
*/ */
public static boolean hasSensitiveWord(String text) { public boolean hasSensitiveWord(String text) {
if (StringUtils.isBlank(text)) return false; if (StringUtils.isBlank(text)) return false;
return !Objects.equals(filter(text), text); return !Objects.equals(filter(text), text);
} }
@@ -48,7 +55,7 @@ public final class SensitiveWordUtils {
* @param text 待替换文本 * @param text 待替换文本
* @return 替换后的文本 * @return 替换后的文本
*/ */
public static String filter(String text) { public String filter(String text) {
StringBuilder result = new StringBuilder(text); StringBuilder result = new StringBuilder(text);
int index = 0; int index = 0;
while (index < result.length()) { while (index < result.length()) {
@@ -93,7 +100,7 @@ public final class SensitiveWordUtils {
* *
* @param words 敏感词数组 * @param words 敏感词数组
*/ */
public static void loadWord(List<String> words) { public void loadWord(List<String> words) {
if (!CollectionUtils.isEmpty(words)) { if (!CollectionUtils.isEmpty(words)) {
Word newRoot = new Word(' '); Word newRoot = new Word(' ');
words.forEach(word -> loadWord(word, newRoot)); words.forEach(word -> loadWord(word, newRoot));
@@ -106,7 +113,7 @@ public final class SensitiveWordUtils {
* *
* @param word * @param word
*/ */
public static void loadWord(String word, Word root) { public void loadWord(String word, Word root) {
if (StringUtils.isBlank(word)) { if (StringUtils.isBlank(word)) {
return; return;
} }
@@ -136,7 +143,7 @@ public final class SensitiveWordUtils {
* *
* @param path 文本文件的绝对路径 * @param path 文本文件的绝对路径
*/ */
public static void loadWordFromFile(String path) { public void loadWordFromFile(String path) {
try (InputStream inputStream = Files.newInputStream(Paths.get(path))) { try (InputStream inputStream = Files.newInputStream(Paths.get(path))) {
loadWord(inputStream); loadWord(inputStream);
} catch (IOException e) { } catch (IOException e) {
@@ -150,7 +157,7 @@ public final class SensitiveWordUtils {
* @param inputStream 文本文件输入流 * @param inputStream 文本文件输入流
* @throws IOException IO异常 * @throws IOException IO异常
*/ */
public static void loadWord(InputStream inputStream) throws IOException { public void loadWord(InputStream inputStream) throws IOException {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
String line; String line;
ArrayList<String> list = new ArrayList<>(); ArrayList<String> list = new ArrayList<>();
@@ -167,7 +174,7 @@ public final class SensitiveWordUtils {
* @param c 待检测字符 * @param c 待检测字符
* @return true: 需要跳过, false: 不需要跳过 * @return true: 需要跳过, false: 不需要跳过
*/ */
private static boolean skip(char c) { private boolean skip(char c) {
return skipSet.contains(c); return skipSet.contains(c);
} }
@@ -186,17 +193,7 @@ public final class SensitiveWordUtils {
public Word(char c) { public Word(char c) {
this.c = c; this.c = c;
this.end = false;
this.next = new HashMap<>(); this.next = new HashMap<>();
} }
} }
public static void main(String[] args) {
String text = "白日,梦";
String filter = filter(text);
System.out.println(filter);
}
} }

View File

@@ -0,0 +1,18 @@
package com.abin.mallchat.common.common.utils.sensitiveWord;
import java.util.List;
/**
* 敏感词
*
* @author zhaoyuhang
* @date 2023/07/09
*/
public interface IWordDeny {
/**
* 获取结果
* @return 结果
* @since 0.0.13
*/
List<String> deny();
}

View File

@@ -0,0 +1,102 @@
package com.abin.mallchat.common.common.utils.sensitiveWord;
import java.util.List;
/**
* 敏感词引导类
*
* @author zhaoyuhang
* @date 2023/07/08
*/
public class SensitiveWordBs {
/**
* 私有化构造器
*/
private SensitiveWordBs() {
}
/**
* 脱敏策略
*/
private SensitiveWordFilter sensitiveWordFilter = DFAFilter.getInstance();
/**
* 敏感词列表
*/
private IWordDeny wordDeny;
public static SensitiveWordBs newInstance() {
return new SensitiveWordBs();
}
/**
* 初始化
*
* 1. 根据配置,初始化对应的 map。比较消耗性能。
* @since 0.0.13
* @return this
*/
public SensitiveWordBs init() {
List<String> words = wordDeny.deny();
loadWord(words);
return this;
}
/**
* 过滤策略
*
* @param filter 过滤器
* @return 结果
* @since 0.7.0
*/
public SensitiveWordBs filterStrategy(SensitiveWordFilter filter) {
if (filter == null) {
throw new IllegalArgumentException("filter can not be null");
}
this.sensitiveWordFilter = filter;
return this;
}
public SensitiveWordBs sensitiveWord(IWordDeny wordDeny) {
if (wordDeny == null) {
throw new IllegalArgumentException("wordDeny can not be null");
}
this.wordDeny = wordDeny;
return this;
}
/**
* 有敏感词
*
* @param text 文本
* @return boolean
*/
public boolean hasSensitiveWord(String text) {
return sensitiveWordFilter.hasSensitiveWord(text);
}
/**
* 过滤
*
* @param text 文本
* @return {@link String}
*/
public String filter(String text) {
return sensitiveWordFilter.filter(text);
}
/**
* 加载敏感词列表
*
* @param words 敏感词数组
*/
private void loadWord(List<String> words) {
sensitiveWordFilter.loadWord(words);
}
}

View File

@@ -0,0 +1,37 @@
package com.abin.mallchat.common.common.utils.sensitiveWord;
import java.util.List;
/**
* 敏感词过滤
*
* @author zhaoyuhang
* @date 2023/07/08
*/
public interface SensitiveWordFilter {
/**
* 有敏感词
*
* @param text 文本
* @return boolean
*/
boolean hasSensitiveWord(String text);
/**
* 过滤
*
* @param text 文本
* @return {@link String}
*/
String filter(String text);
/**
* 加载敏感词列表
*
* @param words 敏感词数组
*/
void loadWord(List<String> words);
}

View File

@@ -0,0 +1,24 @@
package com.abin.mallchat.common.sensitive;
import com.abin.mallchat.common.common.utils.sensitiveWord.IWordDeny;
import com.abin.mallchat.common.sensitive.dao.SensitiveWordDao;
import com.abin.mallchat.common.sensitive.domain.SensitiveWord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.stream.Collectors;
@Component
public class MyWordDeny implements IWordDeny {
@Autowired
private SensitiveWordDao sensitiveWordDao;
@Override
public List<String> deny() {
return sensitiveWordDao.list()
.stream()
.map(SensitiveWord::getWord)
.collect(Collectors.toList());
}
}

View File

@@ -1,5 +0,0 @@
package com.abin.mallchat.common.sensitive.service;
public interface ISensitiveWordService {
}

View File

@@ -1,39 +0,0 @@
package com.abin.mallchat.common.sensitive.service.impl;
import com.abin.mallchat.common.common.utils.SensitiveWordUtils;
import com.abin.mallchat.common.sensitive.dao.SensitiveWordDao;
import com.abin.mallchat.common.sensitive.domain.SensitiveWord;
import com.abin.mallchat.common.sensitive.service.ISensitiveWordService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import javax.annotation.PostConstruct;
import java.util.List;
import java.util.stream.Collectors;
@Service
@Slf4j
public class SensitiveWordServiceImpl implements ISensitiveWordService {
@Autowired
private SensitiveWordDao sensitiveWordDao;
@Autowired
private ThreadPoolTaskExecutor threadPoolTaskExecutor;
@PostConstruct
public void initSensitiveWord() {
threadPoolTaskExecutor.execute(() -> {
log.info("[initSensitiveWord] start");
List<SensitiveWord> list = sensitiveWordDao.list();
if (!CollectionUtils.isEmpty(list)) {
List<String> wordList = list.stream()
.map(SensitiveWord::getWord)
.collect(Collectors.toList());
SensitiveWordUtils.loadWord(wordList);
}
log.info("[initSensitiveWord] end; loading sensitiveWords num:{}", list.size());
});
}
}

View File

@@ -0,0 +1,28 @@
package com.abin.mallchat.common.user.dao;
import com.abin.mallchat.common.user.domain.entity.UserEmoji;
import com.abin.mallchat.common.user.mapper.UserEmojiMapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import org.springframework.stereotype.Service;
import java.util.List;
/**
* <p>
* 用户表情包 服务实现类
* </p>
*
* @author <a href="https://github.com/zongzibinbin">abin</a>
* @since 2023-07-09
*/
@Service
public class UserEmojiDao extends ServiceImpl<UserEmojiMapper, UserEmoji> {
public List<UserEmoji> listByUid(Long uid) {
return lambdaQuery().eq(UserEmoji::getUid, uid).list();
}
public int countByUid(Long uid) {
return lambdaQuery().eq(UserEmoji::getUid, uid).count();
}
}

View File

@@ -0,0 +1,65 @@
package com.abin.mallchat.common.user.domain.entity;
import com.baomidou.mybatisplus.annotation.*;
import lombok.*;
import java.io.Serializable;
import java.util.Date;
/**
* <p>
* 用户表情包
* </p>
*
* @author <a href="https://github.com/zongzibinbin">abin</a>
* @since 2023-07-09
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
@EqualsAndHashCode(callSuper = false)
@TableName("user_emoji")
public class UserEmoji implements Serializable {
private static final long serialVersionUID = 1L;
/**
* id
*/
@TableId(value = "id", type = IdType.AUTO)
private Long id;
/**
* 用户表ID
*/
@TableField("uid")
private Long uid;
/**
* 表情地址
*/
@TableField("expression_url")
private String expressionUrl;
/**
* 逻辑删除(0-正常,1-删除)
*/
@TableField("delete_status")
@TableLogic(value = "0", delval = "1")
private Integer deleteStatus;
/**
* 创建时间
*/
@TableField("create_time")
private Date createTime;
/**
* 修改时间
*/
@TableField("update_time")
private Date updateTime;
}

View File

@@ -0,0 +1,16 @@
package com.abin.mallchat.common.user.mapper;
import com.abin.mallchat.common.user.domain.entity.UserEmoji;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
/**
* <p>
* 用户表情包 Mapper 接口
* </p>
*
* @author <a href="https://github.com/zongzibinbin">abin</a>
* @since 2023-07-09
*/
public interface UserEmojiMapper extends BaseMapper<UserEmoji> {
}

View File

@@ -0,0 +1,13 @@
package com.abin.mallchat.common.user.mapper;
import com.abin.mallchat.common.user.domain.entity.UserEmoji;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
/**
* 用户表情包 Mapper
*
* @author: WuShiJie
* @createTime: 2023/7/3 14:24
*/
public interface UserEmojisMapper extends BaseMapper<UserEmoji> {
}

View File

@@ -0,0 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.abin.mallchat.common.user.mapper.UserEmojiMapper">
</mapper>

View File

@@ -16,6 +16,19 @@
<groupId>com.abin.mallchat</groupId> <groupId>com.abin.mallchat</groupId>
<artifactId>mallchat-common</artifactId> <artifactId>mallchat-common</artifactId>
</dependency> </dependency>
<!-- Used for unit testing -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
<version>5.3.19</version>
<scope>test</scope>
</dependency>
<!-- token计算 --> <!-- token计算 -->
<dependency> <dependency>
<groupId>com.knuddels</groupId> <groupId>com.knuddels</groupId>

View File

@@ -84,7 +84,7 @@ public class ChatController {
@GetMapping("/public/msg/page") @GetMapping("/public/msg/page")
@ApiOperation("消息列表") @ApiOperation("消息列表")
@FrequencyControl(time = 120, count = 20, target = FrequencyControl.Target.IP) @FrequencyControl(time = 120, count = 20, target = FrequencyControl.Target.IP)
public ApiResult<CursorPageBaseResp<ChatMessageResp>> getMsgPage1(@Valid ChatMessagePageReq request) { public ApiResult<CursorPageBaseResp<ChatMessageResp>> getMsgPage(@Valid ChatMessagePageReq request) {
// black(request); // black(request);
CursorPageBaseResp<ChatMessageResp> msgPage = chatService.getMsgPage(request, RequestHolder.get().getUid()); CursorPageBaseResp<ChatMessageResp> msgPage = chatService.getMsgPage(request, RequestHolder.get().getUid());
filterBlackMsg(msgPage); filterBlackMsg(msgPage);
@@ -94,7 +94,6 @@ public class ChatController {
private void filterBlackMsg(CursorPageBaseResp<ChatMessageResp> memberPage) { private void filterBlackMsg(CursorPageBaseResp<ChatMessageResp> memberPage) {
Set<String> blackMembers = getBlackUidSet(); Set<String> blackMembers = getBlackUidSet();
memberPage.getList().removeIf(a -> blackMembers.contains(a.getFromUser().getUid().toString())); memberPage.getList().removeIf(a -> blackMembers.contains(a.getFromUser().getUid().toString()));
System.out.println(1);
} }
@PostMapping("/msg") @PostMapping("/msg")

View File

@@ -1,6 +1,5 @@
package com.abin.mallchat.custom.chat.domain.vo.request; package com.abin.mallchat.custom.chat.domain.vo.request;
import com.abin.mallchat.common.chat.domain.enums.MessageTypeEnum;
import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
@@ -30,7 +29,7 @@ public class ChatMessageReq {
@ApiModelProperty("消息类型") @ApiModelProperty("消息类型")
@NotNull @NotNull
private Integer msgType = MessageTypeEnum.TEXT.getType(); private Integer msgType;
@ApiModelProperty("消息内容类型不同传值不同见https://www.yuque.com/snab/mallcaht/rkb2uz5k1qqdmcmd") @ApiModelProperty("消息内容类型不同传值不同见https://www.yuque.com/snab/mallcaht/rkb2uz5k1qqdmcmd")
@NotNull @NotNull

View File

@@ -7,7 +7,6 @@ import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import java.util.Date; import java.util.Date;
import java.util.Map;
/** /**
* Description: 消息 * Description: 消息
@@ -27,16 +26,8 @@ public class ChatMessageResp {
@Data @Data
public static class UserInfo { public static class UserInfo {
@ApiModelProperty("用户名称")
private String username;
@ApiModelProperty("用户id") @ApiModelProperty("用户id")
private Long uid; private Long uid;
@ApiModelProperty("头像")
private String avatar;
@ApiModelProperty("归属地")
private String locPlace;
@ApiModelProperty("徽章标识如果没有展示null")
private Badge badge;
} }
@Data @Data
@@ -45,36 +36,12 @@ public class ChatMessageResp {
private Long id; private Long id;
@ApiModelProperty("消息发送时间") @ApiModelProperty("消息发送时间")
private Date sendTime; private Date sendTime;
@ApiModelProperty("消息内容-废弃")
@Deprecated
private String content;
@ApiModelProperty("消息链接映射-废弃")
@Deprecated
private Map<String, String> urlTitleMap;
@ApiModelProperty("消息类型 1正常文本 2.撤回消息") @ApiModelProperty("消息类型 1正常文本 2.撤回消息")
private Integer type; private Integer type;
@ApiModelProperty("消息内容不同的消息类型内容体不同见https://www.yuque.com/snab/mallcaht/rkb2uz5k1qqdmcmd") @ApiModelProperty("消息内容不同的消息类型内容体不同见https://www.yuque.com/snab/mallcaht/rkb2uz5k1qqdmcmd")
private Object body; private Object body;
@ApiModelProperty("消息标记") @ApiModelProperty("消息标记")
private MessageMark messageMark; private MessageMark messageMark;
@ApiModelProperty("父消息如果没有父消息返回的是null")
private ReplyMsg reply;
}
@Data
@Deprecated
public static class ReplyMsg {
@ApiModelProperty("消息id")
private Long id;
@ApiModelProperty("用户名称")
private String username;
@ApiModelProperty("消息内容")
private String content;
@ApiModelProperty("是否可消息跳转 0否 1是")
private Integer canCallback;
@ApiModelProperty("跳转间隔的消息条数")
private Integer gapCount;
} }
@Data @Data
@@ -88,12 +55,4 @@ public class ChatMessageResp {
@ApiModelProperty("该用户是否已经举报 0否 1是") @ApiModelProperty("该用户是否已经举报 0否 1是")
private Integer userDislike; private Integer userDislike;
} }
@Data
public static class Badge {
@ApiModelProperty("徽章图像")
private String img;
@ApiModelProperty("徽章说明")
private String describe;
}
} }

View File

@@ -1,5 +1,6 @@
package com.abin.mallchat.custom.chat.domain.vo.response.msg; package com.abin.mallchat.custom.chat.domain.vo.response.msg;
import com.abin.mallchat.common.common.utils.discover.domain.UrlInfo;
import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Builder; import lombok.Builder;
@@ -22,7 +23,7 @@ public class TextMsgResp {
@ApiModelProperty("消息内容") @ApiModelProperty("消息内容")
private String content; private String content;
@ApiModelProperty("消息链接映射") @ApiModelProperty("消息链接映射")
private Map<String, String> urlTitleMap; private Map<String, UrlInfo> urlContentMap;
@ApiModelProperty("艾特的uid") @ApiModelProperty("艾特的uid")
private List<Long> atUidList; private List<Long> atUidList;
@ApiModelProperty("父消息如果没有父消息返回的是null") @ApiModelProperty("父消息如果没有父消息返回的是null")

Some files were not shown because too many files have changed in this diff Show More