diff --git a/.gitignore b/.gitignore index d75b3761..339c4b27 100644 Binary files a/.gitignore and b/.gitignore differ diff --git a/ruoyi-modules/ruoyi-aihuman/pom.xml b/ruoyi-modules/ruoyi-aihuman/pom.xml index d594c634..65327e0e 100644 --- a/ruoyi-modules/ruoyi-aihuman/pom.xml +++ b/ruoyi-modules/ruoyi-aihuman/pom.xml @@ -18,6 +18,8 @@ 3.2.1 + 5.13.0 + 1.5.5 @@ -70,5 +72,24 @@ org.ruoyi ruoyi-common-excel + + + net.java.dev.jna + jna + ${jna.version} + + + + net.java.dev.jna + jna-platform + ${jna.version} + + + + org.java-websocket + Java-WebSocket + ${java-websocket.version} + + \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/config/WebConfig.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/config/WebConfig.java new file mode 100644 index 00000000..5ccaf134 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/config/WebConfig.java @@ -0,0 +1,16 @@ +package org.ruoyi.aihuman.config; + +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +@Configuration +public class WebConfig implements WebMvcConfigurer { + @Override + public void addResourceHandlers(ResourceHandlerRegistry registry) { + // 映射/voice/**路径到classpath:/voice/目录 + registry.addResourceHandler("/voice/**") + .addResourceLocations("classpath:/voice/") + .setCachePeriod(3600); + } +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/controller/AihumanRealConfigController.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/controller/AihumanRealConfigController.java new file mode 100644 index 00000000..e1732db0 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/controller/AihumanRealConfigController.java @@ -0,0 +1,157 @@ +package org.ruoyi.aihuman.controller; + +import java.util.List; + +import cn.dev33.satoken.annotation.SaIgnore; +import lombok.RequiredArgsConstructor; +import jakarta.servlet.http.HttpServletResponse; +import jakarta.validation.constraints.*; +import cn.dev33.satoken.annotation.SaCheckPermission; +import org.ruoyi.common.log.enums.OperatorType; +import org.springframework.web.bind.annotation.*; +import org.springframework.validation.annotation.Validated; +import org.ruoyi.common.idempotent.annotation.RepeatSubmit; +import org.ruoyi.common.log.annotation.Log; +import org.ruoyi.common.web.core.BaseController; +import org.ruoyi.core.page.PageQuery; +import org.ruoyi.common.core.domain.R; +import org.ruoyi.common.core.validate.AddGroup; +import org.ruoyi.common.core.validate.EditGroup; +import org.ruoyi.common.log.enums.BusinessType; +import org.ruoyi.common.excel.utils.ExcelUtil; +import org.ruoyi.aihuman.domain.vo.AihumanRealConfigVo; +import org.ruoyi.aihuman.domain.bo.AihumanRealConfigBo; +import org.ruoyi.aihuman.service.AihumanRealConfigService; +import org.ruoyi.core.page.TableDataInfo; + +/** + * 真人交互数字人配置 + * + * @author ageerle + * @date Tue Oct 21 11:46:52 GMT+08:00 2025 + */ +//临时免登录 +@SaIgnore + +@Validated +@RequiredArgsConstructor +@RestController +@RequestMapping("/aihuman/aihumanRealConfig") +public class AihumanRealConfigController extends BaseController { + + private final AihumanRealConfigService aihumanRealConfigService; + +/** + * 查询真人交互数字人配置列表 + */ +@SaCheckPermission("aihuman:aihumanRealConfig:list") +@GetMapping("/list") + public TableDataInfo list(AihumanRealConfigBo bo, PageQuery pageQuery) { + return aihumanRealConfigService.queryPageList(bo, pageQuery); + } + + /** + * 导出真人交互数字人配置列表 + */ + @SaCheckPermission("aihuman:aihumanRealConfig:export") + @Log(title = "真人交互数字人配置", businessType = BusinessType.EXPORT) + @PostMapping("/export") + public void export(AihumanRealConfigBo bo, HttpServletResponse response) { + List list = aihumanRealConfigService.queryList(bo); + ExcelUtil.exportExcel(list, "真人交互数字人配置", AihumanRealConfigVo.class, response); + } + + /** + * 获取真人交互数字人配置详细信息 + * + * @param id 主键 + */ + @SaCheckPermission("aihuman:aihumanRealConfig:query") + @GetMapping("/{id}") + public R getInfo(@NotNull(message = "主键不能为空") + @PathVariable Integer id) { + return R.ok(aihumanRealConfigService.queryById(id)); + } + + /** + * 新增真人交互数字人配置 + */ + @SaCheckPermission("aihuman:aihumanRealConfig:add") + @Log(title = "真人交互数字人配置", businessType = BusinessType.INSERT) + @RepeatSubmit() + @PostMapping() + public R add(@Validated(AddGroup.class) @RequestBody AihumanRealConfigBo bo) { + return toAjax(aihumanRealConfigService.insertByBo(bo)); + } + + /** + * 修改真人交互数字人配置 + */ + @SaCheckPermission("aihuman:aihumanRealConfig:edit") + @Log(title = "真人交互数字人配置", businessType = BusinessType.UPDATE) + @RepeatSubmit() + @PutMapping() + public R edit(@Validated(EditGroup.class) @RequestBody AihumanRealConfigBo bo) { + return toAjax(aihumanRealConfigService.updateByBo(bo)); + } + + /** + * 删除真人交互数字人配置 + * + * @param ids 主键串 + */ + @SaCheckPermission("aihuman:aihumanRealConfig:remove") + @Log(title = "真人交互数字人配置", businessType = BusinessType.DELETE) + @DeleteMapping("/{ids}") + public R remove(@NotEmpty(message = "主键不能为空") + @PathVariable Integer[] ids) { + return toAjax(aihumanRealConfigService.deleteWithValidByIds(List.of(ids), true)); + } + + /** + * 1.执行以下命令： + * cd F:\Projects\AI-Human\LiveTalking + * conda activate D:\zg117\C\Users\zg117\.conda\envs\livetalking_new + * python app.py --transport webrtc --model wav2lip --avatar_id wav2lip256_avatar1 + * + * 2.监听 python app.py --transport webrtc --model wav2lip --avatar_id wav2lip256_avatar1 执行情况 + * + * 3.返回执行结果并打开页面 + * http://127.0.0.1:8010/webrtcapi-diy.html + */ + @SaCheckPermission("aihuman:aihumanRealConfig:run") + //@Log(title = "真人交互数字人配置", businessType = BusinessType.UPDATE, operatorType = OperatorType.OTHER) + @RepeatSubmit() + @PutMapping("/run") + public R run(@Validated(EditGroup.class) @RequestBody AihumanRealConfigBo bo) { + boolean result = aihumanRealConfigService.runByBo(bo); + if (result) { + // 返回前端页面URL，前端可以根据这个URL跳转或打开新页面 + // http://127.0.0.1:8010/webrtcapi-diy.html 其中的 http://127.0.0.1 获取当前java服务的IP地址 + // return R.ok("http://127.0.0.1:8010/webrtcapi-diy.html"); + // 运行状态 + bo.setRunStatus("1"); + return R.ok("http://127.0.0.1:8010/webrtcapi-diy.html"); + } else { + return R.fail("启动真人交互数字人失败"); + } + } + + /** + * 停止真人交互数字人配置任务 + */ + @SaCheckPermission("aihuman:aihumanRealConfig:stop") + //@Log(title = "真人交互数字人配置", businessType = BusinessType.UPDATE, operatorType = OperatorType.OTHER) + @RepeatSubmit() + @PutMapping("/stop") + public R stop(@Validated(EditGroup.class) @RequestBody AihumanRealConfigBo bo) { + boolean result = aihumanRealConfigService.stopByBo(bo); + if (result) { + // 运行状态 + bo.setRunStatus("0"); + return R.ok("真人交互数字人任务已停止"); + } else { + return R.fail("停止真人交互数字人任务失败或没有正在运行的任务"); + } + } +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/controller/AihumanVolcengineController.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/controller/AihumanVolcengineController.java new file mode 100644 index 00000000..c396ba0a --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/controller/AihumanVolcengineController.java @@ -0,0 +1,508 @@ +package org.ruoyi.aihuman.controller; + +import cn.dev33.satoken.annotation.SaIgnore; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.core.io.ResourceLoader; +import org.springframework.beans.factory.annotation.Autowired; +import org.ruoyi.aihuman.domain.VoiceRequest; +import org.ruoyi.aihuman.protocol.EventType; +import org.ruoyi.aihuman.protocol.Message; +import org.ruoyi.aihuman.protocol.MsgType; +import org.ruoyi.aihuman.protocol.SpeechWebSocketClient; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.HashMap; +import java.util.UUID; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.TimeUnit; + +import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +/** + * 火山引擎相关接口 + * + * @author ruoyi + */ +// 临时免登录 +@SaIgnore + +@Validated +@RequiredArgsConstructor +@Slf4j +@RestController +@RequestMapping("/aihuman/volcengine") +public class AihumanVolcengineController { + + @Autowired + private ResourceLoader resourceLoader; + + private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final Logger logger = LoggerFactory.getLogger(AihumanVolcengineController.class); + + + @PostMapping("/generate-voice-direct") + public ResponseEntity generateVoiceDirect(@RequestBody VoiceRequest request) { + try { + // 生成唯一的语音ID + String voiceId = UUID.randomUUID().toString().replace("-", ""); + + log.info("开始生成语音，voiceId: {}", voiceId); + + // 调用火山引擎TTS API获取音频数据 + byte[] audioData = generateVoiceData(request, voiceId); + + // 设置响应头，返回音频数据 + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.parseMediaType("audio/wav")); + headers.setContentDispositionFormData("attachment", "voice_" + System.currentTimeMillis() + ".wav"); + headers.setContentLength(audioData.length); + + log.info("语音生成成功并返回，长度: {} bytes", audioData.length); + return new ResponseEntity<>(audioData, headers, HttpStatus.OK); + } catch (Exception e) { + log.error("生成语音失败", e); + return new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR); + } + } + + private byte[] generateVoiceData(VoiceRequest request, String voiceId) { + try { + // 这里是调用火山引擎TTS API的核心逻辑 + // 您需要根据火山引擎的API文档实现具体的调用逻辑 + // 注意：这只是一个示例框架，您需要根据实际情况进行实现 + + // 调用火山引擎API并获取音频数据 + // 假设您已经有现有的调用逻辑，这里保留原有的实现 + String endpoint = request.getEndpoint(); + String appId = request.getAppId(); + String accessToken = request.getAccessToken(); + String resourceId = request.getResourceId(); + String voice = request.getVoice(); + String text = request.getText(); + String encoding = request.getEncoding(); + + // 调用原有的火山引擎API调用方法（如果有） + // 或者直接在这里实现API调用逻辑 + byte[] audioData = callVolcengineTtsApiByte(endpoint, appId, accessToken, + resourceId, voice, text, encoding); + + log.info("成功生成语音数据，大小: {} bytes", audioData.length); + return audioData; + } catch (Exception e) { + log.error("生成语音数据失败", e); + throw new RuntimeException("生成语音数据失败", e); + } + + } + + private byte[] callVolcengineTtsApiByte(String endpoint, String appId, String accessToken, + String resourceId, String voice, String text, String encoding) { + try { + // 确保resourceId不为空，如果为空则根据voice类型获取默认值 + if (resourceId == null || resourceId.isEmpty()) { + resourceId = voiceToResourceId(voice); + } + + // 设置请求头 + Map headers = new HashMap<>(); + headers.put("X-Api-App-Key", appId); + headers.put("X-Api-Access-Key", accessToken); + headers.put("X-Api-Resource-Id", resourceId); + headers.put("X-Api-Connect-Id", UUID.randomUUID().toString()); + + // 创建WebSocket客户端 + SpeechWebSocketClient client = new SpeechWebSocketClient(new URI(endpoint), headers); + ByteArrayOutputStream totalAudioStream = new ByteArrayOutputStream(); + boolean audioReceived = false; + + try { + // 连接WebSocket + client.connectBlocking(); + + // 构建请求参数 + Map request = new HashMap<>(); + request.put("user", Map.of("uid", UUID.randomUUID().toString())); + request.put("namespace", "BidirectionalTTS"); + + Map reqParams = new HashMap<>(); + reqParams.put("speaker", voice); + + Map audioParams = new HashMap<>(); + audioParams.put("format", encoding); + audioParams.put("sample_rate", 24000); + audioParams.put("enable_timestamp", true); + + reqParams.put("audio_params", audioParams); + reqParams.put("additions", objectMapper.writeValueAsString(Map.of("disable_markdown_filter", false))); + + request.put("req_params", reqParams); + + // 开始连接 + client.sendStartConnection(); + // 等待连接成功 + client.waitForMessage(MsgType.FULL_SERVER_RESPONSE, EventType.CONNECTION_STARTED); + + // 处理每个句子 + String[] sentences = text.split("。"); + for (int i = 0; i < sentences.length; i++) { + if (sentences[i].trim().isEmpty()) { + continue; + } + + String sessionId = UUID.randomUUID().toString(); + ByteArrayOutputStream sentenceAudioStream = new ByteArrayOutputStream(); + + // 开始会话 + Map startReq = new HashMap<>(); + startReq.put("user", request.get("user")); + startReq.put("namespace", request.get("namespace")); + startReq.put("req_params", request.get("req_params")); + startReq.put("event", EventType.START_SESSION.getValue()); + client.sendStartSession(objectMapper.writeValueAsBytes(startReq), sessionId); + // 等待会话开始 + client.waitForMessage(MsgType.FULL_SERVER_RESPONSE, EventType.SESSION_STARTED); + + // 发送文本内容 + for (char c : sentences[i].toCharArray()) { + @SuppressWarnings("unchecked") + Map currentReqParams = new HashMap<>((Map) request.get("req_params")); + currentReqParams.put("text", String.valueOf(c)); + + Map currentRequest = new HashMap<>(); + currentRequest.put("user", request.get("user")); + currentRequest.put("namespace", request.get("namespace")); + currentRequest.put("req_params", currentReqParams); + currentRequest.put("event", EventType.TASK_REQUEST.getValue()); + + client.sendTaskRequest(objectMapper.writeValueAsBytes(currentRequest), sessionId); + } + + // 结束会话 + client.sendFinishSession(sessionId); + + // 接收响应 + while (true) { + Message msg = client.receiveMessage(); + switch (msg.getType()) { + case FULL_SERVER_RESPONSE: + break; + case AUDIO_ONLY_SERVER: + if (!audioReceived && sentenceAudioStream.size() > 0) { + audioReceived = true; + } + if (msg.getPayload() != null) { + sentenceAudioStream.write(msg.getPayload()); + } + break; + default: + // 不抛出异常，记录日志并继续处理 + log.warn("Unexpected message type: {}", msg.getType()); + } + if (msg.getEvent() == EventType.SESSION_FINISHED) { + break; + } + } + + // 将当前句子的音频追加到总音频流 + if (sentenceAudioStream.size() > 0) { + totalAudioStream.write(sentenceAudioStream.toByteArray()); + } + } + + // 验证是否收到音频数据 + if (totalAudioStream.size() > 0) { + log.info("Audio data generated successfully, size: {} bytes", totalAudioStream.size()); + return totalAudioStream.toByteArray(); + } else { + throw new RuntimeException("No audio data received"); + } + } finally { + // 结束连接 + client.sendFinishConnection(); + client.closeBlocking(); + } + } catch (Exception e) { + log.error("Error calling Volcengine TTS API: {}", e.getMessage(), e); + throw new RuntimeException("Failed to generate voice", e); + } + } + + + /** + * 生成语音文件接口 + * 用户传入JSON参数，返回音频文件的播放地址 + */ + @PostMapping("/generate-voice") + public ResponseEntity generateVoice(@RequestBody VoiceRequest request) { + try { + // 1. 解析请求参数 + String endpoint = request.getEndpoint(); + String appId = request.getAppId(); + String accessToken = request.getAccessToken(); + String resourceId = request.getResourceId(); + String voice = request.getVoice(); + String text = request.getText(); + String encoding = request.getEncoding(); + + // 1.1 验证必要参数 + if (endpoint == null || endpoint.isEmpty()) { + return ResponseEntity.badRequest().body(Map.of("error", "endpoint cannot be null or empty")); + } + if (appId == null || appId.isEmpty()) { + return ResponseEntity.badRequest().body(Map.of("error", "appId cannot be null or empty")); + } + if (accessToken == null || accessToken.isEmpty()) { + return ResponseEntity.badRequest().body(Map.of("error", "accessToken cannot be null or empty")); + } + if (text == null || text.isEmpty()) { + return ResponseEntity.badRequest().body(Map.of("error", "text cannot be null or empty")); + } + + // 1.2 设置默认值 + if (encoding == null || encoding.isEmpty()) { + encoding = "mp3"; + } + + // 2. 调用火山引擎API生成音频文件 + String audioUrl = callVolcengineTtsApi(endpoint, appId, accessToken, resourceId, voice, text, encoding); + + // 3. 构造并返回响应 + Map response = new HashMap<>(); + response.put("audioUrl", audioUrl); + + return ResponseEntity.ok(response); + } catch (Exception e) { + // 处理异常情况 + Map errorResponse = new HashMap<>(); + errorResponse.put("error", "生成音频文件失败: " + e.getMessage()); + return ResponseEntity.status(500).body(errorResponse); + } + } + + /** + * 调用火山引擎TTS API生成音频文件 + */ + private String callVolcengineTtsApi(String endpoint, String appId, String accessToken, + String resourceId, String voice, String text, String encoding) { + try { + // 确保resourceId不为空，如果为空则根据voice类型获取默认值 + if (resourceId == null || resourceId.isEmpty()) { + resourceId = voiceToResourceId(voice); + } + + // 生成唯一的文件名 + String timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss")); + String randomId = UUID.randomUUID().toString().substring(0, 8); + String fileName = "voice_" + timestamp + "_" + randomId + "." + encoding; + + // 获取resources/voice目录路径 + String voiceDirPath = getVoiceDirectoryPath(); + File voiceDir = new File(voiceDirPath); + if (!voiceDir.exists()) { + voiceDir.mkdirs(); + } + + String filePath = voiceDirPath + File.separator + fileName; + + // 设置请求头 + Map headers = new HashMap<>(); + headers.put("X-Api-App-Key", appId); + headers.put("X-Api-Access-Key", accessToken); + headers.put("X-Api-Resource-Id", resourceId); + headers.put("X-Api-Connect-Id", UUID.randomUUID().toString()); + + // 创建WebSocket客户端 + SpeechWebSocketClient client = new SpeechWebSocketClient(new URI(endpoint), headers); + ByteArrayOutputStream totalAudioStream = new ByteArrayOutputStream(); + boolean audioReceived = false; + + try { + // 连接WebSocket + client.connectBlocking(); + + // 构建请求参数 + Map request = new HashMap<>(); + request.put("user", Map.of("uid", UUID.randomUUID().toString())); + request.put("namespace", "BidirectionalTTS"); + + Map reqParams = new HashMap<>(); + reqParams.put("speaker", voice); + + Map audioParams = new HashMap<>(); + audioParams.put("format", encoding); + audioParams.put("sample_rate", 24000); + audioParams.put("enable_timestamp", true); + + reqParams.put("audio_params", audioParams); + reqParams.put("additions", objectMapper.writeValueAsString(Map.of("disable_markdown_filter", false))); + + request.put("req_params", reqParams); + + // 开始连接 + client.sendStartConnection(); + // 等待连接成功 + client.waitForMessage(MsgType.FULL_SERVER_RESPONSE, EventType.CONNECTION_STARTED); + + // 处理每个句子 + String[] sentences = text.split("。"); + for (int i = 0; i < sentences.length; i++) { + if (sentences[i].trim().isEmpty()) { + continue; + } + + String sessionId = UUID.randomUUID().toString(); + ByteArrayOutputStream sentenceAudioStream = new ByteArrayOutputStream(); + + // 开始会话 + Map startReq = new HashMap<>(); + startReq.put("user", request.get("user")); + startReq.put("namespace", request.get("namespace")); + startReq.put("req_params", request.get("req_params")); + startReq.put("event", EventType.START_SESSION.getValue()); + client.sendStartSession(objectMapper.writeValueAsBytes(startReq), sessionId); + // 等待会话开始 + client.waitForMessage(MsgType.FULL_SERVER_RESPONSE, EventType.SESSION_STARTED); + + // 发送文本内容 + for (char c : sentences[i].toCharArray()) { + @SuppressWarnings("unchecked") + Map currentReqParams = new HashMap<>((Map) request.get("req_params")); + currentReqParams.put("text", String.valueOf(c)); + + Map currentRequest = new HashMap<>(); + currentRequest.put("user", request.get("user")); + currentRequest.put("namespace", request.get("namespace")); + currentRequest.put("req_params", currentReqParams); + currentRequest.put("event", EventType.TASK_REQUEST.getValue()); + + client.sendTaskRequest(objectMapper.writeValueAsBytes(currentRequest), sessionId); + } + + // 结束会话 + client.sendFinishSession(sessionId); + + // 接收响应 + while (true) { + Message msg = client.receiveMessage(); + switch (msg.getType()) { + case FULL_SERVER_RESPONSE: + break; + case AUDIO_ONLY_SERVER: + if (!audioReceived && sentenceAudioStream.size() > 0) { + audioReceived = true; + } + if (msg.getPayload() != null) { + sentenceAudioStream.write(msg.getPayload()); + } + break; + default: + // 不抛出异常，记录日志并继续处理 + log.warn("Unexpected message type: {}", msg.getType()); + } + if (msg.getEvent() == EventType.SESSION_FINISHED) { + break; + } + } + + // 将当前句子的音频追加到总音频流 + if (sentenceAudioStream.size() > 0) { + totalAudioStream.write(sentenceAudioStream.toByteArray()); + } + } + + // 保存音频文件 + if (totalAudioStream.size() > 0) { + Files.write(Paths.get(filePath), totalAudioStream.toByteArray(), StandardOpenOption.CREATE); + log.info("Audio saved to file: {}", filePath); + } else { + throw new RuntimeException("No audio data received"); + } + + // 结束连接 + client.sendFinishConnection(); + } finally { + client.closeBlocking(); + } + + // 返回音频文件的访问路径 + return "/voice/" + fileName; + } catch (Exception e) { + log.error("Error calling Volcengine TTS API: {}", e.getMessage(), e); + throw new RuntimeException("Failed to generate voice", e); + } + } + + /** + * 根据voice类型获取resourceId + */ + private String voiceToResourceId(String voice) { + if (voice != null && voice.startsWith("S_")) { + return "volc.megatts.default"; + } + return "volc.service_type.10029"; + } + + /** + * 获取voice目录路径 + */ + private String getVoiceDirectoryPath() { + try { + // 获取当前项目根目录 + String projectRoot = System.getProperty("user.dir"); + + // 构建目标目录路径：ruoyi-ai/ruoyi-modules/ruoyi-aihuman/src/main/resources/voice + File targetDir = new File(projectRoot, "ruoyi-modules/ruoyi-aihuman/src/main/resources/voice"); + + // 确保目录存在 + if (!targetDir.exists()) { + boolean created = targetDir.mkdirs(); + if (created) { + logger.info("成功创建目录: {}", targetDir.getAbsolutePath()); + } else { + logger.warn("无法创建目录: {}", targetDir.getAbsolutePath()); + + // 降级方案：直接使用项目根目录下的voice文件夹 + File fallbackDir = new File(projectRoot, "voice"); + if (!fallbackDir.exists()) { + fallbackDir.mkdirs(); + } + return fallbackDir.getAbsolutePath(); + } + } + + return targetDir.getAbsolutePath(); + } catch (Exception e) { + logger.error("获取音频目录路径失败", e); + + // 异常情况下的安全降级 + File safeDir = new File("voice"); + if (!safeDir.exists()) { + safeDir.mkdirs(); + } + return safeDir.getAbsolutePath(); + } + } +} + + diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/domain/AihumanRealConfig.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/domain/AihumanRealConfig.java new file mode 100644 index 00000000..2c9c207d --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/domain/AihumanRealConfig.java @@ -0,0 +1,101 @@ +package org.ruoyi.aihuman.domain; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.Data; +import java.time.LocalDateTime; +import java.io.Serializable; + +/** + * 真人交互数字人配置对象 aihuman_real_config + * + * @author ageerle + * @date Tue Oct 21 11:46:52 GMT+08:00 2025 + */ +@Data +@TableName("aihuman_real_config") +public class AihumanRealConfig implements Serializable { + + + /** + * 主键id + */ + @TableId(value = "id", type = IdType.AUTO) + private Integer id; + + /** + * 场景名称 + */ + private String name; + + /** + * 真人形象名称 + */ + private String avatars; + + /** + * 模型名称 + */ + private String models; + + /** + * 形象参数（预留） + */ + private String avatarsParams; + + /** + * 模型参数（预留） + */ + private String modelsParams; + + /** + * 智能体参数（扣子） + */ + private String agentParams; + + /** + * 创建时间 + */ + private LocalDateTime createTime; + + /** + * 更新时间 + */ + private LocalDateTime updateTime; + + /** + * 状态 + */ + private Integer status; + + /** + * 发布状态 + */ + private Integer publish; + + /** + * 运行参数 + */ + private String runParams; + + /** + * 运行状态 + */ + private String runStatus; + + /** + * 创建部门 + */ + private String createDept; + + /** + * 创建用户 + */ + private String createBy; + + /** + * 更新用户 + */ + private String updateBy; + + +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/domain/VoiceRequest.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/domain/VoiceRequest.java new file mode 100644 index 00000000..a0ce0c65 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/domain/VoiceRequest.java @@ -0,0 +1,21 @@ +package org.ruoyi.aihuman.domain; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + + +/** + * 语音请求参数实体类 + */ +@Data +public class VoiceRequest { + + @JsonProperty("ENDPOINT") + private String endpoint; + private String appId; + private String accessToken; + private String resourceId; + private String voice; + private String text; + private String encoding; + +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/domain/bo/AihumanRealConfigBo.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/domain/bo/AihumanRealConfigBo.java new file mode 100644 index 00000000..e162f81a --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/domain/bo/AihumanRealConfigBo.java @@ -0,0 +1,87 @@ +package org.ruoyi.aihuman.domain.bo; + +import org.ruoyi.aihuman.domain.AihumanRealConfig; +import io.github.linpeilie.annotations.AutoMapper; +import lombok.Data; + +import java.time.LocalDateTime; +import java.io.Serializable; + +/** + * 真人交互数字人配置业务对象 aihuman_real_config + * + * @author ageerle + * @date Tue Oct 21 11:46:52 GMT+08:00 2025 + */ +@Data + +@AutoMapper(target = AihumanRealConfig.class, reverseConvertGenerate = false) +public class AihumanRealConfigBo implements Serializable { + + private Integer id; + + /** + * 场景名称 + */ + private String name; + /** + * 真人形象名称 + */ + private String avatars; + /** + * 模型名称 + */ + private String models; + /** + * 形象参数（预留） + */ + private String avatarsParams; + /** + * 模型参数（预留） + */ + private String modelsParams; + /** + * 智能体参数（扣子） + */ + private String agentParams; + /** + * 创建时间 + */ + private LocalDateTime createTime; + /** + * 更新时间 + */ + private LocalDateTime updateTime; + /** + * 状态 + */ + private Integer status; + /** + * 发布状态 + */ + private Integer publish; + + /** + * 运行参数 + */ + private String runParams; + + /** + * 运行状态 + */ + private String runStatus; + + /** + * 创建部门 + */ + private String createDept; + /** + * 创建用户 + */ + private String createBy; + /** + * 更新用户 + */ + private String updateBy; + +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/domain/vo/AihumanRealConfigVo.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/domain/vo/AihumanRealConfigVo.java new file mode 100644 index 00000000..da71fc70 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/domain/vo/AihumanRealConfigVo.java @@ -0,0 +1,108 @@ +package org.ruoyi.aihuman.domain.vo; + +import java.time.LocalDateTime; +import java.io.Serializable; +import org.ruoyi.aihuman.domain.AihumanRealConfig; +import com.alibaba.excel.annotation.ExcelIgnoreUnannotated; +import com.alibaba.excel.annotation.ExcelProperty; +import io.github.linpeilie.annotations.AutoMapper; +import lombok.Data; +import org.ruoyi.common.excel.annotation.ExcelDictFormat; +import org.ruoyi.common.excel.convert.ExcelDictConvert; + + +/** + * 真人交互数字人配置视图对象 aihuman_real_config + * + * @author ageerle + * @date Tue Oct 21 11:46:52 GMT+08:00 2025 + */ +@Data +@ExcelIgnoreUnannotated +@AutoMapper(target = AihumanRealConfig.class) +public class AihumanRealConfigVo implements Serializable { + + private Integer id; + /** + * 场景名称 + */ + @ExcelProperty(value = "场景名称") + private String name; + /** + * 真人形象名称 + */ + @ExcelProperty(value = "真人形象名称") + private String avatars; + /** + * 模型名称 + */ + @ExcelProperty(value = "模型名称") + private String models; + /** + * 形象参数（预留） + */ + @ExcelProperty(value = "形象参数", converter = ExcelDictConvert.class) + @ExcelDictFormat(readConverterExp = "$column.readConverterExp()") + private String avatarsParams; + /** + * 模型参数（预留） + */ + @ExcelProperty(value = "模型参数", converter = ExcelDictConvert.class) + @ExcelDictFormat(readConverterExp = "$column.readConverterExp()") + private String modelsParams; + /** + * 智能体参数（扣子） + */ + @ExcelProperty(value = "智能体参数", converter = ExcelDictConvert.class) + @ExcelDictFormat(readConverterExp = "$column.readConverterExp()") + private String agentParams; + /** + * 创建时间 + */ + @ExcelProperty(value = "创建时间") + private LocalDateTime createTime; + /** + * 更新时间 + */ + @ExcelProperty(value = "更新时间") + private LocalDateTime updateTime; + /** + * 状态 + */ + @ExcelProperty(value = "状态") + private Integer status; + /** + * 发布状态 + */ + @ExcelProperty(value = "发布状态") + private Integer publish; + + /** + * 运行参数 + */ + @ExcelProperty(value = "运行参数") + private String runParams; + + /** + * 运行状态 + */ + @ExcelProperty(value = "运行状态") + private String runStatus; + + /** + * 创建部门 + */ + @ExcelProperty(value = "创建部门") + private String createDept; + /** + * 创建用户 + */ + @ExcelProperty(value = "创建用户") + private String createBy; + /** + * 更新用户 + */ + @ExcelProperty(value = "更新用户") + private String updateBy; + +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/mapper/AihumanRealConfigMapper.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/mapper/AihumanRealConfigMapper.java new file mode 100644 index 00000000..9aae74d7 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/mapper/AihumanRealConfigMapper.java @@ -0,0 +1,17 @@ +package org.ruoyi.aihuman.mapper; + +import org.ruoyi.aihuman.domain.AihumanRealConfig; +import org.ruoyi.aihuman.domain.vo.AihumanRealConfigVo; +import org.ruoyi.core.mapper.BaseMapperPlus; +import org.apache.ibatis.annotations.Mapper; + +/** + * 真人交互数字人配置Mapper接口 + * + * @author ageerle + * @date Tue Oct 21 11:46:52 GMT+08:00 2025 + */ +@Mapper +public interface AihumanRealConfigMapper extends BaseMapperPlus { + +} diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/CompressionBits.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/CompressionBits.java new file mode 100644 index 00000000..78794aa4 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/CompressionBits.java @@ -0,0 +1,26 @@ +package org.ruoyi.aihuman.protocol; + +import lombok.Getter; + +@Getter +public enum CompressionBits { + None_((byte) 0), + Gzip((byte) 0b1), + Custom((byte) 0b11), + ; + + private final byte value; + + CompressionBits(byte b) { + this.value = b; + } + + public static CompressionBits fromValue(int value) { + for (CompressionBits type : CompressionBits.values()) { + if (type.value == value) { + return type; + } + } + throw new IllegalArgumentException("Unknown CompressionBits value: " + value); + } +} diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/EventType.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/EventType.java new file mode 100644 index 00000000..ece69af3 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/EventType.java @@ -0,0 +1,90 @@ +package org.ruoyi.aihuman.protocol; + +import lombok.Getter; + +@Getter +public enum EventType { + // Default event + NONE(0), + + // Upstream Connection events (1-49) + START_CONNECTION(1), + START_TASK(1), + FINISH_CONNECTION(2), + FINISH_TASK(2), + + // Downstream Connection events (50-99) + CONNECTION_STARTED(50), + TASK_STARTED(50), + CONNECTION_FAILED(51), + TASK_FAILED(51), + CONNECTION_FINISHED(52), + TASK_FINISHED(52), + + // Upstream Session events (100-149) + START_SESSION(100), + CANCEL_SESSION(101), + FINISH_SESSION(102), + + // Downstream Session events (150-199) + SESSION_STARTED(150), + SESSION_CANCELED(151), + SESSION_FINISHED(152), + SESSION_FAILED(153), + USAGE_RESPONSE(154), + CHARGE_DATA(154), + + // Upstream General events (200-249) + TASK_REQUEST(200), + UPDATE_CONFIG(201), + + // Downstream General events (250-299) + AUDIO_MUTED(250), + + // Upstream TTS events (300-349) + SAY_HELLO(300), + + // Downstream TTS events (350-399) + TTS_SENTENCE_START(350), + TTS_SENTENCE_END(351), + TTS_RESPONSE(352), + TTS_ENDED(359), + PODCAST_ROUND_START(360), + PODCAST_ROUND_RESPONSE(361), + PODCAST_ROUND_END(362), + + // Downstream ASR events (450-499) + ASR_INFO(450), + ASR_RESPONSE(451), + ASR_ENDED(459), + + // Upstream Chat events (500-549) + CHAT_TTS_TEXT(500), + + // Downstream Chat events (550-599) + CHAT_RESPONSE(550), + CHAT_ENDED(559), + + // Subtitle events (650-699) + SOURCE_SUBTITLE_START(650), + SOURCE_SUBTITLE_RESPONSE(651), + SOURCE_SUBTITLE_END(652), + TRANSLATION_SUBTITLE_START(653), + TRANSLATION_SUBTITLE_RESPONSE(654), + TRANSLATION_SUBTITLE_END(655); + + private final int value; + + EventType(int value) { + this.value = value; + } + + public static EventType fromValue(int value) { + for (EventType type : EventType.values()) { + if (type.value == value) { + return type; + } + } + throw new IllegalArgumentException("Unknown EventType value: " + value); + } +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/HeaderSizeBits.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/HeaderSizeBits.java new file mode 100644 index 00000000..6c09a20d --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/HeaderSizeBits.java @@ -0,0 +1,27 @@ +package org.ruoyi.aihuman.protocol; + +import lombok.Getter; + +@Getter +public enum HeaderSizeBits { + HeaderSize4((byte) 1), + HeaderSize8((byte) 2), + HeaderSize12((byte) 3), + HeaderSize16((byte) 4), + ; + + private final byte value; + + HeaderSizeBits(byte b) { + this.value = b; + } + + public static HeaderSizeBits fromValue(int value) { + for (HeaderSizeBits type : HeaderSizeBits.values()) { + if (type.value == value) { + return type; + } + } + throw new IllegalArgumentException("Unknown HeaderSizeBits value: " + value); + } +} diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/Message.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/Message.java new file mode 100644 index 00000000..d3605edc --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/Message.java @@ -0,0 +1,220 @@ +package org.ruoyi.aihuman.protocol; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +@Slf4j +@Data +public class Message { + private byte version = VersionBits.Version1.getValue(); + private byte headerSize = HeaderSizeBits.HeaderSize4.getValue(); + private MsgType type; + private MsgTypeFlagBits flag; + private byte serialization = SerializationBits.JSON.getValue(); + private byte compression = 0; + + private EventType event; + private String sessionId; + private String connectId; + private int sequence; + private int errorCode; + + private byte[] payload; + + public Message(MsgType type, MsgTypeFlagBits flag) { + this.type = type; + this.flag = flag; + } + + public static Message unmarshal(byte[] data) throws Exception { + ByteBuffer buffer = ByteBuffer.wrap(data); + + byte type_and_flag = data[1]; + MsgType type = MsgType.fromValue((type_and_flag >> 4) & 0x0F); + MsgTypeFlagBits flag = MsgTypeFlagBits.fromValue(type_and_flag & 0x0F); + + // Read version and header size + int versionAndHeaderSize = buffer.get(); + VersionBits version = VersionBits.fromValue((versionAndHeaderSize >> 4) & 0x0F); + HeaderSizeBits headerSize = HeaderSizeBits.fromValue(versionAndHeaderSize & 0x0F); + + // Skip second byte + buffer.get(); + + // Read serialization and compression method + int serializationCompression = buffer.get(); + SerializationBits serialization = SerializationBits.fromValue((serializationCompression >> 4) & 0x0F); + CompressionBits compression = CompressionBits.fromValue(serializationCompression & 0x0F); + + // Skip padding bytes + int headerSizeInt = 4 * (int) headerSize.getValue(); + int paddingSize = headerSizeInt - 3; + while (paddingSize > 0) { + buffer.get(); + paddingSize -= 1; + } + + Message message = new Message(type, flag); + message.setVersion(version.getValue()); + message.setHeaderSize(headerSize.getValue()); + message.setSerialization(serialization.getValue()); + message.setCompression(compression.getValue()); + + // Read sequence if present + if (flag == MsgTypeFlagBits.POSITIVE_SEQ || flag == MsgTypeFlagBits.NEGATIVE_SEQ) { + // Read 4 bytes from ByteBuffer and parse as int (big-endian) + byte[] sequeueBytes = new byte[4]; + if (buffer.remaining() >= 4) { + buffer.get(sequeueBytes); // Read 4 bytes into array + ByteBuffer wrapper = ByteBuffer.wrap(sequeueBytes); + wrapper.order(ByteOrder.BIG_ENDIAN); // Set big-endian order + message.setSequence(wrapper.getInt()); + } + } + + // Read event if present + if (flag == MsgTypeFlagBits.WITH_EVENT) { + // Read 4 bytes from ByteBuffer and parse as int (big-endian) + byte[] eventBytes = new byte[4]; + if (buffer.remaining() >= 4) { + buffer.get(eventBytes); // Read 4 bytes into array + ByteBuffer wrapper = ByteBuffer.wrap(eventBytes); + wrapper.order(ByteOrder.BIG_ENDIAN); // Set big-endian order + message.setEvent(EventType.fromValue(wrapper.getInt())); + } + + if (type != MsgType.ERROR && !(message.event == EventType.START_CONNECTION + || message.event == EventType.FINISH_CONNECTION || + message.event == EventType.CONNECTION_STARTED + || message.event == EventType.CONNECTION_FAILED || + message.event == EventType.CONNECTION_FINISHED)) { + // Read sessionId if present + int sessionIdLength = buffer.getInt(); + if (sessionIdLength > 0) { + byte[] sessionIdBytes = new byte[sessionIdLength]; + buffer.get(sessionIdBytes); + message.setSessionId(new String(sessionIdBytes, StandardCharsets.UTF_8)); + } + } + + if (message.event == EventType.CONNECTION_STARTED || message.event == EventType.CONNECTION_FAILED + || message.event == EventType.CONNECTION_FINISHED) { + // Read connectId if present + int connectIdLength = buffer.getInt(); + if (connectIdLength > 0) { + byte[] connectIdBytes = new byte[connectIdLength]; + buffer.get(connectIdBytes); + message.setConnectId(new String(connectIdBytes, StandardCharsets.UTF_8)); + } + } + } + + // Read errorCode if present + if (type == MsgType.ERROR) { + // Read 4 bytes from ByteBuffer and parse as int (big-endian) + byte[] errorCodeBytes = new byte[4]; + if (buffer.remaining() >= 4) { + buffer.get(errorCodeBytes); // Read 4 bytes into array + ByteBuffer wrapper = ByteBuffer.wrap(errorCodeBytes); + wrapper.order(ByteOrder.BIG_ENDIAN); // Set big-endian order + message.setErrorCode(wrapper.getInt()); + } + } + + // Read remaining bytes as payload + if (buffer.remaining() > 0) { + // 4 bytes length + int payloadLength = buffer.getInt(); + if (payloadLength > 0) { + byte[] payloadBytes = new byte[payloadLength]; + buffer.get(payloadBytes); + message.setPayload(payloadBytes); + } + } + + return message; + } + + public byte[] marshal() throws Exception { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + + // Write header + buffer.write((version & 0x0F) << 4 | (headerSize & 0x0F)); + buffer.write((type.getValue() & 0x0F) << 4 | (flag.getValue() & 0x0F)); + buffer.write((serialization & 0x0F) << 4 | (compression & 0x0F)); + + int headerSizeInt = 4 * (int) headerSize; + int padding = headerSizeInt - buffer.size(); + while (padding > 0) { + buffer.write(0); + padding -= 1; + } + + // Write event if present + if (event != null) { + byte[] eventBytes = ByteBuffer.allocate(4).putInt(event.getValue()).array(); + buffer.write(eventBytes); + } + + // Write sessionId if present + if (sessionId != null) { + byte[] sessionIdBytes = sessionId.getBytes(StandardCharsets.UTF_8); + buffer.write(ByteBuffer.allocate(4).putInt(sessionIdBytes.length).array()); + buffer.write(sessionIdBytes); + } + + // Write connectId if present + if (connectId != null) { + byte[] connectIdBytes = connectId.getBytes(StandardCharsets.UTF_8); + buffer.write(ByteBuffer.allocate(4).putInt(connectIdBytes.length).array()); + buffer.write(connectIdBytes); + } + + // Write sequence if present + if (sequence != 0) { + buffer.write(ByteBuffer.allocate(4).putInt(sequence).array()); + } + + // Write errorCode if present + if (errorCode != 0) { + buffer.write(ByteBuffer.allocate(4).putInt(errorCode).array()); + } + + // Write payload if present + if (payload != null && payload.length > 0) { + buffer.write(ByteBuffer.allocate(4).putInt(payload.length).array()); + buffer.write(payload); + } + return buffer.toByteArray(); + } + + @Override + public String toString() { + switch (this.type) { + case AUDIO_ONLY_SERVER: + case AUDIO_ONLY_CLIENT: + if (this.flag == MsgTypeFlagBits.POSITIVE_SEQ || this.flag == MsgTypeFlagBits.NEGATIVE_SEQ) { + return String.format("MsgType: %s, EventType: %s, Sequence: %d, PayloadSize: %d", this.type, this.event, this.sequence, + this.payload != null ? this.payload.length : 0); + } + return String.format("MsgType: %s, EventType: %s, PayloadSize: %d", this.type, this.event, + this.payload != null ? this.payload.length : 0); + case ERROR: + return String.format("MsgType: %s, EventType: %s, ErrorCode: %d, Payload: %s", this.type, this.event, this.errorCode, + this.payload != null ? new String(this.payload) : "null"); + default: + if (this.flag == MsgTypeFlagBits.POSITIVE_SEQ || this.flag == MsgTypeFlagBits.NEGATIVE_SEQ) { + return String.format("MsgType: %s, EventType: %s, Sequence: %d, Payload: %s", + this.type, this.event, this.sequence, + this.payload != null ? new String(this.payload) : "null"); + } + return String.format("MsgType: %s, EventType: %s, Payload: %s", this.type, this.event, + this.payload != null ? new String(this.payload) : "null"); + } + } +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/MsgType.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/MsgType.java new file mode 100644 index 00000000..965f40e8 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/MsgType.java @@ -0,0 +1,29 @@ +package org.ruoyi.aihuman.protocol; + +import lombok.Getter; + +@Getter +public enum MsgType { + INVALID((byte) 0), + FULL_CLIENT_REQUEST((byte) 0b1), + AUDIO_ONLY_CLIENT((byte) 0b10), + FULL_SERVER_RESPONSE((byte) 0b1001), + AUDIO_ONLY_SERVER((byte) 0b1011), + FRONT_END_RESULT_SERVER((byte) 0b1100), + ERROR((byte) 0b1111); + + private final byte value; + + MsgType(byte value) { + this.value = value; + } + + public static MsgType fromValue(int value) { + for (MsgType type : MsgType.values()) { + if (type.value == value) { + return type; + } + } + throw new IllegalArgumentException("Unknown MsgType value: " + value); + } +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/MsgTypeFlagBits.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/MsgTypeFlagBits.java new file mode 100644 index 00000000..393a24dc --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/MsgTypeFlagBits.java @@ -0,0 +1,27 @@ +package org.ruoyi.aihuman.protocol; + +import lombok.Getter; + +@Getter +public enum MsgTypeFlagBits { + NO_SEQ((byte) 0), // Non-terminating packet without sequence number + POSITIVE_SEQ((byte) 0b1), // Non-terminating packet with positive sequence number + LAST_NO_SEQ((byte) 0b10), // Terminating packet without sequence number + NEGATIVE_SEQ((byte) 0b11), // Terminating packet with negative sequence number + WITH_EVENT((byte) 0b100); // Packet containing event number + + private final byte value; + + MsgTypeFlagBits(byte value) { + this.value = value; + } + + public static MsgTypeFlagBits fromValue(int value) { + for (MsgTypeFlagBits flag : MsgTypeFlagBits.values()) { + if (flag.value == value) { + return flag; + } + } + throw new IllegalArgumentException("Unknown MsgTypeFlagBits value: " + value); + } +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/SerializationBits.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/SerializationBits.java new file mode 100644 index 00000000..01e8f102 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/SerializationBits.java @@ -0,0 +1,27 @@ +package org.ruoyi.aihuman.protocol; + +import lombok.Getter; + +@Getter +public enum SerializationBits { + Raw((byte) 0), + JSON((byte) 0b1), + Thrift((byte) 0b11), + Custom((byte) 0b1111), + ; + + private final byte value; + + SerializationBits(byte b) { + this.value = b; + } + + public static SerializationBits fromValue(int value) { + for (SerializationBits type : SerializationBits.values()) { + if (type.value == value) { + return type; + } + } + throw new IllegalArgumentException("Unknown SerializationBits value: " + value); + } +} diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/SpeechWebSocketClient.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/SpeechWebSocketClient.java new file mode 100644 index 00000000..c41e57b3 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/SpeechWebSocketClient.java @@ -0,0 +1,115 @@ +package org.ruoyi.aihuman.protocol; + +import lombok.extern.slf4j.Slf4j; +import org.java_websocket.client.WebSocketClient; +import org.java_websocket.handshake.ServerHandshake; + +import java.net.URI; +import java.nio.ByteBuffer; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +@Slf4j +public class SpeechWebSocketClient extends WebSocketClient { + private final BlockingQueue messageQueue = new LinkedBlockingQueue<>(); + + public SpeechWebSocketClient(URI serverUri, Map headers) { + super(serverUri, headers); + } + + @Override + public void onOpen(ServerHandshake handshakedata) { + log.info("WebSocket connection established, Logid: {}", handshakedata.getFieldValue("x-tt-logid")); + } + + @Override + public void onMessage(String message) { + log.warn("Received unexpected text message: {}", message); + } + + @Override + public void onMessage(ByteBuffer bytes) { + try { + Message message = Message.unmarshal(bytes.array()); + messageQueue.put(message); + } catch (Exception e) { + log.error("Failed to parse message", e); + } + } + + @Override + public void onClose(int code, String reason, boolean remote) { + log.info("WebSocket connection closed: code={}, reason={}, remote={}", code, reason, remote); + } + + @Override + public void onError(Exception ex) { + log.error("WebSocket error", ex); + } + + public void sendStartConnection() throws Exception { + Message message = new Message(MsgType.FULL_CLIENT_REQUEST, MsgTypeFlagBits.WITH_EVENT); + message.setEvent(EventType.START_CONNECTION); + message.setPayload("{}".getBytes()); + sendMessage(message); + } + + public void sendFinishConnection() throws Exception { + Message message = new Message(MsgType.FULL_CLIENT_REQUEST, MsgTypeFlagBits.WITH_EVENT); + message.setEvent(EventType.FINISH_CONNECTION); + sendMessage(message); + } + + public void sendStartSession(byte[] payload, String sessionId) throws Exception { + Message message = new Message(MsgType.FULL_CLIENT_REQUEST, MsgTypeFlagBits.WITH_EVENT); + message.setEvent(EventType.START_SESSION); + message.setSessionId(sessionId); + message.setPayload(payload); + sendMessage(message); + } + + public void sendFinishSession(String sessionId) throws Exception { + Message message = new Message(MsgType.FULL_CLIENT_REQUEST, MsgTypeFlagBits.WITH_EVENT); + message.setEvent(EventType.FINISH_SESSION); + message.setSessionId(sessionId); + message.setPayload("{}".getBytes()); + sendMessage(message); + } + + public void sendTaskRequest(byte[] payload, String sessionId) throws Exception { + Message message = new Message(MsgType.FULL_CLIENT_REQUEST, MsgTypeFlagBits.WITH_EVENT); + message.setEvent(EventType.TASK_REQUEST); + message.setSessionId(sessionId); + message.setPayload(payload); + sendMessage(message); + } + + public void sendFullClientMessage(byte[] payload) throws Exception { + Message message = new Message(MsgType.FULL_CLIENT_REQUEST, MsgTypeFlagBits.NO_SEQ); + message.setPayload(payload); + sendMessage(message); + } + + public void sendMessage(Message message) throws Exception { + log.info("Send: {}", message); + send(message.marshal()); + } + + public Message receiveMessage() throws InterruptedException { + Message message = messageQueue.take(); + log.info("Receive: {}", message); + return message; + } + + public Message waitForMessage(MsgType type, EventType event) throws InterruptedException { + while (true) { + Message message = receiveMessage(); + if (message.getType() == type && message.getEvent() == event) { + return message; + } else { + throw new RuntimeException("Unexpected message: " + message); + } + } + } +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/VersionBits.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/VersionBits.java new file mode 100644 index 00000000..6f4a8174 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/protocol/VersionBits.java @@ -0,0 +1,27 @@ +package org.ruoyi.aihuman.protocol; + +import lombok.Getter; + +@Getter +public enum VersionBits { + Version1((byte) 1), + Version2((byte) 2), + Version3((byte) 3), + Version4((byte) 4), + ; + + private final byte value; + + VersionBits(byte b) { + this.value = b; + } + + public static VersionBits fromValue(int value) { + for (VersionBits type : VersionBits.values()) { + if (type.value == value) { + return type; + } + } + throw new IllegalArgumentException("Unknown VersionBits value: " + value); + } +} diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/service/AihumanRealConfigService.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/service/AihumanRealConfigService.java new file mode 100644 index 00000000..0f258ffa --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/service/AihumanRealConfigService.java @@ -0,0 +1,56 @@ +package org.ruoyi.aihuman.service; + +import org.ruoyi.aihuman.domain.vo.AihumanRealConfigVo; +import org.ruoyi.aihuman.domain.bo.AihumanRealConfigBo; + import org.ruoyi.core.page.TableDataInfo; + import org.ruoyi.core.page.PageQuery; + +import java.util.Collection; +import java.util.List; + +/** + * 真人交互数字人配置Service接口 + * + * @author ageerle + * @date Tue Oct 21 11:46:52 GMT+08:00 2025 + */ +public interface AihumanRealConfigService { + + /** + * 查询真人交互数字人配置 + */ + AihumanRealConfigVo queryById(Integer id); + + /** + * 查询真人交互数字人配置列表 + */ + TableDataInfo queryPageList(AihumanRealConfigBo bo, PageQuery pageQuery); + + /** + * 查询真人交互数字人配置列表 + */ + List queryList(AihumanRealConfigBo bo); + + /** + * 新增真人交互数字人配置 + */ + Boolean insertByBo(AihumanRealConfigBo bo); + + /** + * 修改真人交互数字人配置 + */ + Boolean updateByBo(AihumanRealConfigBo bo); + + /** + * 执行真人交互数字人配置 + */ + Boolean runByBo(AihumanRealConfigBo bo); + + /** + * 校验并批量删除真人交互数字人配置信息 + */ + Boolean deleteWithValidByIds(Collection ids, Boolean isValid); + + // 在AihumanRealConfigService接口中添加 + Boolean stopByBo(AihumanRealConfigBo bo); +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/service/AihumanVolcengineService.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/service/AihumanVolcengineService.java new file mode 100644 index 00000000..6c99f013 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/service/AihumanVolcengineService.java @@ -0,0 +1,4 @@ +package org.ruoyi.aihuman.service; + +public interface AihumanVolcengineService { +} diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/service/impl/AihumanRealConfigServiceImpl.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/service/impl/AihumanRealConfigServiceImpl.java new file mode 100644 index 00000000..e1a09f6f --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/service/impl/AihumanRealConfigServiceImpl.java @@ -0,0 +1,532 @@ +package org.ruoyi.aihuman.service.impl; + +import com.sun.jna.Library; +import com.sun.jna.Native; +import jakarta.annotation.PreDestroy; +import org.ruoyi.common.core.utils.MapstructUtils; +import org.ruoyi.core.page.TableDataInfo; +import org.ruoyi.core.page.PageQuery; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.ruoyi.aihuman.domain.bo.AihumanRealConfigBo; +import org.ruoyi.aihuman.domain.vo.AihumanRealConfigVo; +import org.ruoyi.aihuman.domain.AihumanRealConfig; +import org.ruoyi.aihuman.mapper.AihumanRealConfigMapper; +import org.ruoyi.aihuman.service.AihumanRealConfigService; +import org.ruoyi.common.core.utils.StringUtils; +import org.ruoyi.common.redis.utils.RedisUtils; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.Collection; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.sun.jna.platform.win32.WinNT; +import com.sun.jna.Pointer; +import java.util.concurrent.TimeUnit; + +/** + * 真人交互数字人配置Service业务层处理 + * + * @author ageerle + * @date Tue Oct 21 11:46:52 GMT+08:00 2025 + */ +@RequiredArgsConstructor +@Service +public class AihumanRealConfigServiceImpl implements AihumanRealConfigService { + + private final AihumanRealConfigMapper baseMapper; + // 存储当前运行的进程，用于停止操作 + private volatile Process runningProcess = null; + + /** + * 查询真人交互数字人配置 + */ + @Override + public AihumanRealConfigVo queryById(Integer id) { + return baseMapper.selectVoById(id); + } + + /** + * 查询真人交互数字人配置列表 + */ + @Override + public TableDataInfo queryPageList(AihumanRealConfigBo bo, PageQuery pageQuery) { + LambdaQueryWrapper lqw = buildQueryWrapper(bo); + Page result = baseMapper.selectVoPage(pageQuery.build(), lqw); + return TableDataInfo.build(result); + } + + /** + * 查询真人交互数字人配置列表 + */ + @Override + public List queryList(AihumanRealConfigBo bo) { + LambdaQueryWrapper lqw = buildQueryWrapper(bo); + return baseMapper.selectVoList(lqw); + } + + private LambdaQueryWrapper buildQueryWrapper(AihumanRealConfigBo bo) { + LambdaQueryWrapper lqw = Wrappers.lambdaQuery(); + lqw.like(StringUtils.isNotBlank(bo.getName()), AihumanRealConfig::getName, bo.getName()); + lqw.like(StringUtils.isNotBlank(bo.getAvatars()), AihumanRealConfig::getAvatars, bo.getAvatars()); + lqw.like(StringUtils.isNotBlank(bo.getModels()), AihumanRealConfig::getModels, bo.getModels()); + lqw.eq(StringUtils.isNotBlank(bo.getAvatarsParams()), AihumanRealConfig::getAvatarsParams, bo.getAvatarsParams()); + lqw.eq(StringUtils.isNotBlank(bo.getModelsParams()), AihumanRealConfig::getModelsParams, bo.getModelsParams()); + lqw.eq(StringUtils.isNotBlank(bo.getAgentParams()), AihumanRealConfig::getAgentParams, bo.getAgentParams()); + lqw.eq(bo.getCreateTime() != null, AihumanRealConfig::getCreateTime, bo.getCreateTime()); + lqw.eq(bo.getUpdateTime() != null, AihumanRealConfig::getUpdateTime, bo.getUpdateTime()); + lqw.eq(bo.getStatus() != null, AihumanRealConfig::getStatus, bo.getStatus()); + lqw.eq(bo.getPublish() != null, AihumanRealConfig::getPublish, bo.getPublish()); + lqw.eq(StringUtils.isNotBlank(bo.getRunParams()), AihumanRealConfig::getRunParams, bo.getRunParams()); + // 添加runStatus字段的查询条件 + lqw.eq(StringUtils.isNotBlank(bo.getRunStatus()), AihumanRealConfig::getRunStatus, bo.getRunStatus()); + lqw.eq(StringUtils.isNotBlank(bo.getCreateDept()), AihumanRealConfig::getCreateDept, bo.getCreateDept()); + lqw.eq(StringUtils.isNotBlank(bo.getCreateBy()), AihumanRealConfig::getCreateBy, bo.getCreateBy()); + lqw.eq(StringUtils.isNotBlank(bo.getUpdateBy()), AihumanRealConfig::getUpdateBy, bo.getUpdateBy()); + return lqw; + } + + /** + * 新增真人交互数字人配置 + */ + @Override + public Boolean insertByBo(AihumanRealConfigBo bo) { + AihumanRealConfig add = MapstructUtils.convert(bo, AihumanRealConfig. class); + validEntityBeforeSave(add); + boolean flag = baseMapper.insert(add) > 0; + if (flag) { + bo.setId(add.getId()); + } + return flag; + } + + /** + * 修改真人交互数字人配置 + */ + @Override + public Boolean updateByBo(AihumanRealConfigBo bo) { + AihumanRealConfig update = MapstructUtils.convert(bo, AihumanRealConfig. class); + validEntityBeforeSave(update); + return baseMapper.updateById(update) > 0; + } + + /** + * 保存前的数据校验 + */ + private void validEntityBeforeSave(AihumanRealConfig entity) { + //TODO 做一些数据校验,如唯一约束 + } + + /** + * 批量删除真人交互数字人配置 + */ + @Override + public Boolean deleteWithValidByIds(Collection ids, Boolean isValid) { + if (isValid) { + //TODO 做一些业务上的校验,判断是否需要校验 + } + return baseMapper.deleteBatchIds(ids) > 0; + } + + private static final Logger log = LoggerFactory.getLogger(AihumanRealConfigServiceImpl.class); + + /** + * 执行真人交互数字人配置 + * 通过主键获取数据库记录，然后从run_params字段读取命令并执行 + */ + @Override + public Boolean runByBo(AihumanRealConfigBo bo) { + try { + // 1. 通过主键获取数据库记录 + Integer id = bo.getId(); + if (id == null) { + log.error("执行命令失败：主键ID为空"); + throw new RuntimeException("执行命令失败：主键ID为空"); + } + + // 检查是否已经有对应的进程在运行 + String redisKey = "aihuman:process:" + id; + String existingPid = RedisUtils.getCacheObject(redisKey); + if (StringUtils.isNotEmpty(existingPid) && isProcessRunning(existingPid)) { + log.warn("ID为{}的配置已有进程在运行，进程ID: {}", id, existingPid); + // 刷新run_status状态为运行中 + AihumanRealConfig updateStatus = new AihumanRealConfig(); + updateStatus.setId(id); + updateStatus.setRunStatus("1"); // 1表示运行中 + baseMapper.updateById(updateStatus); + return true; + } + + // 查询数据库记录 + AihumanRealConfig config = baseMapper.selectById(id); + if (config == null) { + log.error("执行命令失败：未找到ID为{}的配置记录", id); + throw new RuntimeException("执行命令失败：未找到对应的配置记录"); + } + + // 2. 从记录中获取run_params字段 + String runParams = config.getRunParams(); + if (StringUtils.isBlank(runParams)) { + log.error("执行命令失败：ID为{}的记录中run_params字段为空", id); + throw new RuntimeException("执行命令失败：run_params字段为空"); + } + + // 3. 解析并执行命令 + // 将多行命令合并为一个命令字符串 + String[] commands = runParams.split("\\r?\\n"); + if (commands.length == 0) { + log.error("执行命令失败：runParams中没有有效的命令"); + throw new RuntimeException("执行命令失败：runParams中没有有效的命令"); + } + + // 将所有命令合并到一个命令字符串中，使用&&连接，确保在同一个进程中执行 + StringBuilder mergedCmd = new StringBuilder(); + for (int i = 0; i < commands.length; i++) { + String command = commands[i].trim(); + if (command.isEmpty()) { + continue; + } + + if (mergedCmd.length() > 0) { + mergedCmd.append(" && "); + } + + mergedCmd.append(command); + } + + String cmd = "cmd.exe /c " + mergedCmd.toString(); + log.info("准备执行合并命令：{}", cmd); + + // 更新数据库中的运行状态为运行中 + AihumanRealConfig updateStatus = new AihumanRealConfig(); + updateStatus.setId(id); + updateStatus.setRunStatus("1"); // 1表示运行中 + baseMapper.updateById(updateStatus); + + // 使用线程池执行命令并监听输出 + ExecutorService executor = Executors.newSingleThreadExecutor(); + executor.submit(() -> { + try { + Process process = Runtime.getRuntime().exec(cmd); + // 保存进程引用，用于后续停止操作 + runningProcess = process; + + // 获取进程ID并保存到Redis + String pid = getProcessId(process); + if (!"unknown".equals(pid)) { + RedisUtils.setCacheObject(redisKey, pid); + log.info("保存进程ID到Redis：key={}, pid={}", redisKey, pid); + } + + // 读取标准输出 + new Thread(() -> { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + log.info("[LiveTalking] {}", line); + } + } catch (IOException e) { + log.error("读取命令输出失败", e); + } + }).start(); + + // 读取debug输出 + new Thread(() -> { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) { + String line; + while ((line = reader.readLine()) != null) { + log.debug("[LiveTalking DEBUG] {}", line); + } + } catch (IOException e) { + log.error("读取命令debug输出失败", e); + } + }).start(); + + // 等待进程结束 + int exitCode = process.waitFor(); + log.info("LiveTalking进程结束，退出码: {}", exitCode); + + // 进程结束后更新数据库状态为已停止 + AihumanRealConfig endStatus = new AihumanRealConfig(); + endStatus.setId(id); + endStatus.setRunStatus("0"); // 0表示已停止 + baseMapper.updateById(endStatus); + + // 进程结束后从Redis中删除进程ID + RedisUtils.deleteObject(redisKey); + log.info("从Redis中删除进程ID：key={}", redisKey); + + // 进程结束后清空引用 + runningProcess = null; + } catch (Exception e) { + log.error("执行命令失败", e); + // 发生异常时更新数据库状态为失败 + try { + AihumanRealConfig errorStatus = new AihumanRealConfig(); + errorStatus.setId(id); + errorStatus.setRunStatus("2"); // 2表示启动失败 + baseMapper.updateById(errorStatus); + } catch (Exception ex) { + log.error("更新状态失败", ex); + } + // 发生异常时从Redis中删除进程ID + RedisUtils.deleteObject(redisKey); + // 发生异常时清空引用 + runningProcess = null; + } + }); + + executor.shutdown(); + return true; + } catch (Exception e) { + log.error("执行命令过程中发生异常", e); + return false; + } + } + + /** + * 检查进程是否正在运行 + * @param pid 进程ID + * @return 是否正在运行 + */ + private boolean isProcessRunning(String pid) { + if (StringUtils.isEmpty(pid) || "unknown".equals(pid)) { + return false; + } + + try { + boolean isWindows = System.getProperty("os.name").toLowerCase().contains("win"); + ProcessBuilder processBuilder; + + if (isWindows) { + processBuilder = new ProcessBuilder("tasklist", "/FI", "PID eq " + pid); + } else { + processBuilder = new ProcessBuilder("ps", "-p", pid); + } + + Process process = processBuilder.start(); + int exitCode = process.waitFor(); + + // 在Windows上，tasklist命令如果找不到进程，退出码也是0，但输出中不会包含PID + if (isWindows) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + if (line.contains(pid)) { + return true; + } + } + } + return false; + } else { + // 在Linux/Mac上，ps命令如果找不到进程，退出码不为0 + return exitCode == 0; + } + } catch (Exception e) { + log.error("检查进程是否运行失败, pid={}", pid, e); + return false; + } + } + + /** + * 停止正在运行的真人交互数字人配置任务 + */ + @Override + public Boolean stopByBo(AihumanRealConfigBo bo) { + try { + Integer id = bo.getId(); + String redisKey = "aihuman:process:" + id; + + // 首先检查Redis中是否有对应的进程ID + String pid = RedisUtils.getCacheObject(redisKey); + if (StringUtils.isNotEmpty(pid)) { + // 如果Redis中有进程ID，先尝试通过进程ID停止进程 + try { + // 根据操作系统类型，使用不同的命令终止进程树 + if (System.getProperty("os.name").toLowerCase().contains("win")) { + // Windows系统使用taskkill命令终止进程树 + log.info("通过Redis中的PID停止进程: taskkill /F /T /PID {}", pid); + Process killProcess = Runtime.getRuntime().exec("taskkill /F /T /PID " + pid); + // 等待kill命令执行完成 + killProcess.waitFor(5, TimeUnit.SECONDS); + } else { + // Linux/Mac系统使用pkill命令终止进程树 + Runtime.getRuntime().exec("pkill -P " + pid); + } + } catch (Exception e) { + log.error("通过Redis中的PID停止进程失败", e); + } + } + + // 然后检查本地runningProcess引用 + if (runningProcess != null && runningProcess.isAlive()) { + log.info("正在停止LiveTalking进程..."); + // 强制销毁进程树，确保完全停止 + destroyProcessTree(runningProcess); + + // 更新数据库中的运行状态为已停止 + AihumanRealConfig updateStatus = new AihumanRealConfig(); + updateStatus.setId(id); + updateStatus.setRunStatus("0"); // 0表示已停止 + baseMapper.updateById(updateStatus); + + runningProcess = null; + log.info("LiveTalking进程已停止"); + } else { + log.warn("没有正在运行的LiveTalking进程"); + // 确保数据库中的状态也是已停止 + AihumanRealConfig updateStatus = new AihumanRealConfig(); + updateStatus.setId(id); + updateStatus.setRunStatus("0"); // 0表示已停止 + baseMapper.updateById(updateStatus); + } + + // 无论如何都从Redis中删除进程ID + RedisUtils.deleteObject(redisKey); + log.info("从Redis中删除进程ID：key={}", redisKey); + + return true; + } catch (Exception e) { + log.error("停止进程时发生异常", e); + // 发生异常时也尝试从Redis中删除进程ID + try { + RedisUtils.deleteObject("aihuman:process:" + bo.getId()); + } catch (Exception ex) { + log.error("从Redis中删除进程ID失败", ex); + } + return false; + } + } + + /** + * 销毁进程及其子进程（进程树） + * @param process 要销毁的进程 + */ + private void destroyProcessTree(Process process) { + try { + if (process.isAlive()) { + // 获取进程ID + String pid = getProcessId(process); + log.info("获取到进程ID: {}", pid); + + // 根据操作系统类型，使用不同的命令终止进程树 + if (System.getProperty("os.name").toLowerCase().contains("win")) { + // Windows系统使用taskkill命令终止进程树 + log.info("执行taskkill命令终止进程树: taskkill /F /T /PID {}", pid); + Process killProcess = Runtime.getRuntime().exec("taskkill /F /T /PID " + pid); + // 等待kill命令执行完成 + killProcess.waitFor(5, TimeUnit.SECONDS); + } else { + // Linux/Mac系统使用pkill命令终止进程树 + Runtime.getRuntime().exec("pkill -P " + pid); + process.destroy(); + } + } + } catch (Exception e) { + log.error("销毁进程树时发生异常", e); + // 如果出现异常，尝试使用普通销毁方法 + process.destroy(); + try { + // 强制销毁 + if (process.isAlive()) { + process.destroyForcibly(); + } + } catch (Exception ex) { + log.error("强制销毁进程失败", ex); + } + } + } + + /** + * 获取进程ID + * @param process 进程对象 + * @return 进程ID + */ + private String getProcessId(Process process) { + try { + // 不同JVM实现可能有所不同，这里尝试通过反射获取 + if (process.getClass().getName().equals("java.lang.Win32Process") || + process.getClass().getName().equals("java.lang.ProcessImpl")) { + Field f = process.getClass().getDeclaredField("handle"); + f.setAccessible(true); + long handl = f.getLong(process); + Kernel32 kernel = Kernel32.INSTANCE; + WinNT.HANDLE handle = new WinNT.HANDLE(); + handle.setPointer(Pointer.createConstant(handl)); + return String.valueOf(kernel.GetProcessId(handle)); + } else if (process.getClass().getName().equals("java.lang.UNIXProcess")) { + Field f = process.getClass().getDeclaredField("pid"); + f.setAccessible(true); + return String.valueOf(f.getInt(process)); + } + } catch (Exception e) { + log.error("获取进程ID失败", e); + } + + // 如果反射获取失败，尝试通过wmic命令获取 + try { + // 对于Windows系统，可以尝试使用wmic命令获取进程ID + if (System.getProperty("os.name").toLowerCase().contains("win")) { + ProcessHandle.Info info = process.toHandle().info(); + return String.valueOf(process.toHandle().pid()); + } + } catch (Exception e) { + log.error("通过ProcessHandle获取进程ID失败", e); + } + + return "unknown"; + } + + // JNA接口定义，用于Windows系统获取进程ID + interface Kernel32 extends Library { + Kernel32 INSTANCE = (Kernel32) Native.loadLibrary("kernel32", Kernel32.class); + int GetProcessId(WinNT.HANDLE hProcess); + } + + @PreDestroy + public void onDestroy() { + if (runningProcess != null && runningProcess.isAlive()) { + try { + log.info("应用关闭，正在停止数字人进程"); + destroyProcessTree(runningProcess); + + // 查找所有运行状态为运行中的配置，并更新为已停止 + LambdaQueryWrapper lqw = Wrappers.lambdaQuery(); + lqw.eq(AihumanRealConfig::getRunStatus, "1"); + List runningConfigs = baseMapper.selectList(lqw); + for (AihumanRealConfig config : runningConfigs) { + config.setRunStatus("0"); + baseMapper.updateById(config); + + // 从Redis中删除对应的进程ID记录 + String redisKey = "aihuman:process:" + config.getId(); + RedisUtils.deleteObject(redisKey); + log.info("应用关闭，从Redis中删除进程ID：key={}", redisKey); + } + } catch (Exception e) { + log.error("停止数字人进程失败", e); + // 即使发生异常，也尝试清理Redis中的进程ID记录 + try { + LambdaQueryWrapper lqw = Wrappers.lambdaQuery(); + lqw.eq(AihumanRealConfig::getRunStatus, "1"); + List runningConfigs = baseMapper.selectList(lqw); + for (AihumanRealConfig config : runningConfigs) { + RedisUtils.deleteObject("aihuman:process:" + config.getId()); + } + } catch (Exception ex) { + log.error("清理Redis中的进程ID记录失败", ex); + } + } + } + } +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/service/impl/AihumanVolcengineServiceImpl.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/service/impl/AihumanVolcengineServiceImpl.java new file mode 100644 index 00000000..aa11aabd --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/service/impl/AihumanVolcengineServiceImpl.java @@ -0,0 +1,4 @@ +package org.ruoyi.aihuman.service.impl; + +public class AihumanVolcengineServiceImpl { +} diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/sql/aihuman_real_config_menu.sql b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/sql/aihuman_real_config_menu.sql new file mode 100644 index 00000000..c1ef85a6 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/sql/aihuman_real_config_menu.sql @@ -0,0 +1,19 @@ +-- 菜单 SQL +insert into sys_menu (menu_id, menu_name, parent_id, order_num, path, component, is_frame, is_cache, menu_type, visible, status, perms, icon, create_dept, create_by, create_time, update_by, update_time, remark) +values(1980480880138051584, '真人交互数字人配置', '2000', '1', 'aihumanRealConfig', 'aihuman/aihumanRealConfig/index', 1, 0, 'C', '0', '0', 'aihuman:aihumanRealConfig:list', '#', 103, 1, sysdate(), null, null, '真人交互数字人配置菜单'); + +-- 按钮 SQL +insert into sys_menu (menu_id, menu_name, parent_id, order_num, path, component, is_frame, is_cache, menu_type, visible, status, perms, icon, create_dept, create_by, create_time, update_by, update_time, remark) +values(1980480880138051585, '真人交互数字人配置查询', 1980480880138051584, '1', '#', '', 1, 0, 'F', '0', '0', 'aihuman:aihumanRealConfig:query', '#', 103, 1, sysdate(), null, null, ''); + +insert into sys_menu (menu_id, menu_name, parent_id, order_num, path, component, is_frame, is_cache, menu_type, visible, status, perms, icon, create_dept, create_by, create_time, update_by, update_time, remark) +values(1980480880138051586, '真人交互数字人配置新增', 1980480880138051584, '2', '#', '', 1, 0, 'F', '0', '0', 'aihuman:aihumanRealConfig:add', '#', 103, 1, sysdate(), null, null, ''); + +insert into sys_menu (menu_id, menu_name, parent_id, order_num, path, component, is_frame, is_cache, menu_type, visible, status, perms, icon, create_dept, create_by, create_time, update_by, update_time, remark) +values(1980480880138051587, '真人交互数字人配置修改', 1980480880138051584, '3', '#', '', 1, 0, 'F', '0', '0', 'aihuman:aihumanRealConfig:edit', '#', 103, 1, sysdate(), null, null, ''); + +insert into sys_menu (menu_id, menu_name, parent_id, order_num, path, component, is_frame, is_cache, menu_type, visible, status, perms, icon, create_dept, create_by, create_time, update_by, update_time, remark) +values(1980480880138051588, '真人交互数字人配置删除', 1980480880138051584, '4', '#', '', 1, 0, 'F', '0', '0', 'aihuman:aihumanRealConfig:remove', '#', 103, 1, sysdate(), null, null, ''); + +insert into sys_menu (menu_id, menu_name, parent_id, order_num, path, component, is_frame, is_cache, menu_type, visible, status, perms, icon, create_dept, create_by, create_time, update_by, update_time, remark) +values(1980480880138051589, '真人交互数字人配置导出', 1980480880138051584, '5', '#', '', 1, 0, 'F', '0', '0', 'aihuman:aihumanRealConfig:export', '#', 103, 1, sysdate(), null, null, ''); diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/volcengine/Bidirection.java b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/volcengine/Bidirection.java new file mode 100644 index 00000000..f4dd19e3 --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/java/org/ruoyi/aihuman/volcengine/Bidirection.java @@ -0,0 +1,160 @@ +package org.ruoyi.aihuman.volcengine; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.ruoyi.aihuman.protocol.EventType; +import org.ruoyi.aihuman.protocol.Message; +import org.ruoyi.aihuman.protocol.MsgType; +import org.ruoyi.aihuman.protocol.SpeechWebSocketClient; +import lombok.extern.slf4j.Slf4j; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.net.URI; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +@Slf4j +public class Bidirection { + private static final String ENDPOINT = "wss://openspeech.bytedance.com/api/v3/tts/bidirection"; + private static final ObjectMapper objectMapper = new ObjectMapper(); + + /** + * Get resource ID based on voice type + * + * @param voice Voice type string + * @return Corresponding resource ID + */ + public static String voiceToResourceId(String voice) { + // Map different voice types to resource IDs based on actual needs + if (voice.startsWith("S_")) { + return "volc.megatts.default"; + } + return "volc.service_type.10029"; + } + + public static void main(String[] args) throws Exception { + // Configure parameters + String appId = System.getProperty("appId", "1055299334"); + String accessToken = System.getProperty("accessToken", "fOHuq4R4dirMYiOruCU3Ek9q75zV0KVW"); + String resourceId = System.getProperty("resourceId", "seed-tts-2.0"); + String voice = System.getProperty("voice", "zh_female_vv_uranus_bigtts"); + String text = System.getProperty("text", "你好呀！我是AI合成的语音，很高兴认识你。"); + String encoding = System.getProperty("encoding", "mp3"); + + if (appId.isEmpty() || accessToken.isEmpty()) { + throw new IllegalArgumentException("Please set appId and accessToken system properties"); + } + + // Set request headers + Map headers = Map.of( + "X-Api-App-Key", appId, + "X-Api-Access-Key", accessToken, + "X-Api-Resource-Id", resourceId.isEmpty() ? voiceToResourceId(voice) : resourceId, + "X-Api-Connect-Id", UUID.randomUUID().toString()); + + // Create WebSocket client + SpeechWebSocketClient client = new SpeechWebSocketClient(new URI(ENDPOINT), headers); + try { + client.connectBlocking(); + Map request = Map.of( + "user", Map.of("uid", UUID.randomUUID().toString()), + "namespace", "BidirectionalTTS", + "req_params", Map.of( + "speaker", voice, + "audio_params", Map.of( + "format", encoding, + "sample_rate", 24000, + "enable_timestamp", true), + // additions requires a JSON string + "additions", objectMapper.writeValueAsString(Map.of( + "disable_markdown_filter", false)))); + + // Start connection + client.sendStartConnection(); + // Wait for connection started + client.waitForMessage(MsgType.FULL_SERVER_RESPONSE, EventType.CONNECTION_STARTED); + + // Process each sentence + String[] sentences = text.split("。"); + boolean audioReceived = false; + for (int i = 0; i < sentences.length; i++) { + if (sentences[i].trim().isEmpty()) { + continue; + } + + String sessionId = UUID.randomUUID().toString(); + ByteArrayOutputStream audioStream = new ByteArrayOutputStream(); + + // Start session + Map startReq = Map.of( + "user", request.get("user"), + "namespace", request.get("namespace"), + "req_params", request.get("req_params"), + "event", EventType.START_SESSION.getValue()); + client.sendStartSession(objectMapper.writeValueAsBytes(startReq), sessionId); + // Wait for session started + client.waitForMessage(MsgType.FULL_SERVER_RESPONSE, EventType.SESSION_STARTED); + + // Send text + for (char c : sentences[i].toCharArray()) { + // Create new req_params with text + @SuppressWarnings("unchecked") + Map currentReqParams = new HashMap<>( + (Map) request.get("req_params")); + currentReqParams.put("text", String.valueOf(c)); + + // Create current request + Map currentRequest = Map.of( + "user", request.get("user"), + "namespace", request.get("namespace"), + "req_params", currentReqParams, + "event", EventType.TASK_REQUEST.getValue()); + + client.sendTaskRequest(objectMapper.writeValueAsBytes(currentRequest), sessionId); + } + + // End session + client.sendFinishSession(sessionId); + + // Receive response + while (true) { + Message msg = client.receiveMessage(); + switch (msg.getType()) { + case FULL_SERVER_RESPONSE: + break; + case AUDIO_ONLY_SERVER: + if (!audioReceived && audioStream.size() > 0) { + audioReceived = true; + } + if (msg.getPayload() != null) { + audioStream.write(msg.getPayload()); + } + break; + default: + throw new RuntimeException("Unexpected message: " + msg); + } + if (msg.getEvent() == EventType.SESSION_FINISHED) { + break; + } + } + + if (audioStream.size() > 0) { + String fileName = String.format("%s_session_%d.%s", voice, i, encoding); + Files.write(new File(fileName).toPath(), audioStream.toByteArray()); + log.info("Audio saved to file: {}", fileName); + } + } + + if (!audioReceived) { + throw new RuntimeException("No audio data received"); + } + + // End connection + client.sendFinishConnection(); + } finally { + client.closeBlocking(); + } + } +} \ No newline at end of file diff --git a/ruoyi-modules/ruoyi-aihuman/src/main/resources/mapper/aihuman/AihumanRealConfigMapper.xml b/ruoyi-modules/ruoyi-aihuman/src/main/resources/mapper/aihuman/AihumanRealConfigMapper.xml new file mode 100644 index 00000000..8c7011ca --- /dev/null +++ b/ruoyi-modules/ruoyi-aihuman/src/main/resources/mapper/aihuman/AihumanRealConfigMapper.xml @@ -0,0 +1,7 @@ + + + + + diff --git a/ruoyi-modules/ruoyi-generator/src/main/resources/generator.yml b/ruoyi-modules/ruoyi-generator/src/main/resources/generator.yml index fea39ec6..7f5955fb 100644 --- a/ruoyi-modules/ruoyi-generator/src/main/resources/generator.yml +++ b/ruoyi-modules/ruoyi-generator/src/main/resources/generator.yml @@ -3,8 +3,8 @@ gen: # 作者 author: ageerle # 默认生成包路径 system 需改成自己的模块名称 如 system monitor tool - packageName: org.ruoyi.system + packageName: org.ruoyi.aihuman # 自动去除表前缀，默认是false autoRemovePre: false # 表前缀（生成类名不会包含表前缀，多个用逗号分隔） - tablePrefix: sys_ + tablePrefix: aihuman_