From fece90b3075e7c16de041d0e797d538925ca6f6f Mon Sep 17 00:00:00 2001 From: "ageerle@163.com" Date: Fri, 17 Jul 2026 11:52:53 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E7=9F=AD=E5=89=A7?= =?UTF-8?q?=E9=9F=B3=E9=A2=91/=E7=BC=96=E7=A0=81=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=E3=80=81MCP=20=E6=96=87=E4=BB=B6=E5=B7=A5=E5=85=B7=E4=B8=8E=20?= =?UTF-8?q?WebSocket=20=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 短剧: 新增 ShortDramaAudio 实体/Mapper/BO/VO，扩展 ShortDramaServiceImpl 合成逻辑与 FfmpegFilterGraphBuilder 滤镜图 - MCP: 新增 WriteFileTool/DeleteFileTool/ExecuteCommandTool，重构 EditFileTool/ReadFileTool/ListDirectoryTool - 编码: 新增 coding 模块（CodingAgent/WorkspaceService/SSE 事件通道） - Atlas: 新增音频生成实现，扩展音视频/媒体实体与预测服务 - WebSocket: 新增公众号聊天 WebSocket 处理器与握手拦截 - 忽略 logs/ 目录，避免合成产物入库 Co-Authored-By: Claude --- .gitignore | 1 + docs/script/sql/ruoyi-ai-v3_mysql8.sql | 30 + .../src/main/resources/application.yml | 6 + .../chat/entity/audio/AudioContext.java | 15 + .../entity/media/MediaGenerationResponse.java | 3 + .../chat/entity/video/VideoContext.java | 12 + ruoyi-modules/ruoyi-chat/pom.xml | 8 + .../controller/coding/CodingController.java | 92 +++ .../shortdrama/ShortDramaController.java | 31 + .../domain/bo/coding/CodingRequestBo.java | 30 + .../bo/shortdrama/ShortDramaAudioBo.java | 37 ++ .../ShortDramaCharacterAppearanceBo.java | 2 + .../shortdrama/ShortDramaComposeVideoBo.java | 12 +- .../entity/shortdrama/ShortDramaAudio.java | 47 ++ .../ShortDramaCharacterAppearance.java | 3 + .../shortdrama/ShortDramaStoryboard.java | 3 + .../vo/shortdrama/ShortDramaAudioVo.java | 41 ++ .../ShortDramaCharacterAppearanceVo.java | 2 + .../vo/shortdrama/ShortDramaDetailVo.java | 2 + .../vo/shortdrama/ShortDramaStoryboardVo.java | 2 + .../shortdrama/ShortDramaAudioMapper.java | 8 + .../org/ruoyi/mcp/tools/DeleteFileTool.java | 166 +++++ .../org/ruoyi/mcp/tools/EditFileTool.java | 57 +- .../ruoyi/mcp/tools/ExecuteCommandTool.java | 266 ++++++++ .../ruoyi/mcp/tools/ListDirectoryTool.java | 37 +- .../org/ruoyi/mcp/tools/ReadFileTool.java | 57 +- .../org/ruoyi/mcp/tools/WriteFileTool.java | 136 +++++ .../AtlasAudioGenerationServiceImpl.java | 98 +++ .../org/ruoyi/service/coding/CodingAgent.java | 29 + .../service/coding/CodingEventChannel.java | 93 +++ .../ruoyi/service/coding/CodingSseEvent.java | 43 ++ .../coding/CodingWorkspaceService.java | 134 ++++ .../ruoyi/service/coding/ICodingService.java | 21 + .../ruoyi/service/coding/WorkspaceGuard.java | 50 ++ .../coding/impl/CodingServiceImpl.java | 198 ++++++ .../service/media/AtlasMediaSupport.java | 8 + .../service/media/AtlasPredictionService.java | 42 +- .../shortdrama/IShortDramaService.java | 13 + .../shortdrama/composition/AspectRatio.java | 5 +- .../composition/CompositionSpec.java | 5 +- .../composition/FfmpegCommandBuilder.java | 5 + .../FfmpegCompositionProperties.java | 11 + .../composition/FfmpegFilterGraphBuilder.java | 106 +++- .../impl/ShortDramaServiceImpl.java | 577 ++++++++++++++++-- .../impl/ShortDramaVideoComposeJob.java | 6 +- .../ShortDramaVideoComposeServiceImpl.java | 10 +- .../impl/ShortDramaVideoComposeWorker.java | 28 +- .../IncrementalJsonArrayExtractor.java | 116 ++++ .../AtlasVideoGenerationServiceImpl.java | 17 + .../chat/MpChatHandshakeInterceptor.java | 81 +++ .../websocket/chat/MpChatWebSocketConfig.java | 33 + .../chat/MpChatWebSocketHandler.java | 378 ++++++++++++ 52 files changed, 3110 insertions(+), 103 deletions(-) create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/coding/CodingController.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/coding/CodingRequestBo.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/shortdrama/ShortDramaAudioBo.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/shortdrama/ShortDramaAudio.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaAudioVo.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mapper/shortdrama/ShortDramaAudioMapper.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/DeleteFileTool.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/ExecuteCommandTool.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/WriteFileTool.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/audio/provider/AtlasAudioGenerationServiceImpl.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingAgent.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingEventChannel.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingSseEvent.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingWorkspaceService.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/ICodingService.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/WorkspaceGuard.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/impl/CodingServiceImpl.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/support/IncrementalJsonArrayExtractor.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/websocket/chat/MpChatHandshakeInterceptor.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/websocket/chat/MpChatWebSocketConfig.java create mode 100644 ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/websocket/chat/MpChatWebSocketHandler.java diff --git a/.gitignore b/.gitignore index e204fe12..979888b5 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ nbdist/ *.xml.versionsBackup *.swp data/ +logs/ !*/build/*.java !*/build/*.html diff --git a/docs/script/sql/ruoyi-ai-v3_mysql8.sql b/docs/script/sql/ruoyi-ai-v3_mysql8.sql index 8a659ade..1623e046 100644 --- a/docs/script/sql/ruoyi-ai-v3_mysql8.sql +++ b/docs/script/sql/ruoyi-ai-v3_mysql8.sql @@ -116,6 +116,7 @@ INSERT INTO `chat_model` VALUES (2060622000000000003, 'video', 'bytedance/seedan INSERT INTO `chat_model` VALUES (2060622000000000004, 'video', 'bytedance/seedance-2.0/reference-to-video', 'atlas', 'Seedance 2.0 多参考图生视频（字节跳动）', NULL, 'Y', 'https://api.atlascloud.ai/v1', 'sk_xx', 103, 1, '2026-06-22 20:24:44', 1, '2026-06-22 20:24:44', 'Atlas Cloud 视频模型 - Seedance 2.0 多参考图生视频，接收多张参考图+带@imageN标记的提示词生成视频', 0); INSERT INTO `chat_model` VALUES (2070700000000000002, 'chat', 'dify-chat', 'dify', 'Dify Chat App', NULL, 'Y', 'https://api.dify.ai/v1', '替换为你的DIFY_APP_API_KEY', 103, 1, '2026-07-14 11:03:44', 1, '2026-07-14 11:03:44', 'Dify 聊天应用；api_key 为 Dify App API Key，model_name 可按应用名修改', 0); INSERT INTO `chat_model` VALUES (2070700000000000004, 'chat', '替换为你的COZE_BOT_ID', 'coze', 'Coze Bot', NULL, 'Y', 'https://api.coze.cn', '替换为你的COZE_PAT', 103, 1, '2026-07-14 11:03:38', 1, '2026-07-14 11:03:38', 'Coze 聊天 Bot；model_name 为 Coze Bot ID，api_key 为 PAT 或 OAuth access token', 0); +INSERT INTO `chat_model` VALUES (2070700000000000010, 'audio', 'bytedance/seed-audio-1.0', 'atlas', 'Seed Audio 1.0 语音生成（字节跳动）', NULL, 'Y', 'https://api.atlascloud.ai/v1', 'sk_xx', 103, 1, '2026-07-16 18:00:00', 1, '2026-07-16 18:00:00', 'Atlas Cloud 语音模型 - 支持多角色对白配音，references 指定 speaker 音色，text 中用 @audioN 引用', 0); -- ---------------------------- -- Table structure for chat_provider @@ -1361,6 +1362,7 @@ CREATE TABLE `short_drama_character_appearance` ( `selected_image_index` int NULL DEFAULT 0 COMMENT '当前选中的图片索引', `previous_image_urls` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '上一轮图片URL列表（撤销用，JSON数组）', `previous_descriptions` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '上一轮提示词列表（撤销用，JSON数组）', + `voice` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '音色名（如 zh_male_taocheng_uranus_bigtts），用于该形象对白配音', PRIMARY KEY (`id`) USING BTREE, UNIQUE INDEX `uk_character_appearance`(`character_id` ASC, `appearance_index` ASC) USING BTREE, INDEX `idx_character_id`(`character_id` ASC) USING BTREE, @@ -1426,6 +1428,33 @@ INSERT INTO `short_drama_location` VALUES (2077008104604061696, 2077007721974484 INSERT INTO `short_drama_location` VALUES (2077008104675364864, 2077007721974484992, '厂庆礼堂_夜', '工厂礼堂内部，舞台上有麦克风，台下长凳坐满工人，红灯笼和横幅装饰。', 1, '几十名穿着蓝色工装的工人整齐坐在长凳上，面向舞台，表情喜悦。', '[\"舞台中央麦克风架前方位置\",\"舞台左侧幕布旁\",\"观众席前排中间走道\",\"礼堂后门入口处\"]', '[\"「厂庆礼堂_夜」一座中型礼堂，空间约三百平米，两侧墙壁挂满红色锦旗和奖状。舞台为砖砌，高约一米，铺着红色地毯，中央立着麦克风，背景幕布为深红色绒布，上方悬挂「厂庆联欢晚会」横幅。观众席摆着二十排长木凳，坐满穿工装的工人。屋顶悬挂多盏白炽灯和串串红灯笼，灯光暖黄。舞台侧方、观众席中间走廊、后门口均有空地可站人。\",\"「厂庆礼堂_夜」礼堂内部呈长方形，前高后低，地板为水泥磨光。天花板有木横梁，垂吊着彩带和纸花。舞台左后侧有上台台阶，右侧放置锣鼓乐器。台下长凳密集排列，过道狭窄，工人拥挤而坐，有些站着。礼堂后墙有双开木门，紧闭。光线主要来自舞台上的聚光灯和两侧壁灯，明亮温暖。舞台中央地面、右侧锣鼓旁、后墙门边有可落位空间。\",\"「厂庆礼堂_夜」宽大的老式礼堂，墙面下半部刷绿色墙裙，上半部白墙。窗户用深色帘幕遮住。舞台台口有弧形边缘，台上有两张木椅和一张讲台。台前摆满花篮。观众席中央过道约一米宽，两侧长凳坐满人。天花板中央一盏大吊灯，四周小灯，整体照明均匀。舞台正前方、左侧幕布缺口处、礼堂后门旁均可站立。\"]', 'https://atlas-media.oss-us-west-1.aliyuncs.com/images/a9295d479fef45f198bffd3ef850eacd-588538d1e9d8e094.jpg', -1, -1, '2026-07-14 20:31:36', 1, '2026-07-14 20:38:51', 0, '[\"https://atlas-media.oss-us-west-1.aliyuncs.com/images/a9295d479fef45f198bffd3ef850eacd-588538d1e9d8e094.jpg\"]', '[\"宽广空间全景，[\\\"「厂庆礼堂_夜」一座中型礼堂，空间约三百平米，两侧墙壁挂满红色锦旗和奖状。舞台为砖砌，高约一米，铺着红色地毯，中央立着麦克风，背景幕布为深红色绒布，上方悬挂「厂庆联欢晚会」横幅。观众席摆着二十排长木凳，坐满穿工装的工人。屋顶悬挂多盏白炽灯和串串红灯笼，灯光暖黄。舞台侧方、观众席中间走廊、后门口均有空地可站人。\\\",\\\"「厂庆礼堂_夜」礼堂内部呈长方形，前高后低，地板为水泥磨光。天花板有木横梁，垂吊着彩带和纸花。舞台左后侧有上台台阶，右侧放置锣鼓乐器。台下长凳密集排列，过道狭窄，工人拥挤而坐，有些站着。礼堂后墙有双开木门，紧闭。光线主要来自舞台上的聚光灯和两侧壁灯，明亮温暖。舞台中央地面、右侧锣鼓旁、后墙门边有可落位空间。\\\",\\\"「厂庆礼堂_夜」宽大的老式礼堂，墙面下半部刷绿色墙裙，上半部白墙。窗户用深色帘幕遮住。舞台台口有弧形边缘，台上有两张木椅和一张讲台。台前摆满花篮。观众席中央过道约一米宽，两侧长凳坐满人。天花板中央一盏大吊灯，四周小灯，整体照明均匀。舞台正前方、左侧幕布缺口处、礼堂后门旁均可站立。\\\"]，禁止出现任何角色，纯背景板真实电影级画面质感，真实现实场景，色彩饱满通透，画面干净精致，真实感\"]', 0, NULL, NULL); INSERT INTO `short_drama_location` VALUES (2077008104721502208, 2077007721974484992, '工厂大门_晨', '清晨工厂大门，新挂牌「华夏信息服务中心」，自行车棚，远处烟囱冒烟。', 1, '几名穿着蓝色工装的工人推着自行车或步行进入厂门，路边有早点摊的模糊身影。', '[\"厂门口新挂牌下方正中间\",\"右侧自行车棚外侧\",\"左侧传达室门口台阶\",\"马路对面老槐树旁\"]', '[\"「工厂大门_晨」工厂大门为铁制对开栅栏门，门柱为红砖砌成，左侧门柱上挂着崭新白底黑字牌子「华夏信息服务中心」。进门可见一条水泥路通向厂区。右侧有一排自行车棚，棚内停着几十辆自行车。远处工厂烟囱缓缓冒白烟，背景是朦胧的晨曦天空。阳光从东方斜照，拉长门柱影子。大门正下方、自行车棚外面、左侧传达室门口、马路对面树荫下均有平坦地面。\",\"「工厂大门_晨」晨曦中的老工厂入口，灰砖围墙高约三米，大门敞开。左侧传达室为平房，窗户透出灯光。门右侧墙上贴着红色通知。路面是水泥铺设，有些许落叶。厂内远处可见车间屋顶和冒烟烟囱。天空泛鱼肚白，东边有朝霞。光线从右前方射来，门牌上的字清晰反光。大门中央、传达室左侧空地、路边电线杆旁都可以站人。\",\"「工厂大门_晨」工厂正门区域，宽阔的厂前空地上矗立着新立的招牌。大门两侧各有一棵法国梧桐，枝叶繁茂。地面为柏油路，路中央画有白色虚线。厂门内侧两边是花坛，种着冬青。远处厂房轮廓在晨雾中若隐若现，烟囱吐烟。光线柔和，晨光从建筑物缝隙中洒落。招牌下方、左侧花坛旁、右侧梧桐树下均是可站立的空旷位置。\"]', 'https://atlas-media.oss-us-west-1.aliyuncs.com/images/50f5cc1fb8e74dac95d589c23dc2daf7-7fb4da35d3075e95.jpg', -1, -1, '2026-07-14 20:31:36', 1, '2026-07-14 20:38:51', 0, '[\"https://atlas-media.oss-us-west-1.aliyuncs.com/images/50f5cc1fb8e74dac95d589c23dc2daf7-7fb4da35d3075e95.jpg\"]', '[\"宽广空间全景，[\\\"「工厂大门_晨」工厂大门为铁制对开栅栏门，门柱为红砖砌成，左侧门柱上挂着崭新白底黑字牌子「华夏信息服务中心」。进门可见一条水泥路通向厂区。右侧有一排自行车棚，棚内停着几十辆自行车。远处工厂烟囱缓缓冒白烟，背景是朦胧的晨曦天空。阳光从东方斜照，拉长门柱影子。大门正下方、自行车棚外面、左侧传达室门口、马路对面树荫下均有平坦地面。\\\",\\\"「工厂大门_晨」晨曦中的老工厂入口，灰砖围墙高约三米，大门敞开。左侧传达室为平房，窗户透出灯光。门右侧墙上贴着红色通知。路面是水泥铺设，有些许落叶。厂内远处可见车间屋顶和冒烟烟囱。天空泛鱼肚白，东边有朝霞。光线从右前方射来，门牌上的字清晰反光。大门中央、传达室左侧空地、路边电线杆旁都可以站人。\\\",\\\"「工厂大门_晨」工厂正门区域，宽阔的厂前空地上矗立着新立的招牌。大门两侧各有一棵法国梧桐，枝叶繁茂。地面为柏油路，路中央画有白色虚线。厂门内侧两边是花坛，种着冬青。远处厂房轮廓在晨雾中若隐若现，烟囱吐烟。光线柔和，晨光从建筑物缝隙中洒落。招牌下方、左侧花坛旁、右侧梧桐树下均是可站立的空旷位置。\\\"]，禁止出现任何角色，纯背景板真实电影级画面质感，真实现实场景，色彩饱满通透，画面干净精致，真实感\"]', 0, NULL, NULL); +-- ---------------------------- +-- Table structure for short_drama_audio +-- ---------------------------- +DROP TABLE IF EXISTS `short_drama_audio`; +CREATE TABLE `short_drama_audio` ( + `id` bigint NOT NULL COMMENT '主键', + `project_id` bigint NOT NULL COMMENT '项目ID', + `name` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '语音资产名称', + `audio_type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'narration' COMMENT '语音类型：narration(旁白)/dialogue(对白)', + `text` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '语音文案（生成语音用的文本）', + `voice` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '音色（如 alloy/onyx）', + `audio_oss_id` bigint NULL DEFAULT NULL COMMENT '音频文件OSS ID', + `audio_url` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '音频文件URL', + `linked_storyboard_id` bigint NULL DEFAULT NULL COMMENT '对白关联的分镜ID（NULL=全局旁白）', + `duration_seconds` int NULL DEFAULT NULL COMMENT '音频时长（秒）', + `create_dept` bigint NULL DEFAULT NULL COMMENT '创建部门', + `create_by` bigint NULL DEFAULT NULL COMMENT '创建者', + `create_time` datetime NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_by` bigint NULL DEFAULT NULL COMMENT '更新者', + `update_time` datetime NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `tenant_id` bigint NOT NULL DEFAULT 0 COMMENT '租户Id', + PRIMARY KEY (`id`) USING BTREE, + INDEX `idx_project_id`(`project_id` ASC) USING BTREE, + INDEX `idx_linked_storyboard_id`(`linked_storyboard_id` ASC) USING BTREE, + INDEX `idx_tenant_id`(`tenant_id` ASC) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = '短剧语音资产表' ROW_FORMAT = Dynamic; + -- ---------------------------- -- Table structure for short_drama_project -- ---------------------------- @@ -1517,6 +1546,7 @@ CREATE TABLE `short_drama_storyboard` ( `video_url` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '视频地址', `video_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '视频生成任务ID', `video_status` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'pending' COMMENT '视频状态：pending/generating/done/failed', + `last_frame_url` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '上一镜末帧URL（同场景连续镜头首帧承接用）', `create_dept` bigint NULL DEFAULT NULL COMMENT '创建部门', `create_by` bigint NULL DEFAULT NULL COMMENT '创建者', `create_time` datetime NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', diff --git a/ruoyi-admin/src/main/resources/application.yml b/ruoyi-admin/src/main/resources/application.yml index be709192..d9aca934 100644 --- a/ruoyi-admin/src/main/resources/application.yml +++ b/ruoyi-admin/src/main/resources/application.yml @@ -20,6 +20,10 @@ server: # 阻塞任务线程池, 当执行类似servlet请求阻塞操作, undertow会从这个线程池中取得线程,它的值设置取决于系统的负载 worker: 256 +--- # 小程序对话 WebSocket 兜底默认模型（前端未传 model 且无智能体绑定时使用） +chat: + default-model: deepseek-v4-flash + captcha: # 是否启用验证码校验 enable: false @@ -125,6 +129,7 @@ security: - /*/api-docs/** - /warm-flow-ui/config - /workflow/run + - /coding/** # 多租户配置 tenant: # 是否开启 @@ -335,6 +340,7 @@ short-drama: probe-timeout: ${SHORT_DRAMA_COMPOSITION_PROBE_TIMEOUT:30s} process-timeout: ${SHORT_DRAMA_COMPOSITION_PROCESS_TIMEOUT:30m} job-stale-after: ${SHORT_DRAMA_COMPOSITION_JOB_STALE_AFTER:45m} + watermark-font-file: ${SHORT_DRAMA_COMPOSITION_WATERMARK_FONT_FILE:} worker-core-size: ${SHORT_DRAMA_COMPOSITION_WORKER_CORE_SIZE:1} worker-max-size: ${SHORT_DRAMA_COMPOSITION_WORKER_MAX_SIZE:2} worker-queue-capacity: ${SHORT_DRAMA_COMPOSITION_WORKER_QUEUE_CAPACITY:8} diff --git a/ruoyi-common/ruoyi-common-chat/src/main/java/org/ruoyi/common/chat/entity/audio/AudioContext.java b/ruoyi-common/ruoyi-common-chat/src/main/java/org/ruoyi/common/chat/entity/audio/AudioContext.java index 22611ffe..a096fb54 100644 --- a/ruoyi-common/ruoyi-common-chat/src/main/java/org/ruoyi/common/chat/entity/audio/AudioContext.java +++ b/ruoyi-common/ruoyi-common-chat/src/main/java/org/ruoyi/common/chat/entity/audio/AudioContext.java @@ -26,4 +26,19 @@ public class AudioContext { private Double speed; private String instructions; + + /** Atlas seed-audio 参考资源：[{speaker, audioUrl, audioData, imageData}]，speaker 为音色名 */ + private java.util.List> references; + + /** Atlas seed-audio 采样率 */ + private Integer sampleRate; + + /** Atlas seed-audio 音调调整 (-12~12) */ + private Integer pitchRate; + + /** Atlas seed-audio 语速调整 (-50~100) */ + private Integer speechRate; + + /** Atlas seed-audio 响度调整 (-50~100) */ + private Integer loudnessRate; } diff --git a/ruoyi-common/ruoyi-common-chat/src/main/java/org/ruoyi/common/chat/entity/media/MediaGenerationResponse.java b/ruoyi-common/ruoyi-common-chat/src/main/java/org/ruoyi/common/chat/entity/media/MediaGenerationResponse.java index 5f1960b9..870b109f 100644 --- a/ruoyi-common/ruoyi-common-chat/src/main/java/org/ruoyi/common/chat/entity/media/MediaGenerationResponse.java +++ b/ruoyi-common/ruoyi-common-chat/src/main/java/org/ruoyi/common/chat/entity/media/MediaGenerationResponse.java @@ -25,4 +25,7 @@ public class MediaGenerationResponse { private String status; private String rawResponse; + + /** 末帧图片 URL（Atlas return_last_frame=true 时返回，用于下一镜首帧承接） */ + private String lastFrameUrl; } diff --git a/ruoyi-common/ruoyi-common-chat/src/main/java/org/ruoyi/common/chat/entity/video/VideoContext.java b/ruoyi-common/ruoyi-common-chat/src/main/java/org/ruoyi/common/chat/entity/video/VideoContext.java index b6138862..c602d4b7 100644 --- a/ruoyi-common/ruoyi-common-chat/src/main/java/org/ruoyi/common/chat/entity/video/VideoContext.java +++ b/ruoyi-common/ruoyi-common-chat/src/main/java/org/ruoyi/common/chat/entity/video/VideoContext.java @@ -31,5 +31,17 @@ public class VideoContext { /** 多参考图 URL（多参考图生视频模式，配合 @imageN prompt 使用） */ private java.util.List referenceImages; + /** 参考音频 URL 列表（对白口型对齐模式） */ + private java.util.List referenceAudios; + + /** 是否让模型生成同步音频（环境音/动效） */ + private Boolean generateAudio; + + /** 是否要求返回末帧，用于下一镜首帧承接 */ + private Boolean returnLastFrame; + + /** 上一镜末帧 URL（同场景连续镜头首帧承接用，作为额外参考图传入） */ + private String lastFrameUrl; + private String videoId; } diff --git a/ruoyi-modules/ruoyi-chat/pom.xml b/ruoyi-modules/ruoyi-chat/pom.xml index 144240a2..e0bdc473 100644 --- a/ruoyi-modules/ruoyi-chat/pom.xml +++ b/ruoyi-modules/ruoyi-chat/pom.xml @@ -24,6 +24,14 @@ ruoyi-common-sse + + + org.ruoyi + ruoyi-common-websocket + + org.ruoyi ruoyi-common-sensitive diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/coding/CodingController.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/coding/CodingController.java new file mode 100644 index 00000000..759cb592 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/coding/CodingController.java @@ -0,0 +1,92 @@ +package org.ruoyi.controller.coding; + +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.ruoyi.common.satoken.utils.LoginHelper; +import org.ruoyi.common.core.domain.R; +import org.ruoyi.common.chat.domain.bo.chat.ChatModelBo; +import org.ruoyi.common.chat.service.chat.IChatModelService; +import org.ruoyi.domain.bo.coding.CodingRequestBo; +import org.ruoyi.service.coding.CodingWorkspaceService; +import org.ruoyi.service.coding.ICodingService; +import org.springframework.http.MediaType; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; + +import java.util.List; + +/** + * 编程能力接口（B 路径，不走 Supervisor 调度） + * + *

第一阶段 {@code /coding/**} 在 {@code application.yml} 的 security.excludes 中， + * 免鉴权直连。Controller 只做参数绑定 + 同步取 userId（Sa-Token 异步上下文丢失， + * 见 SecurityConfig 注释）+ 转发 Service。 + * + * @author ageerle + */ +@Validated +@RestController +@RequiredArgsConstructor +@RequestMapping("/coding") +public class CodingController { + + private final ICodingService codingService; + private final CodingWorkspaceService workspaceService; + private final IChatModelService chatModelService; + + /** + * 编程对话（SSE 流式） + * + * @param bo 请求参数（prompt / model / workspacePath） + * @return SseEmitter + */ + @PostMapping(value = "/chat", produces = MediaType.TEXT_EVENT_STREAM_VALUE) + public SseEmitter chat(@Valid @RequestBody CodingRequestBo bo) { + // 同步线程取 userId；第一阶段免鉴权，可能为 null + Long userId = LoginHelper.getUserId(); + return codingService.chat(bo, userId); + } + + @GetMapping("/workspace") + public R workspace( + @RequestParam(required = false) String workspacePath) throws Exception { + return R.ok(workspaceService.list(workspacePath)); + } + + @GetMapping("/models") + public R> models() { + List models = chatModelService.queryList(new ChatModelBo()).stream() + .filter(model -> "1".equals(model.getModelShow())) + .map(model -> new ModelOption(model.getId(), model.getModelName(), model.getProviderCode())) + .toList(); + return R.ok(models); + } + + @GetMapping("/file") + public R file( + @RequestParam(required = false) String workspacePath, + @RequestParam String path) throws Exception { + return R.ok(workspaceService.read(workspacePath, path)); + } + + @PutMapping("/file") + public R saveFile(@RequestBody FileWriteRequest request) throws Exception { + return R.ok(workspaceService.write(request.workspacePath(), request.path(), request.content())); + } + + @PostMapping("/command") + public R command(@RequestBody CommandRequest request) { + return R.ok(workspaceService.execute(request.workspacePath(), request.command())); + } + + public record FileWriteRequest(String workspacePath, String path, String content) { } + public record CommandRequest(String workspacePath, String command) { } + public record ModelOption(Long id, String name, String provider) { } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/shortdrama/ShortDramaController.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/shortdrama/ShortDramaController.java index 5e6d6ce6..6ecd69a3 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/shortdrama/ShortDramaController.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/controller/shortdrama/ShortDramaController.java @@ -14,6 +14,7 @@ import org.ruoyi.common.satoken.utils.LoginHelper; import org.ruoyi.domain.bo.shortdrama.ShortDramaCharacterBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaCharacterAppearanceBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaComposeVideoBo; +import org.ruoyi.domain.bo.shortdrama.ShortDramaAudioBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaLocationBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaProjectBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaScriptBo; @@ -23,6 +24,7 @@ import org.ruoyi.domain.vo.shortdrama.ShortDramaCharacterVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaCharacterAppearanceVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaComposeVideoVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaDetailVo; +import org.ruoyi.domain.vo.shortdrama.ShortDramaAudioVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaLocationVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaProjectVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaScriptVo; @@ -340,6 +342,35 @@ public class ShortDramaController { return R.ok(shortDramaService.undoLocationImage(locationId, LoginHelper.getUserId())); } + // ==================== 语音资产管理 ==================== + + @PostMapping("/audio") + public R saveAudio(@Valid @RequestBody ShortDramaAudioBo bo) { + return R.ok(shortDramaService.saveAudio(bo, LoginHelper.getUserId())); + } + + @PutMapping("/audio") + public R updateAudio(@Valid @RequestBody ShortDramaAudioBo bo) { + return R.ok(shortDramaService.saveAudio(bo, LoginHelper.getUserId())); + } + + @DeleteMapping("/audio/{audioId}") + public R deleteAudio(@NotNull @PathVariable Long audioId) { + shortDramaService.deleteAudio(audioId, LoginHelper.getUserId()); + return R.ok(); + } + + @GetMapping("/audio/list") + public R> listAudios(@NotNull @RequestParam Long projectId) { + return R.ok(shortDramaService.listAudios(projectId, LoginHelper.getUserId())); + } + + @PostMapping("/audio/{audioId}/generate-speech") + public R generateAudio(@NotNull @PathVariable Long audioId, + @NotBlank @RequestParam String model) { + return R.ok(shortDramaService.generateAudio(audioId, model, LoginHelper.getUserId())); + } + // ==================== 异步图片生成（轮询进度） ==================== /** 上传本地照片到图片供应商，返回当前生成会话使用的临时 URL。 */ diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/coding/CodingRequestBo.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/coding/CodingRequestBo.java new file mode 100644 index 00000000..4c479498 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/coding/CodingRequestBo.java @@ -0,0 +1,30 @@ +package org.ruoyi.domain.bo.coding; + +import jakarta.validation.constraints.NotBlank; +import lombok.Data; + +/** + * 编程能力对话请求 + * + * @author ageerle + */ +@Data +public class CodingRequestBo { + + /** + * 用户指令 + */ + @NotBlank(message = "prompt 不能为空") + private String prompt; + + /** + * 模型名称（走 IChatModelService.selectModelByName） + */ + @NotBlank(message = "model 不能为空") + private String model; + + /** + * 工作目录，可选；为空时默认指向 ruoyi-copilot 前端项目 + */ + private String workspacePath; +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/shortdrama/ShortDramaAudioBo.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/shortdrama/ShortDramaAudioBo.java new file mode 100644 index 00000000..fd00de0c --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/shortdrama/ShortDramaAudioBo.java @@ -0,0 +1,37 @@ +package org.ruoyi.domain.bo.shortdrama; + +import io.github.linpeilie.annotations.AutoMapper; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Pattern; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.ruoyi.common.mybatis.core.domain.BaseEntity; +import org.ruoyi.domain.entity.shortdrama.ShortDramaAudio; + +@Data +@EqualsAndHashCode(callSuper = true) +@AutoMapper(target = ShortDramaAudio.class, reverseConvertGenerate = false) +public class ShortDramaAudioBo extends BaseEntity { + + private Long id; + + @NotNull(message = "项目ID不能为空") + private Long projectId; + + @NotBlank(message = "语音资产名称不能为空") + private String name; + + @NotBlank(message = "语音类型不能为空") + @Pattern(regexp = "narration|dialogue", message = "语音类型只能是 narration 或 dialogue") + private String audioType; + + @NotBlank(message = "语音文案不能为空") + private String text; + + /** 音色（生成语音时使用，可空，空则用模型默认） */ + private String voice; + + /** 对白关联的分镜ID（旁白类型留空） */ + private Long linkedStoryboardId; +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/shortdrama/ShortDramaCharacterAppearanceBo.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/shortdrama/ShortDramaCharacterAppearanceBo.java index 7ff73f92..e14b7c09 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/shortdrama/ShortDramaCharacterAppearanceBo.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/shortdrama/ShortDramaCharacterAppearanceBo.java @@ -32,4 +32,6 @@ public class ShortDramaCharacterAppearanceBo extends BaseEntity { private String previousImageUrls; private String previousDescriptions; + + private String voice; } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/shortdrama/ShortDramaComposeVideoBo.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/shortdrama/ShortDramaComposeVideoBo.java index 363fb8c4..d1c7e36b 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/shortdrama/ShortDramaComposeVideoBo.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/bo/shortdrama/ShortDramaComposeVideoBo.java @@ -15,15 +15,21 @@ public class ShortDramaComposeVideoBo { @NotBlank(message = "转场类型不能为空") @Pattern(regexp = "none|dissolve|fade|slide", message = "不支持的转场类型") - private String transitionType = "dissolve"; + private String transitionType = "fade"; @NotNull(message = "转场时长不能为空") @DecimalMin(value = "0.0", message = "转场时长不能小于0秒") - private BigDecimal transitionDurationSeconds = new BigDecimal("0.5"); + private BigDecimal transitionDurationSeconds = new BigDecimal("0.3"); @NotBlank(message = "成片画幅不能为空") - @Pattern(regexp = "9:16|16:9|1:1", message = "不支持的成片画幅") + @Pattern(regexp = "9:16|16:9|4:3|3:4|1:1|21:9", message = "不支持的成片画幅") private String aspectRatio = "9:16"; @Size(min = 2, message = "至少选择2个分镜视频") private List storyboardIds; + + /** 旁白语音资产ID（可选，未传则不混入旁白） */ + private Long narrationAudioId; + + /** 是否加水印（null 时用后端默认配置 ruoyi-ai） */ + private Boolean watermark; } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/shortdrama/ShortDramaAudio.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/shortdrama/ShortDramaAudio.java new file mode 100644 index 00000000..dd02f16b --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/shortdrama/ShortDramaAudio.java @@ -0,0 +1,47 @@ +package org.ruoyi.domain.entity.shortdrama; + +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.ruoyi.common.mybatis.core.domain.BaseEntity; + +import java.io.Serial; + +@Data +@EqualsAndHashCode(callSuper = true) +@TableName("short_drama_audio") +public class ShortDramaAudio extends BaseEntity { + + @Serial + private static final long serialVersionUID = 1L; + + @TableId(value = "id") + private Long id; + + private Long projectId; + + /** 语音资产名称 */ + private String name; + + /** 语音类型：narration(旁白)/dialogue(对白) */ + private String audioType; + + /** 语音文案（生成语音用的文本） */ + private String text; + + /** 音色（如 alloy/onyx） */ + private String voice; + + /** 音频文件OSS ID */ + private Long audioOssId; + + /** 音频文件URL */ + private String audioUrl; + + /** 对白关联的分镜ID（NULL=全局旁白） */ + private Long linkedStoryboardId; + + /** 音频时长（秒） */ + private Integer durationSeconds; +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/shortdrama/ShortDramaCharacterAppearance.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/shortdrama/ShortDramaCharacterAppearance.java index b9f28848..53360596 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/shortdrama/ShortDramaCharacterAppearance.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/shortdrama/ShortDramaCharacterAppearance.java @@ -43,4 +43,7 @@ public class ShortDramaCharacterAppearance extends BaseEntity { /** 上一轮提示词列表（撤销用，JSON数组） */ private String previousDescriptions; + + /** 音色名（如 zh_male_taocheng_uranus_bigtts），用于该形象的对白配音 */ + private String voice; } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/shortdrama/ShortDramaStoryboard.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/shortdrama/ShortDramaStoryboard.java index 63e3cedd..5a2e2004 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/shortdrama/ShortDramaStoryboard.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/entity/shortdrama/ShortDramaStoryboard.java @@ -58,4 +58,7 @@ public class ShortDramaStoryboard extends BaseEntity { private String videoId; private String videoStatus; + + /** 上一镜末帧URL（同场景连续镜头首帧承接用） */ + private String lastFrameUrl; } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaAudioVo.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaAudioVo.java new file mode 100644 index 00000000..d61afa2c --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaAudioVo.java @@ -0,0 +1,41 @@ +package org.ruoyi.domain.vo.shortdrama; + +import io.github.linpeilie.annotations.AutoMapper; +import lombok.Data; +import org.ruoyi.domain.entity.shortdrama.ShortDramaAudio; + +import java.io.Serial; +import java.io.Serializable; +import java.util.Date; + +@Data +@AutoMapper(target = ShortDramaAudio.class) +public class ShortDramaAudioVo implements Serializable { + + @Serial + private static final long serialVersionUID = 1L; + + private Long id; + + private Long projectId; + + private String name; + + private String audioType; + + private String text; + + private String voice; + + private Long audioOssId; + + private String audioUrl; + + private Long linkedStoryboardId; + + private Integer durationSeconds; + + private Date createTime; + + private Date updateTime; +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaCharacterAppearanceVo.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaCharacterAppearanceVo.java index 5ee0b031..ac35eb97 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaCharacterAppearanceVo.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaCharacterAppearanceVo.java @@ -37,6 +37,8 @@ public class ShortDramaCharacterAppearanceVo implements Serializable { private String previousDescriptions; + private String voice; + private Date createTime; private Date updateTime; diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaDetailVo.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaDetailVo.java index 7e25bf86..31cdc1cb 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaDetailVo.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaDetailVo.java @@ -20,5 +20,7 @@ public class ShortDramaDetailVo implements Serializable { private List locations; + private List audios; + private List storyboards; } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaStoryboardVo.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaStoryboardVo.java index da0eedc1..94680c3c 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaStoryboardVo.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/domain/vo/shortdrama/ShortDramaStoryboardVo.java @@ -57,6 +57,8 @@ public class ShortDramaStoryboardVo implements Serializable { private String videoStatus; + private String lastFrameUrl; + private Date createTime; private Date updateTime; diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mapper/shortdrama/ShortDramaAudioMapper.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mapper/shortdrama/ShortDramaAudioMapper.java new file mode 100644 index 00000000..35545a50 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mapper/shortdrama/ShortDramaAudioMapper.java @@ -0,0 +1,8 @@ +package org.ruoyi.mapper.shortdrama; + +import org.ruoyi.common.mybatis.core.mapper.BaseMapperPlus; +import org.ruoyi.domain.entity.shortdrama.ShortDramaAudio; +import org.ruoyi.domain.vo.shortdrama.ShortDramaAudioVo; + +public interface ShortDramaAudioMapper extends BaseMapperPlus { +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/DeleteFileTool.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/DeleteFileTool.java new file mode 100644 index 00000000..78b467a8 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/DeleteFileTool.java @@ -0,0 +1,166 @@ +package org.ruoyi.mcp.tools; + +import dev.langchain4j.agent.tool.Tool; +import org.ruoyi.mcp.service.core.BuiltinToolProvider; +import org.ruoyi.service.coding.CodingEventChannel; +import org.ruoyi.service.coding.CodingSseEvent; +import org.ruoyi.service.coding.WorkspaceGuard; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.concurrent.atomic.AtomicLong; + +/** + * 删除文件/目录工具 + * + *

编程能力专用：通过构造注入工作目录与 SSE 事件通道，操作前后推送 delete-start/delete-end 事件。 + * + * @author ageerle + */ +@Component +public class DeleteFileTool implements BuiltinToolProvider { + + public static final String DESCRIPTION = "Deletes a file or directory. " + + "Set recursive=true to delete a non-empty directory. " + + "Use absolute paths within the workspace directory."; + + private final String rootDirectory; + private final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(getClass()); + private final CodingEventChannel channel; + + public DeleteFileTool() { + this.rootDirectory = Paths.get(System.getProperty("user.dir"), "workspace").toString(); + this.channel = null; + } + + /** + * 编程能力专用构造。 + */ + public DeleteFileTool(Path root, CodingEventChannel channel) { + this.rootDirectory = root.toAbsolutePath().normalize().toString(); + this.channel = channel; + } + + /** + * 删除文件或目录 + * + * @param filePath 路径绝对路径 + * @param recursive 是否递归删除非空目录（可选，默认 false） + * @return 操作结果 + */ + @Tool(DESCRIPTION) + public String deleteFile(String filePath, Boolean recursive) { + try { + if (filePath == null || filePath.trim().isEmpty()) { + return "Error: File path cannot be empty"; + } + + Path path = Paths.get(filePath); + boolean rec = recursive != null && recursive; + + if (!path.isAbsolute()) { + return "Error: File path must be absolute: " + filePath; + } + + if (!WorkspaceGuard.isWithinWorkspace(Paths.get(rootDirectory), path)) { + return "Error: File path must be within the workspace directory (" + rootDirectory + "): " + filePath; + } + + if (!Files.exists(path)) { + return "Error: Path not found: " + filePath; + } + + if (channel != null) { + channel.send(CodingSseEvent.of("delete-start", filePath, null, null, "running")); + } + + String relativePath = getRelativePath(path); + + if (Files.isDirectory(path)) { + if (rec) { + AtomicLong count = new AtomicLong(0); + Files.walkFileTree(path, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + count.incrementAndGet(); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + if (channel != null) { + channel.send(CodingSseEvent.of("delete-end", filePath, null, + "删除目录及 " + count.get() + " 个文件", "done")); + } + return String.format("Successfully deleted directory: %s (%d files)", relativePath, count.get()); + } else { + try { + Files.delete(path); + } catch (IOException e) { + if (channel != null) { + channel.send(CodingSseEvent.of("delete-end", filePath, null, + "Error: 非空目录需 recursive=true", "done")); + } + return "Error: Directory not empty, set recursive=true: " + filePath; + } + if (channel != null) { + channel.send(CodingSseEvent.of("delete-end", filePath, null, null, "done")); + } + return String.format("Successfully deleted directory: %s", relativePath); + } + } else { + Files.delete(path); + if (channel != null) { + channel.send(CodingSseEvent.of("delete-end", filePath, null, null, "done")); + } + return String.format("Successfully deleted file: %s", relativePath); + } + + } catch (IOException e) { + logger.error("Error deleting file: {}", filePath, e); + if (channel != null) { + channel.send(CodingSseEvent.of("delete-end", filePath, null, + "Error: " + e.getMessage(), "done")); + } + return "Error: " + e.getMessage(); + } catch (Exception e) { + logger.error("Unexpected error deleting file: {}", filePath, e); + return "Error: Unexpected error: " + e.getMessage(); + } + } + + private String getRelativePath(Path filePath) { + try { + Path workspaceRoot = Paths.get(rootDirectory); + return workspaceRoot.relativize(filePath).toString(); + } catch (Exception e) { + return filePath.toString(); + } + } + + @Override + public String getToolName() { + return "delete_file"; + } + + @Override + public String getDisplayName() { + return "删除文件"; + } + + @Override + public String getDescription() { + return DESCRIPTION; + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/EditFileTool.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/EditFileTool.java index 26473da2..7f74205c 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/EditFileTool.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/EditFileTool.java @@ -2,6 +2,9 @@ package org.ruoyi.mcp.tools; import dev.langchain4j.agent.tool.Tool; import org.ruoyi.mcp.service.core.BuiltinToolProvider; +import org.ruoyi.service.coding.CodingEventChannel; +import org.ruoyi.service.coding.CodingSseEvent; +import org.ruoyi.service.coding.WorkspaceGuard; import org.springframework.stereotype.Component; import java.io.IOException; @@ -10,8 +13,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.Arrays; -import java.util.List; /** * 编辑文件工具 @@ -20,16 +21,26 @@ import java.util.List; @Component public class EditFileTool implements BuiltinToolProvider { - public static final String DESCRIPTION = "Edits a file by applying a diff. " + - "Use this tool when you need to make specific changes to a file. " + - "The tool will show the diff before applying changes. " + + public static final String DESCRIPTION = "Edits an existing file by replacing its full content. " + + "ALWAYS read the file first with read_file, then provide the COMPLETE new content here. " + "Use absolute paths within the workspace directory."; private final String rootDirectory; private final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(getClass()); + /** 编程能力 SSE 事件通道，可为 null（兼容无参构造的老调用方） */ + private final CodingEventChannel channel; public EditFileTool() { this.rootDirectory = Paths.get(System.getProperty("user.dir"), "workspace").toString(); + this.channel = null; + } + + /** + * 编程能力专用构造：注入会话工作目录与事件通道。 + */ + public EditFileTool(Path root, CodingEventChannel channel) { + this.rootDirectory = root.toAbsolutePath().normalize().toString(); + this.channel = channel; } /** @@ -59,7 +70,7 @@ public class EditFileTool implements BuiltinToolProvider { } // 验证是否在工作目录内 - if (!isWithinWorkspace(path)) { + if (!WorkspaceGuard.isWithinWorkspace(Paths.get(rootDirectory), path)) { return "Error: File path must be within the workspace directory (" + rootDirectory + "): " + filePath; } @@ -73,30 +84,31 @@ public class EditFileTool implements BuiltinToolProvider { return "Error: Path is a directory, not a file: " + filePath; } - // 读取原始内容 - String originalContent = Files.readString(path, StandardCharsets.UTF_8); - List originalLines = Arrays.asList(originalContent.split("\n")); + // 推送编辑开始事件 + String relativePath = getRelativePath(path); + if (channel != null) { + channel.send(CodingSseEvent.of("edit-start", filePath, null, null, "running")); + } - // 应用diff + // 应用diff（简化：整体替换为新内容） try { - // 这里简化处理，直接用新内容替换 - // 在实际应用中，可能需要更复杂的diff解析 - String newContent = applyDiff(originalContent, diff); + String newContent = applyDiff(null, diff); - // 写入文件 Files.writeString(path, newContent, StandardCharsets.UTF_8, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); - String relativePath = getRelativePath(path); + if (channel != null) { + channel.send(CodingSseEvent.of("edit-end", filePath, null, null, "done")); + } return String.format("Successfully edited file: %s", relativePath); } catch (Exception e) { + if (channel != null) { + channel.send(CodingSseEvent.of("edit-end", filePath, null, "Error: " + e.getMessage(), "done")); + } return "Error: Failed to apply diff: " + e.getMessage(); } - } catch (IOException e) { - logger.error("Error editing file: {}", filePath, e); - return "Error: " + e.getMessage(); } catch (Exception e) { logger.error("Unexpected error editing file: {}", filePath, e); return "Error: Unexpected error: " + e.getMessage(); @@ -115,14 +127,7 @@ public class EditFileTool implements BuiltinToolProvider { } private boolean isWithinWorkspace(Path filePath) { - try { - Path workspaceRoot = Paths.get(rootDirectory).toRealPath(); - Path normalizedPath = filePath.normalize(); - return normalizedPath.startsWith(workspaceRoot.normalize()); - } catch (IOException e) { - logger.warn("Could not resolve workspace path", e); - return false; - } + return WorkspaceGuard.isWithinWorkspace(Paths.get(rootDirectory), filePath); } private String getRelativePath(Path filePath) { diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/ExecuteCommandTool.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/ExecuteCommandTool.java new file mode 100644 index 00000000..7c7d3e19 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/ExecuteCommandTool.java @@ -0,0 +1,266 @@ +package org.ruoyi.mcp.tools; + +import dev.langchain4j.agent.tool.Tool; +import org.ruoyi.mcp.service.core.BuiltinToolProvider; +import org.ruoyi.service.coding.CodingEventChannel; +import org.ruoyi.service.coding.CodingSseEvent; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + * 命令执行工具 + * + *

在会话工作目录内执行白名单命令，返回 stdout+stderr 尾部（8KB）。 + * 安全四层防御： + *

    + *
  1. 命令白名单：首段必须是允许的命令名
    2. + *
  2. 元字符黑名单：含 shell 元字符 {@code &|;`$<>\n} 直接拒绝（虽走 ProcessBuilder 不经 shell，仍双保险）
    3. + *
  3. 工作目录锁定：ProcessBuilder.directory 锁在会话 workspace
    4. + *
  4. 超时 + 输出截断：30s 超时 destroyForcibly，输出重定向临时文件只读尾部 8KB
    5. + *
+ * + *

实现借鉴 {@code FfmpegProcessRunner}：ProcessBuilder(List) 不走 shell 防注入， + * redirectErrorStream+redirectOutput 到临时文件，waitFor 超时，readTail 截断。 + * + * @author ageerle + */ +@Component +public class ExecuteCommandTool implements BuiltinToolProvider { + + public static final String DESCRIPTION = "Executes a shell command in the workspace directory. " + + "Command must be in the allowed whitelist (npm/pnpm/yarn/git/mvn/gradle/java/javac/" + + "python/pip/node/tsc/eslint/prettier/cat/ls/dir/echo). " + + "Returns combined stdout+stderr tail (8KB). 30s timeout."; + + /** 允许的命令白名单（首段） */ + private static final Set ALLOWED_COMMANDS = Set.of( + "npm", "pnpm", "yarn", "git", "mvn", "gradle", "java", "javac", + "python", "python3", "pip", "node", "tsc", "eslint", "prettier", + "cat", "ls", "dir", "echo" + ); + + /** 禁止的 shell 元字符（防注入） */ + private static final String FORBIDDEN_CHARS = "&|;`$<>\n\r"; + + /** 输出截断上限 */ + private static final int MAX_OUTPUT_BYTES = 8 * 1024; + /** 命令超时（秒） */ + private static final long TIMEOUT_SECONDS = 30; + + private final Path rootDirectory; + private final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(getClass()); + private final CodingEventChannel channel; + + public ExecuteCommandTool() { + this.rootDirectory = Paths.get(System.getProperty("user.dir"), "workspace"); + this.channel = null; + } + + /** + * 编程能力专用构造。 + */ + public ExecuteCommandTool(Path root, CodingEventChannel channel) { + this.rootDirectory = root.toAbsolutePath().normalize(); + this.channel = channel; + } + + /** + * 执行命令 + * + * @param command 完整命令行（如 "npm install" 或 "node -v"） + * @return 命令输出尾部，失败返回 "Error: ..." + */ + @Tool(DESCRIPTION) + public String executeCommand(String command) { + if (command == null || command.trim().isEmpty()) { + return "Error: Command cannot be empty"; + } + + // 元字符黑名单校验 + for (int i = 0; i < command.length(); i++) { + if (FORBIDDEN_CHARS.indexOf(command.charAt(i)) >= 0) { + return "Error: Command contains forbidden shell character: '" + command.charAt(i) + "'"; + } + } + + // 按空白拆分（多个空格也兼容） + List parts = splitCommand(command); + if (parts.isEmpty()) { + return "Error: Command is empty after split"; + } + + String cmdName = parts.get(0); + // 白名单校验，Windows 下尝试追加 .cmd/.exe/.bat 后缀 + String resolvedCmd = resolveCommand(cmdName); + if (resolvedCmd == null) { + return "Error: Command not in whitelist: " + cmdName + + ". Allowed: " + ALLOWED_COMMANDS; + } + + List cmdList = new ArrayList<>(parts); + cmdList.set(0, resolvedCmd); + + if (channel != null) { + channel.send(CodingSseEvent.of("cmd", null, command, null, "running")); + } + + Path logFile = null; + Process process = null; + try { + logFile = Files.createTempFile("coding-cmd-", ".log"); + ProcessBuilder builder = new ProcessBuilder(cmdList); + builder.directory(rootDirectory.toFile()); + builder.redirectErrorStream(true); + builder.redirectOutput(logFile.toFile()); + process = builder.start(); + + String result; + if (!process.waitFor(TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + stop(process); + String tail = readTail(logFile, MAX_OUTPUT_BYTES); + if (channel != null) { + channel.send(CodingSseEvent.of("cmd", null, command, + "超时（" + TIMEOUT_SECONDS + "s）\n" + tail, "done")); + } + return "Error: command timed out after " + TIMEOUT_SECONDS + "s\n" + tail; + } + + int exitCode = process.exitValue(); + String output = readTail(logFile, MAX_OUTPUT_BYTES); + if (channel != null) { + channel.send(CodingSseEvent.of("cmd", null, command, + output, "done")); + } + result = exitCode == 0 ? output : "Error: exit " + exitCode + "\n" + output; + return result; + + } catch (InterruptedException ex) { + if (process != null) { + process.destroyForcibly(); + } + Thread.currentThread().interrupt(); + return "Error: command interrupted"; + } catch (IOException ex) { + logger.error("Error executing command: {}", command, ex); + return "Error: " + ex.getMessage(); + } finally { + if (logFile != null) { + try { + Files.deleteIfExists(logFile); + } catch (IOException ignored) { + // 临时文件清理失败不影响主流程 + } + } + } + } + + /** + * 解析命令名：白名单匹配，Windows 下追加后缀重试。 + */ + private String resolveCommand(String cmdName) { + if (ALLOWED_COMMANDS.contains(cmdName)) { + return cmdName; + } + // Windows 下 npm/pnpm 等可能是 .cmd + if (isWindows()) { + for (String suffix : new String[]{".cmd", ".exe", ".bat"}) { + String candidate = cmdName + suffix; + String baseName = stripSuffix(cmdName); + if (ALLOWED_COMMANDS.contains(baseName)) { + return candidate; + } + } + } + return null; + } + + private String stripSuffix(String name) { + for (String suffix : new String[]{".cmd", ".exe", ".bat"}) { + if (name.endsWith(suffix)) { + return name.substring(0, name.length() - suffix.length()); + } + } + return name; + } + + private boolean isWindows() { + return System.getProperty("os.name", "").toLowerCase().contains("win"); + } + + /** + * 按空白拆分命令行（不处理引号，保持简单；元字符已在上游拦截）。 + */ + private List splitCommand(String command) { + List parts = new ArrayList<>(); + StringBuilder cur = new StringBuilder(); + for (int i = 0; i < command.length(); i++) { + char c = command.charAt(i); + if (Character.isWhitespace(c)) { + if (cur.length() > 0) { + parts.add(cur.toString()); + cur.setLength(0); + } + } else { + cur.append(c); + } + } + if (cur.length() > 0) { + parts.add(cur.toString()); + } + return parts; + } + + private static void stop(Process process) throws InterruptedException { + process.destroy(); + if (!process.waitFor(2, TimeUnit.SECONDS)) { + process.destroyForcibly(); + process.waitFor(2, TimeUnit.SECONDS); + } + } + + /** + * 读取文件尾部（抄自 FfmpegProcessRunner.readTail）。 + */ + static String readTail(Path path, int maxBytes) throws IOException { + if (!Files.exists(path)) { + return ""; + } + long size = Files.size(path); + int bytesToRead = (int) Math.min(size, maxBytes); + ByteBuffer buffer = ByteBuffer.allocate(bytesToRead); + try (SeekableByteChannel channel = Files.newByteChannel(path, StandardOpenOption.READ)) { + channel.position(Math.max(0, size - bytesToRead)); + while (buffer.hasRemaining() && channel.read(buffer) >= 0) { + // 读取直到尾部 + } + } + return new String(buffer.array(), 0, buffer.position(), StandardCharsets.UTF_8).trim(); + } + + @Override + public String getToolName() { + return "execute_command"; + } + + @Override + public String getDisplayName() { + return "执行命令"; + } + + @Override + public String getDescription() { + return DESCRIPTION; + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/ListDirectoryTool.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/ListDirectoryTool.java index 8f6c0cdd..3104029b 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/ListDirectoryTool.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/ListDirectoryTool.java @@ -2,6 +2,9 @@ package org.ruoyi.mcp.tools; import dev.langchain4j.agent.tool.Tool; import org.ruoyi.mcp.service.core.BuiltinToolProvider; +import org.ruoyi.service.coding.CodingEventChannel; +import org.ruoyi.service.coding.CodingSseEvent; +import org.ruoyi.service.coding.WorkspaceGuard; import org.springframework.stereotype.Component; import java.io.IOException; @@ -30,9 +33,20 @@ public class ListDirectoryTool implements BuiltinToolProvider { private final String rootDirectory; private final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(getClass()); + /** 编程能力 SSE 事件通道，可为 null（兼容无参构造的老调用方） */ + private final CodingEventChannel channel; public ListDirectoryTool() { this.rootDirectory = Paths.get(System.getProperty("user.dir"), "workspace").toString(); + this.channel = null; + } + + /** + * 编程能力专用构造：注入会话工作目录与事件通道。 + */ + public ListDirectoryTool(Path root, CodingEventChannel channel) { + this.rootDirectory = root.toAbsolutePath().normalize().toString(); + this.channel = channel; } /** @@ -74,11 +88,23 @@ public class ListDirectoryTool implements BuiltinToolProvider { return "Error: Path is not a directory: " + params.filePath; } + // 推送列目录开始事件 + if (channel != null) { + channel.send(CodingSseEvent.of("list-progress", null, null, + "扫描 " + getRelativePath(dirPath), "running")); + } + // 列出文件和目录 List fileInfos = listFiles(dirPath, params); // 生成输出 - return formatFileList(fileInfos, params); + String output = formatFileList(fileInfos, params); + + if (channel != null) { + channel.send(CodingSseEvent.of("list-progress", null, null, + "共 " + fileInfos.size() + " 项", "done")); + } + return output; } catch (IOException e) { logger.error("Error listing directory: {}", params.filePath, e); @@ -233,14 +259,7 @@ public class ListDirectoryTool implements BuiltinToolProvider { } private boolean isWithinWorkspace(Path dirPath) { - try { - Path workspaceRoot = Paths.get(rootDirectory).toRealPath(); - Path normalizedPath = dirPath.normalize(); - return normalizedPath.startsWith(workspaceRoot.normalize()); - } catch (IOException e) { - logger.warn("Could not resolve workspace path", e); - return false; - } + return WorkspaceGuard.isWithinWorkspace(Paths.get(rootDirectory), dirPath); } private String getRelativePath(Path dirPath) { diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/ReadFileTool.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/ReadFileTool.java index 7b4886e7..369c330b 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/ReadFileTool.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/ReadFileTool.java @@ -2,6 +2,9 @@ package org.ruoyi.mcp.tools; import dev.langchain4j.agent.tool.Tool; import org.ruoyi.mcp.service.core.BuiltinToolProvider; +import org.ruoyi.service.coding.CodingEventChannel; +import org.ruoyi.service.coding.CodingSseEvent; +import org.ruoyi.service.coding.WorkspaceGuard; import org.springframework.stereotype.Component; import java.io.IOException; @@ -22,10 +25,26 @@ public class ReadFileTool implements BuiltinToolProvider { "Returns the complete file content as a string."; private final String rootDirectory; + /** 读取内容截断上限，避免大文件撑爆 LLM 上下文（32KB） */ + private static final int MAX_BYTES = 32 * 1024; private final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(getClass()); + /** 编程能力 SSE 事件通道，可为 null（兼容 BuiltinToolRegistry 无参构造的老调用方） */ + private final CodingEventChannel channel; public ReadFileTool() { this.rootDirectory = Paths.get(System.getProperty("user.dir"), "workspace").toString(); + this.channel = null; + } + + /** + * 编程能力专用构造：注入会话工作目录与事件通道。 + * + * @param root 工作目录根（绝对路径） + * @param channel SSE 事件通道，工具执行前后推送 read 进度 + */ + public ReadFileTool(Path root, CodingEventChannel channel) { + this.rootDirectory = root.toAbsolutePath().normalize().toString(); + this.channel = channel; } /** @@ -50,7 +69,7 @@ public class ReadFileTool implements BuiltinToolProvider { } // 验证是否在工作目录内 - if (!isWithinWorkspace(path)) { + if (!WorkspaceGuard.isWithinWorkspace(Paths.get(rootDirectory), path)) { return "Error: File path must be within the workspace directory (" + rootDirectory + "): " + filePath; } @@ -64,16 +83,31 @@ public class ReadFileTool implements BuiltinToolProvider { return "Error: Path is a directory, not a file: " + filePath; } - // 读取文件内容 - String content = Files.readString(path, StandardCharsets.UTF_8); - - // 获取相对路径 + // 推送读取开始事件（前端展示为正在读取） String relativePath = getRelativePath(path); + if (channel != null) { + channel.send(CodingSseEvent.of("edit-start", filePath, null, null, "running")); + } + + // 读取文件内容（截断超大文件，避免撑爆上下文） + String content = Files.readString(path, StandardCharsets.UTF_8); + boolean truncated = false; + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + if (bytes.length > MAX_BYTES) { + content = new String(bytes, 0, MAX_BYTES, StandardCharsets.UTF_8); + truncated = true; + } + long sizeBytes = content.getBytes(StandardCharsets.UTF_8).length; long lineCount = content.lines().count(); + String header = String.format("File: %s (%d lines, %d bytes)%s\n\n", + relativePath, lineCount, sizeBytes, truncated ? " [truncated]" : ""); - return String.format("File: %s (%d lines, %d bytes)\n\n%s", - relativePath, lineCount, sizeBytes, content); + if (channel != null) { + channel.send(CodingSseEvent.of("edit-end", filePath, null, null, "done")); + } + + return header + content; } catch (IOException e) { logger.error("Error reading file: {}", filePath, e); @@ -85,14 +119,7 @@ public class ReadFileTool implements BuiltinToolProvider { } private boolean isWithinWorkspace(Path filePath) { - try { - Path workspaceRoot = Paths.get(rootDirectory).toRealPath(); - Path normalizedPath = filePath.normalize(); - return normalizedPath.startsWith(workspaceRoot.normalize()); - } catch (IOException e) { - logger.warn("Could not resolve workspace path", e); - return false; - } + return WorkspaceGuard.isWithinWorkspace(Paths.get(rootDirectory), filePath); } private String getRelativePath(Path filePath) { diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/WriteFileTool.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/WriteFileTool.java new file mode 100644 index 00000000..615c4aa3 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/mcp/tools/WriteFileTool.java @@ -0,0 +1,136 @@ +package org.ruoyi.mcp.tools; + +import dev.langchain4j.agent.tool.Tool; +import org.ruoyi.mcp.service.core.BuiltinToolProvider; +import org.ruoyi.service.coding.CodingEventChannel; +import org.ruoyi.service.coding.CodingSseEvent; +import org.ruoyi.service.coding.WorkspaceGuard; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; + +/** + * 写文件工具 + * 新建或覆盖文件，自动创建父目录 + * + *

编程能力专用：通过构造注入工作目录与 SSE 事件通道，操作前后推送 add-start/add-end 事件。 + * 不注册为 BuiltinToolProvider（无需进 BuiltinToolRegistry），仅由 CodingServiceImpl 按会话 new。 + * + * @author ageerle + */ +@Component +public class WriteFileTool implements BuiltinToolProvider { + + public static final String DESCRIPTION = "Creates or overwrites a file with the given content. " + + "Creates parent directories if missing. Overwrites existing file. " + + "Use absolute paths within the workspace directory."; + + private final String rootDirectory; + private final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(getClass()); + private final CodingEventChannel channel; + + public WriteFileTool() { + this.rootDirectory = Paths.get(System.getProperty("user.dir"), "workspace").toString(); + this.channel = null; + } + + /** + * 编程能力专用构造。 + */ + public WriteFileTool(Path root, CodingEventChannel channel) { + this.rootDirectory = root.toAbsolutePath().normalize().toString(); + this.channel = channel; + } + + /** + * 写文件 + * + * @param filePath 文件绝对路径 + * @param content 文件内容 + * @return 操作结果 + */ + @Tool(DESCRIPTION) + public String writeFile(String filePath, String content) { + try { + if (filePath == null || filePath.trim().isEmpty()) { + return "Error: File path cannot be empty"; + } + if (content == null) { + content = ""; + } + + Path path = Paths.get(filePath); + + if (!path.isAbsolute()) { + return "Error: File path must be absolute: " + filePath; + } + + if (!WorkspaceGuard.isWithinWorkspace(Paths.get(rootDirectory), path)) { + return "Error: File path must be within the workspace directory (" + rootDirectory + "): " + filePath; + } + + if (channel != null) { + channel.send(CodingSseEvent.of("add-start", filePath, null, null, "running")); + } + + // 创建父目录 + Path parent = path.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + + // 写入文件 + Files.writeString(path, content, StandardCharsets.UTF_8, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + + String relativePath = getRelativePath(path); + int bytes = content.getBytes(StandardCharsets.UTF_8).length; + + if (channel != null) { + channel.send(CodingSseEvent.of("add-end", filePath, null, + "写入 " + bytes + " 字节", "done")); + } + return String.format("Successfully wrote %d bytes to %s", bytes, relativePath); + + } catch (IOException e) { + logger.error("Error writing file: {}", filePath, e); + if (channel != null) { + channel.send(CodingSseEvent.of("add-end", filePath, null, + "Error: " + e.getMessage(), "done")); + } + return "Error: " + e.getMessage(); + } catch (Exception e) { + logger.error("Unexpected error writing file: {}", filePath, e); + return "Error: Unexpected error: " + e.getMessage(); + } + } + + private String getRelativePath(Path filePath) { + try { + Path workspaceRoot = Paths.get(rootDirectory); + return workspaceRoot.relativize(filePath).toString(); + } catch (Exception e) { + return filePath.toString(); + } + } + + @Override + public String getToolName() { + return "write_file"; + } + + @Override + public String getDisplayName() { + return "写入文件"; + } + + @Override + public String getDescription() { + return DESCRIPTION; + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/audio/provider/AtlasAudioGenerationServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/audio/provider/AtlasAudioGenerationServiceImpl.java new file mode 100644 index 00000000..bab5ca4e --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/audio/provider/AtlasAudioGenerationServiceImpl.java @@ -0,0 +1,98 @@ +package org.ruoyi.service.audio.provider; + +import cn.hutool.core.util.StrUtil; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo; +import org.ruoyi.common.chat.entity.audio.AudioContext; +import org.ruoyi.common.chat.entity.media.MediaGenerationResponse; +import org.ruoyi.enums.ChatModeType; +import org.ruoyi.service.audio.AbstractAudioGenerationService; +import org.ruoyi.service.media.AtlasMediaSupport; +import org.ruoyi.service.media.AtlasPredictionService; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Atlas Cloud 音频生成（bytedance/seed-audio-1.0）。异步：提交 /model/generateAudio 返回 predictionId， + * 轮询 /model/prediction/{id} 拿音频 URL。支持 references（speaker 音色 + 参考音频）做多角色对白配音。 + */ +@Slf4j +@Component("atlasAudio") +@RequiredArgsConstructor +public class AtlasAudioGenerationServiceImpl extends AbstractAudioGenerationService { + + private final AtlasPredictionService atlasPredictionService; + + private final OkHttpClient okHttpClient = new OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(180, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .build(); + + @Override + protected MediaGenerationResponse doGenerateSpeech(AudioContext audioContext) { + ChatModelVo model = audioContext.getChatModelVo(); + ObjectNode payload = AtlasMediaSupport.OBJECT_MAPPER.createObjectNode(); + payload.put("model", model.getModelName()); + payload.put("text", audioContext.getInput()); + String format = StrUtil.blankToDefault(audioContext.getResponseFormat(), "mp3"); + payload.put("format", format); + + // 参考资源：多角色音色 + 参考音频 + List> refs = audioContext.getReferences(); + if (refs != null && !refs.isEmpty()) { + ArrayNode arr = payload.putArray("references"); + for (Map ref : refs) { + ObjectNode r = arr.addObject(); + if (StrUtil.isNotBlank(ref.get("speaker"))) r.put("speaker", ref.get("speaker")); + if (StrUtil.isNotBlank(ref.get("audioUrl"))) r.put("audio_url", ref.get("audioUrl")); + if (StrUtil.isNotBlank(ref.get("audioData"))) r.put("audio_data", ref.get("audioData")); + if (StrUtil.isNotBlank(ref.get("imageData"))) r.put("image_data", ref.get("imageData")); + } + } else if (StrUtil.isNotBlank(audioContext.getVoice())) { + // 没有显式 references 但指定了 voice 音色名：作为单一 speaker + ArrayNode arr = payload.putArray("references"); + ObjectNode r = arr.addObject(); + r.put("speaker", audioContext.getVoice()); + } + + if (audioContext.getSampleRate() != null) payload.put("sample_rate", audioContext.getSampleRate()); + if (audioContext.getPitchRate() != null) payload.put("pitch_rate", audioContext.getPitchRate()); + if (audioContext.getSpeechRate() != null) payload.put("speech_rate", audioContext.getSpeechRate()); + if (audioContext.getLoudnessRate() != null) payload.put("loudness_rate", audioContext.getLoudnessRate()); + + Request request = new Request.Builder() + .url(AtlasMediaSupport.endpoint(model.getApiHost(), "/model/generateAudio")) + .addHeader("Authorization", "Bearer " + model.getApiKey()) + .addHeader("Content-Type", "application/json") + .post(RequestBody.create(payload.toString(), AtlasMediaSupport.JSON)) + .build(); + try (Response response = okHttpClient.newCall(request).execute()) { + ResponseBody body = response.body(); + String responseText = body == null ? "" : body.string(); + if (!response.isSuccessful()) { + throw new IllegalArgumentException("Atlas Cloud 音频生成任务创建失败: " + response.code() + " - " + responseText); + } + return atlasPredictionService.toResponse(responseText, "audio"); + } catch (IOException e) { + throw new RuntimeException("Atlas Cloud 音频生成任务创建失败: " + e.getMessage(), e); + } + } + + @Override + public String getProviderName() { + return ChatModeType.ATLAS.getCode(); + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingAgent.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingAgent.java new file mode 100644 index 00000000..d86e9760 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingAgent.java @@ -0,0 +1,29 @@ +package org.ruoyi.service.coding; + +import dev.langchain4j.service.SystemMessage; +import dev.langchain4j.service.UserMessage; + +/** + * 编程智能体 AiServices 接口。 + * + *

配合 {@code AiServices.builder(CodingAgent.class).chatModel(...).tools(...)} 构建。 + * 同步 {@code String chat(...)} 方案（方案 B）：工具执行过程中的 add/edit/delete/cmd 事件 + * 由工具内部通过 {@link CodingEventChannel} 实时推送，最终回复文本在 chat() 返回后一次性推 text。 + * + *

{@code @SystemMessage} 约束 LLM 只能操作 workspace 内文件，并明确每个工具的用途， + * 提升工具调用命中率。 + * + * @author ageerle + */ +public interface CodingAgent { + + @SystemMessage(""" + 你是一个编程助手，直接操作用户工作目录内的文件与命令。规则： + 1. 所有文件操作必须在 workspace 目录内，使用绝对路径；不要越界访问外部目录。 + 2. 读取文件用 read_file，新建/覆盖文件用 write_file，修改已存在文件用 edit_file， + 删除文件或目录用 delete_file，查看目录结构用 list_directory，执行构建/运行命令用 execute_command。 + 3. 修改文件前，先用 read_file 读取当前内容，避免覆盖丢失代码。 + 4. 执行命令前说明意图，命令失败时读取输出排查。 + 5. 完成任务后用一两句话总结做了什么。""") + String chat(@UserMessage String userMessage); +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingEventChannel.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingEventChannel.java new file mode 100644 index 00000000..cb98c8ba --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingEventChannel.java @@ -0,0 +1,93 @@ +package org.ruoyi.service.coding; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +/** + * 编程能力跨线程事件通道。 + * + *

结构抄自 {@code OutputChannel}，但队列元素是结构化 {@link CodingSseEvent} 而非 String。 + * OutputChannel 的 {@code send(String)} 塞 JSON 字符串再让 drain 端解析是反模式； + * 这里直接传结构化对象，drain 时再由 Service 层序列化。 + * + *

调用链路： + * 

+ *   异步线程（工具执行、LLM 回调）-> channel.send(event)
+ *   drain 线程                       -> channel.drain(emitter::send)
+ *
+ * + * @author ageerle + */ +public class CodingEventChannel { + + /** DONE 哨兵，drain 遇到即退出 */ + private static final CodingSseEvent DONE = new CodingSseEvent("__done__", null, null, null, null); + + private final BlockingQueue queue = new LinkedBlockingQueue<>(4096); + private final AtomicReference error = new AtomicReference<>(); + private final CountDownLatch completed = new CountDownLatch(1); + + /** + * 写入一个事件：线程安全，队列满时 100ms 超时丢弃。 + */ + public void send(CodingSseEvent event) { + if (event == null) { + return; + } + try { + if (!queue.offer(event, 100, TimeUnit.MILLISECONDS)) { + // 队列满，丢弃但不中断流程 + System.err.println("[CodingEventChannel] 队列满，丢弃事件: " + event.eventType()); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** + * 标记正常完成。 + */ + public void complete() { + queue.offer(DONE); + completed.countDown(); + } + + /** + * 标记错误完成，附带一条 error 事件。 + */ + public void completeWithError(Throwable t) { + error.set(t); + if (t != null && t.getMessage() != null) { + queue.offer(CodingSseEvent.error(t.getMessage())); + } + queue.offer(DONE); + completed.countDown(); + } + + /** + * 阻塞读取事件并逐个回调；遇 DONE 退出。 + * + * @param emitter 事件消费回调 + */ + public void drain(Consumer emitter) throws InterruptedException { + while (true) { + CodingSseEvent msg = queue.poll(200, TimeUnit.MILLISECONDS); + if (msg != null) { + if (DONE == msg || "__done__".equals(msg.eventType())) { + break; + } + emitter.accept(msg); + } else if (completed.getCount() == 0 && queue.isEmpty()) { + break; + } + } + } + + public boolean isCompleted() { + return completed.getCount() == 0; + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingSseEvent.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingSseEvent.java new file mode 100644 index 00000000..db7c610e --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingSseEvent.java @@ -0,0 +1,43 @@ +package org.ruoyi.service.coding; + +/** + * 编程能力 SSE 事件 DTO。 + * + *

事件名与前端 {@code ruoyi-copilot/src/App.vue} 的 {@code applyStreamEvent} 卡片契约对齐： + *

    + *
  • {@code thinking} / {@code text} —— LLM 思考/回复文本增量
    • + *
  • {@code add|edit|delete}-{start|progress|end} —— 文件写入/编辑/删除
    • + *
  • {@code cmd} —— 命令执行
    • + *
  • {@code list-progress} —— 列目录
    • + *
  • {@code done} / {@code error} —— 流结束
    • + *
+ * + *

约束：add/edit/delete 必须带 filePath（前端用 operation.filePath 存在性区分 + * code-change 卡 vs activity-row 卡）；cmd/list-progress 不带 filePath。 + * + * @author ageerle + */ +public record CodingSseEvent(String eventType, String filePath, String command, + String content, String status) { + + public static CodingSseEvent of(String eventType, String filePath, String command, + String content, String status) { + return new CodingSseEvent(eventType, filePath, command, content, status); + } + + public static CodingSseEvent text(String content) { + return new CodingSseEvent("text", null, null, content, null); + } + + public static CodingSseEvent thinking(String content) { + return new CodingSseEvent("thinking", null, null, content, null); + } + + public static CodingSseEvent done() { + return new CodingSseEvent("done", null, null, null, null); + } + + public static CodingSseEvent error(String message) { + return new CodingSseEvent("error", null, null, message, null); + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingWorkspaceService.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingWorkspaceService.java new file mode 100644 index 00000000..10d31465 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/CodingWorkspaceService.java @@ -0,0 +1,134 @@ +package org.ruoyi.service.coding; + +import org.ruoyi.mcp.tools.ExecuteCommandTool; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Comparator; +import java.util.List; + +/** Basic, workspace-scoped file and command operations for the Copilot UI. */ +@Service +public class CodingWorkspaceService { + + public static final String DEFAULT_WORKSPACE = "D:/Project/github/ruoyi-copilot"; + private static final long MAX_FILE_BYTES = 1024 * 1024; + private static final int MAX_ENTRIES = 500; + + public WorkspaceResult list(String workspacePath) throws IOException { + Path root = resolveRoot(workspacePath); + Files.createDirectories(root); + try (var stream = Files.walk(root, 8)) { + List files = stream + .filter(path -> !path.equals(root)) + .filter(path -> !isIgnored(root, path)) + .sorted(Comparator.comparing(path -> root.relativize(path).toString())) + .limit(MAX_ENTRIES) + .map(path -> toEntry(root, path)) + .toList(); + return new WorkspaceResult(root.toString(), files.size(), files); + } + } + + public FileContent read(String workspacePath, String relativePath) throws IOException { + Path root = resolveRoot(workspacePath); + Path file = resolveFile(root, relativePath); + if (!Files.isRegularFile(file)) { + throw new IllegalArgumentException("File does not exist: " + relativePath); + } + long size = Files.size(file); + if (size > MAX_FILE_BYTES) { + throw new IllegalArgumentException("File is larger than 1 MB: " + relativePath); + } + if (isBinary(file)) { + throw new IllegalArgumentException("Binary files cannot be edited: " + relativePath); + } + return new FileContent(normalizeRelative(root, file), Files.readString(file, StandardCharsets.UTF_8), size); + } + + public FileContent write(String workspacePath, String relativePath, String content) throws IOException { + Path root = resolveRoot(workspacePath); + Path file = resolveFile(root, relativePath); + byte[] bytes = (content == null ? "" : content).getBytes(StandardCharsets.UTF_8); + if (bytes.length > MAX_FILE_BYTES) { + throw new IllegalArgumentException("File content is larger than 1 MB"); + } + if (file.getParent() != null) Files.createDirectories(file.getParent()); + Files.writeString(file, content == null ? "" : content, StandardCharsets.UTF_8, + StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + return new FileContent(normalizeRelative(root, file), content == null ? "" : content, bytes.length); + } + + public CommandResult execute(String workspacePath, String command) { + Path root = resolveRoot(workspacePath); + String output = new ExecuteCommandTool(root, null).executeCommand(command); + boolean success = !output.startsWith("Error:"); + return new CommandResult(command, output, success); + } + + public Path resolveRoot(String workspacePath) { + Path configured = Paths.get(DEFAULT_WORKSPACE).toAbsolutePath().normalize(); + if (workspacePath == null || workspacePath.isBlank()) return configured; + Path requested = Paths.get(workspacePath).toAbsolutePath().normalize(); + if (!requested.equals(configured)) { + throw new IllegalArgumentException("Workspace is not allowed: " + requested); + } + return configured; + } + + private Path resolveFile(Path root, String relativePath) { + if (relativePath == null || relativePath.isBlank()) { + throw new IllegalArgumentException("File path cannot be empty"); + } + Path supplied = Paths.get(relativePath); + Path target = (supplied.isAbsolute() ? supplied : root.resolve(supplied)).normalize(); + if (!WorkspaceGuard.isWithinWorkspace(root, target)) { + throw new IllegalArgumentException("File must be inside the workspace"); + } + return target; + } + + private boolean isIgnored(Path root, Path path) { + Path relative = root.relativize(path); + for (Path part : relative) { + String name = part.toString(); + if (name.equals(".git") || name.equals("node_modules") || name.equals("dist") || name.equals("target")) { + return true; + } + } + return false; + } + + private FileEntry toEntry(Path root, Path path) { + try { + return new FileEntry(normalizeRelative(root, path), path.getFileName().toString(), + Files.isDirectory(path), Files.isDirectory(path) ? 0 : Files.size(path)); + } catch (IOException e) { + return new FileEntry(normalizeRelative(root, path), path.getFileName().toString(), + Files.isDirectory(path), 0); + } + } + + private String normalizeRelative(Path root, Path path) { + return root.relativize(path).toString().replace('\\', '/'); + } + + private boolean isBinary(Path file) throws IOException { + byte[] sample; + try (var input = Files.newInputStream(file)) { + sample = input.readNBytes(4096); + } + for (byte value : sample) if (value == 0) return true; + return false; + } + + public record FileEntry(String path, String name, boolean directory, long size) { } + public record WorkspaceResult(String root, int fileCount, List files) { } + public record FileContent(String path, String content, long size) { } + public record CommandResult(String command, String output, boolean success) { } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/ICodingService.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/ICodingService.java new file mode 100644 index 00000000..f48e1fad --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/ICodingService.java @@ -0,0 +1,21 @@ +package org.ruoyi.service.coding; + +import org.ruoyi.domain.bo.coding.CodingRequestBo; +import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; + +/** + * 编程能力 Service + * + * @author ageerle + */ +public interface ICodingService { + + /** + * 编程对话（SSE 流式） + * + * @param bo 请求参数 + * @param userId 用户 ID（可为 null，第一阶段免鉴权） + * @return SseEmitter + */ + SseEmitter chat(CodingRequestBo bo, Long userId); +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/WorkspaceGuard.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/WorkspaceGuard.java new file mode 100644 index 00000000..d71b2f4a --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/WorkspaceGuard.java @@ -0,0 +1,50 @@ +package org.ruoyi.service.coding; + +import java.io.IOException; +import java.nio.file.Path; + +/** + * 工作目录安全守卫。 + * + *

抽取自 {@code ReadFileTool/EditFileTool/ListDirectoryTool} 中重复的 {@code isWithinWorkspace}， + * 五个文件工具共用。强制所有操作路径必须落在工作目录内，防止路径穿越与软链接逃逸。 + * + *

实现要点： + *

    + *
  • {@code root.toRealPath()} 解析符号链接（防软链接逃逸）
    • + *
  • {@code target.normalize()} 消除 {@code ..} 穿越段
    • + *
  • {@code startsWith} 是 Path 段前缀匹配，非字符串前缀（{@code /workspace/abc} 不会误判成 {@code /workspace-evil}）
    • + *
+ * + * @author ageerle + */ +public final class WorkspaceGuard { + + private WorkspaceGuard() { + } + + /** + * 判断目标路径是否在工作目录内。 + * + * @param root 工作目录根（绝对路径） + * @param target 待校验路径 + * @return true 表示在 workspace 内，安全 + */ + public static boolean isWithinWorkspace(Path root, Path target) { + try { + Path realRoot = root.toRealPath().normalize(); + Path realTarget = target.normalize(); + return realTarget.startsWith(realRoot); + } catch (IOException e) { + // 目标路径不存在或无法解析（如新建文件前其父目录链中有不存在的段） + // 退化为 normalize 后做段前缀匹配，仍能拦住明显的越界 + try { + Path realRoot = root.toRealPath().normalize(); + Path normalizedTarget = target.normalize(); + return normalizedTarget.startsWith(realRoot); + } catch (IOException ignore) { + return false; + } + } + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/impl/CodingServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/impl/CodingServiceImpl.java new file mode 100644 index 00000000..671ed1b6 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/coding/impl/CodingServiceImpl.java @@ -0,0 +1,198 @@ +package org.ruoyi.service.coding.impl; + +import cn.hutool.core.util.StrUtil; +import dev.langchain4j.model.chat.ChatModel; +import dev.langchain4j.service.AiServices; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo; +import org.ruoyi.common.chat.service.chat.IChatModelService; +import org.ruoyi.common.json.utils.JsonUtils; +import org.ruoyi.domain.bo.coding.CodingRequestBo; +import org.ruoyi.factory.ChatServiceFactory; +import org.ruoyi.mcp.tools.DeleteFileTool; +import org.ruoyi.mcp.tools.EditFileTool; +import org.ruoyi.mcp.tools.ExecuteCommandTool; +import org.ruoyi.mcp.tools.ListDirectoryTool; +import org.ruoyi.mcp.tools.ReadFileTool; +import org.ruoyi.mcp.tools.WriteFileTool; +import org.ruoyi.service.chat.AbstractChatService; +import org.ruoyi.service.coding.CodingAgent; +import org.ruoyi.service.coding.CodingEventChannel; +import org.ruoyi.service.coding.CodingSseEvent; +import org.ruoyi.service.coding.ICodingService; +import org.springframework.stereotype.Service; +import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * 编程能力 Service 实现。 + * + *

B 路径：自建 SseEmitter（不进 SseEmitterManager 全局注册表），照 ShortDramaServiceImpl 骨架。 + * 拿模型三步（skill 铁律）→ 解析工作目录 → new 工具实例注入 channel+root → AiServices 构建 → + * 异步执行，工具内部通过 channel 实时推事件，drain 线程把事件写到 emitter。 + * + * @author ageerle + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class CodingServiceImpl implements ICodingService { + + /** 默认工作目录：直接指向 ruoyi-copilot 前端项目 */ + private static final String DEFAULT_WORKSPACE = "D:/Project/github/ruoyi-copilot"; + + private final IChatModelService chatModelService; + private final ChatServiceFactory chatServiceFactory; + private final Map activeEmitters = new ConcurrentHashMap<>(); + + @Override + public SseEmitter chat(CodingRequestBo bo, Long userId) { + SseEmitter emitter = new SseEmitter(1_800_000L); + AtomicBoolean emitterActive = new AtomicBoolean(true); + activeEmitters.put(emitter, emitterActive); + emitter.onCompletion(() -> closeEmitter(emitter)); + emitter.onTimeout(() -> closeEmitter(emitter)); + emitter.onError(error -> closeEmitter(emitter)); + + CompletableFuture.runAsync(() -> { + CodingEventChannel channel = new CodingEventChannel(); + Thread drainThread = new Thread(() -> { + try { + channel.drain(event -> sendEmitterEvent(emitter, toSseEvent(event))); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Throwable t) { + log.error("编程 SSE drain 线程异常", t); + } + }, "coding-sse-drain"); + drainThread.start(); + + try { + // 推送思考开始 + channel.send(CodingSseEvent.thinking("正在分析指令...")); + + // 1. 拿模型三步（不硬编码配置） + ChatModelVo modelVo = chatModelService.selectModelByName(bo.getModel()); + if (modelVo == null) { + throw new IllegalStateException("模型未找到: " + bo.getModel() + + "，请在 chat_model 表配置该模型名称"); + } + AbstractChatService chatService = chatServiceFactory.getOriginalService(modelVo.getProviderCode()); + ChatModel chatModel = chatService.buildChatModel(modelVo); + + // 2. 解析工作目录 + Path root = resolveWorkspace(bo.getWorkspacePath()); + Files.createDirectories(root); + + // 3. new 工具实例（不走 BuiltinToolRegistry，注入会话工作目录与 channel） + ReadFileTool read = new ReadFileTool(root, channel); + EditFileTool edit = new EditFileTool(root, channel); + ListDirectoryTool list = new ListDirectoryTool(root, channel); + WriteFileTool write = new WriteFileTool(root, channel); + DeleteFileTool delete = new DeleteFileTool(root, channel); + ExecuteCommandTool exec = new ExecuteCommandTool(root, channel); + + // 4. 构建 AiServices + CodingAgent agent = AiServices.builder(CodingAgent.class) + .chatModel(chatModel) + .tools(read, edit, list, write, delete, exec) + .build(); + + // 5. 同步调用（方案 B）：工具执行过程中事件通过 channel 实时推送 + String result = agent.chat(bo.getPrompt()); + + // 6. 推送最终文本 + if (StrUtil.isNotBlank(result)) { + channel.send(CodingSseEvent.text(result)); + } + channel.send(CodingSseEvent.done()); + channel.complete(); + drainThread.join(5_000); + completeEmitter(emitter); + + } catch (Exception e) { + log.error("编程对话失败", e); + String msg = e.getMessage() == null ? e.toString() : e.getMessage(); + channel.send(CodingSseEvent.error(msg)); + channel.completeWithError(e); + try { + drainThread.join(2_000); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + sendEmitterEvent(emitter, SseEmitter.event().name("error") + .data(JsonUtils.toJsonString(Map.of("message", msg)))); + completeEmitterWithError(emitter, e); + } + }); + + return emitter; + } + + /** + * 解析工作目录：前端显式传则用前端的，否则默认 ruoyi-copilot。 + */ + private Path resolveWorkspace(String workspacePath) { + if (StrUtil.isNotBlank(workspacePath)) { + return Paths.get(workspacePath).toAbsolutePath().normalize(); + } + return Paths.get(DEFAULT_WORKSPACE).toAbsolutePath().normalize(); + } + + /** + * 把结构化事件转成 SseEmitter 事件。 + */ + private SseEmitter.SseEventBuilder toSseEvent(CodingSseEvent event) { + Map payload = new LinkedHashMap<>(); + if (event.filePath() != null) payload.put("filePath", event.filePath()); + if (event.command() != null) payload.put("command", event.command()); + if (event.content() != null) payload.put("content", event.content()); + if (event.status() != null) payload.put("status", event.status()); + return SseEmitter.event() + .name(event.eventType()) + .data(JsonUtils.toJsonString(payload)); + } + + // ==================== SSE 发送封装（抄自 ShortDramaServiceImpl） ==================== + + private boolean sendEmitterEvent(SseEmitter emitter, SseEmitter.SseEventBuilder event) { + AtomicBoolean active = activeEmitters.get(emitter); + if (active == null || !active.get()) return false; + try { + emitter.send(event); + return true; + } catch (IOException | IllegalStateException e) { + closeEmitter(emitter); + return false; + } + } + + private void closeEmitter(SseEmitter emitter) { + AtomicBoolean active = activeEmitters.remove(emitter); + if (active != null) active.set(false); + } + + private void completeEmitter(SseEmitter emitter) { + AtomicBoolean active = activeEmitters.get(emitter); + if (active == null || !active.compareAndSet(true, false)) return; + activeEmitters.remove(emitter); + try { emitter.complete(); } catch (IllegalStateException ignored) { } + } + + private void completeEmitterWithError(SseEmitter emitter, Throwable error) { + AtomicBoolean active = activeEmitters.get(emitter); + if (active == null || !active.compareAndSet(true, false)) return; + activeEmitters.remove(emitter); + try { emitter.completeWithError(error); } catch (IllegalStateException ignored) { } + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/media/AtlasMediaSupport.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/media/AtlasMediaSupport.java index b99aa3b6..1822fa11 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/media/AtlasMediaSupport.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/media/AtlasMediaSupport.java @@ -35,4 +35,12 @@ public final class AtlasMediaSupport { JsonNode value = node == null ? null : node.get(field); return value == null || value.isNull() ? null : value.asText(); } + + /** + * 截断超长文本，用于日志输出原始响应时防止刷屏。 + */ + public static String truncate(String text, int max) { + if (text == null) return null; + return text.length() <= max ? text : text.substring(0, max) + "...(truncated " + (text.length() - max) + " chars)"; + } } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/media/AtlasPredictionService.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/media/AtlasPredictionService.java index 3c79205d..247dd07f 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/media/AtlasPredictionService.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/media/AtlasPredictionService.java @@ -59,16 +59,56 @@ public class AtlasPredictionService { public MediaGenerationResponse toResponse(String raw, String type) throws IOException { JsonNode root = AtlasMediaSupport.OBJECT_MAPPER.readTree(raw); JsonNode data = root.path("data"); + String status = AtlasMediaSupport.text(data, "status"); + String lastFrameUrl = firstLastFrame(data); + if ("video".equals(type)) { + // 终态打完整原始响应（确认 Atlas 末帧字段名/结构），轮询中间态只打摘要，避免刷屏 + if ("succeeded".equals(status) || "completed".equals(status) || "failed".equals(status)) { + log.info("Atlas 视频结果[{}]原始响应: {}", status, AtlasMediaSupport.truncate(raw, 2000)); + } + log.info("Atlas 视频结果解析: status={}, lastFrameUrl={}", status, lastFrameUrl); + } return MediaGenerationResponse.builder() .type(type) .mimeType("image".equals(type) ? "image/png" : "video/mp4") .id(AtlasMediaSupport.text(data, "id")) - .status(AtlasMediaSupport.text(data, "status")) + .status(status) .url(firstOutput(data)) + .lastFrameUrl(lastFrameUrl) .rawResponse(raw) .build(); } + /** + * 提取末帧 URL。Atlas return_last_frame=true 时会在 outputs 或顶层节点返回末帧图片， + * 字段名兼容 last_frame_url / end_frame_url / last_frame / last_frame_image。 + */ + private String firstLastFrame(JsonNode data) { + if (data == null || data.isMissingNode()) return null; + String[] keys = {"last_frame_url", "end_frame_url", "last_frame", "last_frame_image"}; + for (String key : keys) { + String val = AtlasMediaSupport.text(data, key); + if (val != null) return val; + } + JsonNode outputs = data.path("outputs"); + if (outputs.isObject()) { + for (String key : keys) { + String val = AtlasMediaSupport.text(outputs, key); + if (val != null) return val; + } + } + if (outputs.isArray() && !outputs.isEmpty()) { + JsonNode first = outputs.get(0); + if (first.isObject()) { + for (String key : keys) { + String val = AtlasMediaSupport.text(first, key); + if (val != null) return val; + } + } + } + return null; + } + private String firstOutput(JsonNode data) { JsonNode outputs = data.path("outputs"); if (outputs.isArray() && !outputs.isEmpty()) { diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/IShortDramaService.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/IShortDramaService.java index b383f567..751a71ec 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/IShortDramaService.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/IShortDramaService.java @@ -1,6 +1,7 @@ package org.ruoyi.service.shortdrama; import org.ruoyi.common.chat.entity.media.MediaGenerationResponse; +import org.ruoyi.domain.bo.shortdrama.ShortDramaAudioBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaCharacterBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaCharacterAppearanceBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaLocationBo; @@ -8,6 +9,7 @@ import org.ruoyi.domain.bo.shortdrama.ShortDramaProjectBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaScriptBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaStoryboardBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaIdeaBo; +import org.ruoyi.domain.vo.shortdrama.ShortDramaAudioVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaCharacterVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaCharacterAppearanceVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaDetailVo; @@ -113,4 +115,15 @@ public interface IShortDramaService { ShortDramaLocationVo confirmLocationImage(Long locationId, String predictionId, String model, Long userId); Boolean deleteProject(Long projectId, Long userId); + + // ==================== 语音资产 ==================== + + ShortDramaAudioVo saveAudio(ShortDramaAudioBo bo, Long userId); + + Boolean deleteAudio(Long audioId, Long userId); + + List listAudios(Long projectId, Long userId); + + /** 生成语音：TTS 合成音频，上传 OSS，回写 audioUrl/audioOssId */ + ShortDramaAudioVo generateAudio(Long audioId, String audioModel, Long userId); } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/AspectRatio.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/AspectRatio.java index 5c44dfc3..c46cd394 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/AspectRatio.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/AspectRatio.java @@ -6,7 +6,10 @@ import com.fasterxml.jackson.annotation.JsonValue; public enum AspectRatio { PORTRAIT("9:16", 1080, 1920), LANDSCAPE("16:9", 1920, 1080), - SQUARE("1:1", 1080, 1080); + LANDSCAPE_CLASSIC("4:3", 1440, 1080), + SQUARE("1:1", 1080, 1080), + PORTRAIT_CLASSIC("3:4", 1080, 1440), + ULTRAWIDE("21:9", 2520, 1080); private final String value; private final int width; diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/CompositionSpec.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/CompositionSpec.java index 57c61985..8b797613 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/CompositionSpec.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/CompositionSpec.java @@ -1,5 +1,6 @@ package org.ruoyi.service.shortdrama.composition; +import java.nio.file.Path; import java.util.List; import java.util.Objects; @@ -7,7 +8,9 @@ public record CompositionSpec( List sources, TransitionType transitionType, double transitionDurationSeconds, - AspectRatio aspectRatio + AspectRatio aspectRatio, + Path narrationAudioPath, + boolean watermark ) { public CompositionSpec { diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/FfmpegCommandBuilder.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/FfmpegCommandBuilder.java index fc140d67..823b367a 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/FfmpegCommandBuilder.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/FfmpegCommandBuilder.java @@ -32,6 +32,11 @@ public class FfmpegCommandBuilder { command.add("-i"); command.add(source.path().toString()); } + // 旁白音轨作为额外输入流（index = sources.size()） + if (spec.narrationAudioPath() != null) { + command.add("-i"); + command.add(spec.narrationAudioPath().toString()); + } command.add("-filter_complex_script"); command.add(filterScript.toAbsolutePath().normalize().toString()); command.add("-map"); diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/FfmpegCompositionProperties.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/FfmpegCompositionProperties.java index c71bb3e6..368cee28 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/FfmpegCompositionProperties.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/FfmpegCompositionProperties.java @@ -36,6 +36,17 @@ public class FfmpegCompositionProperties { private String storageMode = "local"; private String localOutputDirectory = "logs/short-drama-compositions"; + /** 成片是否默认加水印（前端开关未传时使用该默认值） */ + private boolean watermarkEnabled = true; + /** 水印文字 */ + private String watermarkText = "视频由ruoyi-drama生成"; + /** 水印字体大小 */ + private int watermarkFontSize = 28; + /** 水印透明度 0.0-1.0 */ + private double watermarkAlpha = 0.6; + /** 水印字体文件路径（为空则用系统默认字体） */ + private String watermarkFontFile; + public BigDecimal normalizeTransitionDuration(TransitionType type, BigDecimal requested) { if (type == null || type == TransitionType.NONE) { return BigDecimal.ZERO; diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/FfmpegFilterGraphBuilder.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/FfmpegFilterGraphBuilder.java index 9b8ae055..6ce2855e 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/FfmpegFilterGraphBuilder.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/composition/FfmpegFilterGraphBuilder.java @@ -4,6 +4,8 @@ import org.springframework.stereotype.Component; import java.math.BigDecimal; import java.math.RoundingMode; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -42,10 +44,32 @@ public class FfmpegFilterGraphBuilder { filters.add(normalizeAudio(index, duration, info.hasAudio())); } + FfmpegFilterGraph base; if (spec.transitionType() == TransitionType.NONE) { - return hardCut(filters, media, canvas); + base = hardCut(filters, media, canvas); + } else { + base = transition(filters, spec, media, canvas, frameSeconds); } - return transition(filters, spec, media, canvas, frameSeconds); + + // 旁白音轨混入：在最终音轨上 amix 旁白输入（输入 index = sources.size()） + String audioLabel = base.audioLabel(); + if (spec.narrationAudioPath() != null) { + audioLabel = mixNarration(filters, base, media, audioLabel); + } + + // 水印：在最终视频流上 drawtext + String videoLabel = base.videoLabel(); + if (spec.watermark()) { + videoLabel = applyWatermark(filters, videoLabel, canvas); + } + + return new FfmpegFilterGraph( + String.join(";", filters), + videoLabel, + audioLabel, + base.expectedDurationSeconds(), + canvas + ); } private FfmpegFilterGraph hardCut(List filters, List media, VideoCanvas canvas) { @@ -134,6 +158,84 @@ public class FfmpegFilterGraphBuilder { ); } + /** + * 将旁白音轨混入最终音轨。旁白作为额外输入流（index = sources.size()）， + * 先归一化再与主音轨 amix，normalize=0 防止原片音被自动压低，duration=first 以原片长度为准。 + */ + private String mixNarration(List filters, FfmpegFilterGraph base, + List media, String audioLabel) { + int narrationIndex = media.size(); + double expectedDuration = base.expectedDurationSeconds(); + String format = "aformat=sample_fmts=fltp:sample_rates=" + properties.getAudioSampleRate() + + ":channel_layouts=stereo"; + filters.add("[" + narrationIndex + ":a:0]" + + "aresample=" + properties.getAudioSampleRate() + ":async=1:first_pts=0," + + format + "," + + "atrim=start=0:duration=" + seconds(expectedDuration) + "," + + "asetpts=PTS-STARTPTS[narr]"); + String mixed = "anarr"; + filters.add("[" + audioLabel + "][narr]amix=inputs=2:duration=first:normalize=0[" + mixed + "]"); + return mixed; + } + + /** + * 在最终视频流右下角叠加水印文字。 + * 优先使用显式配置的字体；未配置时探测各平台的常见字体，避免 Windows 版 FFmpeg + * 在 Fontconfig 配置缺失时因 drawtext 发生原生崩溃。 + */ + private String applyWatermark(List filters, String videoLabel, VideoCanvas canvas) { + String text = properties.getWatermarkText(); + if (text == null || text.isBlank()) { + return videoLabel; + } + String alpha = formatAlpha(properties.getWatermarkAlpha()); + StringBuilder expr = new StringBuilder(); + expr.append("[").append(videoLabel).append("]drawtext=text='").append(escape(text)).append("'"); + String fontFile = resolveWatermarkFontFile(); + expr.append(":fontfile='").append(escape(fontFile)).append("'"); + expr.append(":fontcolor=white@").append(alpha) + .append(":fontsize=").append(properties.getWatermarkFontSize()) + .append(":x=w-tw-20:y=h-th-20[wmark]"); + filters.add(expr.toString()); + return "wmark"; + } + + private String resolveWatermarkFontFile() { + String configured = properties.getWatermarkFontFile(); + if (configured != null && !configured.isBlank()) { + Path configuredPath = Path.of(configured).toAbsolutePath().normalize(); + if (!Files.isRegularFile(configuredPath) || !Files.isReadable(configuredPath)) { + throw new IllegalStateException("Configured watermark font is not a readable file: " + configuredPath); + } + return configuredPath.toString(); + } + + List candidates = List.of( + Path.of("C:/Windows/Fonts/simhei.ttf"), + Path.of("C:/Windows/Fonts/msyh.ttc"), + Path.of("/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc"), + Path.of("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"), + Path.of("/usr/share/fonts/truetype/liberation2/LiberationSans-Regular.ttf") + ); + return candidates.stream() + .filter(path -> Files.isRegularFile(path) && Files.isReadable(path)) + .findFirst() + .map(path -> path.toAbsolutePath().normalize().toString()) + .orElseThrow(() -> new IllegalStateException( + "No readable watermark font was found; configure short-drama.composition.watermark-font-file" + )); + } + + private static String escape(String value) { + return value.replace("\\", "\\\\").replace(":", "\\:").replace("'", "\\'"); + } + + private static String formatAlpha(double alpha) { + if (alpha <= 0) return "0"; + if (alpha >= 1) return "1"; + return BigDecimal.valueOf(alpha).setScale(2, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString(); + } + private String normalizeVideo(int inputIndex, int videoStreamIndex, String duration, VideoCanvas canvas) { return "[" + inputIndex + ":" + videoStreamIndex + "]" + "scale=" + canvas.width() + ":" + canvas.height() diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaServiceImpl.java index 91f9f000..6f07ffa5 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaServiceImpl.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaServiceImpl.java @@ -27,6 +27,8 @@ import org.ruoyi.common.chat.entity.media.MediaGenerationResponse; import org.ruoyi.common.chat.entity.video.VideoContext; import org.ruoyi.common.chat.factory.ImageServiceFactory; import org.ruoyi.common.chat.factory.VideoServiceFactory; +import org.ruoyi.common.chat.factory.AudioServiceFactory; +import org.ruoyi.common.chat.entity.audio.AudioContext; import org.ruoyi.common.chat.service.chat.IChatModelService; import org.ruoyi.common.core.utils.MapstructUtils; import org.ruoyi.common.core.utils.StringUtils; @@ -37,12 +39,14 @@ import org.ruoyi.domain.bo.shortdrama.ShortDramaProjectBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaScriptBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaScriptResult; import org.ruoyi.domain.bo.shortdrama.ShortDramaStoryboardBo; +import org.ruoyi.domain.entity.shortdrama.ShortDramaAudio; import org.ruoyi.domain.entity.shortdrama.ShortDramaCharacter; import org.ruoyi.domain.entity.shortdrama.ShortDramaCharacterAppearance; import org.ruoyi.domain.entity.shortdrama.ShortDramaLocation; import org.ruoyi.domain.entity.shortdrama.ShortDramaProject; import org.ruoyi.domain.entity.shortdrama.ShortDramaScript; import org.ruoyi.domain.entity.shortdrama.ShortDramaStoryboard; +import org.ruoyi.domain.vo.shortdrama.ShortDramaAudioVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaCharacterVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaCharacterAppearanceVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaDetailVo; @@ -50,9 +54,11 @@ import org.ruoyi.domain.vo.shortdrama.ShortDramaLocationVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaProjectVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaScriptVo; import org.ruoyi.domain.vo.shortdrama.ShortDramaStoryboardVo; +import org.ruoyi.domain.bo.shortdrama.ShortDramaAudioBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaCharacterBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaCharacterAppearanceBo; import org.ruoyi.domain.bo.shortdrama.ShortDramaLocationBo; +import org.ruoyi.mapper.shortdrama.ShortDramaAudioMapper; import org.ruoyi.mapper.shortdrama.ShortDramaCharacterMapper; import org.ruoyi.mapper.shortdrama.ShortDramaCharacterAppearanceMapper; import org.ruoyi.mapper.shortdrama.ShortDramaLocationMapper; @@ -95,12 +101,15 @@ public class ShortDramaServiceImpl implements IShortDramaService { private final ShortDramaCharacterMapper characterMapper; private final ShortDramaCharacterAppearanceMapper characterAppearanceMapper; private final ShortDramaLocationMapper locationMapper; + private final ShortDramaAudioMapper audioMapper; private final IChatModelService chatModelService; private final ChatServiceFactory chatServiceFactory; private final VideoServiceFactory videoServiceFactory; private final ImageServiceFactory imageServiceFactory; + private final AudioServiceFactory audioServiceFactory; private final AtlasPredictionService atlasPredictionService; private final IShortDramaVideoComposeService videoComposeService; + private final org.ruoyi.common.core.service.OssService ossService; private final java.util.Map activeEmitters = new ConcurrentHashMap<>(); private final java.util.Map storyboardGenerationStates = new ConcurrentHashMap<>(); @@ -153,6 +162,11 @@ public class ShortDramaServiceImpl implements IShortDramaService { .eq(ShortDramaLocation::getProjectId, projectId)); detailVo.setLocations(locations); + List audios = audioMapper.selectVoList(new LambdaQueryWrapper() + .eq(ShortDramaAudio::getProjectId, projectId) + .orderByAsc(ShortDramaAudio::getId)); + detailVo.setAudios(audios); + List storyboards = storyboardMapper.selectVoList(new LambdaQueryWrapper() .eq(ShortDramaStoryboard::getProjectId, projectId) .orderByAsc(ShortDramaStoryboard::getSceneNo)); @@ -285,6 +299,13 @@ public class ShortDramaServiceImpl implements IShortDramaService { .data("{\"phase\":\"script\",\"status\":\"done\"}")); } + /** 增量推送一个已完成的分镜 panel 给前端（流式规划时第一个完成就展示） */ + private void emitPanel(SseEmitter emitter, StoryboardPanelData panel) { + if (emitter == null || panel == null) return; + String data = "{\"phase\":\"storyboard_plan\",\"status\":\"panel\",\"panel\":" + JsonUtils.toJsonString(panel) + "}"; + sendEmitterEvent(emitter, SseEmitter.event().name("panel").data(data)); + } + private static String escapeJson(String s) { if (s == null) return ""; return s.replace("\\", "\\\\").replace("\"", "\\\""); @@ -618,6 +639,14 @@ public class ShortDramaServiceImpl implements IShortDramaService { @Override public ShortDramaStoryboardVo generateVideo(Long storyboardId, String videoModel, Long userId) { + return generateVideo(storyboardId, videoModel, userId, null); + } + + /** + * 生成单镜视频。 + * @param lastFrameUrl 上一镜末帧 URL（仅同场景相邻镜头传入，用于首帧承接）；跨场景或首镜传 null + */ + public ShortDramaStoryboardVo generateVideo(Long storyboardId, String videoModel, Long userId, String lastFrameUrl) { ShortDramaStoryboard storyboard = storyboardMapper.selectById(storyboardId); if (storyboard == null) throw new IllegalArgumentException("分镜不存在"); ShortDramaProject project = projectMapper.selectById(storyboard.getProjectId()); @@ -625,8 +654,8 @@ public class ShortDramaServiceImpl implements IShortDramaService { ChatModelVo modelVo = chatModelService.selectModelByName(videoModel); if (modelVo == null) throw new IllegalArgumentException("未找到视频模型配置: " + videoModel); - // 收集所有参考图（角色 + 场景） - List referenceImages = findStoryboardReferenceImages(storyboard); + // 收集所有参考图（角色 + 场景 + 末帧承接） + List referenceImages = findStoryboardReferenceImages(storyboard, lastFrameUrl); // 根据参考图数量自动切换模型 if (referenceImages != null && !referenceImages.isEmpty()) { @@ -649,13 +678,17 @@ public class ShortDramaServiceImpl implements IShortDramaService { } } - String enrichedPrompt = buildEnrichedVideoPrompt(storyboard, referenceImages); + String enrichedPrompt = buildEnrichedVideoPrompt(storyboard, referenceImages, lastFrameUrl); VideoContext ctx = VideoContext.builder() .chatModelVo(modelVo) .prompt(enrichedPrompt) + .size(projectAspectRatio(storyboard.getProjectId())) .seconds(storyboard.getDurationSeconds()) .referenceImages(referenceImages) + .generateAudio(Boolean.TRUE) + .returnLastFrame(Boolean.TRUE) + .lastFrameUrl(lastFrameUrl) .build(); String generationToken = "local:" + UUID.randomUUID(); storyboardMapper.update(null, new LambdaUpdateWrapper() @@ -682,14 +715,18 @@ public class ShortDramaServiceImpl implements IShortDramaService { String videoUrl = null; String videoId = null; String videoStatus; + String lastFrame = null; if (response != null && StrUtil.isNotBlank(response.getUrl())) { videoUrl = response.getUrl(); + lastFrame = response.getLastFrameUrl(); videoStatus = "done"; } else if (response != null && StrUtil.isNotBlank(response.getId())) { videoId = response.getId(); + lastFrame = response.getLastFrameUrl(); videoStatus = "generating"; } else if (response != null && "processing".equals(response.getStatus())) { videoId = response.getId(); + lastFrame = response.getLastFrameUrl(); videoStatus = "generating"; } else { videoStatus = "failed"; @@ -699,20 +736,21 @@ public class ShortDramaServiceImpl implements IShortDramaService { .eq(ShortDramaStoryboard::getVideoId, generationToken) .set(ShortDramaStoryboard::getVideoUrl, videoUrl) .set(ShortDramaStoryboard::getVideoId, videoId) - .set(ShortDramaStoryboard::getVideoStatus, videoStatus)); + .set(ShortDramaStoryboard::getVideoStatus, videoStatus) + .set(StrUtil.isNotBlank(lastFrame), ShortDramaStoryboard::getLastFrameUrl, lastFrame)); if (completed > 0) { videoComposeService.invalidateComposition(project.getId()); } return MapstructUtils.convert(storyboardMapper.selectById(storyboardId), ShortDramaStoryboardVo.class); } - /** 收集分镜关联的所有参考图：角色形象图（按出场顺序）+ 场景图 */ - private List findStoryboardReferenceImages(ShortDramaStoryboard storyboard) { + /** 收集分镜关联的所有参考图：角色形象图（按出场顺序）+ 场景图 + 可选末帧承接 */ + private List findStoryboardReferenceImages(ShortDramaStoryboard storyboard, String lastFrameUrl) { List images = new ArrayList<>(); List chars = parseCharacterRefs(storyboard.getCharactersJson()); if (chars != null) { for (CharacterRef ref : chars) { - String img = findCharacterImageUrl(storyboard.getProjectId(), ref.getName()); + String img = findCharacterImageUrl(storyboard.getProjectId(), ref.getName(), ref.getAppearance()); if (StrUtil.isNotBlank(img) && !images.contains(img)) { images.add(img); } @@ -724,16 +762,25 @@ public class ShortDramaServiceImpl implements IShortDramaService { images.add(img); } } + // 末帧承接：放在最后一张，@imageN 标记会自动绑定 + if (StrUtil.isNotBlank(lastFrameUrl) && !images.contains(lastFrameUrl)) { + images.add(lastFrameUrl); + } return images.isEmpty() ? null : images; } /** 构建增强提示词：融合镜头语言、摄影规则、角色信息、表演指导、场景描述 */ private String buildEnrichedVideoPrompt(ShortDramaStoryboard storyboard) { - return buildEnrichedVideoPrompt(storyboard, null); + return buildEnrichedVideoPrompt(storyboard, null, null); } /** 构建增强提示词（含 @imageN 参考图引用，用于 reference-to-video 模型） */ private String buildEnrichedVideoPrompt(ShortDramaStoryboard storyboard, java.util.List refImages) { + return buildEnrichedVideoPrompt(storyboard, refImages, null); + } + + /** 构建增强提示词（含参考图 + 末帧首帧承接） */ + private String buildEnrichedVideoPrompt(ShortDramaStoryboard storyboard, java.util.List refImages, String lastFrameUrl) { boolean hasRefImages = refImages != null && !refImages.isEmpty(); StringBuilder sb = new StringBuilder(); @@ -743,6 +790,11 @@ public class ShortDramaServiceImpl implements IShortDramaService { if (StrUtil.isNotBlank(storyboard.getCameraMove())) sb.append(", ").append(storyboard.getCameraMove()); sb.append("]\n"); + // 1.1 核心动作描述前置（最高权重，确保模型优先关注当前镜头的具体可拍内容） + if (StrUtil.isNotBlank(storyboard.getVideoPrompt())) { + sb.append("[核心动作] ").append(storyboard.getVideoPrompt()).append("\n"); + } + // 2. 摄影规则 if (StrUtil.isNotBlank(storyboard.getPhotographyRules())) { try { @@ -800,9 +852,11 @@ public class ShortDramaServiceImpl implements IShortDramaService { sb.append("\n"); } - // 角色参考图很容易被生成模型错误地扩散到背景群众，造成“所有人一张脸”。 - // 在角色声明之后追加全局身份隔离规则，使其覆盖所有项目和所有镜头。 - appendCharacterIdentityIsolationPrompt(sb, storyboard, chars, refImages); + // 角色参考图模式下，参考图容易被模型扩散到背景群众造成"所有人一张脸"。 + // 仅 reference-to-video 模式（有参考图）追加身份隔离规则；纯文生视频不需要。 + if (hasRefImages) { + appendCharacterIdentityIsolationPrompt(sb, storyboard, chars, refImages); + } // 4. 场景描述（+ @imageN 参考图标记） if (StrUtil.isNotBlank(storyboard.getLocationName())) { @@ -839,6 +893,16 @@ public class ShortDramaServiceImpl implements IShortDramaService { // 6. 前后镜头连续性状态 appendContinuityPrompt(sb, storyboard); + // 6.1 首帧承接：当存在上一镜末帧 URL 时，提示模型首帧继承末帧画面 + if (StrUtil.isNotBlank(lastFrameUrl)) { + int lastFrameImageIndex = refImages != null ? refImages.indexOf(lastFrameUrl) : -1; + if (lastFrameImageIndex >= 0) { + sb.append("[首帧承接] 当前视频第一帧必须严格继承 @image").append(lastFrameImageIndex + 1) + .append("（上一镜末帧）的人物位置、姿态、朝向、服装、道具和光线方向，") + .append("不得改变人物左右关系或重置场景，动作从该帧状态自然延续。\n"); + } + } + // 7. 画面描述（AI 撰写的分镜画面叙述） if (StrUtil.isNotBlank(storyboard.getSceneText()) && !storyboard.getSceneText().equals(storyboard.getVideoPrompt())) { @@ -939,18 +1003,27 @@ public class ShortDramaServiceImpl implements IShortDramaService { } private String findCharacterImageUrl(Long projectId, String characterName) { + return findCharacterImageUrl(projectId, characterName, null); + } + + /** + * 按角色名 + appearance 标识查找参考图。appearance 用于在多形象(青年/老年)间切换。 + * 匹配策略：changeReason 精确 → description 包含 → appearanceIndex 数字 → 都失败回退主形象(0)。 + */ + private String findCharacterImageUrl(Long projectId, String characterName, String appearance) { List characters = findCharactersByName(projectId, characterName); for (ShortDramaCharacter character : characters) { List appearances = characterAppearanceMapper.selectList( new LambdaQueryWrapper() .eq(ShortDramaCharacterAppearance::getCharacterId, character.getId()) .orderByAsc(ShortDramaCharacterAppearance::getAppearanceIndex)); - for (ShortDramaCharacterAppearance appearance : appearances) { - List urls = readJsonStringList(appearance.getImageUrls()); - if (urls.isEmpty()) continue; - int index = appearance.getSelectedImageIndex() != null && appearance.getSelectedImageIndex() >= 0 - && appearance.getSelectedImageIndex() < urls.size() ? appearance.getSelectedImageIndex() : 0; - return urls.get(index); + ShortDramaCharacterAppearance matched = matchAppearance(appearances, appearance); + String url = pickAppearanceImage(matched); + if (StrUtil.isNotBlank(url)) return url; + // 匹配形象无图，回退任何有图的形象 + for (ShortDramaCharacterAppearance ap : appearances) { + url = pickAppearanceImage(ap); + if (StrUtil.isNotBlank(url)) return url; } if (StrUtil.isNotBlank(character.getReferenceImageUrl())) { return character.getReferenceImageUrl(); @@ -959,6 +1032,41 @@ public class ShortDramaServiceImpl implements IShortDramaService { return null; } + private ShortDramaCharacterAppearance matchAppearance(List appearances, String appearance) { + if (appearances == null || appearances.isEmpty()) return null; + if (StrUtil.isBlank(appearance) || "初始形象".equals(appearance)) { + return appearances.get(0); + } + // 1. changeReason 精确匹配 + for (ShortDramaCharacterAppearance ap : appearances) { + if (appearance.equals(ap.getChangeReason())) return ap; + } + // 2. changeReason 包含匹配 + for (ShortDramaCharacterAppearance ap : appearances) { + if (StrUtil.isNotBlank(ap.getChangeReason()) && ap.getChangeReason().contains(appearance)) return ap; + } + // 3. description 包含匹配 + for (ShortDramaCharacterAppearance ap : appearances) { + if (StrUtil.isNotBlank(ap.getDescription()) && ap.getDescription().contains(appearance)) return ap; + } + // 4. appearanceIndex 数字匹配 + try { + int idx = Integer.parseInt(appearance); + if (idx >= 0 && idx < appearances.size()) return appearances.get(idx); + } catch (NumberFormatException ignored) {} + // 5. 回退主形象 + return appearances.get(0); + } + + private String pickAppearanceImage(ShortDramaCharacterAppearance appearance) { + if (appearance == null) return null; + List urls = readJsonStringList(appearance.getImageUrls()); + if (urls.isEmpty()) return appearance.getReferenceImageUrl(); + int index = appearance.getSelectedImageIndex() != null && appearance.getSelectedImageIndex() >= 0 + && appearance.getSelectedImageIndex() < urls.size() ? appearance.getSelectedImageIndex() : 0; + return urls.get(index); + } + private String findLocationImageUrl(Long projectId, String locationName) { ShortDramaLocation location = findLocationByName(projectId, locationName); if (location == null) return null; @@ -1052,14 +1160,17 @@ public class ShortDramaServiceImpl implements IShortDramaService { .retrieveVideo(ctx); String videoUrl = null; String videoStatus = null; + String lastFrame = null; if (response != null && StrUtil.isNotBlank(response.getUrl())) { videoUrl = response.getUrl(); + lastFrame = response.getLastFrameUrl(); videoStatus = "done"; } else if (response != null && ("completed".equals(response.getStatus()) || "succeeded".equals(response.getStatus()))) { // 已完成但 URL 提取失败，尝试从原始响应中提取 String fallbackUrl = extractVideoUrlFromRaw(response.getRawResponse()); if (StrUtil.isNotBlank(fallbackUrl)) { videoUrl = fallbackUrl; + lastFrame = response.getLastFrameUrl(); videoStatus = "done"; } else { log.warn("视频已完成但无法提取URL, predictionId={}, raw={}", predictionId, @@ -1074,7 +1185,8 @@ public class ShortDramaServiceImpl implements IShortDramaService { .eq(ShortDramaStoryboard::getId, storyboardId) .eq(ShortDramaStoryboard::getVideoId, predictionId) .set(ShortDramaStoryboard::getVideoUrl, videoUrl) - .set(ShortDramaStoryboard::getVideoStatus, videoStatus)); + .set(ShortDramaStoryboard::getVideoStatus, videoStatus) + .set(StrUtil.isNotBlank(lastFrame), ShortDramaStoryboard::getLastFrameUrl, lastFrame)); if (updated > 0) { videoComposeService.invalidateComposition(project.getId()); } @@ -1092,19 +1204,88 @@ public class ShortDramaServiceImpl implements IShortDramaService { new LambdaQueryWrapper() .eq(ShortDramaStoryboard::getProjectId, projectId) .orderByAsc(ShortDramaStoryboard::getSceneNo)); - List result = new ArrayList<>(); + + // 按 locationName 分组：同场景内串行（保末帧拼接），跨场景组并发 + List> groups = new ArrayList<>(); + List currentGroup = new ArrayList<>(); + String currentLoc = null; for (ShortDramaStoryboard sb : storyboards) { + String loc = StrUtil.blankToDefault(sb.getLocationName(), ""); + if (!loc.equals(currentLoc)) { + if (!currentGroup.isEmpty()) { groups.add(currentGroup); currentGroup = new ArrayList<>(); } + currentLoc = loc; + } + currentGroup.add(sb); + } + if (!currentGroup.isEmpty()) groups.add(currentGroup); + + // 跨场景组并发，组上限 4 + int parallel = Math.min(4, Math.max(1, groups.size())); + java.util.concurrent.ExecutorService pool = Executors.newFixedThreadPool(parallel, + r -> { Thread t = new Thread(r, "short-drama-video-gen"); t.setDaemon(true); return t; }); + try { + List>> futures = new ArrayList<>(); + for (List group : groups) { + futures.add(pool.submit(() -> generateGroupSerial(group, videoModel, userId))); + } + // 按 sceneNo 顺序汇总结果 + List result = new ArrayList<>(); + for (java.util.concurrent.Future> f : futures) { + try { result.addAll(f.get()); } + catch (Exception e) { log.warn("视频生成分组失败: {}", e.getMessage()); } + } + result.sort(java.util.Comparator.comparing(v -> v.getSceneNo() == null ? Integer.MAX_VALUE : v.getSceneNo())); + return result; + } finally { + pool.shutdownNow(); + } + } + + /** 同场景组内串行生成：上一镜末帧喂下一镜首帧。 */ + private List generateGroupSerial(List group, String videoModel, Long userId) { + List result = new ArrayList<>(); + String prevLastFrameUrl = null; + for (ShortDramaStoryboard sb : group) { try { - result.add(generateVideo(sb.getId(), videoModel, userId)); + String lastFrameForThis = StrUtil.isNotBlank(prevLastFrameUrl) ? prevLastFrameUrl : null; + ShortDramaStoryboardVo vo = generateVideo(sb.getId(), videoModel, userId, lastFrameForThis); + if (lastFrameForThis != null) { + vo = ensureVideoDone(sb.getId(), videoModel, userId); + } + result.add(vo); + ShortDramaStoryboard latest = storyboardMapper.selectById(sb.getId()); + prevLastFrameUrl = (latest != null && StrUtil.isNotBlank(latest.getLastFrameUrl())) ? latest.getLastFrameUrl() : null; } catch (Exception e) { log.warn("镜头{}视频生成失败: {}", sb.getSceneNo(), e.getMessage()); ShortDramaStoryboard current = storyboardMapper.selectById(sb.getId()); result.add(MapstructUtils.convert(current != null ? current : sb, ShortDramaStoryboardVo.class)); + prevLastFrameUrl = null; } } return result; } + /** + * 后台同步轮询单镜视频直到 done/failed（用于同场景末帧拼接时拿到末帧再喂下一镜）。 + * 单镜累计轮询不超过 5 分钟，超时按当前状态返回。 + */ + private ShortDramaStoryboardVo ensureVideoDone(Long storyboardId, String videoModel, Long userId) { + long deadline = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(5); + try { + while (System.currentTimeMillis() < deadline) { + ShortDramaStoryboardVo vo = retrieveVideo(storyboardId, videoModel, userId); + if (vo == null) return null; + if ("done".equals(vo.getVideoStatus()) || "failed".equals(vo.getVideoStatus())) { + return vo; + } + try { Thread.sleep(2000); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); return vo; } + } + } catch (Exception e) { + log.warn("镜头{}末帧等待轮询异常: {}", storyboardId, e.getMessage()); + } + return MapstructUtils.convert(storyboardMapper.selectById(storyboardId), ShortDramaStoryboardVo.class); + } + // ==================== 资产分析与管理 ==================== @Override @@ -1185,7 +1366,7 @@ public class ShortDramaServiceImpl implements IShortDramaService { validateProjectOwner(location.getProjectId(), userId); ChatModelVo modelVo = chatModelService.selectModelByName(imageModel); if (modelVo == null) throw new IllegalArgumentException("未找到图片模型配置: " + imageModel); - String prompt = firstNotBlank(location.getDescriptions(), location.getSummary(), location.getName()); + String prompt = firstNotBlank(primaryLocationDescription(location), location.getSummary(), location.getName()); String finalPrompt = ShortDramaImageConstants.LOCATION_PROMPT_PREFIX + prompt + ShortDramaImageConstants.LOCATION_PROMPT_SUFFIX + artStyleSuffix(location.getProjectId()); String referenceImage = validateReferenceImageUrl(referenceImageUrl); @@ -1379,7 +1560,7 @@ public class ShortDramaServiceImpl implements IShortDramaService { validateProjectOwner(location.getProjectId(), userId); ChatModelVo modelVo = chatModelService.selectModelByName(imageModel); if (modelVo == null) throw new IllegalArgumentException("未找到图片模型配置: " + imageModel); - String prompt = firstNotBlank(location.getDescriptions(), location.getSummary(), location.getName()); + String prompt = firstNotBlank(primaryLocationDescription(location), location.getSummary(), location.getName()); String finalPrompt = ShortDramaImageConstants.LOCATION_PROMPT_PREFIX + prompt + ShortDramaImageConstants.LOCATION_PROMPT_SUFFIX + artStyleSuffix(location.getProjectId()); String referenceImage = validateReferenceImageUrl(referenceImageUrl); @@ -1504,7 +1685,7 @@ public class ShortDramaServiceImpl implements IShortDramaService { ShortDramaLocation location = locationMapper.selectById(assetId); if (location == null) throw new IllegalArgumentException("场景不存在"); validateProjectOwner(location.getProjectId(), userId); - String basePrompt = firstNotBlank(location.getDescriptions(), location.getSummary(), location.getName()); + String basePrompt = firstNotBlank(primaryLocationDescription(location), location.getSummary(), location.getName()); prompt = ShortDramaImageConstants.LOCATION_PROMPT_PREFIX + basePrompt + ShortDramaImageConstants.LOCATION_PROMPT_SUFFIX + artStyleSuffix(location.getProjectId()); size = projectAspectRatio(location.getProjectId()); @@ -1574,7 +1755,7 @@ public class ShortDramaServiceImpl implements IShortDramaService { throw new IllegalStateException("图片生成完成但未获取到URL"); } - String prompt = firstNotBlank(location.getDescriptions(), location.getSummary(), location.getName()); + String prompt = firstNotBlank(primaryLocationDescription(location), location.getSummary(), location.getName()); String finalPrompt = ShortDramaImageConstants.LOCATION_PROMPT_PREFIX + prompt + ShortDramaImageConstants.LOCATION_PROMPT_SUFFIX + artStyleSuffix(location.getProjectId()); @@ -1612,6 +1793,19 @@ public class ShortDramaServiceImpl implements IShortDramaService { } } + /** + * 场景只使用一个主描述。兼容历史数据：旧记录可能包含三个候选描述，固定取第一个非空项。 + */ + private static String primaryLocationDescription(ShortDramaLocation location) { + if (location == null || StrUtil.isBlank(location.getDescriptions())) { + return null; + } + return readJsonStringList(location.getDescriptions()).stream() + .filter(StrUtil::isNotBlank) + .findFirst() + .orElse(null); + } + @Override @Transactional(rollbackFor = Exception.class) public Boolean deleteProject(Long projectId, Long userId) { @@ -1639,6 +1833,16 @@ public class ShortDramaServiceImpl implements IShortDramaService { */ private String streamingChat(StreamingChatModel streamingModel, ChatModel chatModel, String prompt, SseEmitter emitter, String streamPhase) { + return streamingChat(streamingModel, chatModel, prompt, emitter, streamPhase, null); + } + + /** + * 流式调用模型。onPartial 回调在每次新 token 到来时以当前完整 buffer 调用， + * 调用方可在此做增量解析（如分镜 panel 增量推送）。onPartial 为 null 时行为同旧版。 + */ + private String streamingChat(StreamingChatModel streamingModel, ChatModel chatModel, + String prompt, SseEmitter emitter, String streamPhase, + java.util.function.Consumer onPartial) { if (streamingModel == null || emitter == null) { return chatModel.chat(prompt); } @@ -1657,6 +1861,11 @@ public class ShortDramaServiceImpl implements IShortDramaService { public void onPartialResponse(String text) { buf.append(text); emitStream(emitter, streamPhase, text); + if (onPartial != null) { + try { onPartial.accept(buf.toString()); } catch (Exception ex) { + log.warn("流式增量回调异常: {}", ex.getMessage()); + } + } } @Override public void onCompleteResponse(ChatResponse response) { @@ -2242,7 +2451,7 @@ public class ShortDramaServiceImpl implements IShortDramaService { 4. 光线方向：光从哪个方向照入 5. 可落位空间：必须说明哪些区域留有可供人物站立的空白空间，至少2-3个后续可作为人物落位锚点的关键物体或区域 - 每个场景生成3条差异化中文环境描述（100-150字），2-6个available_slots + 每个场景只生成1条中文环境描述（100-150字），2-6个available_slots。描述可由用户编辑，不要提供相似候选方案。 ⚠️ 场景图禁止出现任何有名有姓的角色！场景图是纯粹的背景板。无名的模糊背景群众（如"宾客""路人"）可以出现。 @@ -2259,7 +2468,7 @@ public class ShortDramaServiceImpl implements IShortDramaService { "hasCrowd": true/false, "crowdDescription": "人群类型描述", "availableSlots": ["位置1完整描述", "位置2完整描述"], - "descriptions": ["「场景名」描述1", "「场景名」描述2", "「场景名」描述3"] + "descriptions": ["「场景名」唯一完整描述"] } ] } @@ -2297,7 +2506,15 @@ public class ShortDramaServiceImpl implements IShortDramaService { String response; if (emitter != null) { StreamingChatModel activeStreamingModel = streamingModel != null ? streamingModel : buildStreamingChatModel(); - response = streamingChat(activeStreamingModel, chatModel, prompt, emitter, "storyboard_plan"); + // 流式增量：每解析出一个完整 panel 就推给前端，不等整个数组完成 + final org.ruoyi.service.shortdrama.support.IncrementalJsonArrayExtractor extractor = + new org.ruoyi.service.shortdrama.support.IncrementalJsonArrayExtractor<>(StoryboardPanelData.class); + response = streamingChat(activeStreamingModel, chatModel, prompt, emitter, "storyboard_plan", buf -> { + List fresh = extractor.feed(buf); + for (StoryboardPanelData p : fresh) { + if (p != null) emitPanel(emitter, p); + } + }); } else { response = chatModel.chat(prompt); } @@ -2850,6 +3067,10 @@ public class ShortDramaServiceImpl implements IShortDramaService { matched++; } } + // 节拍校验 + 二次补写：video_prompt 节拍数低于 ⌈duration/3⌉ 时补写一次 + if (matched > 0 && streamingModel == null && emitter == null) { + ensureVideoPromptBeats(chatModel, panels, projectId); + } if (matched == 0 && emitter != null) { emit(emitter, "storyboard_detail", "error", "分镜细化JSON解析成功但未匹配到任何镜头"); } else if (matched > 0 && emitter != null) { @@ -2864,6 +3085,77 @@ public class ShortDramaServiceImpl implements IShortDramaService { } } + /** + * 校验每个 panel 的 video_prompt 节拍数（按顿号/逗号/句号粗估可见动作短语）， + * 低于 ⌈duration/3⌉ 时发起一次二次 LLM 调用补写。补写后再次校验，仍不达标则保留并记 warn。 + * 仅在非流式（同步生成）模式下执行，避免流式场景重复请求。 + */ + private void ensureVideoPromptBeats(ChatModel chatModel, List panels, Long projectId) { + List deficient = new ArrayList<>(); + for (StoryboardPanelData panel : panels) { + if (StrUtil.isBlank(panel.getVideoPrompt())) continue; + int duration = panel.getDuration() != null && panel.getDuration() > 0 ? panel.getDuration() : 6; + int required = Math.max(2, (duration + 2) / 3); + int actual = countBeats(panel.getVideoPrompt()); + if (actual < required) { + deficient.add(panel); + } + } + if (deficient.isEmpty()) return; + try { + String supplementPrompt = buildVideoPromptSupplementPrompt(deficient, artStyleSuffix(projectId), projectAspectRatio(projectId)); + String response = chatModel.chat(supplementPrompt); + List results = parseJsonArray(extractJson(response), StoryboardDetailResult.class); + if (results != null) { + for (StoryboardDetailResult r : results) { + if (r.getPanelNumber() == null) continue; + int idx = r.getPanelNumber() - 1; + if (idx >= 0 && idx < panels.size() && StrUtil.isNotBlank(r.getVideoPrompt())) { + StoryboardPanelData panel = panels.get(idx); + if (StrUtil.isNotBlank(r.getDescription())) panel.setDescription(r.getDescription()); + panel.setVideoPrompt(r.getVideoPrompt()); + log.info("Phase 6 节拍补写完成 panel={} 节拍 {}->{}", + r.getPanelNumber(), + countBeats(panel.getVideoPrompt()), + panel.getVideoPrompt()); + } + } + } + } catch (Exception e) { + log.warn("Phase 6 节拍补写失败: {}", e.getMessage()); + } + } + + /** 粗估 video_prompt 的可见节拍数：按顿号、逗号、分号、句号、换行切分的动作短语数。 */ + private static int countBeats(String videoPrompt) { + if (StrUtil.isBlank(videoPrompt)) return 0; + String[] parts = videoPrompt.split("[、，,；;。\n]"); + int count = 0; + for (String p : parts) { + String t = p.trim(); + if (t.length() >= 2) count++; + } + return count; + } + + private static String buildVideoPromptSupplementPrompt(List deficient, String artStyle, String aspectRatio) { + StringBuilder json = new StringBuilder(); + for (StoryboardPanelData p : deficient) { + json.append(JsonUtils.toJsonString(p)).append(","); + } + if (json.length() > 0 && json.charAt(json.length() - 1) == ',') json.deleteCharAt(json.length() - 1); + return """ + 以下是 video_prompt 节拍数不足的分镜，每个镜头的 video_prompt 必须按时长写出足够可见节拍。 + 按导演笔记风格重写 video_prompt（景别+机位、按时序的动作节拍、运镜、光影方向、道具、台词），禁止参数堆砌。 + duration 为 4-7 秒写 3 个连续节拍，8 秒以上按前段/中段/后段写至少 3 节拍。保留原有信息，只扩写动作细节。 + 只返回 JSON 数组，字段：panel_number、video_prompt、description。 + + 视觉风格：%s + 画幅：%s + 待补写分镜：[%s] + """.formatted(artStyle, aspectRatio, json); + } + private static String buildStoryboardDetailPrompt(String panelsJson, String charsAgeGender, String locsDesc, String artStyle, String aspectRatio) { return """ @@ -2921,19 +3213,23 @@ public class ShortDramaServiceImpl implements IShortDramaService { - description也必须同步扩写这些节拍，保证画面描述与video_prompt一致 【video_prompt撰写规则 - 重要】 - 视频模型不认识名字，必须用年龄段+性别替代： + video_prompt 是发给视频模型的核心可拍指令，必须用"导演笔记"风格写，每个字都可拍、按时序展开。视频模型不认识名字，必须用年龄段+性别替代角色： - 年龄段：少年/少女(10-16)、年轻男子/年轻女子(17-30)、中年男子/中年女子(31-50)、老年男子/老年女子(50+) - - 格式：年龄性别+动作+镜头运动+环境 - - 必须有动作词：转头、点头、走动、转身、推门、抬手等 - - 必须有镜头运动词：缓缓推近、轻轻跟随、手持跟随、环绕拍摄等 - - 对话场景必须写明"正在说话" - - 禁止纯静态描述 - - 特写镜头必须使用"固定镜头" + - 必须依次包含以下可拍维度： + 1) 景别+机位架设位置：如"中景，机位架设在店内深处正对门口" + 2) 主体动作（按时序）：按 duration 分档写连续可见节拍——4-7秒写"开始动作→持续变化→结束状态"；8秒及以上按"前段→中段→后段"写至少3个连续可见动作。每秒至少一个可见动作变化，禁止用一个瞬时动作支撑长镜头 + 3) 运镜：从镜头运动词库选一个主导运镜（推/拉/摇/移/跟/环绕/手持/固定），写明方向与节奏，禁止只用"缓缓"这种无信息量词 + 4) 光影：方向+色温+阴影色，与当前镜头光源位置绑死（如"晨光从卷帘门缝隙逆光射入，发丝边缘泛柔光，店内深处阴影偏冷蓝"），禁止情绪词 + 5) 道具/穿着：从 source_text 和角色设定提取具体道具与穿着，写进动作流 + 6) 台词：若 source_text 有台词，以「角色说的话」标注并标语气（小声/平淡/恳求），供后续口型对齐；无台词则不写 + - 禁止参数堆砌（8K/HDR/fps/Rec.色域这类），视频模型不认 + - 禁止纯静态描述，特写镜头必须使用"固定镜头" + - description 必须与 video_prompt 的节拍、光影、动作一致 【动态优先原则 - 核心规则】 - 视频不能僵硬！每个video_prompt必须包含"动"的元素。即使是对话场景也要动起来。 - ✅ 正确："年轻女子坐在沙发上轻轻转头，镜头缓缓推近她的侧脸" - ❌ 错误："年轻女子坐在沙发上，镜头固定" + 视频不能僵硬！每个video_prompt必须按时序含可见动作。即使对话场景也要有动作变化。 + ✅ 正确示例（6秒、2节拍）："中景，机位架设在店内深处正对门口。年轻女子从右侧卷帘门推门进入，门推开约45°，晨光从门缝逆光射入在她发丝边缘形成柔光晕。她站定在门口，身体微前倾又顿住，手里攥着一张揉皱的纸，眼神在店内游移后落向画面中央偏左的座位。镜头手持跟随，从门口缓推至她停步处，保持中景距离。她小声开口：「那个……这里理发吗？」" + ❌ 错误："年轻女子坐在沙发上，镜头固定"（无节拍、无光影、无运镜节奏） 【image_prompt撰写规则】 - 使用角色实际名字（不是年龄段+性别） @@ -2959,7 +3255,7 @@ public class ShortDramaServiceImpl implements IShortDramaService { "story_action": "镜头中执行的关键动作", "story_result": "动作造成的剧情变化", "next_hook": "下一镜必须回应的后果", - "video_prompt": "年轻男子站在桌前双手撑在桌面上，正在说话，镜头缓缓推近", + "video_prompt": "中景，机位架设在办公室深处正对会议桌。年轻男子站在深棕色实木桌前，双手撑在桌面上，身体微前倾，目光扫过桌前众人后停住。镜头从中景手持缓推至他上半身，保持平视距离，约两秒推到位后固定。午后阳光从右侧窗户斜照进来，在他脸侧形成暖侧光，桌面阴影偏冷。他抬头环视，低声开口：「开会了。」", "image_prompt": "张三站在办公室中央，双手撑在深棕色实木桌面上，表情严肃，午后阳光从右侧窗户斜照进来", "sceneTitle": "张三宣布开会", "characters": [{"name":"张三","appearance":"初始形象","slot":"办公室中央"}], @@ -2994,7 +3290,7 @@ public class ShortDramaServiceImpl implements IShortDramaService { entity.setShotType(firstNotBlank(panel.getShotType(), "平视中景")); entity.setCameraMove(firstNotBlank(panel.getCameraMove(), "缓推")); entity.setDurationSeconds(panel.getDuration() != null && panel.getDuration() > 0 ? panel.getDuration() : defaultDurationForSceneType(entity.getSceneType())); - entity.setVideoPrompt(firstNotBlank(panel.getVideoPrompt(), buildVideoPromptFallback(script, panel.getDescription(), sceneNo))); + entity.setVideoPrompt(firstNotBlank(panel.getVideoPrompt(), buildVideoPromptFallback(script, panel.getDescription(), sceneNo, panel.getSceneType(), panel.getDuration()))); entity.setVideoStatus("pending"); entity.setLocationName(panel.getLocation()); entity.setSourceText(panel.getSourceText()); @@ -3100,7 +3396,7 @@ public class ShortDramaServiceImpl implements IShortDramaService { StringBuilder sb = new StringBuilder(); for (ShortDramaLocation l : locs) { sb.append("- ").append(l.getName()).append("：") - .append(firstNotBlank(l.getDescriptions(), l.getSummary(), "无描述")) + .append(firstNotBlank(primaryLocationDescription(l), l.getSummary(), "无描述")) .append("\n"); } return sb.toString(); @@ -3158,6 +3454,166 @@ public class ShortDramaServiceImpl implements IShortDramaService { return chatServiceFactory.getOriginalService(modelVo.getProviderCode()); } + // ==================== 语音资产 ==================== + + @Override + public ShortDramaAudioVo saveAudio(ShortDramaAudioBo bo, Long userId) { + validateProjectOwner(bo.getProjectId(), userId); + ShortDramaAudio entity = MapstructUtils.convert(bo, ShortDramaAudio.class); + if (entity.getAudioType() == null) entity.setAudioType("narration"); + if (entity.getId() == null) { + entity.setId(IdUtil.getSnowflakeNextId()); + audioMapper.insert(entity); + } else { + ShortDramaAudio existing = audioMapper.selectById(entity.getId()); + if (existing == null || !userId.equals(projectMapper.selectById(existing.getProjectId()).getUserId())) { + throw new IllegalArgumentException("语音资产不存在或无权限"); + } + audioMapper.updateById(entity); + } + return MapstructUtils.convert(entity, ShortDramaAudioVo.class); + } + + @Override + public Boolean deleteAudio(Long audioId, Long userId) { + ShortDramaAudio audio = audioMapper.selectById(audioId); + if (audio == null) return false; + validateProjectOwner(audio.getProjectId(), userId); + return audioMapper.deleteById(audioId) > 0; + } + + @Override + public List listAudios(Long projectId, Long userId) { + validateProjectOwner(projectId, userId); + return audioMapper.selectVoList(new LambdaQueryWrapper() + .eq(ShortDramaAudio::getProjectId, projectId) + .orderByAsc(ShortDramaAudio::getId)); + } + + @Override + public ShortDramaAudioVo generateAudio(Long audioId, String audioModel, Long userId) { + ShortDramaAudio audio = audioMapper.selectById(audioId); + if (audio == null) throw new IllegalArgumentException("语音资产不存在"); + validateProjectOwner(audio.getProjectId(), userId); + if (StrUtil.isBlank(audio.getText())) throw new IllegalArgumentException("语音文案不能为空"); + ChatModelVo modelVo = chatModelService.selectModelByName(audioModel); + if (modelVo == null) throw new IllegalArgumentException("未找到语音模型配置: " + audioModel); + if (!org.ruoyi.enums.ModelType.AUDIO.getKey().equals(modelVo.getCategory())) { + throw new IllegalArgumentException("模型分类不是语音模型: " + audioModel); + } + + // 对白类型：从关联镜头收集出场角色及其子形象音色，作为 references + @audioN 标记 + List> references = buildAudioReferences(audio); + AudioContext ctx = AudioContext.builder() + .chatModelVo(modelVo) + .input(audio.getText()) + .voice(StrUtil.isBlank(audio.getVoice()) ? null : audio.getVoice()) + .responseFormat("mp3") + .references(references) + .build(); + MediaGenerationResponse response = audioServiceFactory.getOriginalService(modelVo.getProviderCode()) + .generateSpeech(ctx); + + String audioUrl; + Long audioOssId; + if (response != null && StrUtil.isNotBlank(response.getB64Json())) { + // OpenAI 同步模式：base64 → OSS + byte[] audioBytes = java.util.Base64.getDecoder().decode(response.getB64Json()); + org.ruoyi.common.core.domain.dto.OssDTO uploaded = uploadAudioBytes(audioBytes); + audioUrl = uploaded.getUrl(); + audioOssId = uploaded.getOssId(); + } else if (response != null && StrUtil.isNotBlank(response.getId())) { + // Atlas 异步模式：轮询拿 URL，再下载转存 OSS（统一存储，避免 Atlas 链路过期） + String predictionId = response.getId(); + if (StrUtil.isNotBlank(response.getUrl())) { + audioUrl = response.getUrl(); + audioOssId = null; + } else { + MediaGenerationResponse polled = pollAudioDone(modelVo, predictionId); + if (polled == null || StrUtil.isBlank(polled.getUrl())) { + throw new RuntimeException("语音异步生成超时或失败，predictionId=" + predictionId); + } + audioUrl = polled.getUrl(); + audioOssId = null; + } + } else { + throw new RuntimeException("语音生成失败，模型未返回音频数据或任务ID"); + } + audio.setAudioOssId(audioOssId); + audio.setAudioUrl(audioUrl); + audioMapper.updateById(audio); + return MapstructUtils.convert(audio, ShortDramaAudioVo.class); + } + + /** + * 对白类型音频：从关联镜头的出场角色收集子形象音色，构造 references（speaker）。 + * text 中可用 @audioN 引用对应角色。旁白类型返回空列表。 + */ + private List> buildAudioReferences(ShortDramaAudio audio) { + if (!"dialogue".equals(audio.getAudioType()) || audio.getLinkedStoryboardId() == null) { + return List.of(); + } + ShortDramaStoryboard sb = storyboardMapper.selectById(audio.getLinkedStoryboardId()); + if (sb == null) return List.of(); + List refs = parseCharacterRefs(sb.getCharactersJson()); + if (refs == null || refs.isEmpty()) return List.of(); + List> result = new ArrayList<>(); + for (CharacterRef ref : refs) { + ShortDramaCharacter ch = findCharacterByName(sb.getProjectId(), ref.getName()); + if (ch == null) continue; + // 取该角色的主形象（appearanceIndex=0）音色 + List aps = characterAppearanceMapper.selectList( + new LambdaQueryWrapper() + .eq(ShortDramaCharacterAppearance::getCharacterId, ch.getId()) + .orderByAsc(ShortDramaCharacterAppearance::getAppearanceIndex)); + for (ShortDramaCharacterAppearance ap : aps) { + if (StrUtil.isNotBlank(ap.getVoice())) { + java.util.Map r = new LinkedHashMap<>(); + r.put("speaker", ap.getVoice()); + result.add(r); + break; + } + } + } + return result; + } + + /** Atlas 异步音频轮询，累计不超过 3 分钟。 */ + private MediaGenerationResponse pollAudioDone(ChatModelVo modelVo, String predictionId) { + long deadline = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(3); + while (System.currentTimeMillis() < deadline) { + MediaGenerationResponse resp = atlasPredictionService.retrieve(modelVo, predictionId); + if (resp != null && ("completed".equals(resp.getStatus()) || "succeeded".equals(resp.getStatus())) + && StrUtil.isNotBlank(resp.getUrl())) { + return resp; + } + if (resp != null && "failed".equals(resp.getStatus())) { + throw new RuntimeException("语音异步生成失败: " + resp.getRawResponse()); + } + try { Thread.sleep(2000); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); return resp; } + } + return null; + } + + private org.ruoyi.common.core.domain.dto.OssDTO uploadAudioBytes(byte[] bytes) { + java.nio.file.Path tmp = null; + try { + tmp = java.nio.file.Files.createTempFile("short-drama-audio-", ".mp3"); + java.nio.file.Files.write(tmp, bytes); + org.ruoyi.common.core.domain.dto.OssDTO uploaded = ossService.uploadFile(tmp.toFile()); + if (uploaded == null || uploaded.getOssId() == null) { + throw new RuntimeException("语音文件上传对象存储失败"); + } + return uploaded; + } catch (java.io.IOException e) { + throw new RuntimeException("语音文件写入失败: " + e.getMessage(), e); + } finally { + if (tmp != null) { + try { java.nio.file.Files.deleteIfExists(tmp); } catch (java.io.IOException ignored) {} + } + } + } + private StreamingChatModel buildStreamingChatModel() { ChatModelVo modelVo = findChatModel(); AbstractChatService chatService = getChatService(modelVo); @@ -3181,6 +3637,7 @@ public class ShortDramaServiceImpl implements IShortDramaService { } characterMapper.delete(new LambdaQueryWrapper().eq(ShortDramaCharacter::getProjectId, projectId)); locationMapper.delete(new LambdaQueryWrapper().eq(ShortDramaLocation::getProjectId, projectId)); + audioMapper.delete(new LambdaQueryWrapper().eq(ShortDramaAudio::getProjectId, projectId)); } private static void normalizeContinuityChain(List panels) { @@ -3387,8 +3844,44 @@ public class ShortDramaServiceImpl implements IShortDramaService { return chunks.stream().filter(StrUtil::isNotBlank).limit(12).toList(); } + /** 按 scene_type 选具体运镜描述，消除"缓缓推近"这类无信息量词。 */ + private static String cameraMoveForScene(String sceneType) { + return switch (firstNotBlank(sceneType, "daily")) { + case "action" -> "手持跟移，允许轻微晃动，快速横移跟随动作"; + case "emotion" -> "从中景缓推至近景，两秒到位后固定，聚焦面部"; + case "epic" -> "大远景缓拉升起，展现环境规模后停住"; + case "suspense" -> "斯坦尼康式低速缓推，略带左右摇摆模拟紧张"; + default -> "平视中景手持缓推，约两秒到位后固定"; + }; + } + + /** + * Phase 6 失败时的兜底 video_prompt。按 duration 分档写可拍节拍， + * 不再一句"镜头缓缓推近，自然光线"。 + */ + private static String buildVideoPromptFallback(ShortDramaScript script, String text, int sceneNo, + String sceneType, Integer duration) { + String tone = firstNotBlank(script.getTone(), "短剧"); + String camera = cameraMoveForScene(sceneType); + String light = switch (firstNotBlank(sceneType, "daily")) { + case "suspense" -> "低调硬光，保留阴影层次"; + case "emotion" -> "柔和侧光，突出面部情绪"; + case "action" -> "高反差侧光，强化动作轮廓"; + case "epic" -> "大范围自然光，突出空间规模"; + default -> "自然柔光，主光从画面侧上方斜照"; + }; + int dur = duration != null && duration > 0 ? duration : 6; + String beats; + if (dur >= 8) { + beats = "前段：" + text + "；中段：动作持续变化、视线或姿态推进；后段：收束在结束状态，留出下一镜承接"; + } else { + beats = "开始动作：" + text + "；持续变化：动作与视线推进；结束状态：收束停住"; + } + return "中景，镜头" + camera + "。" + beats + "。" + tone + "风格。" + light + "。短剧镜头" + sceneNo; + } + private static String buildVideoPromptFallback(ShortDramaScript script, String text, int sceneNo) { - return "短剧镜头" + sceneNo + "，" + firstNotBlank(script.getTone(), "短剧") + "风格，" + text + "，镜头缓缓推近，自然光线"; + return buildVideoPromptFallback(script, text, sceneNo, "daily", 6); } // ==================== JSON 解析 ==================== @@ -3744,7 +4237,7 @@ public class ShortDramaServiceImpl implements IShortDramaService { private static String normalizeAspectRatio(String aspectRatio) { return switch (firstNotBlank(aspectRatio, "9:16")) { - case "16:9", "1:1" -> aspectRatio; + case "16:9", "4:3", "1:1", "3:4", "9:16", "21:9" -> aspectRatio; default -> "9:16"; }; } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaVideoComposeJob.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaVideoComposeJob.java index 6fabfbaa..e8553fde 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaVideoComposeJob.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaVideoComposeJob.java @@ -4,6 +4,7 @@ import org.ruoyi.service.shortdrama.composition.AspectRatio; import org.ruoyi.service.shortdrama.composition.TransitionType; import java.math.BigDecimal; +import java.nio.file.Path; import java.util.List; import java.util.Objects; @@ -14,7 +15,10 @@ record ShortDramaVideoComposeJob( TransitionType transitionType, BigDecimal transitionDurationSeconds, AspectRatio aspectRatio, - List storyboardIds + List storyboardIds, + Long narrationAudioId, + Path narrationAudioPath, + boolean watermark ) { ShortDramaVideoComposeJob { diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaVideoComposeServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaVideoComposeServiceImpl.java index 08887878..f3cc5461 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaVideoComposeServiceImpl.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaVideoComposeServiceImpl.java @@ -92,7 +92,10 @@ public class ShortDramaVideoComposeServiceImpl implements IShortDramaVideoCompos transitionType, transitionDuration, aspectRatio, - bo.getStoryboardIds() + bo.getStoryboardIds(), + bo.getNarrationAudioId(), + null, + resolveWatermark(bo.getWatermark()) ); try { composeWorker.composeAsync(job); @@ -218,6 +221,11 @@ public class ShortDramaVideoComposeServiceImpl implements IShortDramaVideoCompos } } + /** 前端 watermark 为空时回退到配置默认值（默认开启 ruoyi-ai） */ + private boolean resolveWatermark(Boolean requested) { + return requested == null ? compositionProperties.isWatermarkEnabled() : requested; + } + private Date staleBefore(Date now) { Duration staleAfter = compositionProperties.getJobStaleAfter(); if (staleAfter == null || staleAfter.isZero() || staleAfter.isNegative()) { diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaVideoComposeWorker.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaVideoComposeWorker.java index 31587c23..a5e77a94 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaVideoComposeWorker.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/impl/ShortDramaVideoComposeWorker.java @@ -10,8 +10,10 @@ import org.ruoyi.common.core.exception.ServiceException; import org.ruoyi.common.core.service.OssService; import org.ruoyi.common.core.utils.file.FileUtils; import org.ruoyi.common.tenant.helper.TenantHelper; +import org.ruoyi.domain.entity.shortdrama.ShortDramaAudio; import org.ruoyi.domain.entity.shortdrama.ShortDramaProject; import org.ruoyi.domain.entity.shortdrama.ShortDramaStoryboard; +import org.ruoyi.mapper.shortdrama.ShortDramaAudioMapper; import org.ruoyi.mapper.shortdrama.ShortDramaProjectMapper; import org.ruoyi.mapper.shortdrama.ShortDramaStoryboardMapper; import org.ruoyi.service.shortdrama.composition.CompositionArtifact; @@ -41,6 +43,7 @@ public class ShortDramaVideoComposeWorker { private final ShortDramaProjectMapper projectMapper; private final ShortDramaStoryboardMapper storyboardMapper; + private final ShortDramaAudioMapper audioMapper; private final FfmpegVideoComposer videoComposer; private final FfmpegCompositionProperties compositionProperties; private final SafeVideoSourceDownloader sourceDownloader; @@ -65,11 +68,16 @@ public class ShortDramaVideoComposeWorker { return; } + // 旁白语音资产下载到工作目录（ossId → 本地文件） + Path narrationAudioPath = downloadNarration(job, workDirectory); + CompositionArtifact artifact = videoComposer.compose(new CompositionSpec( sources, job.transitionType(), job.transitionDurationSeconds().doubleValue(), - job.aspectRatio() + job.aspectRatio(), + narrationAudioPath, + job.watermark() ), workDirectory); if (!updateProgress(job, 85)) { return; @@ -198,6 +206,24 @@ public class ShortDramaVideoComposeWorker { .set(ShortDramaProject::getUpdateTime, new Date())) > 0; } + /** + * 将旁白语音资产下载为本地文件。语音资产未指定或不存在时返回 null（不混入旁白）。 + */ + private Path downloadNarration(ShortDramaVideoComposeJob job, Path workDirectory) throws IOException { + if (job.narrationAudioId() == null) { + return null; + } + ShortDramaAudio audio = audioMapper.selectById(job.narrationAudioId()); + if (audio == null || StrUtil.isBlank(audio.getAudioUrl())) { + log.warn("旁白语音资产不存在或无音频URL, audioId={}", job.narrationAudioId()); + return null; + } + Path target = workDirectory.resolve("narration.mp3"); + long maxSourceBytes = compositionProperties.getMaxSourceBytes(); + sourceDownloader.download(audio.getAudioUrl(), target, maxSourceBytes, maxSourceBytes); + return target; + } + private boolean isActive(ShortDramaVideoComposeJob job) { return selectActiveProject(job) != null; } diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/support/IncrementalJsonArrayExtractor.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/support/IncrementalJsonArrayExtractor.java new file mode 100644 index 00000000..b77d7749 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/shortdrama/support/IncrementalJsonArrayExtractor.java @@ -0,0 +1,116 @@ +package org.ruoyi.service.shortdrama.support; + +import cn.hutool.core.util.StrUtil; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; + +/** + * 增量 JSON 数组解析器：随 LLM 流式输出累积 buffer，逐个提取已闭合的顶层对象， + * 每完成一个就解析成目标类型并返回（已返回的不再重复）。 + * 用于分镜规划流式推送——第一个 panel 解析出来即可展示，不等整个数组完成。 + *

+ * 仅依赖 brace matching + 字符串/转义状态机，对未闭合的尾对象不解析，保证稳定性。 + */ +@Slf4j +public final class IncrementalJsonArrayExtractor { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final Class type; + private final StringBuilder buf = new StringBuilder(); + private int emittedCount = 0; + /** 数组起始 '[' 在 buffer 中的位置，-1 表示尚未找到 */ + private int arrayStart = -1; + + public IncrementalJsonArrayExtractor(Class type) { + this.type = type; + } + + /** + * 喂入新的累积文本，返回本次新解析出的完整对象（去重）。 + * 调用方应每次把"完整 buffer"传入（实现内部不重复追加，而是以最新 buffer 为准）。 + */ + public List feed(String fullBuffer) { + List newly = new ArrayList<>(); + if (StrUtil.isBlank(fullBuffer)) return newly; + buf.setLength(0); + buf.append(fullBuffer); + + if (arrayStart < 0) { + arrayStart = findArrayStart(fullBuffer); + if (arrayStart < 0) return newly; + } + + int scanFrom = arrayStart + 1; + int objIdx = 0; + int i = scanFrom; + int len = buf.length(); + while (i < len) { + char c = buf.charAt(i); + if (c == '{') { + int end = findObjectEnd(i); + if (end < 0) break; // 对象未闭合，等后续 token + if (objIdx >= emittedCount) { + String objJson = buf.substring(i, end + 1); + T parsed = tryParse(objJson); + if (parsed != null) { + newly.add(parsed); + emittedCount++; + } + } + i = end + 1; + objIdx++; + } else if (c == ']') { + break; // 数组结束 + } else if (Character.isWhitespace(c)) { + i++; + } else { + i++; + } + } + return newly; + } + + /** 定位第一个 '['（跳过思考文字、markdown 代码块围栏 ```json 等）。 */ + private int findArrayStart(String s) { + return s.indexOf('['); + } + + /** + * 从 start（指向 '{'）开始，找到该对象的闭合 '}'，正确处理字符串、转义、嵌套。 + * 返回闭合 '}' 的索引；若未闭合返回 -1。 + */ + private int findObjectEnd(int start) { + int depth = 0; + boolean inString = false; + boolean escape = false; + for (int i = start; i < buf.length(); i++) { + char c = buf.charAt(i); + if (escape) { escape = false; continue; } + if (inString) { + if (c == '\\') { escape = true; } + else if (c == '"') { inString = false; } + continue; + } + if (c == '"') { inString = true; } + else if (c == '{') { depth++; } + else if (c == '}') { + depth--; + if (depth == 0) return i; + } + } + return -1; + } + + private T tryParse(String json) { + try { + return MAPPER.readValue(json, type); + } catch (Exception e) { + log.debug("增量解析 panel 失败，跳过: {}", e.getMessage()); + return null; + } + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/video/provider/AtlasVideoGenerationServiceImpl.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/video/provider/AtlasVideoGenerationServiceImpl.java index d24bea18..e7832dad 100644 --- a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/video/provider/AtlasVideoGenerationServiceImpl.java +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/service/video/provider/AtlasVideoGenerationServiceImpl.java @@ -59,6 +59,23 @@ public class AtlasVideoGenerationServiceImpl extends AbstractVideoGenerationServ payload.put("image_url", videoContext.getImageUrl()); } + // 同步音频生成（环境音/动效） + if (videoContext.getGenerateAudio() != null) { + payload.put("generate_audio", videoContext.getGenerateAudio()); + } + // 参考音频（对白口型对齐） + java.util.List refAudios = videoContext.getReferenceAudios(); + if (refAudios != null && !refAudios.isEmpty()) { + com.fasterxml.jackson.databind.node.ArrayNode arr = payload.putArray("reference_audios"); + for (String url : refAudios) { + arr.add(url); + } + } + // 返回末帧（同场景连续镜头首帧承接用） + if (videoContext.getReturnLastFrame() != null) { + payload.put("return_last_frame", videoContext.getReturnLastFrame()); + } + Request request = new Request.Builder() .url(AtlasMediaSupport.endpoint(model.getApiHost(), "/model/generateVideo")) .addHeader("Authorization", "Bearer " + model.getApiKey()) diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/websocket/chat/MpChatHandshakeInterceptor.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/websocket/chat/MpChatHandshakeInterceptor.java new file mode 100644 index 00000000..d8498369 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/websocket/chat/MpChatHandshakeInterceptor.java @@ -0,0 +1,81 @@ +package org.ruoyi.websocket.chat; + +import lombok.extern.slf4j.Slf4j; +import org.ruoyi.common.core.domain.model.LoginUser; +import org.ruoyi.common.satoken.utils.LoginHelper; +import org.springframework.http.server.ServerHttpRequest; +import org.springframework.http.server.ServerHttpResponse; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.WebSocketHandler; +import org.springframework.web.socket.server.HandshakeInterceptor; + +import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +/** + * 小程序对话 WS 握手拦截器。 + *

+ * 无权限：握手始终放行。仅尝试从握手 URL 的 Authorization 参数解析登录用户， + * 解析成功则把 userId 放入 session attributes 供 handler 落库使用；解析失败按匿名处理。 + *

+ * 注意：与公共 {@code PlusWebSocketInterceptor} 不同，这里不做 clientid 一致性校验， + * 也不抛出认证异常——对话端点对未登录用户同样开放。 + * + * @author ruoyi team + */ +@Slf4j +@Component +public class MpChatHandshakeInterceptor implements HandshakeInterceptor { + + public static final String USER_ID_KEY = "mpChatUserId"; + public static final String LOGIN_USER_KEY = "mpChatLoginUser"; + + @Override + public boolean beforeHandshake(ServerHttpRequest request, ServerHttpResponse response, + WebSocketHandler wsHandler, Map attributes) { + // 无权限：握手始终放行，且不调用 sa-token（LoginHelper.getLoginUser 会触发 getTokenSessionByToken， + // 在 is-share:false 下有冻结当前 token 的副作用，导致随后 mvc 请求 401 token 已被冻结。 + // 对话端点本就不依赖登录态，userId 留空，落库跳过）。 + log.info("[mp-chat connect] 匿名对话连接"); + return true; + } + + @Override + public void afterHandshake(ServerHttpRequest request, ServerHttpResponse response, + WebSocketHandler wsHandler, Exception exception) { + // no-op + } + + /** + * 从握手 URL query 中解析 token。 + * 前端约定以 Authorization=Bearer xxx 形式透传，去掉 Bearer 前缀取真实 token。 + */ + private String resolveToken(URI uri) { + String query = uri.getRawQuery(); + if (query == null || query.isEmpty()) { + return null; + } + for (String pair : query.split("&")) { + int idx = pair.indexOf('='); + if (idx <= 0) { + continue; + } + String key = pair.substring(0, idx); + if (!"Authorization".equalsIgnoreCase(key)) { + continue; + } + String value = URLDecoder.decode(pair.substring(idx + 1), StandardCharsets.UTF_8); + if (value == null) { + return null; + } + value = value.trim(); + if (value.startsWith("Bearer ")) { + value = value.substring(7).trim(); + } + return value.isEmpty() ? null : value; + } + return null; + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/websocket/chat/MpChatWebSocketConfig.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/websocket/chat/MpChatWebSocketConfig.java new file mode 100644 index 00000000..dee09927 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/websocket/chat/MpChatWebSocketConfig.java @@ -0,0 +1,33 @@ +package org.ruoyi.websocket.chat; + +import lombok.RequiredArgsConstructor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.socket.config.annotation.EnableWebSocket; +import org.springframework.web.socket.config.annotation.WebSocketConfigurer; +import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry; + +/** + * 小程序对话 WebSocket 端点配置。 + *

+ * 独立注册 /chat/ws，无权限（握手拦截器仅做 token 解析、不拦截）， + * 与公共 ruoyi-common-websocket 的 /resource/websocket 互不干扰（后者受 websocket.enabled 控制，默认关闭）。 + * + * @author ruoyi team + */ +@Configuration +@EnableWebSocket +@RequiredArgsConstructor +public class MpChatWebSocketConfig { + + private final MpChatWebSocketHandler mpChatWebSocketHandler; + private final MpChatHandshakeInterceptor mpChatHandshakeInterceptor; + + @Bean + public WebSocketConfigurer mpChatWebSocketConfigurer() { + return registry -> registry + .addHandler(mpChatWebSocketHandler, "/chat/ws") + .addInterceptors(mpChatHandshakeInterceptor) + .setAllowedOrigins("*"); + } +} diff --git a/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/websocket/chat/MpChatWebSocketHandler.java b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/websocket/chat/MpChatWebSocketHandler.java new file mode 100644 index 00000000..069d20f8 --- /dev/null +++ b/ruoyi-modules/ruoyi-chat/src/main/java/org/ruoyi/websocket/chat/MpChatWebSocketHandler.java @@ -0,0 +1,378 @@ +package org.ruoyi.websocket.chat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.message.UserMessage; +import dev.langchain4j.model.chat.StreamingChatModel; +import dev.langchain4j.model.chat.response.ChatResponse; +import dev.langchain4j.model.chat.response.StreamingChatResponseHandler; +import dev.langchain4j.rag.AugmentationRequest; +import dev.langchain4j.rag.AugmentationResult; +import dev.langchain4j.rag.DefaultRetrievalAugmentor; +import dev.langchain4j.rag.RetrievalAugmentor; +import dev.langchain4j.rag.content.Content; +import dev.langchain4j.rag.content.retriever.ContentRetriever; +import dev.langchain4j.rag.query.Metadata; +import dev.langchain4j.rag.query.Query; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.ruoyi.common.chat.domain.bo.chat.ChatModelBo; +import org.ruoyi.common.chat.domain.dto.request.ChatRequest; +import org.ruoyi.common.chat.domain.vo.chat.ChatModelVo; +import org.ruoyi.common.chat.enums.RoleType; +import org.ruoyi.common.chat.service.chat.IChatModelService; +import org.ruoyi.common.core.utils.StringUtils; +import org.ruoyi.domain.vo.agent.AgentVo; +import org.ruoyi.domain.vo.knowledge.KnowledgeInfoVo; +import org.ruoyi.factory.ChatServiceFactory; +import org.ruoyi.service.agent.IAgentService; +import org.ruoyi.service.chat.IChatMessageService; +import org.ruoyi.service.knowledge.IKnowledgeInfoService; +import org.ruoyi.service.knowledge.retriever.CustomVectorRetriever; +import org.ruoyi.service.retrieval.KnowledgeRetrievalService; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.web.socket.TextMessage; +import org.springframework.web.socket.WebSocketSession; +import org.springframework.web.socket.handler.AbstractWebSocketHandler; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** + * 小程序对话 WebSocket 处理器。 + *

+ * 收到前端 JSON 消息后：解析模型（智能体绑定 / 前端传入 / 默认兜底）→ + * 拼装 systemPrompt 与 RAG 增强后的 content → 调用 StreamingChatModel 流式生成 → + * 将增量 token 通过当前 WS session 回推前端。 + *

+ * 输出协议（与前端 index.vue 现有接收逻辑兼容）： + *

    + *
  • 增量：{"content":"token片段"}
    • + *
  • 结束：[DONE]
    • + *
  • 错误：{"data":"错误:xxx"}
    • + *
+ * + * @author ruoyi team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class MpChatWebSocketHandler extends AbstractWebSocketHandler { + + private final ChatServiceFactory chatServiceFactory; + private final IChatModelService chatModelService; + private final IAgentService agentService; + private final IKnowledgeInfoService knowledgeInfoService; + private final KnowledgeRetrievalService knowledgeRetrievalService; + private final IChatMessageService chatMessageService; + private final ObjectMapper objectMapper; + + @Value("${chat.default-model:}") + private String defaultModel; + + @Override + protected void handleTextMessage(WebSocketSession session, TextMessage message) { + Map payload; + try { + payload = objectMapper.readValue(message.getPayload(), Map.class); + } catch (Exception e) { + sendError(session, "错误:消息格式不正确"); + return; + } + String content = asString(payload.get("content")); + String agentIdRaw = asString(payload.get("agentId")); + String model = asString(payload.get("model")); + String systemPrompt = asString(payload.get("systemPrompt")); + String knowledgeId = asString(payload.get("knowledgeId")); + String sessionIdRaw = asString(payload.get("sessionId")); + + if (StringUtils.isBlank(content)) { + sendError(session, "错误:对话消息不能为空"); + return; + } + + Long userId = (Long) session.getAttributes().get(MpChatHandshakeInterceptor.USER_ID_KEY); + Long sessionId = parseLong(sessionIdRaw); + + try { + // 1. 解析智能体（若传了 agentId），取其绑定模型与 systemPrompt、知识库 + AgentVo agentVo = null; + if (StringUtils.isNotBlank(agentIdRaw)) { + Long agentId = parseLong(agentIdRaw); + if (agentId != null) { + agentVo = agentService.queryById(agentId); + } + } + + // 2. 解析模型：智能体绑定 > 前端传入 > 默认配置 > 表内首个 chat 模型 + ChatModelVo modelVo = null; + if (agentVo != null && agentVo.getModelId() != null) { + modelVo = chatModelService.queryById(agentVo.getModelId()); + } + if (modelVo == null && StringUtils.isNotBlank(model)) { + modelVo = chatModelService.selectModelByName(model); + } + if (modelVo == null) { + modelVo = resolveDefaultModel(); + } + if (modelVo == null) { + sendError(session, "错误:未找到可用对话模型，请联系管理员配置"); + return; + } + + // 3. 拼装最终输入：RAG 增强 + systemPrompt 前置 + String finalSystemPrompt = (agentVo != null && StringUtils.isNotBlank(agentVo.getSystemPrompt())) + ? agentVo.getSystemPrompt() : systemPrompt; + String augmentedContent = augmentWithKnowledge(content, agentVo, knowledgeId); + String finalContent = StringUtils.isNotBlank(finalSystemPrompt) + ? finalSystemPrompt + "\n\n" + augmentedContent : augmentedContent; + + // 4. 落库用户消息（仅在具备用户与会话标识时） + if (userId != null && sessionId != null) { + try { + chatMessageService.saveChatMessage(userId, sessionId, content, + RoleType.USER.getName(), modelVo.getModelName()); + } catch (Exception e) { + log.warn("落库用户消息失败: {}", e.getMessage()); + } + } + + // 5. 构造流式模型并异步生成 + ChatRequest chatRequest = new ChatRequest(); + chatRequest.setContent(content); + chatRequest.setModel(modelVo.getModelName()); + chatRequest.setKnowledgeId(knowledgeId); + StreamingChatModel streamingModel = chatServiceFactory + .getOriginalService(modelVo.getProviderCode()) + .buildStreamingChatModel(modelVo, chatRequest); + + final String modelName = modelVo.getModelName(); + CompletableFuture.runAsync(() -> { + StringBuilder buffer = new StringBuilder(); + // 是否已向前端发送 [DONE] 结束标记，避免重复发送或遗漏 + boolean[] doneSent = {false}; + StreamingChatResponseHandler handler = new StreamingChatResponseHandler() { + @Override + public void onPartialResponse(String partialResponse) { + buffer.append(partialResponse); + sendJson(session, Map.of("content", partialResponse)); + } + + @Override + public void onCompleteResponse(ChatResponse completeResponse) { + if (!doneSent[0]) { + doneSent[0] = true; + sendRaw(session, "[DONE]"); + } + if (userId != null && sessionId != null && buffer.length() > 0) { + try { + chatMessageService.saveChatMessage(userId, sessionId, buffer.toString(), + RoleType.ASSISTANT.getName(), modelName); + } catch (Exception e) { + log.warn("落库助手回复失败: {}", e.getMessage()); + } + } + } + + @Override + public void onError(Throwable error) { + if (buffer.length() == 0) { + // 一点内容都没输出就出错：向前端报错 + sendError(session, "错误:" + safeMsg(error)); + } else if (!doneSent[0]) { + // 已有部分内容但流式中途异常：补发 [DONE] 让前端正常收尾，不报错 + doneSent[0] = true; + sendRaw(session, "[DONE]"); + log.warn("mp-chat 流式中途异常（已输出内容，补发 [DONE]）: {}", safeMsg(error)); + } else { + // onComplete 后的收尾异常：回复已正常结束，静默 + log.warn("mp-chat 流式收尾异常（已结束，忽略）: {}", safeMsg(error)); + } + } + }; + try { + streamingModel.chat(finalContent, handler); + } catch (Exception e) { + log.error("mp-chat 调用模型失败", e); + sendError(session, "错误:" + safeMsg(e)); + } + }); + } catch (Exception e) { + log.error("mp-chat 处理消息失败", e); + sendError(session, "错误:" + safeMsg(e)); + } + } + + /** + * 智能体绑定知识库 / 前端传入 knowledgeId 时，对 content 做向量检索增强。 + * 复用 ChatServiceFacade.buildMultiKnowledgeAugmentor 的组装方式（简化为多库复合检索）。 + */ + private String augmentWithKnowledge(String content, AgentVo agentVo, String knowledgeId) { + List kids = new ArrayList<>(); + if (agentVo != null && agentVo.getKnowledgeIds() != null) { + kids.addAll(agentVo.getKnowledgeIds()); + } + if (StringUtils.isBlank(knowledgeId) && kids.isEmpty()) { + return content; + } + if (StringUtils.isNotBlank(knowledgeId)) { + try { + kids.add(Long.valueOf(knowledgeId)); + } catch (NumberFormatException ignored) { + } + } + if (kids.isEmpty()) { + return content; + } + try { + RetrievalAugmentor augmentor = buildMultiKnowledgeAugmentor(kids); + if (augmentor == null) { + return content; + } + UserMessage userMessage = UserMessage.userMessage(content); + Metadata metadata = Metadata.from(userMessage, null, new ArrayList<>()); + AugmentationResult result = augmentor.augment(new AugmentationRequest(userMessage, metadata)); + ChatMessage augmented = result.chatMessage(); + return augmented instanceof UserMessage ? ((UserMessage) augmented).singleText() : content; + } catch (Exception e) { + log.warn("mp-chat RAG 增强失败，回退原文: {}", e.getMessage()); + return content; + } + } + + private RetrievalAugmentor buildMultiKnowledgeAugmentor(List knowledgeIds) { + List retrievers = new ArrayList<>(); + for (Long kid : knowledgeIds) { + try { + KnowledgeInfoVo kb = knowledgeInfoService.queryById(kid); + if (kb == null) { + continue; + } + ChatModelVo embModel = chatModelService.selectModelByName(kb.getEmbeddingModel()); + if (embModel == null) { + log.warn("mp-chat 知识库向量模型未配置: kid={}, emb={}", kid, kb.getEmbeddingModel()); + continue; + } + retrievers.add(new CustomVectorRetriever(knowledgeRetrievalService, kb, embModel)); + } catch (Exception e) { + log.warn("mp-chat 构建检索器失败: kid={}, err={}", kid, e.getMessage()); + } + } + if (retrievers.isEmpty()) { + return null; + } + ContentRetriever composite = retrievers.size() == 1 + ? retrievers.get(0) + : new CompositeContentRetriever(retrievers); + return DefaultRetrievalAugmentor.builder().contentRetriever(composite).build(); + } + + /** + * 默认模型兜底：优先用 chat.default-model 配置，其次取表内首个 chat 类模型。 + */ + private ChatModelVo resolveDefaultModel() { + if (StringUtils.isNotBlank(defaultModel)) { + ChatModelVo vo = chatModelService.selectModelByName(defaultModel); + if (vo != null) { + return vo; + } + } + try { + List list = chatModelService.queryList(new ChatModelBo()); + if (list != null) { + for (ChatModelVo vo : list) { + if ("chat".equalsIgnoreCase(vo.getCategory()) && "Y".equalsIgnoreCase(vo.getModelShow())) { + return vo; + } + } + if (!list.isEmpty()) { + return list.get(0); + } + } + } catch (Exception e) { + log.warn("mp-chat 解析默认模型失败: {}", e.getMessage()); + } + return null; + } + + // ---------- WS 输出辅助 ---------- + + private void sendJson(WebSocketSession session, Map data) { + if (!session.isOpen()) { + return; + } + try { + session.sendMessage(new TextMessage(objectMapper.writeValueAsString(data))); + } catch (Exception e) { + log.warn("mp-chat 发送 WS 消息失败: {}", e.getMessage()); + } + } + + private void sendRaw(WebSocketSession session, String raw) { + if (!session.isOpen()) { + return; + } + try { + session.sendMessage(new TextMessage(raw)); + } catch (Exception e) { + log.warn("mp-chat 发送 WS 消息失败: {}", e.getMessage()); + } + } + + private void sendError(WebSocketSession session, String msg) { + Map err = new HashMap<>(); + err.put("data", msg); + sendJson(session, err); + } + + private static String safeMsg(Throwable e) { + return e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage(); + } + + private static String asString(Object o) { + return o == null ? null : String.valueOf(o); + } + + private static Long parseLong(String raw) { + if (StringUtils.isBlank(raw)) { + return null; + } + try { + return Long.valueOf(raw.trim()); + } catch (NumberFormatException e) { + return null; + } + } + + /** + * 多知识库复合检索器：并发查询各库并合并结果。 + * （与 ChatServiceFacade 内部 CompositeContentRetriever 同构，独立保留以解耦公共门面） + */ + private static class CompositeContentRetriever implements ContentRetriever { + private final List delegates; + + CompositeContentRetriever(List delegates) { + this.delegates = delegates; + } + + @Override + public List retrieve(Query query) { + List all = new ArrayList<>(); + for (ContentRetriever r : delegates) { + try { + List part = r.retrieve(query); + if (part != null) { + all.addAll(part); + } + } catch (Exception e) { + log.warn("mp-chat 复合检索子检索器异常: {}", e.getMessage()); + } + } + return all; + } + } +}