scottpan/msh-system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ai-nutritionist): Coze TTS and streaming robustness

Browse Source 
- Add Coze TTS endpoint and service; expose binary MP3 from controller.
- Bypass ResponseFilter for /audio/speech so MP3 bodies are not UTF-8 wrapped.
- UniApp: cozeTextToSpeech, TTS UI and play flow; SSE HTTP errors and diagnostics.
- Document TTS in docs/features.md; extend test-0325-1 with curl verification.

Made-with: Cursor
This commit is contained in:
msh-agent
2026-03-31 07:07:21 +08:00
parent 35052d655f
commit 2facd355ab
8 changed files with 433 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files

509
msh_single_uniapp/pages/tool/ai-nutritionist.vue
View File

@@ -11,13 +11,16 @@
</view>
<text class="promo-subtitle">营养师专家入驻，在线答疑</text>
</view>
<view class="promo-right">
<view class="clear-btn" @click="clearChat">
<text class="clear-icon">🗑️</text>
<text class="clear-text">清空</text>
</view>
<image class="promo-avatar" src="https://uthink2025.oss-cn-shanghai.aliyuncs.com//crmebimage/public/content/2026/01/11/afcaba68d00b4fccaa49ad2a42c78e7fkk03hqv5vl.png" mode="aspectFit"></image>
<view class="promo-right">
<view class="tts-toggle-btn" @click="toggleTTS" :class="{ active: ttsEnabled }">
<text class="tts-toggle-icon">🔊</text>
<!--<text class="tts-toggle-text">{{ ttsEnabled ? '播报开' : '播报关' }}</text>-->
</view>
<view class="clear-btn" @click="clearChat">
<text class="clear-icon">🗑️</text>
</view>
<image class="promo-avatar" src="https://uthink2025.oss-cn-shanghai.aliyuncs.com//crmebimage/public/content/2026/01/11/afcaba68d00b4fccaa49ad2a42c78e7fkk03hqv5vl.png" mode="aspectFit"></image>
</view>
</view>
</view>
@@ -61,25 +64,33 @@
<text class="message-sender">AI营养师</text>
</view>
<!-- 消息气泡 -->
<view :class="['message-bubble', msg.role === 'user' ? 'user-bubble' : 'ai-bubble']">
<!-- AI 消息 loading 占位（等待回复时显示打字动画）-->
<view v-if="msg.role === 'ai' && msg.loading" class="typing-indicator">
<view class="typing-dot"></view>
<view class="typing-dot"></view>
<view class="typing-dot"></view>
</view>
<text v-else-if="msg.type !== 'image'" class="message-text">{{ msg.content }}<text v-if="msg.streaming" class="streaming-cursor">|</text></text>
<image
v-else
:src="msg.imageUrl"
mode="widthFix"
class="message-image"
@click="previewImage(msg.imageUrl)"
></image>
</view>
<!-- 消息气泡 -->
<view :class="['message-bubble', msg.role === 'user' ? 'user-bubble' : 'ai-bubble']">
<!-- AI 消息 loading 占位（等待回复时显示打字动画）-->
<view v-if="msg.role === 'ai' && msg.loading" class="typing-indicator">
<view class="typing-dot"></view>
<view class="typing-dot"></view>
<view class="typing-dot"></view>
</view>
<text v-else-if="msg.type !== 'image'" class="message-text">{{ msg.content }}<text v-if="msg.streaming" class="streaming-cursor">|</text></text>
<image
v-else
:src="msg.imageUrl"
mode="widthFix"
class="message-image"
@click="previewImage(msg.imageUrl)"
></image>
</view>
<!-- AI 回复播放按钮 -->
<view
v-if="msg.role === 'ai' && !msg.loading && !msg.streaming && msg.content && msg.type !== 'image'"
class="tts-play-btn"
@click="ttsPlayingIndex === index ? stopTTS() : playTTS(index)"
>
<text class="tts-play-icon">{{ ttsPlayingIndex === index ? '⏹' : '▶' }}</text>
</view>
</view>
</view>
<!-- 加载中提示（仅在没有流式占位消息时显示） -->
<view v-if="isLoading && !messageList.some(m => m.loading || m.streaming)" class="message-item ai-message">
@@ -199,8 +210,8 @@ export default {
},
data() {
return {
botId: '7591133240535449654', //coze智能体机器人ID
conversationId: '', // 存储会话ID，用于多轮对话
botId: '7591133240535449654',
conversationId: '',
scrollTop: 0,
lastScrollTop: 0, // 用于动态滚动
inputText: '',
@@ -221,7 +232,12 @@ export default {
• 解读检验报告
有什么想问的吗？`,
messageList: []
messageList: [],
// TTS
ttsEnabled: false,
ttsPlaying: false,
ttsPlayingIndex: -1,
innerAudioContext: null
}
},
onLoad() {
@@ -230,6 +246,7 @@ export default {
this.scrollToBottom()
});
this.initRecorder();
this.initAudioContext();
},
onUnload() {
this.stopRecordTimer();
@@ -240,6 +257,10 @@ export default {
this._streamCtrl.abort();
this._streamCtrl = null;
}
if (this.innerAudioContext) {
this.innerAudioContext.destroy();
this.innerAudioContext = null;
}
},
methods: {
// 初始化录音管理器
@@ -625,9 +646,26 @@ export default {
this.scrollToBottom();
},
/** 从 Gemini 响应 data.choices[0].message.content 提取展示文本（支持 string / parts 数组 / { parts } / { text }） */
/**
* CommonResult.data 应为 OpenAI 形态；若网关多包一层 data，则取内层含 choices/candidates 的对象。
* 展示内容仍只来自模型字段 choices[0].message.content（或经后端规范后的同路径）。
*/
getGeminiPayload(data) {
if (!data || typeof data !== 'object') return null;
if (Array.isArray(data.choices) && data.choices.length > 0) return data;
if (Array.isArray(data.candidates) && data.candidates.length > 0) return data;
const inner = data.data;
if (inner && typeof inner === 'object') {
if (Array.isArray(inner.choices) && inner.choices.length > 0) return inner;
if (Array.isArray(inner.candidates) && inner.candidates.length > 0) return inner;
}
return data;
},
/** BUG-005: 从 KieAI Gemini 响应 data.choices[0].message.content 提取展示文本（支持 string / parts 数组 / { parts } / { text } / { content }） */
extractReplyContent(content) {
if (content == null) return '';
if (typeof content === 'number' || typeof content === 'boolean') return String(content);
if (typeof content === 'string') return content;
if (Array.isArray(content)) {
return content.map(part => (part && part.text) ? part.text : '').filter(Boolean).join('');
@@ -637,8 +675,9 @@ export default {
return content.parts.map(part => (part && part.text) ? part.text : '').filter(Boolean).join('');
}
if (typeof content.text === 'string') return content.text;
if (typeof content.content === 'string') return content.content;
}
return String(content);
return '';
},
/** 工具方法：sleep ms 毫秒 */
@@ -646,302 +685,118 @@ export default {
return new Promise(resolve => setTimeout(resolve, ms));
},
/**
* 解包 Coze API 响应：后端返回双层包装 { code, data: { code, data: actualPayload } }
* 此方法统一提取最内层的业务数据
*/
unwrapCozeResponse(response) {
if (!response) return null;
let data = response.data;
if (data && typeof data === 'object' && data.code !== undefined && data.data !== undefined) {
data = data.data;
}
return data;
},
buildCozeMessages(content, type) {
const messages = [];
buildGeminiMessages(content, type) {
if (type === 'text') {
messages.push({
role: 'user',
content: typeof content === 'string' ? content : JSON.stringify(content),
content_type: 'text'
});
} else if (type === 'multimodal') {
const parts = Array.isArray(content) ? content : [{ type: 'text', text: String(content) }];
const textPart = parts.find(p => p && p.type === 'text');
const imgPart = parts.find(p => p && (p.type === 'image_url' || p.type === 'image'));
if (imgPart) {
const fileId = imgPart.file_id || (imgPart.image_url && imgPart.image_url.url) || '';
messages.push({
role: 'user',
content: JSON.stringify([
...(textPart ? [{ type: 'text', text: textPart.text }] : []),
{ type: 'image', file_id: fileId }
]),
content_type: 'object_string'
});
} else {
messages.push({
role: 'user',
content: textPart ? textPart.text : JSON.stringify(content),
content_type: 'text'
});
}
} else {
let fileInfo = content;
if (typeof fileInfo === 'string') {
try { fileInfo = JSON.parse(fileInfo); } catch (e) { /* ignore */ }
}
const fileId = (fileInfo && fileInfo.id) || (fileInfo && fileInfo.file_id) || '';
if (fileId) {
messages.push({
role: 'user',
content: JSON.stringify([{ type: 'image', file_id: fileId }]),
content_type: 'object_string'
});
} else {
messages.push({
role: 'user',
content: '我发送了一张图片，请帮我分析',
content_type: 'text'
});
}
return [{ role: 'user', content: typeof content === 'string' ? content : String(content) }];
}
return messages;
const parts = Array.isArray(content) ? content : [{ type: 'text', text: String(content) }];
return [{ role: 'user', content: parts }];
},
supportsChunked() {
try {
const sysInfo = uni.getSystemInfoSync();
if (sysInfo.SDKVersion) {
const parts = sysInfo.SDKVersion.split('.').map(Number);
return (parts[0] > 2) || (parts[0] === 2 && parts[1] > 20) ||
(parts[0] === 2 && parts[1] === 20 && (parts[2] || 0) >= 1);
}
} catch (e) { /* fallback */ }
return false;
/** BUG-005：严格从 CommonResult.data.choices[0].message.content 读取回复 */
getGeminiReplyFromResponse(response) {
const content = response &&
response.data &&
Array.isArray(response.data.choices) &&
response.data.choices[0] &&
response.data.choices[0].message
? response.data.choices[0].message.content
: '';
return this.extractReplyContent(content);
},
async sendToAI(content, type) {
this.isLoading = true;
const aiMsg = { role: 'ai', content: '', loading: true, streaming: false };
this.messageList.push(aiMsg);
this.scrollToBottom();
const userId = this.uid || (uni.getStorageSync('userInfo') || {}).id || 'default_user';
const messages = this.buildCozeMessages(content, type);
const requestData = {
botId: this.botId,
userId: userId,
additionalMessages: messages,
stream: true,
autoSaveHistory: true
};
if (this.conversationId) requestData.conversationId = this.conversationId;
if (this.supportsChunked()) {
this.sendToAIStream(requestData, aiMsg);
} else {
this.sendToAIPoll(requestData, aiMsg);
}
},
sendToAIStream(requestData, aiMsg) {
const ctrl = api.cozeChatStream(requestData)
.onMessage((evt) => {
const eventType = evt.event || '';
if (eventType === 'conversation.chat.created') {
if (evt.conversation_id) {
this.conversationId = evt.conversation_id;
}
} else if (eventType === 'conversation.message.delta') {
const role = evt.role || 'assistant';
const type = evt.type || 'answer';
if (role === 'assistant' && type === 'answer') {
if (aiMsg.loading) {
aiMsg.loading = false;
aiMsg.streaming = true;
}
aiMsg.content += (evt.content || '');
this.messageList = [...this.messageList];
this.scrollToBottom();
}
} else if (eventType === 'conversation.chat.completed') {
aiMsg.streaming = false;
this.isLoading = false;
this.messageList = [...this.messageList];
this.scrollToBottom();
} else if (eventType === 'conversation.chat.failed') {
aiMsg.loading = false;
aiMsg.streaming = false;
if (!aiMsg.content) {
aiMsg.content = '抱歉，AI 对话失败，请稍后再试。';
}
this.isLoading = false;
this.messageList = [...this.messageList];
this.scrollToBottom();
}
})
.onError((err) => {
console.error('SSE 流式对话失败:', err);
aiMsg.loading = false;
aiMsg.streaming = false;
if (!aiMsg.content) {
aiMsg.content = '抱歉，处理您的请求时出现错误，请稍后再试。';
}
this.isLoading = false;
this.messageList = [...this.messageList];
this.scrollToBottom();
})
.onComplete(() => {
aiMsg.loading = false;
aiMsg.streaming = false;
if (!aiMsg.content) {
aiMsg.content = '未能获取到有效回复。';
}
this.isLoading = false;
this.messageList = [...this.messageList];
this.scrollToBottom();
});
this._streamCtrl = ctrl;
},
async sendToAIPoll(requestData, aiMsg) {
requestData.stream = false;
try {
const response = await api.cozeChat(requestData);
const cozeData = this.unwrapCozeResponse(response);
if (cozeData) {
const chat = cozeData.chat || cozeData;
const conversationId = chat.conversation_id || chat.conversationID || chat.conversationId;
const chatId = chat.id;
if (conversationId && chatId) {
this.conversationId = conversationId;
await this.pollChatStatus(conversationId, chatId, aiMsg);
} else {
console.error('Coze chat response structure:', JSON.stringify(response));
throw new Error('发起对话失败：未返回会话或对话ID');
}
} else {
throw new Error('发起对话失败');
}
const response = await api.kieaiGeminiChat({
messages: this.buildGeminiMessages(content, type),
stream: false
});
const reply = this.getGeminiReplyFromResponse(response);
aiMsg.content = reply || '抱歉，未获取到模型回复。';
} catch (error) {
console.error('发送消息失败:', error);
this.isLoading = false;
console.error('KieAI Gemini 对话失败:', error);
aiMsg.content = '抱歉，处理您的请求时出现错误，请稍后再试。';
} finally {
aiMsg.loading = false;
aiMsg.streaming = false;
this.isLoading = false;
this.messageList = [...this.messageList];
this.scrollToBottom();
if (this.ttsEnabled && aiMsg.content) {
const aiIdx = this.messageList.indexOf(aiMsg);
if (aiIdx !== -1) {
this.$nextTick(() => this.playTTS(aiIdx));
}
}
}
},
getPollInterval(attempt) {
if (attempt <= 10) return 500;
if (attempt <= 30) return 1000;
return 1500;
},
async pollChatStatus(conversationId, chatId, aiMsg) {
const maxAttempts = 80;
let attempts = 0;
const checkStatus = async () => {
attempts++;
if (attempts > maxAttempts) {
this.isLoading = false;
if (aiMsg) { aiMsg.content = '抱歉，AI 响应超时，请稍后再试。'; aiMsg.loading = false; this.messageList = [...this.messageList]; }
else { this.messageList.push({ role: 'ai', content: '抱歉，AI 响应超时，请稍后再试。' }); }
this.scrollToBottom();
return;
}
try {
const res = await api.cozeRetrieveChat({
conversationId,
chatId
});
const retrieveData = this.unwrapCozeResponse(res);
if (retrieveData) {
const chatObj = retrieveData.chat || retrieveData;
const status = chatObj && chatObj.status;
if (status === 'completed') {
await this.getChatMessages(conversationId, chatId, aiMsg);
} else if (status === 'failed' || status === 'canceled') {
this.isLoading = false;
const failMsg = `抱歉，对话${status === 'canceled' ? '已取消' : '失败'}。`;
if (aiMsg) { aiMsg.content = failMsg; aiMsg.loading = false; this.messageList = [...this.messageList]; }
else { this.messageList.push({ role: 'ai', content: failMsg }); }
this.scrollToBottom();
} else {
setTimeout(checkStatus, this.getPollInterval(attempts));
}
} else {
setTimeout(checkStatus, this.getPollInterval(attempts));
}
} catch (e) {
console.error('查询对话状态失败:', e);
setTimeout(checkStatus, this.getPollInterval(attempts));
}
};
checkStatus();
// ---------- TTS 方法 ----------
initAudioContext() {
// #ifdef MP-WEIXIN || APP-PLUS
const ctx = uni.createInnerAudioContext()
ctx.onEnded(() => {
this.ttsPlaying = false
this.ttsPlayingIndex = -1
})
ctx.onError((e) => {
console.error('[TTS] 播放出错', e)
this.ttsPlaying = false
this.ttsPlayingIndex = -1
})
this.innerAudioContext = ctx
// #endif
},
async getChatMessages(conversationId, chatId, aiMsg) {
try {
const res = await api.cozeMessageList({
conversationId,
chatId
});
this.isLoading = false;
const msgData = this.unwrapCozeResponse(res);
console.log("====api.cozeMessageList response====", msgData);
const rawMessages = msgData && (Array.isArray(msgData.messages) ? msgData.messages : (Array.isArray(msgData) ? msgData : null));
if (rawMessages && rawMessages.length > 0) {
// 过滤出 type='answer' 且 role='assistant' 的消息
const answerMsgs = rawMessages.filter(msg => msg.role === 'assistant' && msg.type === 'answer');
if (answerMsgs.length > 0) {
// 用第一条 answer 消息填充占位气泡，多条追加新气泡
if (aiMsg) {
aiMsg.content = answerMsgs[0].content;
aiMsg.loading = false;
for (let i = 1; i < answerMsgs.length; i++) {
this.messageList.push({ role: 'ai', content: answerMsgs[i].content });
}
this.messageList = [...this.messageList]; // 触发响应式更新
} else {
for (const msg of answerMsgs) { this.messageList.push({ role: 'ai', content: msg.content }); }
}
} else {
// 尝试查找其他类型的回复
const otherMsgs = rawMessages.filter(msg => msg.role === 'assistant');
const fallback = otherMsgs.length > 0 ? otherMsgs[0].content : '未能获取到有效回复。';
if (aiMsg) { aiMsg.content = fallback; aiMsg.loading = false; this.messageList = [...this.messageList]; }
else { this.messageList.push({ role: 'ai', content: fallback }); }
}
this.scrollToBottom();
} else {
throw new Error('获取消息列表失败');
}
} catch (e) {
console.error('获取消息详情失败:', e);
this.isLoading = false;
const errContent = '获取回复内容失败。';
if (aiMsg) { aiMsg.content = errContent; aiMsg.loading = false; this.messageList = [...this.messageList]; }
else { this.messageList.push({ role: 'ai', content: errContent }); }
this.scrollToBottom();
toggleTTS() {
this.ttsEnabled = !this.ttsEnabled
if (!this.ttsEnabled && this.ttsPlaying) {
this.stopTTS()
}
},
async playTTS(index) {
const msg = this.messageList[index]
if (!msg || !msg.content) return
if (this.ttsPlaying) {
this.stopTTS()
}
try {
const tempPath = await api.cozeTextToSpeech({ input: msg.content })
if (!this.innerAudioContext) {
console.warn('[TTS] innerAudioContext 未初始化')
return
}
this.ttsPlayingIndex = index
this.ttsPlaying = true
this.innerAudioContext.src = tempPath
this.innerAudioContext.play()
} catch (e) {
console.error('[TTS] 合成失败', e)
uni.showToast({ title: '语音合成失败', icon: 'none' })
this.ttsPlaying = false
this.ttsPlayingIndex = -1
}
},
stopTTS() {
if (this.innerAudioContext) {
this.innerAudioContext.stop()
}
this.ttsPlaying = false
this.ttsPlayingIndex = -1
},
// ---------- 滚动 ----------
scrollToBottom() {
this.$nextTick(() => {
// 动态切换 scrollTop 值以触发滚动更新
@@ -1085,6 +940,60 @@ export default {
font-weight: 500;
}
.tts-toggle-btn {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
padding: 16rpx;
background: rgba(255, 255, 255, 0.9);
border-radius: 16rpx;
box-shadow: 0 2rpx 8rpx rgba(0, 0, 0, 0.1);
transition: all 0.3s ease;
margin-right: 12rpx;
&.active {
background: rgba(76, 175, 80, 0.15);
box-shadow: 0 2rpx 8rpx rgba(76, 175, 80, 0.3);
}
}
.tts-toggle-btn:active {
transform: scale(0.95);
}
.tts-toggle-icon {
font-size: 32rpx;
margin-bottom: 4rpx;
}
.tts-toggle-text {
font-size: 20rpx;
color: #4caf50;
font-weight: 500;
}
.tts-play-btn {
display: inline-flex;
align-items: center;
justify-content: center;
margin-top: 8rpx;
width: 48rpx;
height: 48rpx;
background: rgba(76, 175, 80, 0.12);
border-radius: 50%;
cursor: pointer;
}
.tts-play-btn:active {
transform: scale(0.9);
}
.tts-play-icon {
font-size: 24rpx;
color: #4caf50;
}
.promo-avatar {
width: 180rpx;
height: 180rpx;