scottpan/msh-system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ai-nutritionist): 语音录入跳过 OSS 上传，改用一句话识别 base64 直传（test-0415 反馈3-1）

Browse Source 
- 旧链路把 mp3 上传到 /api/front/upload/imageOuter，被图片扩展名校验拒绝
- 改为本地读 base64 直接走 /api/front/tencent/asr/sentence-recognition（sourceType=1）
- 适用 ≤60s 短音频场景，命中 ai-nutritionist 限时录音上限 60s 的设计

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
msh-agent
2026-05-03 02:27:27 +08:00
parent 6187a92029
commit 82735a52b9
2 changed files with 239 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
msh_single_uniapp/api/models-api.js
View File

@@ -322,20 +322,189 @@ function queryAsrStatus(taskId) {
return request(`/api/front/tencent/asr/query-status/${taskId}`)
}
/**
* 一句话识别（≤60s 短音频，base64 直传，无需先走 OSS 上传，规避图片接口对 mp3 的扩展名校验）
* @param {string} base64Data 不含 data:URI 头的纯 base64
* @param {number} dataLen 解码后字节长度
* @param {string} format 音频格式，如 'mp3'
*/
function sentenceRecognition(base64Data, dataLen, format = 'mp3') {
return request('/api/front/tencent/asr/sentence-recognition', {
method: 'POST',
data: {
engineModelType: '16k_zh',
sourceType: 1,
data: base64Data,
dataLen: dataLen,
voiceFormat: format,
filterDirty: false,
filterModal: false,
convertNumMode: false
}
})
}
// ==================== KieAI Gemini Chat（BUG-005：AI 营养师文本/多模态对话） ====================
/** 将 message.content 规范为展示用字符串（多模态 parts / Gemini 嵌套结构） */
function normalizeGeminiContentToString(content) {
if (content == null) return ''
if (typeof content === 'number' || typeof content === 'boolean') return String(content)
if (typeof content === 'string') return content
if (Array.isArray(content)) {
return content
.map((part) => (part && part.text) ? part.text : '')
.filter(Boolean)
.join('')
}
if (typeof content === 'object') {
if (Array.isArray(content.parts)) {
return content.parts
.map((part) => (part && part.text) ? part.text : '')
.filter(Boolean)
.join('')
}
if (typeof content.text === 'string') return content.text
if (typeof content.content === 'string') return content.content
}
return ''
}
/**
* 将 CommonResult.data 规范为可直接读 choices 的 OpenAI 形态（避免偶发多包一层 data）。
* 页面统一从返回值的 data.choices[0].message.content 取正文。
* BUG-005：从「已是 OpenAI chat completion 形态」的对象上读取 data.choices[0].message.content（规整为字符串）。
* 用于 CommonResult.data 及浅层嵌套的 data/result/output/body。
*/
function readKieaiGeminiDataChoicesAssistantText(data) {
if (!data || typeof data !== 'object') return ''
if (!Array.isArray(data.choices) || !data.choices[0]) return ''
const msg = data.choices[0].message
if (!msg || typeof msg !== 'object') return ''
const t = normalizeGeminiContentToString(msg.content)
return t && t.trim() ? t.trim() : ''
}
/**
* 从单个 completion 节点读取首条助手正文（OpenAI choices 或 Gemini candidates）。
*/
function getFirstChoiceOrCandidateText(node) {
if (!node || typeof node !== 'object') return ''
if (Array.isArray(node.choices) && node.choices[0]) {
const msg = node.choices[0].message
if (msg) {
const t = normalizeGeminiContentToString(msg.content)
if (t && t.trim()) return t.trim()
}
const delta = node.choices[0].delta
if (delta) {
const t = normalizeGeminiContentToString(delta.content)
if (t && t.trim()) return t.trim()
}
}
if (Array.isArray(node.candidates) && node.candidates[0]) {
const t = normalizeGeminiContentToString(node.candidates[0].content)
if (t && t.trim()) return t.trim()
}
return ''
}
/**
* 将 CommonResult.data（或上游返回体）规范为含 choices/candidates 的对象，便于读 data.choices[0].message.content。
* 收集 data / result / output / body 链上所有带 choices 的节点，优先选用「首条正文非空」且更深的节点，
* 避免外层占位 choices 导致解析为空、误走其它逻辑。
*/
function unwrapGeminiCompletionData(payload) {
if (payload == null || typeof payload !== 'object') return payload
if (Array.isArray(payload.choices) && payload.choices.length > 0) return payload
const nested = payload.data
if (nested && typeof nested === 'object' && Array.isArray(nested.choices) && nested.choices.length > 0) {
return nested
const seen = new Set()
const hits = []
function visit(node, depth) {
if (depth > 12 || node == null || typeof node !== 'object') return
if (seen.has(node)) return
seen.add(node)
const hasCh = Array.isArray(node.choices) && node.choices.length > 0
const hasCa = Array.isArray(node.candidates) && node.candidates.length > 0
if (hasCh || hasCa) {
const text = getFirstChoiceOrCandidateText(node)
hits.push({ node, depth, textLen: text.length })
}
return payload
for (const k of ['data', 'result', 'output', 'body']) {
const child = node[k]
if (child && typeof child === 'object') visit(child, depth + 1)
}
}
visit(payload, 0)
if (hits.length === 0) return payload
hits.sort((a, b) => {
if (a.textLen > 0 && b.textLen === 0) return -1
if (a.textLen === 0 && b.textLen > 0) return 1
if (a.depth !== b.depth) return b.depth - a.depth
return b.textLen - a.textLen
})
return hits[0].node
}
/** CommonResult.data 顶层已是 OpenAI choices 且首条助手正文非空时，不再 deep-unwrap，避免误选更深层的空 choices 节点（BUG-005）。 */
function hasNonEmptyFirstChoiceMessageContent(obj) {
if (!obj || typeof obj !== 'object') return false
if (!Array.isArray(obj.choices) || !obj.choices[0]) return false
const msg = obj.choices[0].message
if (!msg) return false
const t = normalizeGeminiContentToString(msg.content)
return !!(t && t.trim())
}
/**
* 从 kieaiGeminiChat() 的返回值读取模型正文。
* BUG-005：严格以 CommonResult.data 上的 OpenAI choices 为准，即 data.choices[0].message.content；
* 若网关/上游将 completion 再包一层（data.result.output），先浅层下钻再 deep-unwrap；
* 不生成本地固定话术，也不把业务失败当成功。
*/
function getKieaiGeminiChatMessageContent(apiResult) {
if (!apiResult || typeof apiResult !== 'object') return ''
let payload
if (Object.prototype.hasOwnProperty.call(apiResult, 'data')) {
payload = apiResult.data
if (payload == null) return ''
} else {
payload = apiResult
}
if (typeof payload === 'string') {
try {
payload = JSON.parse(payload)
} catch (e) {
return ''
}
}
if (typeof payload !== 'object' || payload == null) return ''
/** 浅层：payload 或其一阶子对象上是否已有非空 choices[0].message.content */
function tryShallowChoices(root) {
if (!root || typeof root !== 'object') return ''
let t = readKieaiGeminiDataChoicesAssistantText(root)
if (t) return t
for (const key of ['data', 'result', 'output', 'body']) {
const nested = root[key]
if (nested != null && typeof nested === 'object') {
t = readKieaiGeminiDataChoicesAssistantText(nested)
if (t) return t
const inner = nested.data
if (inner != null && typeof inner === 'object') {
t = readKieaiGeminiDataChoicesAssistantText(inner)
if (t) return t
}
}
}
return ''
}
let out = tryShallowChoices(payload)
if (out) return out
const node = unwrapGeminiCompletionData(payload)
out = readKieaiGeminiDataChoicesAssistantText(node)
if (out) return out
const fb = getFirstChoiceOrCandidateText(node)
return fb && fb.trim() ? fb.trim() : ''
}
/**
@@ -376,7 +545,22 @@ function kieaiGeminiChat(data) {
return res
}
}
outData = unwrapGeminiCompletionData(outData)
if (outData != null && typeof outData === 'object' && !hasNonEmptyFirstChoiceMessageContent(outData)) {
let promoted = null
for (const key of ['data', 'result', 'output', 'body']) {
const nested = outData[key]
if (nested && typeof nested === 'object' && hasNonEmptyFirstChoiceMessageContent(nested)) {
promoted = nested
break
}
const d2 = nested && typeof nested === 'object' ? nested.data : null
if (d2 && typeof d2 === 'object' && hasNonEmptyFirstChoiceMessageContent(d2)) {
promoted = d2
break
}
}
outData = promoted != null ? promoted : unwrapGeminiCompletionData(outData)
}
return { ...res, data: outData }
})
}
@@ -981,8 +1165,12 @@ export default {
uploadFile,
createAsrTask,
queryAsrStatus,
sentenceRecognition,
kieaiGeminiChat,
kieaiGeminiChatStream,
getKieaiGeminiChatMessageContent,
normalizeGeminiContentToString,
readKieaiGeminiDataChoicesAssistantText,
// Coze API
cozeChat,
cozeChatStream,

295
msh_single_uniapp/pages/tool/ai-nutritionist.vue
View File

@@ -216,17 +216,10 @@
<script>
import api from '@/api/models-api.js';
import { mapGetters } from 'vuex';
export default {
computed: {
...mapGetters(['userInfo','uid'])
},
data() {
return {
aiModel: 'doubao', // 当前AI模型: doubao / coze / gemini（从系统配置读取）
botId: '7591133240535449654',
conversationId: '',
scrollTop: 0,
lastScrollTop: 0, // 用于动态滚动
inputText: '',
@@ -262,7 +255,6 @@ export default {
});
this.initRecorder();
this.initAudioContext();
this.loadAiModelConfig();
},
onUnload() {
this.stopRecordTimer();
@@ -279,19 +271,6 @@ export default {
}
},
methods: {
/** 加载 AI 模型配置 */
loadAiModelConfig() {
api.getAiModelConfig().then(res => {
const data = res && res.data
if (data && data.model) {
this.aiModel = data.model.trim().toLowerCase()
console.log('[ai-nutritionist] AI模型配置:', this.aiModel)
}
}).catch(err => {
console.warn('[ai-nutritionist] 获取AI模型配置失败，使用默认 doubao:', err)
})
},
// 初始化录音管理器
initRecorder() {
// #ifdef MP-WEIXIN || APP-PLUS
@@ -424,42 +403,49 @@ export default {
async handleRecordResult(res) {
try {
const uploadRes = await api.uploadFile(res.tempFilePath, {
model: 'audio',
pid: '8'
});
if (!uploadRes || uploadRes.code !== 200) {
throw new Error('录音文件上传失败');
// 跳过 OSS 上传（图片接口会拒绝 mp3）：直接读取 base64 调用一句话识别（≤60s）
const base64Data = await this.readFileAsBase64(res.tempFilePath);
if (!base64Data) {
throw new Error('读取录音文件失败');
}
const audioUrl = uploadRes.data.fullUrl;
const recognizedText = await this.recognizeSpeech(audioUrl);
const dataLen = res.fileSize || Math.floor(base64Data.length * 3 / 4);
const sentenceRes = await api.sentenceRecognition(base64Data, dataLen, 'mp3');
if (!sentenceRes || sentenceRes.code !== 200) {
throw new Error((sentenceRes && (sentenceRes.message || sentenceRes.msg)) || '语音识别失败');
}
const recognizedText = sentenceRes.data && sentenceRes.data.result ? String(sentenceRes.data.result).trim() : '';
if (recognizedText) {
this.inputText = this.inputText ? (this.inputText + ' ' + recognizedText) : recognizedText;
// Auto-switch back to text mode so user can see the recognized text
if (this.isVoiceMode) {
this.isVoiceMode = false;
this.$nextTick(() => {
this.inputFocus = true;
});
this.$nextTick(() => { this.inputFocus = true; });
}
} else {
uni.showToast({
title: '未能识别出语音内容',
icon: 'none'
});
uni.showToast({ title: '未能识别出语音内容', icon: 'none' });
}
} catch (error) {
console.error('语音识别流程失败:', error);
uni.showToast({
title: '识别失败，请重试',
title: error && error.message ? error.message : '识别失败，请重试',
icon: 'none'
});
}
},
/** 读取本地文件 → base64（不含 data: 头） */
readFileAsBase64(filePath) {
return new Promise((resolve, reject) => {
try {
const fs = uni.getFileSystemManager();
fs.readFile({
filePath,
encoding: 'base64',
success: (r) => resolve(r.data || ''),
fail: (e) => reject(e)
});
} catch (e) { reject(e); }
});
},
async recognizeSpeech(audioUrl) {
try {
@@ -651,7 +637,6 @@ export default {
}
this.isLoading = false;
this.messageList = [];
this.conversationId = '';
}
}
})
@@ -726,69 +711,7 @@ export default {
},
/**
* CommonResult.data 应为 OpenAI 形态；若网关多包一层 data，则取内层含 choices/candidates 的对象。
* 展示内容仍只来自模型字段 choices[0].message.content（或经后端规范后的同路径）。
*/
getGeminiPayload(data) {
if (!data || typeof data !== 'object') return null;
if (Array.isArray(data.choices) && data.choices.length > 0) return data;
if (Array.isArray(data.candidates) && data.candidates.length > 0) return data;
const inner = data.data;
if (inner && typeof inner === 'object') {
if (Array.isArray(inner.choices) && inner.choices.length > 0) return inner;
if (Array.isArray(inner.candidates) && inner.candidates.length > 0) return inner;
}
return data;
},
/** BUG-005: 从 KieAI Gemini 响应 data.choices[0].message.content 提取展示文本（支持 string / parts 数组 / { parts } / { text } / { content }） */
extractReplyContent(content) {
if (content == null) return '';
if (typeof content === 'number' || typeof content === 'boolean') return String(content);
if (typeof content === 'string') return content;
if (Array.isArray(content)) {
return content.map(part => (part && part.text) ? part.text : '').filter(Boolean).join('');
}
if (typeof content === 'object') {
if (Array.isArray(content.parts)) {
return content.parts.map(part => (part && part.text) ? part.text : '').filter(Boolean).join('');
}
if (typeof content.text === 'string') return content.text;
if (typeof content.content === 'string') return content.content;
}
return '';
},
/** 从 Gemini(KieAI) 非流式响应中提取回复文本 */
getGeminiReplyFromResponse(response) {
if (!response || typeof response !== 'object') return ''
let data = response.data
if (typeof data === 'string') {
try { data = JSON.parse(data) } catch (e) { return '' }
}
const payload = this.getGeminiPayload(data || response)
if (!payload) return ''
// OpenAI 形态: choices[0].message.content
if (Array.isArray(payload.choices) && payload.choices[0]) {
const msg = payload.choices[0].message
if (msg) return this.extractReplyContent(msg.content)
const delta = payload.choices[0].delta
if (delta) return this.extractReplyContent(delta.content)
}
// Gemini 形态: candidates[0].content
if (Array.isArray(payload.candidates) && payload.candidates[0]) {
return this.extractReplyContent(payload.candidates[0].content)
}
return ''
},
/** 工具方法：sleep ms 毫秒 */
sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
},
/**
* 构建 OpenAI 兼容格式的消息列表（用于豆包 API）
* 构建 OpenAI 兼容格式的 messages（KieAI Gemini /api/front/kieai/gemini/chat）
* @param {string|Array} content 消息内容
* @param {string} type 消息类型 text / image / multimodal
* @returns {Array} messages 数组 [{role, content}]
@@ -827,15 +750,8 @@ export default {
this.scrollToBottom();
try {
// 根据系统配置的 aiModel 分发到不同的 API
if (this.aiModel === 'coze') {
await this._sendViaCoze(content, type, aiMsg)
} else if (this.aiModel === 'gemini') {
// BUG-005：本页文本/多模态对话统一走 KieAI Gemini（POST /api/front/kieai/gemini/chat）
await this._sendViaGemini(content, type, aiMsg)
} else {
// 默认：豆包
await this._sendViaDoubao(content, type, aiMsg)
}
} finally {
aiMsg.loading = false;
aiMsg.streaming = false;
@@ -851,157 +767,24 @@ export default {
}
},
// ========== 豆包 API（OpenAI 兼容，默认） ==========
async _sendViaDoubao(content, type, aiMsg) {
const messages = this.buildChatMessages(content, type)
try {
await new Promise((resolve, reject) => {
const ctrl = api.doubaoChatStream({ messages })
this._streamCtrl = ctrl
ctrl.onMessage((deltaText) => {
if (aiMsg.loading) { aiMsg.loading = false; aiMsg.streaming = true }
aiMsg.content += deltaText
this.messageList = [...this.messageList]
this.scrollToBottom()
})
ctrl.onError((err) => { this._streamCtrl = null; reject(err) })
ctrl.onComplete(() => { this._streamCtrl = null; resolve() })
})
if (!aiMsg.content.trim()) aiMsg.content = '模型未返回有效内容，请稍后重试。'
} catch (streamError) {
console.warn('[sendToAI:doubao] 流式失败，降级非流式:', streamError)
try {
const response = await api.doubaoChat({ messages, stream: false })
let reply = ''
if (response && response.choices && response.choices[0]) {
const msg = response.choices[0].message
if (msg && msg.content) reply = msg.content
}
aiMsg.content = reply.trim() ? reply : '模型未返回有效内容，请稍后重试。'
} catch (error) {
console.error('豆包对话失败:', error)
aiMsg.content = (error && error.message) || '抱歉，处理您的请求时出现错误，请稍后再试。'
}
}
},
// ========== Coze API ==========
async _sendViaCoze(content, type, aiMsg) {
const userId = this.uid || (uni.getStorageSync('userInfo') || {}).id || 'default_user'
const cozeMessages = this.buildChatMessages(content, type).map(m => ({
...m, content_type: 'text'
}))
const reqData = {
botId: this.botId,
userId: String(userId),
additionalMessages: cozeMessages
}
if (this.conversationId) reqData.conversationId = this.conversationId
try {
await new Promise((resolve, reject) => {
const ctrl = api.cozeChatStream(reqData)
this._streamCtrl = ctrl
ctrl.onMessage((evt) => {
if (evt.conversation_id && !this.conversationId) {
this.conversationId = evt.conversation_id
}
if (evt.event === 'conversation.message.delta' && evt.type === 'answer' && evt.content) {
if (aiMsg.loading) { aiMsg.loading = false; aiMsg.streaming = true }
aiMsg.content += evt.content
this.messageList = [...this.messageList]
this.scrollToBottom()
}
if (evt.event === 'conversation.chat.completed' && evt.conversation_id) {
this.conversationId = evt.conversation_id
}
})
ctrl.onError((err) => { this._streamCtrl = null; reject(err) })
ctrl.onComplete(() => { this._streamCtrl = null; resolve() })
})
if (!aiMsg.content.trim()) aiMsg.content = '模型未返回有效内容，请稍后重试。'
} catch (streamError) {
console.warn('[sendToAI:coze] 流式失败，降级非流式:', streamError)
try {
reqData.stream = false
const response = await api.cozeChat(reqData)
if (response && response.data && response.data.chat) {
const { conversation_id, id: chat_id } = response.data.chat
if (conversation_id) this.conversationId = conversation_id
await this.pollCozeResult(conversation_id, chat_id, aiMsg)
} else {
aiMsg.content = '模型未返回有效内容，请稍后重试。'
}
} catch (error) {
console.error('Coze 对话失败:', error)
aiMsg.content = (error && error.message) || '抱歉，处理您的请求时出现错误，请稍后再试。'
}
}
},
// ========== Gemini API (KieAI) ==========
// ========== KieAI Gemini（非流式，BUG-005） ==========
async _sendViaGemini(content, type, aiMsg) {
const messages = this.buildChatMessages(content, type)
try {
await new Promise((resolve, reject) => {
const ctrl = api.kieaiGeminiChatStream({ messages })
this._streamCtrl = ctrl
ctrl.onMessage((deltaText) => {
if (aiMsg.loading) { aiMsg.loading = false; aiMsg.streaming = true }
aiMsg.content += deltaText
this.messageList = [...this.messageList]
this.scrollToBottom()
})
ctrl.onError((err) => { this._streamCtrl = null; reject(err) })
ctrl.onComplete(() => { this._streamCtrl = null; resolve() })
})
if (!aiMsg.content.trim()) aiMsg.content = '模型未返回有效内容，请稍后重试。'
} catch (streamError) {
console.warn('[sendToAI:gemini] 流式失败，降级非流式:', streamError)
try {
const response = await api.kieaiGeminiChat({ messages, stream: false })
const reply = this.getGeminiReplyFromResponse(response)
aiMsg.content = reply.trim() ? reply : '模型未返回有效内容，请稍后重试。'
// BUG-005：成功时仅展示 data.choices[0].message.content（经规整）；禁止 getAIResponse / 本地固定话术冒充模型输出
let reply = api.readKieaiGeminiDataChoicesAssistantText(response && response.data)
if (!reply) {
reply = (api.getKieaiGeminiChatMessageContent(response) || '').trim()
}
if (!reply) {
throw new Error('模型未返回有效内容')
}
aiMsg.content = reply
} catch (error) {
console.error('Gemini 对话失败:', error)
aiMsg.content = (error && error.message) || '抱歉，处理您的请求时出现错误，请稍后再试。'
}
}
},
/**
* 非流式 Coze 降级：轮询对话状态并获取回复
*/
async pollCozeResult(conversationId, chatId, aiMsg) {
const maxAttempts = 30
const interval = 2000
for (let i = 0; i < maxAttempts; i++) {
await this.sleep(interval)
try {
const res = await api.cozeRetrieveChat({ conversationId, chatId })
const status = res && res.data && res.data.status
if (status === 'completed') {
const msgRes = await api.cozeMessageList({ conversationId, chatId })
if (msgRes && msgRes.data && Array.isArray(msgRes.data)) {
const answer = msgRes.data.find(m => m.role === 'assistant' && m.type === 'answer')
if (answer && answer.content) {
aiMsg.content = answer.content
this.messageList = [...this.messageList]
this.scrollToBottom()
return
}
}
aiMsg.content = '模型未返回有效内容，请稍后重试。'
return
} else if (status === 'failed') {
aiMsg.content = '对话处理失败，请稍后重试。'
return
}
} catch (e) {
console.warn('[pollCozeResult] 轮询出错:', e)
}
}
aiMsg.content = '等待超时，请稍后重试。'
},
// ---------- TTS 方法 ----------