scottpan/msh-system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ai-nutritionist): 图片与文字合并为一次 KieAI 多模态请求

Browse Source 
- 后端: buildGeminiRequestBody 支持 content[] 中 Map 形式的多模态项
- 前端: sendMessage 将多图+文字合并为一条 content 数组，一次 sendToAI(multimodal)
- 仅发图时补默认文案「请描述或分析这张图片」，统一走 KieAI

Made-with: Cursor
This commit is contained in:
scottpan
2026-03-03 00:36:28 +08:00
parent 51d2016988
commit 1ddb051977
2 changed files with 186 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

260
msh_single_uniapp/pages/tool/ai-nutritionist.vue
View File

@@ -10,9 +10,15 @@
<text class="promo-sparkle">✦</text>
</view>
<text class="promo-subtitle">营养师专家入驻，在线答疑</text>
</view>
<view class="promo-right">
<view class="clear-btn" @click="clearChat">
<text class="clear-icon">🗑️</text>
<text class="clear-text">清空</text>
</view>
<image class="promo-avatar" src="https://uthink2025.oss-cn-shanghai.aliyuncs.com//crmebimage/public/content/2026/01/11/afcaba68d00b4fccaa49ad2a42c78e7fkk03hqv5vl.png" mode="aspectFit"></image>
</view>
</view>
<image class="promo-avatar" src="https://uthink2025.oss-cn-shanghai.aliyuncs.com//crmebimage/public/content/2026/01/11/afcaba68d00b4fccaa49ad2a42c78e7fkk03hqv5vl.png" mode="aspectFit"></image>
</view>
</view>
<!-- 聊天消息列表 -->
@@ -36,7 +42,7 @@
<text class="message-text">{{ welcomeMessage }}</text>
</view>
</view>
</view>
</view>
<!-- 消息列表 -->
<view
@@ -67,7 +73,7 @@
></image>
</view>
</view>
</view>
</view>
<!-- 加载中提示 -->
<view v-if="isLoading" class="message-item ai-message">
@@ -86,18 +92,18 @@
</view>
</view>
</view>
</view>
</view>
</view>
</scroll-view>
<!-- 快捷问题区域 -->
<view class="quick-questions">
<view class="quick-btn" @click="sendQuickQuestion('透析患者可以喝牛奶吗？')">
<text>透析患者可以喝牛奶吗？</text>
</view>
</view>
<view class="quick-btn" @click="sendQuickQuestion('什么食物含磷比较低？')">
<text>什么食物含磷比较低？</text>
</view>
</view>
</view>
<!-- 输入区域 -->
@@ -109,24 +115,24 @@
<view class="delete-btn" @click="removeImage(index)">
<text class="delete-icon">×</text>
</view>
</view>
</view>
<!-- 继续上传按钮 -->
<view class="preview-item add-btn" @click="chooseImage" v-if="pendingImages.length < 3">
<text class="add-icon">+</text>
</view>
</view>
</view>
<view class="input-wrapper">
<!-- 照片上传按钮 -->
<view class="action-btn" @click="chooseImage">
<image class="action-icon-svg" src="/static/images/icon-camera.svg" mode="aspectFit"></image>
</view>
</view>
<!-- 语音切换按钮 -->
<view class="action-btn" @click="toggleVoiceMode">
<image v-if="!isVoiceMode" class="action-icon-svg mic" src="/static/images/icon-mic.svg" mode="aspectFit"></image>
<text v-else class="action-icon">⌨️</text>
</view>
</view>
<!-- 文本输入框 -->
<input
@@ -151,7 +157,7 @@
@touchend="stopRecord"
>
<text>{{ isRecording ? '松开 结束' : '按住 说话' }}</text>
</view>
</view>
<!-- 发送按钮 -->
<view
@@ -171,8 +177,8 @@
src="/static/images/icon-send.svg"
mode="aspectFit"
></image>
</view>
</view>
</view>
</view>
</view>
</template>
@@ -490,6 +496,23 @@ export default {
this.pendingImages.splice(index, 1);
},
/** 将本地图片转为 data URL，用于 KieAI 多模态合并请求 */
readFileAsDataUrl(filePath) {
return new Promise((resolve, reject) => {
const fs = uni.getFileSystemManager();
const isJpeg = /\.(jpe?g|jfif)$/i.test(filePath);
fs.readFile({
filePath,
encoding: 'base64',
success: (res) => {
const mime = isJpeg ? 'image/jpeg' : 'image/png';
resolve(`data:${mime};base64,${res.data}`);
},
fail: reject
});
});
},
sendImageMessage(imageUrl, fileInfo) {
// 该方法已废弃，逻辑合并到 sendMessage
},
@@ -539,89 +562,106 @@ export default {
return
}
// 先处理图片消息
const imagesToSend = [...this.pendingImages];
this.pendingImages = []; // 清空待发送图片
const text = this.inputText.trim();
this.pendingImages = [];
this.inputText = '';
// 先展示用户消息到界面
for (const img of imagesToSend) {
// 添加用户图片消息到界面
this.messageList.push({
role: 'user',
type: 'image',
imageUrl: img.path,
content: '[图片]'
});
// 发送图片给AI
// 注意：这里我们不等待图片发送完成，而是让它并行处理，
// 或者如果需要严格顺序，可以 await this.sendToAI(...)
// 但考虑到用户体验，我们先显示，后台发送
await this.sendToAI(img.fileInfo || img.path, 'image');
}
if (text) {
this.messageList.push({ role: 'user', content: text, type: 'text' });
}
this.scrollToBottom();
// 合并为一次多模态请求：图片(base64) + 文字，统一走 KieAI Gemini
const contentParts = [];
try {
for (const img of imagesToSend) {
const dataUrl = await this.readFileAsDataUrl(img.path);
contentParts.push({ type: 'image_url', image_url: { url: dataUrl } });
}
if (text) {
contentParts.push({ type: 'text', text });
} else if (contentParts.length > 0) {
contentParts.push({ type: 'text', text: '请描述或分析这张图片' });
}
} catch (e) {
console.error('读取图片失败:', e);
this.messageList.push({ role: 'ai', content: '读取图片失败，请重试。' });
this.scrollToBottom();
return;
}
// 处理文本消息
if (this.inputText.trim()) {
const text = this.inputText.trim();
// 添加用户文本消息到界面
const userMessage = {
role: 'user',
content: text,
type: 'text'
};
this.messageList.push(userMessage);
// 清空输入框
this.inputText = ''
// 滚动到底部
this.scrollToBottom()
await this.sendToAI(text, 'text');
if (contentParts.length === 0) return;
// 仅文字时走纯文字接口；否则走多模态（图+文）一次请求
if (contentParts.length === 1 && contentParts[0].type === 'text') {
await this.sendToAI(contentParts[0].text, 'text');
} else {
await this.sendToAI(contentParts, 'multimodal');
}
this.scrollToBottom();
},
async sendToAI(content, type) {
// 显示加载中
this.isLoading = true
// 获取用户信息，提供 fallback
const userId = this.uid || (uni.getStorageSync('userInfo') || {}).id || 'default_user';
this.isLoading = true;
// 纯文字 或 多模态(图+文) 均走 KieAI Gemini，一次请求
if (type === 'text' || type === 'multimodal') {
try {
const messages = [{ role: 'user', content: content }];
const response = await api.kieaiGeminiChat({ messages, stream: false });
this.isLoading = false;
if (response && response.code === 200 && response.data) {
const data = response.data;
const choice = data.choices && data.choices[0];
const text = choice && choice.message && (choice.message.content || choice.message.text);
const reply = (typeof text === 'string' ? text : (text && String(text))) || '未能获取到有效回复。';
this.messageList.push({ role: 'ai', content: reply });
} else {
const msg = (response && response.message) || '发起对话失败';
this.messageList.push({ role: 'ai', content: '请求失败：' + msg });
}
this.scrollToBottom();
} catch (error) {
console.error('KieAI 对话失败:', error);
this.isLoading = false;
const errMsg = (error && (error.message || error.msg)) || '请稍后再试';
this.messageList.push({ role: 'ai', content: '请求失败：' + errMsg });
this.scrollToBottom();
}
return;
}
// 仅图片且未合并（旧 Coze 路径，保留兼容）
const userId = this.uid || (uni.getStorageSync('userInfo') || {}).id || 'default_user';
try {
// 构建 Coze 接口参数
const messages = [];
if (type === 'text') {
let fileInfo = content;
if (typeof fileInfo === 'string') {
try { fileInfo = JSON.parse(fileInfo); } catch (e) { /* 非JSON */ }
}
const fileId = (fileInfo && fileInfo.id) || (fileInfo && fileInfo.file_id) || '';
if (fileId) {
messages.push({
role: 'user',
type: 'question',
content: content,
content: JSON.stringify([{ type: 'image', file_id: fileId }]),
content_type: 'object_string'
});
} else {
messages.push({
role: 'user',
content: '我发送了一张图片，请帮我分析',
content_type: 'text'
});
} else if (type === 'image') {
// 解析 fileInfo，可能是对象或 JSON 字符串
let fileInfo = content;
if (typeof fileInfo === 'string') {
try { fileInfo = JSON.parse(fileInfo); } catch (e) { /* 非JSON，保持原值 */ }
}
const fileId = (fileInfo && fileInfo.id) || (fileInfo && fileInfo.file_id) || '';
if (fileId) {
messages.push({
role: 'user',
content: JSON.stringify([{ type: 'image', file_id: fileId }]),
content_type: 'object_string'
});
} else {
// 降级处理
messages.push({
role: 'user',
content: '我发送了一张图片，请帮我分析',
content_type: 'text'
});
}
}
const requestData = {
botId: this.botId,
userId: userId,
@@ -629,25 +669,18 @@ export default {
stream: false,
autoSaveHistory: true
};
// 如果有已存在的会话ID，传入以保持多轮对话上下文
if (this.conversationId) {
requestData.conversationId = this.conversationId;
}
// 调用 Coze Chat 接口
if (this.conversationId) requestData.conversationId = this.conversationId;
const response = await api.cozeChat(requestData);
// 处理响应
// Coze 非流式返回会包含 data 字段，其中有 conversation_id 和 id (chat_id)
// 以及 status (created, in_progress, completed, failed, requires_action)
if (response && response.data) {
console.log("====api.cozeChat response====", response.data);
const { conversation_id, id: chat_id } = response.data.chat;
// 存储会话ID用于后续多轮对话
if (conversation_id) {
this.conversationId = conversation_id;
const chat = response.data.chat || response.data;
const conversationId = chat.conversation_id || chat.conversationID || chat.conversationId;
const chatId = chat.id;
if (conversationId && chatId) {
this.conversationId = conversationId;
await this.pollChatStatus(conversationId, chatId);
} else {
throw new Error('发起对话失败：未返回会话或对话ID');
}
// 轮询查询对话状态
await this.pollChatStatus(conversation_id, chat_id);
} else {
throw new Error('发起对话失败');
}
@@ -656,7 +689,7 @@ export default {
this.isLoading = false;
this.messageList.push({
role: 'ai',
content: type === 'text' ? this.getAIResponse(content) : '抱歉，处理您的请求时出现错误，请稍后再试。'
content: '抱歉，处理您的请求时出现错误，请稍后再试。'
});
this.scrollToBottom();
}
@@ -686,7 +719,8 @@ export default {
if (res && res.data) {
console.log("====api.cozeRetrieveChat response====", res.data);
const status = res.data.chat.status;
const chatObj = res.data.chat || res.data;
const status = chatObj && chatObj.status;
if (status === 'completed') {
// 对话完成，获取消息详情
@@ -725,9 +759,10 @@ export default {
this.isLoading = false;
console.log("====api.cozeMessageList response====", res.data);
if (res && res.data && Array.isArray(res.data.messages)) {
const rawMessages = res && res.data && (Array.isArray(res.data.messages) ? res.data.messages : (Array.isArray(res.data) ? res.data : null));
if (rawMessages && rawMessages.length > 0) {
// 过滤出 type='answer' 且 role='assistant' 的消息
const answerMsgs = res.data.messages.filter(msg => msg.role === 'assistant' && msg.type === 'answer');
const answerMsgs = rawMessages.filter(msg => msg.role === 'assistant' && msg.type === 'answer');
if (answerMsgs.length > 0) {
// 可能有多条回复，依次显示
@@ -739,7 +774,7 @@ export default {
}
} else {
// 尝试查找其他类型的回复
const otherMsgs = res.data.messages.filter(msg => msg.role === 'assistant');
const otherMsgs = rawMessages.filter(msg => msg.role === 'assistant');
if (otherMsgs.length > 0) {
for (const msg of otherMsgs) {
this.messageList.push({
@@ -895,12 +930,45 @@ export default {
font-weight: 400;
}
.promo-right {
display: flex;
align-items: center;
gap: 20rpx;
z-index: 1;
}
.clear-btn {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
padding: 16rpx;
background: rgba(255, 255, 255, 0.9);
border-radius: 16rpx;
box-shadow: 0 2rpx 8rpx rgba(0, 0, 0, 0.1);
transition: all 0.3s ease;
}
.clear-btn:active {
transform: scale(0.95);
background: rgba(255, 255, 255, 1);
box-shadow: 0 4rpx 12rpx rgba(0, 0, 0, 0.15);
}
.clear-icon {
font-size: 32rpx;
margin-bottom: 4rpx;
}
.clear-text {
font-size: 22rpx;
color: #ff6b35;
font-weight: 500;
}
.promo-avatar {
width: 180rpx;
height: 180rpx;
position: absolute;
right: 20rpx;
bottom: 0;
}
/* 聊天容器 */