feat: vision model (#489)

* mongo init

* perf: mongo connect

* perf: tts

perf: whisper and tts

peref: tts whisper permission

log

reabase (#488)

* perf: modal

* i18n

* perf: schema lean

* feat: vision model format

* perf: tts loading

* perf: static data

* perf: tts

* feat: image

* perf: image

* perf: upload image and title

* perf: image size

* doc

* perf: color

* doc

* speaking can not select file

* doc

Author: c121914yu

docSite/content/docs/development/configuration.md

@@ -36,6 +36,7 @@ weight: 520
       "quoteMaxToken": 2000, // 最大引用内容长度
       "maxTemperature": 1.2, // 最大温度值
       "censor": false, // 是否开启敏感词过滤(商业版)
+      "vision": false, // 支持图片输入
       "defaultSystemChatPrompt": ""
     },
     {
@@ -47,6 +48,7 @@ weight: 520
       "quoteMaxToken": 8000,
       "maxTemperature": 1.2,
       "censor": false,
+      "vision": false,
       "defaultSystemChatPrompt": ""
     },
     {
@@ -58,6 +60,19 @@ weight: 520
       "quoteMaxToken": 4000,
       "maxTemperature": 1.2,
       "censor": false,
+      "vision": false,
+      "defaultSystemChatPrompt": ""
+    },
+    {
+      "model": "gpt-4-vision-preview",
+      "name": "GPT4-Vision",
+      "maxContext": 128000,
+      "maxResponse": 4000,
+      "price": 0,
+      "quoteMaxToken": 100000,
+      "maxTemperature": 1.2,
+      "censor": false,
+      "vision": true,
       "defaultSystemChatPrompt": ""
     }
   ],

docSite/content/docs/installation/upgrading/461.md

@@ -0,0 +1,16 @@
+---
+title: 'V4.6.1'
+description: 'FastGPT V4.6 .1'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 835
+---
+
+
+## V4.6.1 功能介绍
+
+1. 新增 - GPT4-v 模型支持
+2. 新增 - whisper 语音输入
+3. 优化 - TTS 流传输
+4. 优化 - TTS 缓存

packages/global/common/string/tools.ts

@@ -24,7 +24,7 @@ export const simpleText = (text: string) => {
 };
 
 /* 
-    replace {{variable}} to value
+  replace {{variable}} to value
 */
 export function replaceVariable(text: string, obj: Record<string, string | number>) {
   for (const key in obj) {

packages/global/core/ai/model.d.ts

@@ -9,6 +9,7 @@ export type ChatModelItemType = LLMModelItemType & {
   quoteMaxToken: number;
   maxTemperature: number;
   censor?: boolean;
+  vision?: boolean;
   defaultSystemChatPrompt?: string;
 };
 

packages/global/core/ai/model.ts

@@ -17,6 +17,7 @@ export const defaultChatModels: ChatModelItemType[] = [
     quoteMaxToken: 2000,
     maxTemperature: 1.2,
     censor: false,
+    vision: false,
     defaultSystemChatPrompt: ''
   },
   {
@@ -28,6 +29,7 @@ export const defaultChatModels: ChatModelItemType[] = [
     quoteMaxToken: 8000,
     maxTemperature: 1.2,
     censor: false,
+    vision: false,
     defaultSystemChatPrompt: ''
   },
   {
@@ -39,6 +41,19 @@ export const defaultChatModels: ChatModelItemType[] = [
     quoteMaxToken: 4000,
     maxTemperature: 1.2,
     censor: false,
+    vision: false,
+    defaultSystemChatPrompt: ''
+  },
+  {
+    model: 'gpt-4-vision-preview',
+    name: 'GPT4-Vision',
+    maxContext: 128000,
+    maxResponse: 4000,
+    price: 0,
+    quoteMaxToken: 100000,
+    maxTemperature: 1.2,
+    censor: false,
+    vision: true,
     defaultSystemChatPrompt: ''
   }
 ];

packages/global/core/ai/type.d.ts

@@ -5,12 +5,14 @@ import type {
   ChatCompletionMessageParam,
   ChatCompletionContentPart
 } from 'openai/resources';
+
 export type ChatCompletionContentPart = ChatCompletionContentPart;
 export type ChatCompletionCreateParams = ChatCompletionCreateParams;
-export type ChatMessageItemType = Omit<ChatCompletionMessageParam> & {
+export type ChatMessageItemType = Omit<ChatCompletionMessageParam, 'name'> & {
+  name?: any;
   dataId?: string;
   content: any;
-};
+} & any;
 
 export type ChatCompletion = ChatCompletion;
 export type StreamChatType = Stream<ChatCompletionChunk>;

packages/global/core/chat/constants.ts

@@ -54,3 +54,6 @@ export const ChatSourceMap = {
 
 export const HUMAN_ICON = `/icon/human.svg`;
 export const LOGO_ICON = `/icon/logo.svg`;
+
+export const IMG_BLOCK_KEY = 'img-block';
+export const FILE_BLOCK_KEY = 'file-block';

packages/global/core/chat/utils.ts

@@ -0,0 +1,6 @@
+import { IMG_BLOCK_KEY, FILE_BLOCK_KEY } from './constants';
+
+export function chatContentReplaceBlock(content: string = '') {
+  const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
+  return content.replace(regex, '').trim();
+}

packages/service/common/buffer/tts/schema.ts

@@ -33,3 +33,4 @@ try {
 
 export const MongoTTSBuffer: Model<TTSBufferSchemaType> =
   models[collectionName] || model(collectionName, TTSBufferSchema);
+MongoTTSBuffer.syncIndexes();

packages/service/common/file/image/controller.ts

@@ -5,12 +5,26 @@ export function getMongoImgUrl(id: string) {
   return `${imageBaseUrl}${id}`;
 }
 
-export async function uploadMongoImg({ base64Img, userId }: { base64Img: string; userId: string }) {
+export const maxImgSize = 1024 * 1024 * 12;
+export async function uploadMongoImg({
+  base64Img,
+  teamId,
+  expiredTime
+}: {
+  base64Img: string;
+  teamId: string;
+  expiredTime?: Date;
+}) {
+  if (base64Img.length > maxImgSize) {
+    return Promise.reject('Image too large');
+  }
+
   const base64Data = base64Img.split(',')[1];
 
   const { _id } = await MongoImage.create({
-    userId,
-    binary: Buffer.from(base64Data, 'base64')
+    teamId,
+    binary: Buffer.from(base64Data, 'base64'),
+    expiredTime
   });
 
   return getMongoImgUrl(String(_id));