4.7.1 production (#1173)

Co-authored-by: heheer <[email protected]>

Author: c121914yu

docSite/content/docs/development/upgrading/471.md

@@ -1,5 +1,5 @@
 ---
-title: 'V4.7.1(进行中)'
+title: 'V4.7.1(需要初始化)'
 description: 'FastGPT V4.7.1 更新说明'
 icon: 'upgrade'
 draft: false
@@ -32,8 +32,10 @@ curl --location --request POST 'https://{{host}}/api/admin/clearInvalidData' \
 3. 新增 - 集成 Laf 云函数，可以读取 Laf 账号中的云函数作为 HTTP 模块。
 4. 新增 - 定时器，清理垃圾数据。（采用小范围清理，会清理最近n个小时的，所以请保证服务持续运行，长时间不允许，可以继续执行 clearInvalidData 的接口进行全量清理。）
 5. 商业版新增 - 后台配置系统通知。
-6. 修改 - csv导入模板，取消 header 校验，自动获取前两列。
-7. 修复 - 工具调用模块连线数据类型校验错误。
-8. 修复 - 自定义索引输入时，解构数据失败。
-9. 修复 - rerank 模型数据格式。
-10. 修复 - 问题补全历史记录BUG
+6. 优化 - 支持ip模式导出知识库。
+7. 修改 - csv导入模板，取消 header 校验，自动获取前两列。
+8. 修复 - 工具调用模块连线数据类型校验错误。
+9. 修复 - 自定义索引输入时，解构数据失败。
+10. 修复 - rerank 模型数据格式。
+11. 修复 - 问题补全历史记录BUG
+12. 修复 - 分享页面特殊情况下加载缓慢问题（由于ssr时候数据库不会触发连接）

packages/global/common/file/tools.ts

@@ -10,6 +10,6 @@ export const formatFileSize = (bytes: number): string => {
   return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
 };
 
-export const detectFileEncoding = (buffers: string | Buffer) => {
-  return (detect(buffers)?.encoding || 'utf-8') as BufferEncoding;
+export const detectFileEncoding = (buffer: Buffer) => {
+  return detect(buffer.slice(0, 200))?.encoding?.toLocaleLowerCase();
 };

packages/service/common/file/gridfs/controller.ts

@@ -9,6 +9,7 @@ import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import { ReadFileByBufferParams } from '../read/type';
 import { MongoRwaTextBuffer } from '../../buffer/rawText/schema';
 import { readFileRawContent } from '../read/utils';
+import { PassThrough } from 'stream';
 
 export function getGFSCollection(bucket: `${BucketNameEnum}`) {
   MongoFileSchema;
@@ -113,31 +114,39 @@ export async function getDownloadStream({
   fileId: string;
 }) {
   const bucket = getGridBucket(bucketName);
+  const stream = bucket.openDownloadStream(new Types.ObjectId(fileId));
+  const copyStream = stream.pipe(new PassThrough());
 
-  return bucket.openDownloadStream(new Types.ObjectId(fileId));
-}
-
-export const readFileEncode = async ({
-  bucketName,
-  fileId
-}: {
-  bucketName: `${BucketNameEnum}`;
-  fileId: string;
-}) => {
-  const encodeStream = await getDownloadStream({ bucketName, fileId });
-  let buffers: Buffer = Buffer.from([]);
-  for await (const chunk of encodeStream) {
-    buffers = Buffer.concat([buffers, chunk]);
-    if (buffers.length > 10) {
-      encodeStream.abort();
-      break;
-    }
-  }
+  /* get encoding */
+  const buffer = await (() => {
+    return new Promise<Buffer>((resolve, reject) => {
+      let tmpBuffer: Buffer = Buffer.from([]);
+
+      stream.on('data', (chunk) => {
+        if (tmpBuffer.length < 20) {
+          tmpBuffer = Buffer.concat([tmpBuffer, chunk]);
+        }
+        if (tmpBuffer.length >= 20) {
+          resolve(tmpBuffer);
+        }
+      });
+      stream.on('end', () => {
+        resolve(tmpBuffer);
+      });
+      stream.on('error', (err) => {
+        reject(err);
+      });
+    });
+  })();
 
-  const encoding = detectFileEncoding(buffers);
+  const encoding = detectFileEncoding(buffer);
 
-  return encoding as BufferEncoding;
-};
+  return {
+    fileStream: copyStream,
+    encoding
+    // encoding: 'utf-8'
+  };
+}
 
 export const readFileContentFromMongo = async ({
   teamId,
@@ -162,9 +171,8 @@ export const readFileContentFromMongo = async ({
     };
   }
 
-  const [file, encoding, fileStream] = await Promise.all([
+  const [file, { encoding, fileStream }] = await Promise.all([
     getFileById({ bucketName, fileId }),
-    readFileEncode({ bucketName, fileId }),
     getDownloadStream({ bucketName, fileId })
   ]);
 
@@ -176,12 +184,12 @@ export const readFileContentFromMongo = async ({
 
   const fileBuffers = await (() => {
     return new Promise<Buffer>((resolve, reject) => {
-      let buffers = Buffer.from([]);
+      let buffer = Buffer.from([]);
       fileStream.on('data', (chunk) => {
-        buffers = Buffer.concat([buffers, chunk]);
+        buffer = Buffer.concat([buffer, chunk]);
       });
       fileStream.on('end', () => {
-        resolve(buffers);
+        resolve(buffer);
       });
       fileStream.on('error', (err) => {
         reject(err);

packages/service/common/file/read/pptx.ts

@@ -6,7 +6,11 @@ export const readPptxRawText = async ({
   buffer,
   encoding
 }: ReadFileByBufferParams): Promise<ReadFileResponse> => {
-  const result = await parseOffice({ buffer, encoding, extension: 'pptx' });
+  const result = await parseOffice({
+    buffer,
+    encoding: encoding as BufferEncoding,
+    extension: 'pptx'
+  });
 
   return {
     rawText: result

packages/service/common/file/read/rawText.ts

@@ -1,8 +1,26 @@
 import { ReadFileByBufferParams, ReadFileResponse } from './type.d';
+import iconv from 'iconv-lite';
+
+const rawEncodingList = [
+  'ascii',
+  'utf8',
+  'utf-8',
+  'utf16le',
+  'utf-16le',
+  'ucs2',
+  'ucs-2',
+  'base64',
+  'base64url',
+  'latin1',
+  'binary',
+  'hex'
+];
 
 // 加载源文件内容
 export const readFileRawText = ({ buffer, encoding }: ReadFileByBufferParams): ReadFileResponse => {
-  const content = buffer.toString(encoding);
+  const content = rawEncodingList.includes(encoding)
+    ? buffer.toString(encoding as BufferEncoding)
+    : iconv.decode(buffer, 'gbk');
 
   return {
     rawText: content

packages/service/common/file/read/type.d.ts

@@ -1,7 +1,7 @@
 export type ReadFileByBufferParams = {
   teamId: string;
   buffer: Buffer;
-  encoding: BufferEncoding;
+  encoding: string;
   metadata?: Record<string, any>;
 };
 

packages/service/common/vectorStore/pg/controller.ts

@@ -103,7 +103,7 @@ export const deleteDatasetDataVector = async (
     }
     return Promise.reject('deleteDatasetData: no where');
   })();
-  console.log(where, '===');
+
   try {
     await PgClient.delete(PgDatasetTableName, {
       where: [where]

packages/service/common/vectorStore/pg/index.ts

@@ -13,8 +13,11 @@ export const connectPg = async (): Promise<Pool> => {
     max: Number(process.env.DB_MAX_LINK || 20),
     min: 10,
     keepAlive: true,
-    idleTimeoutMillis: 60000,
-    connectionTimeoutMillis: 20000
+    idleTimeoutMillis: 600000,
+    connectionTimeoutMillis: 20000,
+    query_timeout: 30000,
+    statement_timeout: 40000,
+    idle_in_transaction_session_timeout: 60000
   });
 
   global.pgClient.on('error', async (err) => {

packages/service/package.json

@@ -13,6 +13,7 @@
     "decompress": "^4.2.1",
     "encoding": "^0.1.13",
     "file-type": "^19.0.0",
+    "iconv-lite": "^0.6.3",
     "json5": "^2.2.3",
     "jsonwebtoken": "^9.0.2",
     "mammoth": "^1.6.0",

packages/web/components/common/Textarea/PromptEditor/utils.ts

@@ -173,7 +173,6 @@ export function registerLexicalTextEntity<T extends TextNode>(
 
 export function textToEditorState(text: string = '') {
   const paragraph = text?.split('\n');
-
   return JSON.stringify({
     root: {
       children: paragraph.map((p) => {
@@ -206,11 +205,23 @@ export function textToEditorState(text: string = '') {
 }
 
 export function editorStateToText(editor: LexicalEditor) {
-  const stringifiedEditorState = JSON.stringify(editor.getEditorState().toJSON());
-  const parsedEditorState = editor.parseEditorState(stringifiedEditorState);
-  const editorStateTextString = parsedEditorState.read(() => $getRoot().getTextContent());
-
-  return editorStateTextString;
+  const editorStateTextString: string[] = [];
+  const paragraphs = editor.getEditorState().toJSON().root.children;
+  paragraphs.forEach((paragraph: any) => {
+    const children = paragraph.children;
+    const paragraphText: string[] = [];
+    children.forEach((child: any) => {
+      if (child.type === 'linebreak') {
+        paragraphText.push(`
+`);
+      } else if (child.text) {
+        paragraphText.push(child.text);
+      }
+    });
+    editorStateTextString.push(paragraphText.join(''));
+  });
+  return editorStateTextString.join(`
+`);
 }
 
 const varRegex = /\{\{([a-zA-Z_][a-zA-Z0-9_]*)\}\}/g;