add image input (#486)

newfish-cmyk · web-flow · commit 70f337324652 · 2023-11-17T18:22:29.000+08:00
* add image input

* use json
diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json
@@ -233,10 +233,11 @@
     },
     "chat": {
       "Audio Speech Error": "Audio Speech Error",
-      "Speaking": "I'm listening...",
       "Record": "Speech",
       "Restart": "Restart",
+      "Select File": "Select file",
       "Send Message": "Send Message",
+      "Speaking": "I'm listening...",
       "Stop Speak": "Stop Speak",
       "Type a message": "Input problem",
       "tts": {
@@ -589,8 +590,8 @@
   "wallet": {
     "bill": {
       "Audio Speech": "Audio Speech",
-      "bill username": "User",
-      "Whisper": "Whisper"
+      "Whisper": "Whisper",
+      "bill username": "User"
     }
   }
 }
diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json
@@ -235,6 +235,7 @@
       "Audio Speech Error": "语音播报异常",
       "Record": "语音输入",
       "Restart": "重开对话",
+      "Select File": "选择文件",
       "Send Message": "发送",
       "Speaking": "我在听，请说...",
       "Stop Speak": "停止录音",
@@ -589,8 +590,8 @@
   "wallet": {
     "bill": {
       "Audio Speech": "语音播报",
-      "bill username": "用户",
-      "Whisper": "语音输入"
+      "Whisper": "语音输入",
+      "bill username": "用户"
     }
   }
 }
diff --git a/projects/app/src/components/ChatBox/MessageInput.tsx b/projects/app/src/components/ChatBox/MessageInput.tsx
@@ -1,12 +1,15 @@
 import { useSpeech } from '@/web/common/hooks/useSpeech';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
-import { Box, Flex, Spinner, Textarea } from '@chakra-ui/react';
-import React, { useRef, useEffect } from 'react';
+import { Box, Flex, Image, Spinner, Textarea } from '@chakra-ui/react';
+import React, { useRef, useEffect, useCallback, useState, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 import MyTooltip from '../MyTooltip';
 import MyIcon from '../Icon';
 import styles from './index.module.scss';
 import { useRouter } from 'next/router';
+import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
+import { compressImgAndUpload } from '@/web/common/file/controller';
+import { useToast } from '@/web/common/hooks/useToast';
 
 const MessageInput = ({
   onChange,
@@ -38,6 +41,60 @@ const MessageInput = ({
   const { t } = useTranslation();
   const textareaMinH = '22px';
   const havInput = !!TextareaDom.current?.value;
+  const { toast } = useToast();
+  const [imgBase64Array, setImgBase64Array] = useState<string[]>([]);
+  const [fileList, setFileList] = useState<File[]>([]);
+  const [imgSrcArray, setImgSrcArray] = useState<string[]>([]);
+
+  const { File, onOpen: onOpenSelectFile } = useSelectFile({
+    fileType: '.jpg,.png',
+    multiple: true
+  });
+
+  useEffect(() => {
+    fileList.forEach((file) => {
+      const reader = new FileReader();
+      reader.readAsDataURL(file);
+      reader.onload = async () => {
+        setImgBase64Array((prev) => [...prev, reader.result as string]);
+      };
+    });
+  }, [fileList]);
+
+  const onSelectFile = useCallback((e: File[]) => {
+    if (!e || e.length === 0) {
+      return;
+    }
+    setFileList(e);
+  }, []);
+
+  const handleSend = useCallback(async () => {
+    try {
+      for (const file of fileList) {
+        const src = await compressImgAndUpload({
+          file,
+          maxW: 1000,
+          maxH: 1000,
+          maxSize: 1024 * 1024 * 2
+        });
+        imgSrcArray.push(src);
+      }
+    } catch (err: any) {
+      toast({
+        title: typeof err === 'string' ? err : '文件上传异常',
+        status: 'warning'
+      });
+    }
+
+    const textareaValue = TextareaDom.current?.value || '';
+    const inputMessage =
+      imgSrcArray.length === 0
+        ? textareaValue
+        : `\`\`\`img-block\n${JSON.stringify(imgSrcArray)}\n\`\`\`\n${textareaValue}`;
+    onSendMessage(inputMessage);
+    setImgBase64Array([]);
+    setImgSrcArray([]);
+  }, [TextareaDom, fileList, imgSrcArray, onSendMessage, toast]);
 
   useEffect(() => {
     if (!stream) {
@@ -60,7 +117,7 @@ const MessageInput = ({
     <>
       <Box m={['0 auto', '10px auto']} w={'100%'} maxW={['auto', 'min(800px, 100%)']} px={[0, 5]}>
         <Box
-          py={'18px'}
+          py={imgBase64Array.length > 0 ? '8px' : '18px'}
           position={'relative'}
           boxShadow={isSpeaking ? `0 0 10px rgba(54,111,255,0.4)` : `0 0 10px rgba(0,0,0,0.2)`}
           {...(isPc
@@ -93,11 +150,74 @@ const MessageInput = ({
             <Spinner size={'sm'} mr={4} />
             {t('chat.Converting to text')}
           </Box>
+          {/* file uploader */}
+          <Flex
+            position={'absolute'}
+            alignItems={'center'}
+            left={['12px', '14px']}
+            bottom={['15px', '13px']}
+            h={['26px', '32px']}
+            zIndex={10}
+            cursor={'pointer'}
+            onClick={onOpenSelectFile}
+          >
+            <MyTooltip label={t('core.chat.Select File')}>
+              <MyIcon name={'core/chat/fileSelect'} />
+            </MyTooltip>
+            <File onSelect={onSelectFile} />
+          </Flex>
+          {/* file preview */}
+          <Flex w={'96%'} wrap={'wrap'} ml={4}>
+            {imgBase64Array.length > 0 &&
+              imgBase64Array.map((src, index) => (
+                <Box
+                  key={index}
+                  border={'1px solid rgba(0,0,0,0.12)'}
+                  mr={2}
+                  mb={2}
+                  rounded={'md'}
+                  position={'relative'}
+                  _hover={{
+                    '.close-icon': { display: 'block' }
+                  }}
+                >
+                  <MyIcon
+                    name={'closeSolid'}
+                    w={'16px'}
+                    h={'16px'}
+                    color={'myGray.700'}
+                    cursor={'pointer'}
+                    _hover={{ color: 'myBlue.600' }}
+                    position={'absolute'}
+                    right={-2}
+                    top={-2}
+                    onClick={() => {
+                      setImgBase64Array((prev) => {
+                        prev.splice(index, 1);
+                        return [...prev];
+                      });
+                    }}
+                    className="close-icon"
+                    display={['', 'none']}
+                  />
+                  <Image
+                    alt={'img'}
+                    src={src}
+                    w={'80px'}
+                    h={'80px'}
+                    rounded={'md'}
+                    objectFit={'cover'}
+                  />
+                </Box>
+              ))}
+          </Flex>
           {/* input area */}
           <Textarea
             ref={TextareaDom}
             py={0}
             pr={['45px', '55px']}
+            pl={['36px', '40px']}
+            mt={imgBase64Array.length > 0 ? 4 : 0}
             border={'none'}
             _focusVisible={{
               border: 'none'
@@ -124,13 +244,28 @@ const MessageInput = ({
             onKeyDown={(e) => {
               // enter send.(pc or iframe && enter and unPress shift)
               if ((isPc || window !== parent) && e.keyCode === 13 && !e.shiftKey) {
-                onSendMessage(TextareaDom.current?.value || '');
+                handleSend();
                 e.preventDefault();
               }
               // 全选内容
               // @ts-ignore
               e.key === 'a' && e.ctrlKey && e.target?.select();
             }}
+            onPaste={(e) => {
+              const clipboardData = e.clipboardData;
+              if (clipboardData) {
+                const items = clipboardData.items;
+                const files: File[] = [];
+                for (let i = 0; i < items.length; i++) {
+                  const item = items[i];
+                  if (item.kind === 'file') {
+                    const file = item.getAsFile();
+                    files.push(file as File);
+                  }
+                }
+                setFileList(files);
+              }
+            }}
           />
           <Flex
             position={'absolute'}
@@ -195,7 +330,7 @@ const MessageInput = ({
                     return onStop();
                   }
                   if (havInput) {
-                    onSendMessage(TextareaDom.current?.value || '');
+                    return handleSend();
                   }
                 }}
               >
diff --git a/projects/app/src/components/ChatBox/index.tsx b/projects/app/src/components/ChatBox/index.tsx
@@ -17,18 +17,7 @@ import { useToast } from '@/web/common/hooks/useToast';
 import { useAudioPlay } from '@/web/common/utils/voice';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useCopyData } from '@/web/common/hooks/useCopyData';
-import {
-  Box,
-  Card,
-  Flex,
-  Input,
-  Textarea,
-  Button,
-  useTheme,
-  BoxProps,
-  FlexProps,
-  Spinner
-} from '@chakra-ui/react';
+import { Box, Card, Flex, Input, Button, useTheme, BoxProps, FlexProps } from '@chakra-ui/react';
 import { feConfigs } from '@/web/common/system/staticData';
 import { eventBus } from '@/web/common/utils/eventbus';
 import { adaptChat2GptMessages } from '@fastgpt/global/core/chat/adapt';
@@ -633,7 +622,7 @@ const ChatBox = (
                         borderRadius={'8px 0 8px 8px'}
                         textAlign={'left'}
                       >
-                        <Box as={'p'}>{item.value}</Box>
+                        <Markdown source={item.value} isChatting={false} />
                       </Card>
                     </Box>
                   </>
diff --git a/projects/app/src/components/Icon/icons/core/chat/fileSelect.svg b/projects/app/src/components/Icon/icons/core/chat/fileSelect.svg
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="17" height="18" viewBox="0 0 19 20" fill="none">
+  <path fill-rule="evenodd" clip-rule="evenodd" d="M13.15 1.98982C12.4865 1.98982 11.8501 2.2534 11.3809 2.72259L3.72259 10.3809C2.94066 11.1628 2.50138 12.2234 2.50138 13.3292C2.50138 14.435 2.94066 15.4955 3.72259 16.2774C4.50451 17.0593 5.56502 17.4986 6.67083 17.4986C7.77664 17.4986 8.83715 17.0593 9.61907 16.2774L17.2774 8.61908C17.6028 8.29364 18.1305 8.29364 18.4559 8.61908C18.7814 8.94452 18.7814 9.47216 18.4559 9.79759L10.7976 17.4559C9.7031 18.5504 8.21866 19.1653 6.67083 19.1653C5.123 19.1653 3.63856 18.5504 2.54407 17.4559C1.44959 16.3614 0.834717 14.877 0.834717 13.3292C0.834717 11.7813 1.44959 10.2969 2.54407 9.20242L10.2024 1.54408C10.9842 0.762333 12.0444 0.323151 13.15 0.323151C14.2556 0.323151 15.3158 0.762332 16.0976 1.54408C16.8793 2.32583 17.3185 3.38611 17.3185 4.49167C17.3185 5.59723 16.8793 6.65751 16.0976 7.43926L8.43092 15.0976C7.96191 15.5666 7.32579 15.8301 6.6625 15.8301C5.99921 15.8301 5.36309 15.5666 4.89407 15.0976C4.42506 14.6286 4.16157 13.9925 4.16157 13.3292C4.16157 12.6659 4.42506 12.0298 4.89407 11.5607L11.9694 4.49373C12.2951 4.16849 12.8227 4.1688 13.1479 4.49443C13.4732 4.82006 13.4729 5.34769 13.1472 5.67294L6.07259 12.7393C5.91635 12.8957 5.82824 13.1081 5.82824 13.3292C5.82824 13.5504 5.91613 13.7626 6.07259 13.9191C6.22904 14.0755 6.44124 14.1634 6.6625 14.1634C6.88376 14.1634 7.09595 14.0755 7.25241 13.9191L14.9191 6.26075C15.3881 5.79159 15.6519 5.15505 15.6519 4.49167C15.6519 3.82814 15.3883 3.19178 14.9191 2.72259C14.4499 2.2534 13.8135 1.98982 13.15 1.98982Z" fill="#485058"/>
+</svg>
diff --git a/projects/app/src/components/Icon/index.tsx b/projects/app/src/components/Icon/index.tsx
@@ -112,7 +112,8 @@ const iconPaths = {
   'core/chat/recordFill': () => import('./icons/core/chat/recordFill.svg'),
   'core/chat/stopSpeechFill': () => import('./icons/core/chat/stopSpeechFill.svg'),
   'core/chat/stopSpeech': () => import('./icons/core/chat/stopSpeech.svg'),
-  'core/chat/speaking': () => import('./icons/core/chat/speaking.svg')
+  'core/chat/speaking': () => import('./icons/core/chat/speaking.svg'),
+  'core/chat/fileSelect': () => import('./icons/core/chat/fileSelect.svg')
 };
 
 export type IconName = keyof typeof iconPaths;
diff --git a/projects/app/src/components/Markdown/chat/Image.tsx b/projects/app/src/components/Markdown/chat/Image.tsx
@@ -0,0 +1,18 @@
+import { Box, Flex } from '@chakra-ui/react';
+import MdImage from '../img/Image';
+
+const ImageBlock = ({ images }: { images: string }) => {
+  return (
+    <Flex w={'100%'} wrap={'wrap'}>
+      {JSON.parse(images).map((src: string) => {
+        return (
+          <Box key={src} mr={2} mb={2} rounded={'md'} flex={'0 0 auto'} w={'100px'} h={'100px'}>
+            <MdImage src={src} />
+          </Box>
+        );
+      })}
+    </Flex>
+  );
+};
+
+export default ImageBlock;
diff --git a/projects/app/src/components/Markdown/index.tsx b/projects/app/src/components/Markdown/index.tsx
@@ -16,12 +16,14 @@ const MdImage = dynamic(() => import('./img/Image'));
 const ChatGuide = dynamic(() => import('./chat/Guide'));
 const EChartsCodeBlock = dynamic(() => import('./img/EChartsCodeBlock'));
 const QuoteBlock = dynamic(() => import('./chat/Quote'));
+const ImageBlock = dynamic(() => import('./chat/Image'));
 
 export enum CodeClassName {
   guide = 'guide',
   mermaid = 'mermaid',
   echarts = 'echarts',
-  quote = 'quote'
+  quote = 'quote',
+  img = 'img'
 }
 
 function Code({ inline, className, children }: any) {
@@ -41,6 +43,9 @@ function Code({ inline, className, children }: any) {
   if (codeType === CodeClassName.quote) {
     return <QuoteBlock code={String(children)} />;
   }
+  if (codeType === CodeClassName.img) {
+    return <ImageBlock images={String(children)} />;
+  }
   return (
     <CodeLight className={className} inline={inline} match={match}>
       {children}
diff --git a/projects/app/src/web/common/hooks/useSpeech.ts b/projects/app/src/web/common/hooks/useSpeech.ts
@@ -12,7 +12,7 @@ export const useSpeech = (props?: { shareId?: string }) => {
   const { toast } = useToast();
   const [isSpeaking, setIsSpeaking] = useState(false);
   const [isTransCription, setIsTransCription] = useState(false);
-  const [audioSecond, setAudioSecone] = useState(0);
+  const [audioSecond, setAudioSecond] = useState(0);
   const intervalRef = useRef<any>();
   const startTimestamp = useRef(0);
 
@@ -59,11 +59,11 @@ export const useSpeech = (props?: { shareId?: string }) => {
 
       mediaRecorder.current.onstart = () => {
         startTimestamp.current = Date.now();
-        setAudioSecone(0);
+        setAudioSecond(0);
         intervalRef.current = setInterval(() => {
           const currentTimestamp = Date.now();
           const duration = (currentTimestamp - startTimestamp.current) / 1000;
-          setAudioSecone(duration);
+          setAudioSecond(duration);
         }, 1000);
       };
 

Original file line number	Diff line number	Diff line change
`@@ -235,6 +235,7 @@`
`235`	`235`	`"Audio Speech Error": "语音播报异常",`
`236`	`236`	`"Record": "语音输入",`
`237`	`237`	`"Restart": "重开对话",`
	`238`	`+ "Select File": "选择文件",`
`238`	`239`	`"Send Message": "发送",`
`239`	`240`	`"Speaking": "我在听，请说...",`
`240`	`241`	`"Stop Speak": "停止录音",`
`@@ -589,8 +590,8 @@`
`589`	`590`	`"wallet": {`
`590`	`591`	`"bill": {`
`591`	`592`	`"Audio Speech": "语音播报",`
`592`		`- "bill username": "用户",`
`593`		`- "Whisper": "语音输入"`
	`593`	`+ "Whisper": "语音输入",`
	`594`	`+ "bill username": "用户"`
`594`	`595`	`}`
`595`	`596`	`}`
`596`	`597`	`}`