Feat: Upload file UI/UX enhancements (#5359)

### What problem does this PR solve? Modifies the UX for uploading process on the website. - Adds option to parse on creation the files - Adds progress bar to display progress of chunk - Adds per file feedback on uploading operation #### Screenshots: - Show files uploading: ![image](https://github.com/user-attachments/assets/a5693f42-8232-4d5c-a240-20ed343634a5) - Errors on specific files ![image](https://github.com/user-attachments/assets/986a7f54-ab32-4634-89ab-a098fe1954aa) ### Type of change - [X] New Feature (non-breaking change which adds functionality)
2025-03-05 02:20:32 -05:00 · 2025-03-05 02:20:32 -05:00 · 11e3f5e8b2
commit 11e3f5e8b2
parent f65c3ae62b
13 changed files with 189 additions and 92 deletions
--- a/api/apps/document_app.py
+++ b/api/apps/document_app.py
@ -71,11 +71,13 @@ def upload():
    if not e:
        raise LookupError("Can't find this knowledgebase!")

-    err, _ = FileService.upload_document(kb, file_objs, current_user.id)
+    err, files = FileService.upload_document(kb, file_objs, current_user.id)
+    files = [f[0] for f in files] # remove the blob
+    
    if err:
        return get_json_result(
-            data=False, message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
-    return get_json_result(data=True)
+            data=files, message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
+    return get_json_result(data=files)


@manager.route('/web_crawl', methods=['POST'])  # noqa: F821
--- a/web/src/components/file-upload-modal/index.tsx
+++ b/web/src/components/file-upload-modal/index.tsx
@ -2,8 +2,10 @@ import { useTranslate } from '@/hooks/common-hooks';
 import { IModalProps } from '@/interfaces/common';
 import { InboxOutlined } from '@ant-design/icons';
 import {
+  Checkbox,
  Flex,
  Modal,
+  Progress,
  Segmented,
  Tabs,
  TabsProps,
@ -21,10 +23,12 @@ const FileUpload = ({
  directory,
  fileList,
  setFileList,
+  uploadProgress,
 }: {
  directory: boolean;
  fileList: UploadFile[];
  setFileList: Dispatch<SetStateAction<UploadFile[]>>;
+  uploadProgress: number;
 }) => {
  const { t } = useTranslate('fileManager');
  const props: UploadProps = {
@ -35,7 +39,7 @@ const FileUpload = ({
      newFileList.splice(index, 1);
      setFileList(newFileList);
    },
-    beforeUpload: (file) => {
+    beforeUpload: (file: UploadFile) => {
      setFileList((pre) => {
        return [...pre, file];
      });
@ -44,38 +48,59 @@ const FileUpload = ({
    },
    directory,
    fileList,
+    progress: {
+      strokeWidth: 2,
+    },
  };

  return (
-    <Dragger {...props} className={styles.uploader}>
-      <p className="ant-upload-drag-icon">
-        <InboxOutlined />
-      </p>
-      <p className="ant-upload-text">{t('uploadTitle')}</p>
-      <p className="ant-upload-hint">{t('uploadDescription')}</p>
-      {false && <p className={styles.uploadLimit}>{t('uploadLimit')}</p>}
-    </Dragger>
+    <>
+      <Progress percent={uploadProgress} showInfo={false} />
+      <Dragger {...props} className={styles.uploader}>
+        <p className="ant-upload-drag-icon">
+          <InboxOutlined />
+        </p>
+        <p className="ant-upload-text">{t('uploadTitle')}</p>
+        <p className="ant-upload-hint">{t('uploadDescription')}</p>
+        {false && <p className={styles.uploadLimit}>{t('uploadLimit')}</p>}
+      </Dragger>
+    </>
  );
 };

+interface IFileUploadModalProps extends IModalProps<boolean> {
+  uploadFileList: UploadFile[];
+  setUploadFileList: Dispatch<SetStateAction<UploadFile[]>>;
+  uploadProgress: number;
+  setUploadProgress: Dispatch<SetStateAction<number>>;
+}
+
 const FileUploadModal = ({
  visible,
  hideModal,
  loading,
  onOk: onFileUploadOk,
-}: IModalProps<UploadFile[]>) => {
+  uploadFileList: fileList,
+  setUploadFileList: setFileList,
+  uploadProgress,
+  setUploadProgress,
+}: IFileUploadModalProps) => {
  const { t } = useTranslate('fileManager');
  const [value, setValue] = useState<string | number>('local');
-  const [fileList, setFileList] = useState<UploadFile[]>([]);
-  const [directoryFileList, setDirectoryFileList] = useState<UploadFile[]>([]);
+  const [parseOnCreation, setParseOnCreation] = useState(false);

  const clearFileList = () => {
    setFileList([]);
-    setDirectoryFileList([]);
+    setUploadProgress(0);
  };

  const onOk = async () => {
-    const ret = await onFileUploadOk?.([...fileList, ...directoryFileList]);
+    if (uploadProgress === 100) {
+      hideModal?.();
+      return;
+    }
+
+    const ret = await onFileUploadOk?.(parseOnCreation);
    return ret;
  };

@ -92,6 +117,7 @@ const FileUploadModal = ({
          directory={false}
          fileList={fileList}
          setFileList={setFileList}
+          uploadProgress={uploadProgress}
        ></FileUpload>
      ),
    },
@ -101,8 +127,9 @@ const FileUploadModal = ({
      children: (
        <FileUpload
          directory
-          fileList={directoryFileList}
-          setFileList={setDirectoryFileList}
+          fileList={fileList}
+          setFileList={setFileList}
+          uploadProgress={uploadProgress}
        ></FileUpload>
      ),
    },
@ -129,7 +156,15 @@ const FileUploadModal = ({
            onChange={setValue}
          />
          {value === 'local' ? (
-            <Tabs defaultActiveKey="1" items={items} />
+            <>
+              <Checkbox
+                checked={parseOnCreation}
+                onChange={(e) => setParseOnCreation(e.target.checked)}
+              >
+                {t('parseOnCreation')}
+              </Checkbox>
+              <Tabs defaultActiveKey="1" items={items} />
+            </>
          ) : (
            t('comingSoon', { keyPrefix: 'common' })
          )}
--- a/web/src/hooks/document-hooks.ts
+++ b/web/src/hooks/document-hooks.ts
@ -248,60 +248,27 @@ export const useUploadNextDocument = () => {
  } = useMutation({
    mutationKey: ['uploadDocument'],
    mutationFn: async (fileList: UploadFile[]) => {
-      const partitionedFileList = fileList.reduce<UploadFile[][]>(
-        (acc, cur, index) => {
-          const partIndex = Math.floor(index / 20); // Uploads 20 documents at a time
-          if (!acc[partIndex]) {
-            acc[partIndex] = [];
-          }
-          acc[partIndex].push(cur);
-          return acc;
-        },
-        [],
-      );
+      const formData = new FormData();
+      formData.append('kb_id', knowledgeId);
+      fileList.forEach((file: any) => {
+        formData.append('file', file);
+      });

-      let allRet = [];
-      for (const listPart of partitionedFileList) {
-        const formData = new FormData();
-        formData.append('kb_id', knowledgeId);
-        listPart.forEach((file: any) => {
-          formData.append('file', file);
-        });
+      try {
+        const ret = await kbService.document_upload(formData);
+        const code = get(ret, 'data.code');

-        try {
-          const ret = await kbService.document_upload(formData);
-          allRet.push(ret);
-        } catch (error) {
-          allRet.push({ data: { code: 500 } });
-
-          const filenames = listPart.map((file: any) => file.name).join(', ');
-          console.warn(error);
-          console.warn('Error uploading files:', filenames);
+        if (code === 0 || code === 500) {
+          queryClient.invalidateQueries({ queryKey: ['fetchDocumentList'] });
        }
+        return ret?.data;
+      } catch (error) {
+        console.warn(error);
+        return {
+          code: 500,
+          message: error + '',
+        };
      }
-
-      const succeed = allRet.every((ret) => get(ret, 'data.code') === 0);
-      const any500 = allRet.some((ret) => get(ret, 'data.code') === 500);
-
-      if (succeed) {
-        message.success(i18n.t('message.uploaded'));
-      }
-
-      if (succeed || any500) {
-        queryClient.invalidateQueries({ queryKey: ['fetchDocumentList'] });
-      }
-
-      const allData = {
-        code: any500
-          ? 500
-          : succeed
-            ? 0
-            : allRet.filter((ret) => get(ret, 'data.code') !== 0)[0]?.data
-                ?.code,
-        data: succeed,
-        message: allRet.map((ret) => get(ret, 'data.message')).join('/n'),
-      };
-      return allData;
    },
  });

--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@ -126,9 +126,9 @@ export default {
      filesSelected: 'Files selected',
      upload: 'Upload',
      run: 'Parse',
-      runningStatus0: 'UNParsed',
-      runningStatus1: 'Parsing',
-      runningStatus2: 'CANCEL',
+      runningStatus0: 'PENDING',
+      runningStatus1: 'PARSING',
+      runningStatus2: 'CANCELED',
      runningStatus3: 'SUCCESS',
      runningStatus4: 'FAIL',
      pageRanges: 'Page Ranges',
@ -743,6 +743,7 @@ This auto-tag feature enhances retrieval by adding another layer of domain-speci
      newFolder: 'New Folder',
      file: 'File',
      uploadFile: 'Upload File',
+      parseOnCreation: 'Parse on creation',
      directory: 'Directory',
      uploadTitle: 'Drag and drop your file here to upload',
      uploadDescription:
--- a/web/src/locales/es.ts
+++ b/web/src/locales/es.ts
@ -480,6 +480,7 @@ export default {
      newFolder: 'Nueva carpeta',
      file: 'Archivo',
      uploadFile: 'Subir archivo',
+      parseOnCreation: 'Ejecutar en la creación',
      directory: 'Directorio',
      uploadTitle: 'Haz clic o arrastra el archivo a esta área para subir',
      uploadDescription:
--- a/web/src/locales/id.ts
+++ b/web/src/locales/id.ts
@ -648,6 +648,7 @@ export default {
      newFolder: 'Folder Baru',
      file: 'File',
      uploadFile: 'Unggah File',
+      parseOnCreation: 'Memparsing saat dibuat',
      directory: 'Direktori',
      uploadTitle: 'Klik atau seret file ke area ini untuk mengunggah',
      uploadDescription:
--- a/web/src/locales/ja.ts
+++ b/web/src/locales/ja.ts
@ -653,6 +653,7 @@ export default {
      newFolder: '新しいフォルダ',
      file: 'ファイル',
      uploadFile: 'ファイルをアップロード',
+      parseOnCreation: '作成時に解析',
      directory: 'ディレクトリ',
      uploadTitle: 'クリックまたはドラッグしてファイルをアップロード',
      uploadDescription:
--- a/web/src/locales/pt-br.ts
+++ b/web/src/locales/pt-br.ts
@ -639,6 +639,7 @@ export default {
      newFolder: 'Nova Pasta',
      file: 'Arquivo',
      uploadFile: 'Carregar Arquivo',
+      parseOnCreation: 'Executar na criação',
      directory: 'Diretório',
      uploadTitle:
        'Clique ou arraste o arquivo para esta área para fazer o upload',
--- a/web/src/locales/vi.ts
+++ b/web/src/locales/vi.ts
@ -707,6 +707,7 @@ export default {
      newFolder: 'Thư mục mới',
      file: 'Tệp',
      uploadFile: 'Tải tệp lên',
+      parseOnCreation: 'Phân tích khi tạo',
      directory: 'Thư mục',
      uploadTitle: 'Nhấp hoặc kéo thả tệp vào khu vực này để tải lên',
      uploadDescription:
--- a/web/src/locales/zh-traditional.ts
+++ b/web/src/locales/zh-traditional.ts
@ -708,6 +708,7 @@ export default {
      pleaseSelect: '請選擇',
      newFolder: '新建文件夾',
      uploadFile: '上傳文件',
+      parseOnCreation: '創建時解析',
      uploadTitle: '點擊或拖拽文件至此區域即可上傳',
      uploadDescription:
        '支持單次或批量上傳。單個檔案大小不超過10MB，最多上傳128份檔案。',
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@ -726,6 +726,7 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于
      pleaseSelect: '请选择',
      newFolder: '新建文件夹',
      uploadFile: '上传文件',
+      parseOnCreation: '创建时解析',
      uploadTitle: '点击或拖拽文件至此区域即可上传',
      uploadDescription:
        '支持单次或批量上传。 单个文件大小不超过10MB，最多上传128份文件。严禁上传违禁文件。',
--- a/web/src/pages/add-knowledge/components/knowledge-file/hooks.ts
+++ b/web/src/pages/add-knowledge/components/knowledge-file/hooks.ts
@ -10,7 +10,6 @@ import {
 } from '@/hooks/document-hooks';
 import { useGetKnowledgeSearchParams } from '@/hooks/route-hook';
 import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
-import { getUnSupportedFilesCount } from '@/utils/document-util';
 import { UploadFile } from 'antd';
 import { useCallback, useState } from 'react';
 import { useNavigate } from 'umi';
@ -143,29 +142,103 @@ export const useHandleUploadDocument = () => {
    hideModal: hideDocumentUploadModal,
    showModal: showDocumentUploadModal,
  } = useSetModalState();
+  const [fileList, setFileList] = useState<UploadFile[]>([]);
+  const [uploadProgress, setUploadProgress] = useState<number>(0);
  const { uploadDocument, loading } = useUploadNextDocument();
+  const { runDocumentByIds, loading: _ } = useRunNextDocument();

  const onDocumentUploadOk = useCallback(
-    async (fileList: UploadFile[]): Promise<number | undefined> => {
-      if (fileList.length > 0) {
-        const ret: any = await uploadDocument(fileList);
-        if (typeof ret?.message !== 'string') {
-          return;
-        }
-        const count = getUnSupportedFilesCount(ret?.message);
-        /// 500 error code indicates that some file types are not supported
-        let code = ret?.code;
-        if (
-          ret?.code === 0 ||
-          (ret?.code === 500 && count !== fileList.length) // Some files were not uploaded successfully, but some were uploaded successfully.
-        ) {
-          code = 0;
-          hideDocumentUploadModal();
-        }
-        return code;
+    async (parseOnCreation: boolean): Promise<number | undefined> => {
+      const processFileGroup = async (filesPart: UploadFile[]) => {
+        // set status to uploading on files
+        setFileList(
+          fileList.map((file) => {
+            if (!filesPart.includes(file)) {
+              return file;
+            }
+
+            let newFile = file;
+            newFile.status = 'uploading';
+            newFile.percent = 1;
+            return newFile;
+          }),
+        );
+
+        const ret = await uploadDocument(filesPart);
+
+        const files = ret?.data || [];
+        const succesfulFilenames = files.map((file: any) => file.name);
+
+        // set status to done or error on files (based on response)
+        setFileList(
+          fileList.map((file) => {
+            if (!filesPart.includes(file)) {
+              return file;
+            }
+
+            let newFile = file;
+            newFile.status = succesfulFilenames.includes(file.name)
+              ? 'done'
+              : 'error';
+            newFile.percent = 100;
+            newFile.response = ret.message;
+            return newFile;
+          }),
+        );
+
+        return {
+          code: ret?.code,
+          fileIds: files.map((file: any) => file.id),
+          totalSuccess: succesfulFilenames.length,
+        };
+      };
+
+      const totalFiles = fileList.length;
+
+      if (totalFiles === 0) {
+        console.log('No files to upload');
+        hideDocumentUploadModal();
+        return 0;
      }
+
+      let totalSuccess = 0;
+      let codes = [];
+      let toRunFileIds: any[] = [];
+      for (let i = 0; i < totalFiles; i += 10) {
+        setUploadProgress(Math.floor((i / totalFiles) * 100));
+        const files = fileList.slice(i, i + 10);
+        const {
+          code,
+          totalSuccess: count,
+          fileIds,
+        } = await processFileGroup(files);
+        codes.push(code);
+        totalSuccess += count;
+        toRunFileIds = toRunFileIds.concat(fileIds);
+      }
+
+      const allSuccess = codes.every((code) => code === 0);
+      const any500 = codes.some((code) => code === 500);
+
+      let code = 500;
+      if (allSuccess || (any500 && totalSuccess === totalFiles)) {
+        code = 0;
+        hideDocumentUploadModal();
+      }
+
+      if (parseOnCreation) {
+        await runDocumentByIds({
+          documentIds: toRunFileIds,
+          run: 1,
+          shouldDelete: false,
+        });
+      }
+
+      setUploadProgress(100);
+
+      return code;
    },
-    [uploadDocument, hideDocumentUploadModal],
+    [uploadDocument, hideDocumentUploadModal, fileList],
  );

  return {
@ -174,6 +247,10 @@ export const useHandleUploadDocument = () => {
    documentUploadVisible,
    hideDocumentUploadModal,
    showDocumentUploadModal,
+    uploadFileList: fileList,
+    setUploadFileList: setFileList,
+    uploadProgress,
+    setUploadProgress,
  };
 };

--- a/web/src/pages/add-knowledge/components/knowledge-file/index.tsx
+++ b/web/src/pages/add-knowledge/components/knowledge-file/index.tsx
@ -69,6 +69,10 @@ const KnowledgeFile = () => {
    showDocumentUploadModal,
    onDocumentUploadOk,
    documentUploadLoading,
+    uploadFileList,
+    setUploadFileList,
+    uploadProgress,
+    setUploadProgress,
  } = useHandleUploadDocument();
  const {
    webCrawlUploadVisible,
@ -229,6 +233,10 @@ const KnowledgeFile = () => {
        hideModal={hideDocumentUploadModal}
        loading={documentUploadLoading}
        onOk={onDocumentUploadOk}
+        uploadFileList={uploadFileList}
+        setUploadFileList={setUploadFileList}
+        uploadProgress={uploadProgress}
+        setUploadProgress={setUploadProgress}
      ></FileUploadModal>
      <WebCrawlModal
        visible={webCrawlUploadVisible}