feat: fetch knowledge detail on KnowledgeUploadFile mount and add category column to chunk table and set initial value for the model field of chat setting (#104)

* feat: set initial value for the model field of chat setting

* feat: add category column to chunk table

* feat: fetch knowledge detail on KnowledgeUploadFile mount
This commit is contained in:
balibabu 2024-03-06 19:17:45 +08:00 committed by GitHub
parent b89ac3c4be
commit aaf3084324
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 230 additions and 320 deletions

View File

@ -1,6 +1,6 @@
import showDeleteConfirm from '@/components/deleting-confirm';
import { KnowledgeSearchParams } from '@/constants/knowledge';
import { IKnowledge, ITenantInfo } from '@/interfaces/database/knowledge';
import { IKnowledge } from '@/interfaces/database/knowledge';
import { useCallback, useEffect, useMemo } from 'react';
import { useDispatch, useSearchParams, useSelector } from 'umi';
@ -11,6 +11,17 @@ export const useKnowledgeBaseId = (): string => {
return knowledgeBaseId || '';
};
export const useGetKnowledgeSearchParams = () => {
const [currentQueryParameters] = useSearchParams();
return {
documentId:
currentQueryParameters.get(KnowledgeSearchParams.DocumentId) || '',
knowledgeId:
currentQueryParameters.get(KnowledgeSearchParams.KnowledgeId) || '',
};
};
export const useDeleteDocumentById = (): {
removeDocument: (documentId: string) => Promise<number>;
} => {
@ -36,12 +47,37 @@ export const useDeleteDocumentById = (): {
};
};
export const useGetDocumentDefaultParser = (knowledgeBaseId: string) => {
const data: IKnowledge[] = useSelector(
(state: any) => state.knowledgeModel.data,
export const useFetchKnowledgeDetail = () => {
const dispatch = useDispatch();
const { knowledgeId } = useGetKnowledgeSearchParams();
const fetchKnowledgeDetail = useCallback(
(knowledgeId: string) => {
dispatch({
type: 'knowledgeModel/getKnowledgeDetail',
payload: { kb_id: knowledgeId },
});
},
[dispatch],
);
const item = data.find((x) => x.id === knowledgeBaseId);
useEffect(() => {
fetchKnowledgeDetail(knowledgeId);
}, [fetchKnowledgeDetail, knowledgeId]);
return fetchKnowledgeDetail;
};
export const useSelectKnowledgeDetail = () => {
const knowledge: IKnowledge = useSelector(
(state: any) => state.knowledgeModel.knowledge,
);
return knowledge;
};
export const useGetDocumentDefaultParser = () => {
const item = useSelectKnowledgeDetail();
return {
defaultParserId: item?.parser_id ?? '',
@ -79,35 +115,6 @@ export const useDeleteChunkByIds = (): {
};
};
export const useSelectParserList = (): Array<{
value: string;
label: string;
}> => {
const tenantIfo: Nullable<ITenantInfo> = useSelector(
(state: any) => state.settingModel.tenantIfo,
);
const parserList = useMemo(() => {
const parserArray: Array<string> = tenantIfo?.parser_ids.split(',') ?? [];
return parserArray.map((x) => {
const arr = x.split(':');
return { value: arr[0], label: arr[1] };
});
}, [tenantIfo]);
return parserList;
};
export const useFetchParserList = () => {
const dispatch = useDispatch();
useEffect(() => {
dispatch({
type: 'settingModel/getTenantInfo',
});
}, [dispatch]);
};
export const useFetchKnowledgeBaseConfiguration = () => {
const dispatch = useDispatch();
const knowledgeBaseId = useKnowledgeBaseId();
@ -182,14 +189,3 @@ export const useFetchFileThumbnails = (docIds?: Array<string>) => {
return { fileThumbnails, fetchFileThumbnails };
};
export const useGetKnowledgeSearchParams = () => {
const [currentQueryParameters] = useSearchParams();
return {
documentId:
currentQueryParameters.get(KnowledgeSearchParams.DocumentId) || '',
knowledgeId:
currentQueryParameters.get(KnowledgeSearchParams.KnowledgeId) || '',
};
};

View File

@ -1,5 +1,6 @@
import { ITenantInfo } from '@/interfaces/database/knowledge';
import { IUserInfo } from '@/interfaces/database/userSetting';
import { useCallback, useEffect } from 'react';
import { useCallback, useEffect, useMemo } from 'react';
import { useDispatch, useSelector } from 'umi';
export const useFetchUserInfo = () => {
@ -20,3 +21,46 @@ export const useSelectUserInfo = () => {
return userInfo;
};
export const useSelectTenantInfo = () => {
const tenantInfo: ITenantInfo = useSelector(
(state: any) => state.settingModel.tenantIfo,
);
return tenantInfo;
};
export const useFetchTenantInfo = (isOnMountFetching: boolean = true) => {
const dispatch = useDispatch();
const fetchTenantInfo = useCallback(() => {
dispatch({
type: 'settingModel/getTenantInfo',
});
}, [dispatch]);
useEffect(() => {
if (isOnMountFetching) {
fetchTenantInfo();
}
}, [fetchTenantInfo, isOnMountFetching]);
return fetchTenantInfo;
};
export const useSelectParserList = (): Array<{
value: string;
label: string;
}> => {
const tenantInfo: ITenantInfo = useSelectTenantInfo();
const parserList = useMemo(() => {
const parserArray: Array<string> = tenantInfo?.parser_ids.split(',') ?? [];
return parserArray.map((x) => {
const arr = x.split(':');
return { value: arr[0], label: arr[1] };
});
}, [tenantInfo]);
return parserList;
};

View File

@ -23,19 +23,13 @@ const App: React.FC = () => {
return [
{
key: '1',
label: (
<Button type="text" onClick={logout}>
{t('header.logout')}
</Button>
),
onClick: logout,
label: <Button type="text">{t('header.logout')}</Button>,
},
{
key: '2',
label: (
<Button type="text" onClick={toSetting}>
{t('header.setting')}
</Button>
),
onClick: toSetting,
label: <Button type="text">{t('header.setting')}</Button>,
},
];
}, [t]);

View File

@ -2,11 +2,15 @@ import { ReactComponent as SelectFilesEndIcon } from '@/assets/svg/select-files-
import { ReactComponent as SelectFilesStartIcon } from '@/assets/svg/select-files-start.svg';
import {
useDeleteDocumentById,
useFetchParserList,
useFetchKnowledgeDetail,
useGetDocumentDefaultParser,
useKnowledgeBaseId,
useSelectParserList,
} from '@/hooks/knowledgeHook';
import {
useFetchTenantInfo,
useSelectParserList,
} from '@/hooks/userSettingHook';
import uploadService from '@/services/uploadService';
import {
ArrowLeftOutlined,
@ -29,10 +33,18 @@ import {
UploadProps,
} from 'antd';
import classNames from 'classnames';
import { ReactElement, useEffect, useRef, useState } from 'react';
import {
ReactElement,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from 'react';
import { Link, useDispatch, useNavigate } from 'umi';
import { KnowledgeRouteKey } from '@/constants/knowledge';
import { isFileUploadDone } from '@/utils/documentUtils';
import styles from './index.less';
const { Dragger } = Upload;
@ -43,18 +55,16 @@ type UploadRequestOption = Parameters<
const UploaderItem = ({
file,
actions,
isUpload,
remove,
}: {
isUpload: boolean;
originNode: ReactElement;
file: UploadFile;
fileList: object[];
actions: { download: Function; preview: Function; remove: any };
remove: (id: string) => void;
}) => {
const { parserConfig, defaultParserId } = useGetDocumentDefaultParser(
file?.response?.kb_id,
);
const { parserConfig, defaultParserId } = useGetDocumentDefaultParser();
const { removeDocument } = useDeleteDocumentById();
const [value, setValue] = useState(defaultParserId);
const dispatch = useDispatch();
@ -97,9 +107,13 @@ const UploaderItem = ({
);
const handleRemove = async () => {
const ret: any = await removeDocument(documentId);
if (ret === 0) {
actions?.remove();
if (file.status === 'error') {
remove(documentId);
} else {
const ret: any = await removeDocument(documentId);
if (ret === 0) {
remove(documentId);
}
}
};
@ -147,40 +161,67 @@ const KnowledgeUploadFile = () => {
const knowledgeBaseId = useKnowledgeBaseId();
const [isUpload, setIsUpload] = useState(true);
const dispatch = useDispatch();
const navigate = useNavigate();
const [uploadedFileIds, setUploadedFileIds] = useState<string[]>([]);
const fileListRef = useRef<UploadFile[]>([]);
const navigate = useNavigate();
const enabled = useMemo(() => {
if (isUpload) {
return (
uploadedFileIds.length > 0 &&
fileListRef.current.filter((x) => isFileUploadDone(x)).length ===
uploadedFileIds.length
);
}
return true;
}, [uploadedFileIds, isUpload]);
const createRequest: (props: UploadRequestOption) => void = async function ({
file,
onSuccess,
onError,
onProgress,
// onProgress,
}) {
const { data } = await uploadService.uploadFile(file, knowledgeBaseId);
if (data.retcode === 0) {
const ret = await uploadService.uploadFile(file, knowledgeBaseId);
const data = ret?.data;
if (data?.retcode === 0) {
setUploadedFileIds((pre) => {
return pre.concat(data.data.id);
});
if (onSuccess) {
onSuccess(data.data);
}
} else {
if (onError) {
onError(data.data);
onError(data?.data);
}
}
};
const removeIdFromUploadedIds = useCallback((id: string) => {
setUploadedFileIds((pre) => {
return pre.filter((x) => x !== id);
});
}, []);
const props: UploadProps = {
name: 'file',
multiple: true,
itemRender(originNode, file, fileList, actions) {
fileListRef.current = fileList;
const remove = (id: string) => {
if (isFileUploadDone(file)) {
removeIdFromUploadedIds(id);
}
actions.remove();
};
return (
<UploaderItem
isUpload={isUpload}
file={file}
fileList={fileList}
originNode={originNode}
actions={actions}
remove={remove}
></UploaderItem>
);
},
@ -207,7 +248,8 @@ const KnowledgeUploadFile = () => {
}
};
useFetchParserList();
useFetchTenantInfo();
useFetchKnowledgeDetail();
return (
<div className={styles.uploadWrapper}>
@ -263,8 +305,9 @@ const KnowledgeUploadFile = () => {
<section className={styles.footer}>
<Button
type="primary"
className={styles.nextButton}
// className={styles.nextButton}
onClick={handleNextClick}
disabled={!enabled}
>
Next
</Button>

View File

@ -1,5 +1,9 @@
import { KnowledgeRouteKey } from '@/constants/knowledge';
import { useKnowledgeBaseId } from '@/hooks/knowledgeHook';
import {
useFetchTenantInfo,
useSelectParserList,
} from '@/hooks/userSettingHook';
import { Pagination } from '@/interfaces/common';
import { IKnowledgeFile } from '@/interfaces/database/knowledge';
import { getOneNamespaceEffectsLoading } from '@/utils/storeUtil';
@ -45,6 +49,7 @@ const KnowledgeFile = () => {
const [doc_id, setDocId] = useState('0');
const [parser_id, setParserId] = useState('0');
let navigate = useNavigate();
const parserList = useSelectParserList();
const getKfList = useCallback(() => {
const payload = {
@ -214,6 +219,14 @@ const KnowledgeFile = () => {
dataIndex: 'create_date',
key: 'create_date',
},
{
title: 'Category',
dataIndex: 'parser_id',
key: 'parser_id',
render: (text) => {
return parserList.find((x) => x.value === text)?.label;
},
},
{
title: 'Parsing Status',
dataIndex: 'run',
@ -255,6 +268,8 @@ const KnowledgeFile = () => {
className: `${styles.column}`,
}));
useFetchTenantInfo();
return (
<div className={styles.datasetWrapper}>
<h3>Dataset</h3>

View File

@ -62,7 +62,7 @@ const ParsingActionCell = ({
label: (
<div>
<Button type="link" onClick={showSegmentSetModal}>
Parser type
Category
</Button>
</div>
),

View File

@ -1,4 +1,7 @@
import { useFetchParserList, useSelectParserList } from '@/hooks/knowledgeHook';
import {
useFetchTenantInfo,
useSelectParserList,
} from '@/hooks/userSettingHook';
import { Modal, Space, Tag } from 'antd';
import React, { useEffect, useState } from 'react';
import { useDispatch, useSelector } from 'umi';
@ -20,7 +23,7 @@ const SegmentSetModal: React.FC<kFProps> = ({
const { isShowSegmentSetModal } = kFModel;
const parserList = useSelectParserList();
useFetchParserList();
useFetchTenantInfo();
useEffect(() => {
setSelectedTag(parser_id);
@ -57,7 +60,7 @@ const SegmentSetModal: React.FC<kFProps> = ({
return (
<Modal
title="Parser Type"
title="Category"
open={isShowSegmentSetModal}
onOk={handleOk}
onCancel={handleCancel}

View File

@ -1,9 +1,12 @@
import {
useFetchKnowledgeBaseConfiguration,
useFetchParserList,
useKnowledgeBaseId,
useSelectParserList,
} from '@/hooks/knowledgeHook';
import {
useFetchTenantInfo,
useSelectParserList,
} from '@/hooks/userSettingHook';
import {
Button,
Divider,
@ -93,7 +96,7 @@ const Configuration = () => {
});
}, [form, knowledgeDetails]);
useFetchParserList();
useFetchTenantInfo();
useFetchKnowledgeBaseConfiguration();
useFetchLlmList(LlmModelType.Embedding);

View File

@ -1,165 +1,3 @@
import { KnowledgeRouteKey } from '@/constants/knowledge';
import { useKnowledgeBaseId } from '@/hooks/knowledgeHook';
import { Button, Form, Input, Radio, Select, Space, Tag } from 'antd';
import { useCallback, useEffect, useState } from 'react';
import { useDispatch, useNavigate, useSelector } from 'umi';
import Configuration from './configuration';
import styles from './index.less';
const { CheckableTag } = Tag;
const layout = {
labelCol: { span: 8 },
wrapperCol: { span: 16 },
labelAlign: 'left' as const,
};
const { Option } = Select;
const KnowledgeSetting = () => {
const dispatch = useDispatch();
const settingModel = useSelector((state: any) => state.settingModel);
let navigate = useNavigate();
const { tenantIfo = {} } = settingModel;
const parser_ids = tenantIfo?.parser_ids ?? '';
const embd_id = tenantIfo?.embd_id ?? '';
const [form] = Form.useForm();
const [selectedTag, setSelectedTag] = useState('');
const values = Form.useWatch([], form);
const knowledgeBaseId = useKnowledgeBaseId();
const getTenantInfo = useCallback(async () => {
dispatch({
type: 'settingModel/getTenantInfo',
payload: {},
});
if (knowledgeBaseId) {
const data = await dispatch<any>({
type: 'kSModel/getKbDetail',
payload: {
kb_id: knowledgeBaseId,
},
});
if (data.retcode === 0) {
const { description, name, permission, embd_id } = data.data;
form.setFieldsValue({ description, name, permission, embd_id });
setSelectedTag(data.data.parser_id);
}
}
}, [knowledgeBaseId, dispatch, form]);
const onFinish = async () => {
try {
await form.validateFields();
if (knowledgeBaseId) {
dispatch({
type: 'kSModel/updateKb',
payload: {
...values,
parser_id: selectedTag,
kb_id: knowledgeBaseId,
embd_id: undefined,
},
});
} else {
const retcode = await dispatch<any>({
type: 'kSModel/createKb',
payload: {
...values,
parser_id: selectedTag,
},
});
if (retcode === 0) {
navigate(
`/knowledge/${KnowledgeRouteKey.Dataset}?id=${knowledgeBaseId}`,
);
}
}
} catch (error) {
console.warn(error);
}
};
useEffect(() => {
getTenantInfo();
}, [getTenantInfo]);
const handleChange = (tag: string, checked: boolean) => {
const nextSelectedTag = checked ? tag : selectedTag;
console.log('You are interested in: ', nextSelectedTag);
setSelectedTag(nextSelectedTag);
};
return (
<Form
{...layout}
form={form}
name="validateOnly"
style={{ maxWidth: 1000, padding: 14 }}
>
<Form.Item name="name" label="知识库名称" rules={[{ required: true }]}>
<Input />
</Form.Item>
<Form.Item name="description" label="知识库描述">
<Input.TextArea />
</Form.Item>
<Form.Item name="permission" label="可见权限">
<Radio.Group>
<Radio value="me"></Radio>
<Radio value="team"></Radio>
</Radio.Group>
</Form.Item>
<Form.Item
name="embd_id"
label="Embedding 模型"
hasFeedback
rules={[{ required: true, message: 'Please select your country!' }]}
>
<Select placeholder="Please select a country">
{embd_id.split(',').map((item: string) => {
return (
<Option value={item} key={item}>
{item}
</Option>
);
})}
</Select>
</Form.Item>
<div style={{ marginTop: '5px' }}>
Embedding <span style={{ color: '#1677ff' }}></span>
</div>
<Space size={[0, 8]} wrap>
<div className={styles.tags}>
{parser_ids.split(',').map((tag: string) => {
return (
<CheckableTag
key={tag}
checked={selectedTag === tag}
onChange={(checked) => handleChange(tag, checked)}
>
{tag}
</CheckableTag>
);
})}
</div>
</Space>
<Space size={[0, 8]} wrap></Space>
<div className={styles.preset}>
<div className={styles.left}>xxxxx文章</div>
<div className={styles.right}></div>
</div>
<Form.Item wrapperCol={{ ...layout.wrapperCol, offset: 8 }}>
<Button type="primary" onClick={onFinish}>
</Button>
<Button htmlType="button" style={{ marginLeft: '20px' }}>
</Button>
</Form.Item>
</Form>
);
};
// export default KnowledgeSetting;
export default Configuration;

View File

@ -0,0 +1,18 @@
import {
useFetchTenantInfo,
useSelectTenantInfo,
} from '@/hooks/userSettingHook';
import { useEffect } from 'react';
export const useFetchModelId = (visible: boolean) => {
const fetchTenantInfo = useFetchTenantInfo(false);
const tenantInfo = useSelectTenantInfo();
useEffect(() => {
if (visible) {
fetchTenantInfo();
}
}, [visible, fetchTenantInfo]);
return tenantInfo?.llm_id ?? '';
};

View File

@ -13,6 +13,7 @@ import { variableEnabledFieldMap } from '../constants';
import { useFetchDialog, useResetCurrentDialog, useSetDialog } from '../hooks';
import { IPromptConfigParameters } from '../interface';
import { excludeUnEnabledVariables } from '../utils';
import { useFetchModelId } from './hooks';
import styles from './index.less';
enum ConfigurationSegmented {
@ -54,6 +55,7 @@ const ChatConfigurationModal = ({ visible, hideModal, id }: IProps) => {
);
const promptEngineRef = useRef<Array<IPromptConfigParameters>>([]);
const loading = useOneNamespaceEffectsLoading('chatModel', ['setDialog']);
const modelId = useFetchModelId(visible);
const setDialog = useSetDialog();
const currentDialog = useFetchDialog(id, visible);
@ -128,9 +130,13 @@ const ChatConfigurationModal = ({ visible, hideModal, id }: IProps) => {
if (icon) {
fileList = [{ uid: '1', name: 'file', thumbUrl: icon, status: 'done' }];
}
form.setFieldsValue({ ...currentDialog, icon: fileList });
form.setFieldsValue({
...currentDialog,
icon: fileList,
llm_id: currentDialog.llm_id ?? modelId,
});
}
}, [currentDialog, form, visible]);
}, [currentDialog, form, visible, modelId]);
return (
<Modal

View File

@ -10,7 +10,7 @@ import omit from 'lodash/omit';
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useDispatch, useSearchParams, useSelector } from 'umi';
import { v4 as uuid } from 'uuid';
import { ChatSearchParams, EmptyConversationId } from './constants';
import { ChatSearchParams } from './constants';
import {
IClientConversation,
IMessage,
@ -233,75 +233,6 @@ export const useHandleItemHover = () => {
//#region conversation
export const useCreateTemporaryConversation = () => {
const dispatch = useDispatch();
const { dialogId } = useGetChatSearchParams();
const { handleClickConversation } = useClickConversationCard();
let chatModel = useSelector((state: any) => state.chatModel);
const currentConversation: Pick<
IClientConversation,
'id' | 'message' | 'name' | 'dialog_id'
> = chatModel.currentConversation;
const conversationList: IClientConversation[] = chatModel.conversationList;
const currentDialog: IDialog = chatModel.currentDialog;
const setCurrentConversation = useSetCurrentConversation();
const createTemporaryConversation = useCallback(() => {
const firstConversation = conversationList[0];
const messages = [...(firstConversation?.message ?? [])];
if (messages.some((x) => x.id === EmptyConversationId)) {
return;
}
messages.push({
id: EmptyConversationId,
content: currentDialog?.prompt_config?.prologue ?? '',
role: MessageType.Assistant,
});
let nextCurrentConversation = currentConversation;
// Its the back-end data.
if ('id' in currentConversation) {
nextCurrentConversation = { ...currentConversation, message: messages };
} else {
// client data
nextCurrentConversation = {
id: EmptyConversationId,
name: 'New conversation',
dialog_id: dialogId,
message: messages,
};
}
const nextConversationList = [...conversationList];
nextConversationList.unshift(
nextCurrentConversation as IClientConversation,
);
setCurrentConversation(nextCurrentConversation as IClientConversation);
dispatch({
type: 'chatModel/setConversationList',
payload: nextConversationList,
});
handleClickConversation(EmptyConversationId);
}, [
dispatch,
currentConversation,
dialogId,
setCurrentConversation,
handleClickConversation,
conversationList,
currentDialog,
]);
return { createTemporaryConversation };
};
export const useFetchConversationList = () => {
const dispatch = useDispatch();
const conversationList: any[] = useSelector(
@ -412,7 +343,7 @@ export const useSelectCurrentConversation = () => {
(state: any) => state.chatModel.currentConversation,
);
const dialog = useSelectCurrentDialog();
const { conversationId } = useGetChatSearchParams();
const { conversationId, dialogId } = useGetChatSearchParams();
const addNewestConversation = useCallback((message: string) => {
setCurrentConversation((pre) => {
@ -448,12 +379,12 @@ export const useSelectCurrentConversation = () => {
setCurrentConversation({
id: '',
dialog_id: dialog.id,
dialog_id: dialogId,
reference: [],
message: [nextMessage],
} as any);
}
}, [conversationId, dialog]);
}, [conversationId, dialog, dialogId]);
useEffect(() => {
addPrologue();

View File

@ -1,14 +1,17 @@
import { IKnowledge } from '@/interfaces/database/knowledge';
import kbService from '@/services/kbService';
import { DvaModel } from 'umi';
export interface KnowledgeModelState {
data: any[];
knowledge: IKnowledge;
}
const model: DvaModel<KnowledgeModelState> = {
namespace: 'knowledgeModel',
state: {
data: [],
knowledge: {} as IKnowledge,
},
reducers: {
updateState(state, { payload }) {
@ -17,6 +20,12 @@ const model: DvaModel<KnowledgeModelState> = {
...payload,
};
},
setKnowledge(state, { payload }) {
return {
...state,
knowledge: payload,
};
},
},
effects: {
*rmKb({ payload = {} }, { call, put }) {
@ -42,6 +51,13 @@ const model: DvaModel<KnowledgeModelState> = {
});
}
},
*getKnowledgeDetail({ payload = {} }, { call, put }) {
const { data } = yield call(kbService.get_kb_detail, payload);
if (data.retcode === 0) {
yield put({ type: 'setKnowledge', payload: data.data });
}
return data.retcode;
},
},
};
export default model;

View File

@ -1,4 +1,5 @@
import { IChunk } from '@/interfaces/database/knowledge';
import { UploadFile } from 'antd';
import { v4 as uuid } from 'uuid';
export const buildChunkHighlights = (selectedChunk: IChunk) => {
@ -32,3 +33,5 @@ export const buildChunkHighlights = (selectedChunk: IChunk) => {
})
: [];
};
export const isFileUploadDone = (file: UploadFile) => file.status === 'done';