From af938e9d45b26e71ad13b1c4faa40456d5146620 Mon Sep 17 00:00:00 2001 From: fredzhu Date: Tue, 26 Dec 2023 16:25:15 +0800 Subject: [PATCH] feat(annotator): update annotator component --- packages/app/src/models/dataset/common.tsx | 153 ++-- packages/app/src/pages/Annotator/index.tsx | 4 +- packages/app/src/pages/Dataset/index.tsx | 9 +- packages/app/src/pages/Lab/FlagTool/index.tsx | 9 +- .../app/src/pages/Project/Workspace/index.tsx | 9 +- .../app/src/pages/Project/models/workspace.ts | 10 + packages/app/src/types/dataset.ts | 2 + .../src/Annotator/assets/add-prompt.svg | 11 + .../src/Annotator/assets/attribute.svg | 6 + .../components/src/Annotator/assets/brush.svg | 1 - .../src/Annotator/assets/delete_all.svg | 2 +- .../components/src/Annotator/assets/docs.svg | 10 + .../components/src/Annotator/assets/drag.svg | 2 +- .../components/src/Annotator/assets/house.svg | 10 + .../src/Annotator/assets/keyboard-down.svg | 5 + .../components/src/Annotator/assets/label.svg | 12 + .../components/src/Annotator/assets/layer.svg | 11 + .../components/src/Annotator/assets/logo.svg | 10 + .../src/Annotator/assets/mask-ai.svg | 12 + .../components/src/Annotator/assets/mask.svg | 14 + .../src/Annotator/assets/play-next.svg | 6 + .../src/Annotator/assets/play-pre.svg | 6 + .../src/Annotator/assets/play-stop.svg | 10 + .../components/src/Annotator/assets/play.svg | 10 + .../components/src/Annotator/assets/point.svg | 13 +- .../src/Annotator/assets/polygon-ai.svg | 11 + .../src/Annotator/assets/polygon.svg | 11 +- .../src/Annotator/assets/rectangle-ai.svg | 4 + .../src/Annotator/assets/rectangle.svg | 6 +- .../src/Annotator/assets/remove-prompt.svg | 17 + .../src/Annotator/assets/review.svg | 5 + .../src/Annotator/assets/settings-sliders.svg | 12 + .../src/Annotator/assets/skeleton-ai.svg | 11 + .../src/Annotator/assets/skeleton.svg | 10 + .../src/Annotator/assets/text-prompt.svg | 16 + .../src/Annotator/assets/visual-prompt.svg | 23 + .../src/Annotator/assets/zoomResize.svg | 2 +- .../components/AnnotationEditor/index.tsx | 187 ----- .../components/AttributeEditor/index.less | 54 ++ .../components/AttributeEditor/index.tsx | 97 +++ .../components/AttributesForm/index.less | 77 ++ .../components/AttributesForm/index.tsx | 130 +++ .../components/Classification/index.less | 54 ++ .../components/Classification/index.tsx | 194 +++++ .../components/DisplaySettings/index.less | 58 ++ .../components/DisplaySettings/index.tsx | 109 +++ .../components/EditorStatus/index.less | 28 + .../components/EditorStatus/index.tsx | 40 + .../components/LabelSelector/index.less | 77 ++ .../components/LabelSelector/index.tsx | 103 +++ .../components/MainToolBar/index.less | 83 -- .../components/MainToolBar/index.tsx | 304 ------- .../components/ModelSelectModal/index.less | 61 ++ .../components/ModelSelectModal/index.tsx | 96 +++ .../components/ModelSelector/index.less | 77 ++ .../components/ModelSelector/index.tsx | 56 ++ .../components/ObjectList/index.less | 75 +- .../Annotator/components/ObjectList/index.tsx | 321 +++++--- .../PointItem/{PointItem.tsx => index.tsx} | 4 +- .../components/PointsEditModal/index.less | 65 ++ .../components/PointsEditModal/index.tsx | 114 +++ .../components/ScaleToolBar/index.less | 110 --- .../components/ScaleToolBar/index.tsx | 209 ----- .../index.less | 16 +- .../components/SegConfirmModal/index.tsx | 76 ++ .../components/ShortcutsInfo/index.less | 13 + .../components/ShortcutsInfo/index.tsx | 49 +- .../components/SliderToolBar/index.less | 200 +++++ .../components/SliderToolBar/index.tsx | 422 ++++++++++ .../SmartAnnotationControl/index.less | 8 + .../SmartAnnotationControl/index.tsx | 138 ++-- .../components/SubToolBar/index.less | 11 +- .../Annotator/components/SubToolBar/index.tsx | 135 +--- .../components/TopPagination/index.tsx | 4 +- .../Annotator/components/TopTools/index.less | 11 +- .../src/Annotator/constants/index.ts | 110 ++- .../src/Annotator/constants/render.ts | 13 +- .../src/Annotator/constants/shortcuts.ts | 2 +- packages/components/src/Annotator/editor.tsx | 683 ++++++++-------- .../hooks/{useActions.ts => useActions.tsx} | 747 ++++++++++++------ .../src/Annotator/hooks/useAttributes.ts | 72 ++ .../Annotator/hooks/useCanvasContainer.tsx | 115 ++- ...useCanvasRender.ts => useCanvasRender.tsx} | 93 ++- .../src/Annotator/hooks/useColor.ts | 19 +- .../src/Annotator/hooks/useDataEffect.ts | 75 +- .../src/Annotator/hooks/useHistory.ts | 92 ++- .../src/Annotator/hooks/useLabels.ts | 113 ++- .../src/Annotator/hooks/useMouseEvents.tsx | 38 +- .../src/Annotator/hooks/useObjects.ts | 282 ++++--- .../src/Annotator/hooks/useShortcuts.ts | 36 +- .../src/Annotator/hooks/useSubtools.tsx | 281 +++++++ .../src/Annotator/hooks/useToolActions.ts | 236 ++++-- .../src/Annotator/hooks/useTopTools.tsx | 279 +++++++ .../src/Annotator/hooks/useTranslate.ts | 403 ++++++++++ packages/components/src/Annotator/index.less | 98 ++- packages/components/src/Annotator/preview.tsx | 93 +-- .../components/src/Annotator/sevices/index.ts | 174 +++- .../components/src/Annotator/tools/base.ts | 12 +- .../components/src/Annotator/tools/useMask.ts | 147 ++-- .../src/Annotator/tools/usePoint.ts | 77 ++ .../src/Annotator/tools/usePolygon.ts | 572 +++++++++----- .../src/Annotator/tools/useRectangle.ts | 230 +++++- .../src/Annotator/tools/useSkeleton.ts | 2 +- packages/components/src/Annotator/type.ts | 116 ++- .../components/src/Annotator/utils/base64.ts | 5 + .../components/src/Annotator/utils/color.ts | 11 +- .../components/src/Annotator/utils/compute.ts | 346 +++++--- packages/components/src/Annotator/view.tsx | 51 +- packages/components/src/locales/en-US.ts | 44 ++ packages/components/src/locales/zh-CN.ts | 39 + packages/components/tsconfig.json | 2 + 111 files changed, 7125 insertions(+), 2924 deletions(-) create mode 100644 packages/components/src/Annotator/assets/add-prompt.svg create mode 100644 packages/components/src/Annotator/assets/attribute.svg delete mode 100644 packages/components/src/Annotator/assets/brush.svg create mode 100644 packages/components/src/Annotator/assets/docs.svg create mode 100644 packages/components/src/Annotator/assets/house.svg create mode 100644 packages/components/src/Annotator/assets/keyboard-down.svg create mode 100644 packages/components/src/Annotator/assets/label.svg create mode 100644 packages/components/src/Annotator/assets/layer.svg create mode 100644 packages/components/src/Annotator/assets/logo.svg create mode 100644 packages/components/src/Annotator/assets/mask-ai.svg create mode 100644 packages/components/src/Annotator/assets/mask.svg create mode 100644 packages/components/src/Annotator/assets/play-next.svg create mode 100644 packages/components/src/Annotator/assets/play-pre.svg create mode 100644 packages/components/src/Annotator/assets/play-stop.svg create mode 100644 packages/components/src/Annotator/assets/play.svg create mode 100644 packages/components/src/Annotator/assets/polygon-ai.svg create mode 100644 packages/components/src/Annotator/assets/rectangle-ai.svg create mode 100644 packages/components/src/Annotator/assets/remove-prompt.svg create mode 100644 packages/components/src/Annotator/assets/review.svg create mode 100644 packages/components/src/Annotator/assets/settings-sliders.svg create mode 100644 packages/components/src/Annotator/assets/skeleton-ai.svg create mode 100644 packages/components/src/Annotator/assets/skeleton.svg create mode 100644 packages/components/src/Annotator/assets/text-prompt.svg create mode 100644 packages/components/src/Annotator/assets/visual-prompt.svg delete mode 100644 packages/components/src/Annotator/components/AnnotationEditor/index.tsx create mode 100644 packages/components/src/Annotator/components/AttributeEditor/index.less create mode 100644 packages/components/src/Annotator/components/AttributeEditor/index.tsx create mode 100644 packages/components/src/Annotator/components/AttributesForm/index.less create mode 100644 packages/components/src/Annotator/components/AttributesForm/index.tsx create mode 100644 packages/components/src/Annotator/components/Classification/index.less create mode 100644 packages/components/src/Annotator/components/Classification/index.tsx create mode 100644 packages/components/src/Annotator/components/DisplaySettings/index.less create mode 100644 packages/components/src/Annotator/components/DisplaySettings/index.tsx create mode 100644 packages/components/src/Annotator/components/EditorStatus/index.less create mode 100644 packages/components/src/Annotator/components/EditorStatus/index.tsx create mode 100644 packages/components/src/Annotator/components/LabelSelector/index.less create mode 100644 packages/components/src/Annotator/components/LabelSelector/index.tsx delete mode 100644 packages/components/src/Annotator/components/MainToolBar/index.less delete mode 100644 packages/components/src/Annotator/components/MainToolBar/index.tsx create mode 100644 packages/components/src/Annotator/components/ModelSelectModal/index.less create mode 100644 packages/components/src/Annotator/components/ModelSelectModal/index.tsx create mode 100644 packages/components/src/Annotator/components/ModelSelector/index.less create mode 100644 packages/components/src/Annotator/components/ModelSelector/index.tsx rename packages/components/src/Annotator/components/PointItem/{PointItem.tsx => index.tsx} (97%) create mode 100644 packages/components/src/Annotator/components/PointsEditModal/index.less create mode 100644 packages/components/src/Annotator/components/PointsEditModal/index.tsx delete mode 100644 packages/components/src/Annotator/components/ScaleToolBar/index.less delete mode 100644 packages/components/src/Annotator/components/ScaleToolBar/index.tsx rename packages/components/src/Annotator/components/{AnnotationEditor => SegConfirmModal}/index.less (78%) create mode 100644 packages/components/src/Annotator/components/SegConfirmModal/index.tsx create mode 100644 packages/components/src/Annotator/components/SliderToolBar/index.less create mode 100644 packages/components/src/Annotator/components/SliderToolBar/index.tsx rename packages/components/src/Annotator/hooks/{useActions.ts => useActions.tsx} (55%) create mode 100644 packages/components/src/Annotator/hooks/useAttributes.ts rename packages/components/src/Annotator/hooks/{useCanvasRender.ts => useCanvasRender.tsx} (76%) create mode 100644 packages/components/src/Annotator/hooks/useSubtools.tsx create mode 100644 packages/components/src/Annotator/hooks/useTopTools.tsx create mode 100644 packages/components/src/Annotator/hooks/useTranslate.ts create mode 100644 packages/components/src/Annotator/tools/usePoint.ts diff --git a/packages/app/src/models/dataset/common.tsx b/packages/app/src/models/dataset/common.tsx index 72ed741..ba915a7 100644 --- a/packages/app/src/models/dataset/common.tsx +++ b/packages/app/src/models/dataset/common.tsx @@ -30,7 +30,6 @@ import { PageState, } from './type'; import { isNumber } from 'lodash'; -import { IAnnotationObject } from 'dds-components/Annotator'; import { NsDataSet } from '@/types/dataset'; export default () => { @@ -275,6 +274,40 @@ export default () => { displayLabelIds, isTiledDiff, }; + + // filter displayType + const displayType = pageState.filterValues.displayAnnotationType; + objects = objects + .filter((obj) => { + return ( + (obj.mask && displayType === AnnotationType.Mask) || + (obj.alpha && displayType === AnnotationType.Matting) || + (obj.points && displayType === AnnotationType.KeyPoints) || + (obj.segmentation && displayType === AnnotationType.Segmentation) || + (obj.boundingBox && displayType === AnnotationType.Detection) + ); + }) + .map((obj) => { + return { + ...obj, + mask: displayType === AnnotationType.Mask ? obj.mask : undefined, + alpha: + displayType === AnnotationType.Matting ? obj.alpha : undefined, + points: + displayType === AnnotationType.KeyPoints ? obj.points : undefined, + segmentation: + displayType === AnnotationType.Segmentation + ? obj.segmentation + : undefined, + boundingBox: [ + AnnotationType.Detection, + AnnotationType.KeyPoints, + ].includes(displayType!) + ? obj.boundingBox + : undefined, + }; + }); + // Analysis mode -> filter fn/fp to display if (analysisMode) { const predObjects = objects.filter( @@ -314,37 +347,69 @@ export default () => { }); } - return objects.filter((item) => { - const { showAnnotations, showAllCategory } = displayOptionsResult; - const categoryId = pageState.filterValues.categoryId || ''; - if ( - !showAnnotations || - (!showAllCategory && item.categoryId !== categoryId) || - (diffMode && - item.labelId && - !diffMode.displayLabelIds.includes(item.labelId)) || - (diffMode && - diffMode.isTiledDiff && - item.labelId !== imageData.curLabelId) - ) { - return false; - } - if (!analysisMode && diffMode) { - const label = diffMode.labels.find( - (label) => label.id === item.labelId, - ); - if (!label) return false; - if (label.source === LABEL_SOURCE.gt) return true; - return ( - item.conf !== undefined && - item.conf >= label?.confidenceRange[0] && - item.conf <= label?.confidenceRange[1] + return objects + .filter((item) => { + const { showAnnotations, showAllCategory } = displayOptionsResult; + const categoryId = pageState.filterValues.categoryId || ''; + if ( + !showAnnotations || + (!showAllCategory && item.categoryId !== categoryId) || + (diffMode && + item.labelId && + !diffMode.displayLabelIds.includes(item.labelId)) || + (diffMode && + diffMode.isTiledDiff && + item.labelId !== imageData.curLabelId) + ) { + return false; + } + if (!analysisMode && diffMode) { + const label = diffMode.labels.find( + (label) => label.id === item.labelId, + ); + if (!label) return false; + if (label.source === LABEL_SOURCE.gt) return true; + return ( + item.conf !== undefined && + item.conf >= label?.confidenceRange[0] && + item.conf <= label?.confidenceRange[1] + ); + } + return true; + }) + .map((item) => { + // get custom style + const newItem = { ...item }; + const { + colorAplha: pointAplha, + strokeDash, + lineWidth: thickness, + } = getLabelCustomStyles( + item.labelId, + displayLabelIds, + isTiledDiff || Boolean(pageState.comparisons), ); - } - return true; - }); + if (analysisMode && item.compareResult) { + newItem.customStyles = { + pointAplha, + strokeDash, + thickness, + fillColor: + // @ts-ignore + COMPARE_RESULT_FILL_COLORS[item.compareResult] || 'transparent', + }; + } else { + newItem.customStyles = { + pointAplha, + strokeDash, + thickness, + }; + } + return newItem; + }); }, [ + pageState.filterValues.displayAnnotationType, pageState.comparisons, pageData.filters.labels, displayLabelIds, @@ -353,35 +418,6 @@ export default () => { ], ); - const getCustomObjectStyles = useCallback( - (object: IAnnotationObject) => { - const { - colorAplha: pointAplha, - strokeDash, - lineWidth: thickness, - } = getLabelCustomStyles( - object.labelId, - displayLabelIds, - isTiledDiff || Boolean(pageState.comparisons), - ); - if (Boolean(pageState.comparisons) && object.compareResult) { - return { - pointAplha, - strokeDash, - thickness, - fillColor: - COMPARE_RESULT_FILL_COLORS[object.compareResult] || 'transparent', - }; - } - return { - pointAplha, - strokeDash, - thickness, - }; - }, - [displayLabelIds, isTiledDiff, Boolean(pageState.comparisons)], - ); - return { // page var pageState, @@ -405,6 +441,5 @@ export default () => { // common render displayObjectsFilter, - getCustomObjectStyles, }; }; diff --git a/packages/app/src/pages/Annotator/index.tsx b/packages/app/src/pages/Annotator/index.tsx index af1d5d7..6bee1b9 100644 --- a/packages/app/src/pages/Annotator/index.tsx +++ b/packages/app/src/pages/Annotator/index.tsx @@ -1,5 +1,5 @@ import React, { useEffect, useState } from 'react'; -import { useModel } from '@umijs/max'; +import { history, useModel } from '@umijs/max'; import styles from './index.less'; import { AnnotateEditor, EditorMode } from 'dds-components/Annotator'; import { ImageList } from './components/ImageList'; @@ -102,6 +102,7 @@ const Page: React.FC = () => {
{ } }); }} + onCancel={() => history.push('/')} />
{ onPreviewIndexChange, exitPreview, displayObjectsFilter, - getCustomObjectStyles, } = useModel('dataset.common'); const { onPageDidMount, @@ -111,17 +110,14 @@ const Page: React.FC = () => { }} >
{item.flag > 0 && ( @@ -157,6 +153,7 @@ const Page: React.FC = () => { )} {/* Preview */} = 0 && !isSingleAnnotation} categories={pageData.filters.categories} list={imgList} @@ -171,9 +168,7 @@ const Page: React.FC = () => { onPreviewIndexChange(pageState.previewIndex - 1); }} objectsFilter={displayObjectsFilter} - getCustomObjectStyles={getCustomObjectStyles} displayOptionsResult={displayOptionsResult} - displayAnnotationType={pageState.filterValues.displayAnnotationType} /> {/* Screen loading */} {pageData.screenLoading ? ( diff --git a/packages/app/src/pages/Lab/FlagTool/index.tsx b/packages/app/src/pages/Lab/FlagTool/index.tsx index 785b638..72b01b7 100644 --- a/packages/app/src/pages/Lab/FlagTool/index.tsx +++ b/packages/app/src/pages/Lab/FlagTool/index.tsx @@ -27,7 +27,6 @@ const Page: React.FC = () => { onPreviewIndexChange, exitPreview, displayObjectsFilter, - getCustomObjectStyles, } = useModel('dataset.common'); const { onPageDidMount, @@ -108,17 +107,14 @@ const Page: React.FC = () => { }} > {item.flag > 0 && ( @@ -154,6 +150,7 @@ const Page: React.FC = () => { )} {/* Preview */} = 0 && !isSingleAnnotation} categories={pageData.filters.categories} list={imgList} @@ -168,9 +165,7 @@ const Page: React.FC = () => { onPreviewIndexChange(pageState.previewIndex - 1); }} objectsFilter={displayObjectsFilter} - getCustomObjectStyles={getCustomObjectStyles} displayOptionsResult={displayOptionsResult} - displayAnnotationType={pageState.filterValues.displayAnnotationType} /> {/* Screen loading */} {pageData.screenLoading ? ( diff --git a/packages/app/src/pages/Project/Workspace/index.tsx b/packages/app/src/pages/Project/Workspace/index.tsx index 1064f9f..24c14bb 100644 --- a/packages/app/src/pages/Project/Workspace/index.tsx +++ b/packages/app/src/pages/Project/Workspace/index.tsx @@ -44,7 +44,8 @@ const Page: React.FC = () => { onNextImage, onPrevImage, onLabelSave, - onReviewResult, + onReviewAccept, + onReviewReject, onEnterEdit, onStartLabel, onStartRework, @@ -239,6 +240,7 @@ const Page: React.FC = () => { }} > { {isEditorVisible && (
{ actionElements={actionElements} onCancel={onExitEditor} onSave={onLabelSave} - onReviewResult={onReviewResult} - onEnterEdit={onEnterEdit} + onReviewAccept={onReviewAccept} + onReviewReject={onReviewReject} onNext={onNextImage} onPrev={onPrevImage} /> diff --git a/packages/app/src/pages/Project/models/workspace.ts b/packages/app/src/pages/Project/models/workspace.ts index fb1217a..b309825 100644 --- a/packages/app/src/pages/Project/models/workspace.ts +++ b/packages/app/src/pages/Project/models/workspace.ts @@ -419,6 +419,14 @@ export default () => { setLoading(false); }; + const onReviewAccept = async (imageId: string) => { + return await onReviewResult(imageId, EQaAction.Accept); + }; + + const onReviewReject = async (imageId: string) => { + return await onReviewResult(imageId, EQaAction.Reject); + }; + /** * Initialize page parameters from the URL. * @param urlPageState @@ -487,6 +495,8 @@ export default () => { onNextImage, onLabelSave, onReviewResult, + onReviewAccept, + onReviewReject, onEnterEdit, onStartLabel, onStartRework, diff --git a/packages/app/src/types/dataset.ts b/packages/app/src/types/dataset.ts index c62a284..14dd4a5 100644 --- a/packages/app/src/types/dataset.ts +++ b/packages/app/src/types/dataset.ts @@ -24,6 +24,8 @@ export namespace NsDataSet { compareResult: COMPARE_RESULT; /** Pred index matched in GT analysis mode. */ matchedDetIdx?: number; + /** render styles */ + customStyles?: Record; } export interface DataSetImg extends BaseImage { diff --git a/packages/components/src/Annotator/assets/add-prompt.svg b/packages/components/src/Annotator/assets/add-prompt.svg new file mode 100644 index 0000000..0fbd494 --- /dev/null +++ b/packages/components/src/Annotator/assets/add-prompt.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/attribute.svg b/packages/components/src/Annotator/assets/attribute.svg new file mode 100644 index 0000000..1670dd4 --- /dev/null +++ b/packages/components/src/Annotator/assets/attribute.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/packages/components/src/Annotator/assets/brush.svg b/packages/components/src/Annotator/assets/brush.svg deleted file mode 100644 index 060f420..0000000 --- a/packages/components/src/Annotator/assets/brush.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/packages/components/src/Annotator/assets/delete_all.svg b/packages/components/src/Annotator/assets/delete_all.svg index 02f7ee1..f9f19b8 100644 --- a/packages/components/src/Annotator/assets/delete_all.svg +++ b/packages/components/src/Annotator/assets/delete_all.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/packages/components/src/Annotator/assets/docs.svg b/packages/components/src/Annotator/assets/docs.svg new file mode 100644 index 0000000..dae6d8b --- /dev/null +++ b/packages/components/src/Annotator/assets/docs.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/drag.svg b/packages/components/src/Annotator/assets/drag.svg index c7ea314..ffed696 100644 --- a/packages/components/src/Annotator/assets/drag.svg +++ b/packages/components/src/Annotator/assets/drag.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/packages/components/src/Annotator/assets/house.svg b/packages/components/src/Annotator/assets/house.svg new file mode 100644 index 0000000..5ad39a4 --- /dev/null +++ b/packages/components/src/Annotator/assets/house.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/keyboard-down.svg b/packages/components/src/Annotator/assets/keyboard-down.svg new file mode 100644 index 0000000..36a1fb4 --- /dev/null +++ b/packages/components/src/Annotator/assets/keyboard-down.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/packages/components/src/Annotator/assets/label.svg b/packages/components/src/Annotator/assets/label.svg new file mode 100644 index 0000000..f43d829 --- /dev/null +++ b/packages/components/src/Annotator/assets/label.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/layer.svg b/packages/components/src/Annotator/assets/layer.svg new file mode 100644 index 0000000..39e64bf --- /dev/null +++ b/packages/components/src/Annotator/assets/layer.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/logo.svg b/packages/components/src/Annotator/assets/logo.svg new file mode 100644 index 0000000..7e948f7 --- /dev/null +++ b/packages/components/src/Annotator/assets/logo.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/mask-ai.svg b/packages/components/src/Annotator/assets/mask-ai.svg new file mode 100644 index 0000000..669da2f --- /dev/null +++ b/packages/components/src/Annotator/assets/mask-ai.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/mask.svg b/packages/components/src/Annotator/assets/mask.svg new file mode 100644 index 0000000..a7c1377 --- /dev/null +++ b/packages/components/src/Annotator/assets/mask.svg @@ -0,0 +1,14 @@ + + + + + + + + \ No newline at end of file diff --git a/packages/components/src/Annotator/assets/play-next.svg b/packages/components/src/Annotator/assets/play-next.svg new file mode 100644 index 0000000..efbc00b --- /dev/null +++ b/packages/components/src/Annotator/assets/play-next.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/packages/components/src/Annotator/assets/play-pre.svg b/packages/components/src/Annotator/assets/play-pre.svg new file mode 100644 index 0000000..9ea7700 --- /dev/null +++ b/packages/components/src/Annotator/assets/play-pre.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/packages/components/src/Annotator/assets/play-stop.svg b/packages/components/src/Annotator/assets/play-stop.svg new file mode 100644 index 0000000..ffe088a --- /dev/null +++ b/packages/components/src/Annotator/assets/play-stop.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/play.svg b/packages/components/src/Annotator/assets/play.svg new file mode 100644 index 0000000..ac5e066 --- /dev/null +++ b/packages/components/src/Annotator/assets/play.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/point.svg b/packages/components/src/Annotator/assets/point.svg index 62545c3..1d90f98 100644 --- a/packages/components/src/Annotator/assets/point.svg +++ b/packages/components/src/Annotator/assets/point.svg @@ -1,11 +1,4 @@ - - - - - - - - - - + + + diff --git a/packages/components/src/Annotator/assets/polygon-ai.svg b/packages/components/src/Annotator/assets/polygon-ai.svg new file mode 100644 index 0000000..532e768 --- /dev/null +++ b/packages/components/src/Annotator/assets/polygon-ai.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/polygon.svg b/packages/components/src/Annotator/assets/polygon.svg index dd0eeed..c77e4fe 100644 --- a/packages/components/src/Annotator/assets/polygon.svg +++ b/packages/components/src/Annotator/assets/polygon.svg @@ -1 +1,10 @@ - \ No newline at end of file + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/rectangle-ai.svg b/packages/components/src/Annotator/assets/rectangle-ai.svg new file mode 100644 index 0000000..b3b29c9 --- /dev/null +++ b/packages/components/src/Annotator/assets/rectangle-ai.svg @@ -0,0 +1,4 @@ + + + + diff --git a/packages/components/src/Annotator/assets/rectangle.svg b/packages/components/src/Annotator/assets/rectangle.svg index 75e01aa..593e638 100644 --- a/packages/components/src/Annotator/assets/rectangle.svg +++ b/packages/components/src/Annotator/assets/rectangle.svg @@ -1 +1,5 @@ - \ No newline at end of file + + + + \ No newline at end of file diff --git a/packages/components/src/Annotator/assets/remove-prompt.svg b/packages/components/src/Annotator/assets/remove-prompt.svg new file mode 100644 index 0000000..73c75c5 --- /dev/null +++ b/packages/components/src/Annotator/assets/remove-prompt.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/review.svg b/packages/components/src/Annotator/assets/review.svg new file mode 100644 index 0000000..ab5f45b --- /dev/null +++ b/packages/components/src/Annotator/assets/review.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/packages/components/src/Annotator/assets/settings-sliders.svg b/packages/components/src/Annotator/assets/settings-sliders.svg new file mode 100644 index 0000000..c470e17 --- /dev/null +++ b/packages/components/src/Annotator/assets/settings-sliders.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/skeleton-ai.svg b/packages/components/src/Annotator/assets/skeleton-ai.svg new file mode 100644 index 0000000..53c6262 --- /dev/null +++ b/packages/components/src/Annotator/assets/skeleton-ai.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/skeleton.svg b/packages/components/src/Annotator/assets/skeleton.svg new file mode 100644 index 0000000..48237c9 --- /dev/null +++ b/packages/components/src/Annotator/assets/skeleton.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/text-prompt.svg b/packages/components/src/Annotator/assets/text-prompt.svg new file mode 100644 index 0000000..fd6550d --- /dev/null +++ b/packages/components/src/Annotator/assets/text-prompt.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/visual-prompt.svg b/packages/components/src/Annotator/assets/visual-prompt.svg new file mode 100644 index 0000000..0bec666 --- /dev/null +++ b/packages/components/src/Annotator/assets/visual-prompt.svg @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/components/src/Annotator/assets/zoomResize.svg b/packages/components/src/Annotator/assets/zoomResize.svg index 99e9add..29180f0 100644 --- a/packages/components/src/Annotator/assets/zoomResize.svg +++ b/packages/components/src/Annotator/assets/zoomResize.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/packages/components/src/Annotator/components/AnnotationEditor/index.tsx b/packages/components/src/Annotator/components/AnnotationEditor/index.tsx deleted file mode 100644 index 27324a3..0000000 --- a/packages/components/src/Annotator/components/AnnotationEditor/index.tsx +++ /dev/null @@ -1,187 +0,0 @@ -import { CloseOutlined } from '@ant-design/icons'; -import { Button, Card, Select } from 'antd'; -import classNames from 'classnames'; -import { FloatWrapper } from '../FloatWrapper'; -import { memo, useEffect, useMemo, useState } from 'react'; -import { useKeyPress } from 'ahooks'; -import { EDITOR_SHORTCUTS, EShortcuts } from '../../constants/shortcuts'; -import { useLocale } from 'dds-utils/locale'; -import CategoryCreator from '../CategoryCreator'; -import { Category, IAnnotationObject } from '../../type'; -import PointItem from '../PointItem/PointItem'; -import { EElementType, EObjectType, KEYPOINTS_VISIBLE_TYPE } from '../../constants'; -import './index.less'; - -interface IProps { - hideTitle: boolean; - allowAddCategory: boolean; - latestLabel: string; - categories: Category[]; - currEditObject: IAnnotationObject | undefined; - currObjectIndex: number; - focusObjectIndex: number; - focusEleType: EElementType; - focusEleIndex: number; - onCreateCategory: (name: string) => void; - onCloseAnnotationEditor: () => void; - onFinishCurrCreate: (label: string) => void; - onDeleteCurrObject: () => void; - onChangePointVisible: (index: number, visible: KEYPOINTS_VISIBLE_TYPE) => void; -} - -export const AnnotationEditor: React.FC = memo( - ({ - hideTitle, - allowAddCategory, - latestLabel, - categories, - currEditObject, - currObjectIndex, - focusEleIndex, - focusObjectIndex, - focusEleType, - onCreateCategory, - onFinishCurrCreate, - onDeleteCurrObject, - onCloseAnnotationEditor, - onChangePointVisible - }) => { - const { localeText } = useLocale(); - - const defaultObjectLabel = currEditObject?.label || latestLabel; - const [objLabel, setObjLabel] = useState(defaultObjectLabel); - - useEffect(() => { - setObjLabel(currEditObject?.label || latestLabel); - }, [currEditObject]); - - useKeyPress( - EDITOR_SHORTCUTS[EShortcuts.SaveCurrObject].shortcut, - (event: KeyboardEvent) => { - if (currEditObject) { - event.preventDefault(); - onFinishCurrCreate(objLabel); - } - }, - { - exactMatch: true, - }, - ); - - const showKeypointsList = useMemo(() => { - return currEditObject?.type === EObjectType.Skeleton; - }, [currEditObject]); - - return ( - - - {localeText('DDSAnnotator.annotsEditor.title')} - } - shape="circle" - size="small" - onClick={onCloseAnnotationEditor} - > -
- ) - } - > -
-
- -
- { - showKeypointsList && -
-
- { - currEditObject && currEditObject.keypoints && - currEditObject.keypoints.points.map((ele, eleIndex) => ( - { onChangePointVisible(eleIndex, visible); }} - /> - )) - } -
-
- } -
-
- - -
-
-
- - - ); - }, -); diff --git a/packages/components/src/Annotator/components/AttributeEditor/index.less b/packages/components/src/Annotator/components/AttributeEditor/index.less new file mode 100644 index 0000000..4b0fe8f --- /dev/null +++ b/packages/components/src/Annotator/components/AttributeEditor/index.less @@ -0,0 +1,54 @@ +.dds-annotator-attribute-editor { + position: absolute; + right: 1rem; + top: 1rem; + width: 320px; + box-shadow: 2px 2px 12px 3px rgba(0, 0, 0, 0.6); + opacity: 1; + pointer-events: all; + z-index: 99; + + .ant-card-head { + background-color: @colorPrimary; + color: #fff; + font-size: 15px; + padding: 0 15px; + min-height: 45px; + } + + .ant-card-body { + padding: 0 4px; + } + + &-title { + display: flex; + align-items: center; + justify-content: space-between; + + &-btn { + border: 0; + + &:hover { + background-color: rgba(255, 255, 255, 0.2) !important; + + svg { + color: #fff; + } + } + } + } + + &-content { + display: flex; + flex-direction: column; + align-items: flex-end; + } + + &-actions { + padding: 0 12px 12px; + } + + &:hover { + box-shadow: 2px 2px 12px 3px rgba(0, 0, 0, 0.6); + } +} diff --git a/packages/components/src/Annotator/components/AttributeEditor/index.tsx b/packages/components/src/Annotator/components/AttributeEditor/index.tsx new file mode 100644 index 0000000..2edafac --- /dev/null +++ b/packages/components/src/Annotator/components/AttributeEditor/index.tsx @@ -0,0 +1,97 @@ +import { Button, Card, message } from 'antd'; +import { useImmer } from 'use-immer'; +import { FloatWrapper } from '../FloatWrapper'; +import { memo, useEffect } from 'react'; +import { useLocale } from 'dds-utils/locale'; +import { IAttributeValue, IEditingAttribute } from '../../type'; +import './index.less'; +import AttributesForm from '../AttributesForm'; +import { CloseOutlined } from '@ant-design/icons'; + +interface IProps { + data: IEditingAttribute; + supportEdit?: boolean; + onConfirmAttibuteEdit: (values: IAttributeValue[]) => void; + onCancelAttibuteEdit: () => void; +} + +const AttributeEditor: React.FC = memo( + ({ data, supportEdit, onConfirmAttibuteEdit, onCancelAttibuteEdit }) => { + const { localeText } = useLocale(); + const [values, setValues] = useImmer([]); + + useEffect(() => { + setValues(data?.values || []); + }, [data.values]); + + const onChangeValue = (index: number, value: IAttributeValue) => { + setValues((s) => { + s[index] = value; + }); + }; + + const onConfirm = () => { + if ( + data.attributes.find( + (item, index) => + item.required && + (values[index] === undefined || values[index] === null), + ) + ) { + message.error(localeText('DDSAnnotator.attribute.required')); + return; + } + const results: IAttributeValue[] = []; + data.attributes.forEach((_item, index) => { + results.push(values[index] === undefined ? null : values[index]); + }); + onConfirmAttibuteEdit(results); + }; + + return ( + + +
{localeText('DDSAnnotator.attribute.add')}
+ + + } + > +
+ + {supportEdit && ( +
+ +
+ )} +
+ + + ); + }, +); + +export default AttributeEditor; diff --git a/packages/components/src/Annotator/components/AttributesForm/index.less b/packages/components/src/Annotator/components/AttributesForm/index.less new file mode 100644 index 0000000..359fb8a --- /dev/null +++ b/packages/components/src/Annotator/components/AttributesForm/index.less @@ -0,0 +1,77 @@ +.dds-annotator-attributes-form { + width: 100%; + padding: 12px 12px 0; + + .ant-form-item { + margin-bottom: 14px; + } + + .ant-form-item-label { + font-weight: 500; + } + + .ant-input { + border-color: @colorPrimary; + } + + &-item-title { + display: flex; + align-items: center; + + &-btn { + margin-left: 12px; + height: 24px; + border: 0; + display: flex; + align-items: center; + justify-items: center; + + svg { + width: 22px; + height: 22px; + fill: @colorPrimary; + } + } + + .attribute-warn { + fill: #f53f3f; + } + } +} + +.dds-annotator-attributes-form-dark { + .ant-form-item-label { + & > label { + color: #fff; + } + } + + .ant-radio-wrapper { + color: #fff; + + .ant-radio-disabled .ant-radio-inner { + background-color: #fff; + } + } + + .ant-checkbox-wrapper { + color: #fff; + + .ant-checkbox-disabled .ant-checkbox-inner { + background-color: #fff; + } + + .ant-checkbox-disabled + span { + color: #fff; + } + } + + .ant-input { + color: #fff; + background-color: transparent; + + &::placeholder { + color: rgba(255, 255, 255, 0.3); + } + } +} diff --git a/packages/components/src/Annotator/components/AttributesForm/index.tsx b/packages/components/src/Annotator/components/AttributesForm/index.tsx new file mode 100644 index 0000000..2dbd10a --- /dev/null +++ b/packages/components/src/Annotator/components/AttributesForm/index.tsx @@ -0,0 +1,130 @@ +import React, { memo } from 'react'; +import classNames from 'classnames'; +import { Button, Checkbox, Form, Input, Radio, Tooltip } from 'antd'; +import { EActionType, IAttribute, IAttributeValue } from '../../type'; +import { isEqual } from 'lodash'; +import './index.less'; +import { ReactComponent as Attribute } from '../../assets/attribute.svg'; +import { useLocale } from 'dds-utils/locale'; + +export interface IProps { + isDarkTheme?: boolean; + disabled?: boolean; + data: (IAttribute & { + hasAttributes?: boolean; + requireAttribute?: boolean; + })[]; + values: IAttributeValue[]; + onChangeValue: (index: number, value: IAttributeValue) => void; + onFocusInput?: ( + index: number, + event: React.FocusEvent, + ) => void; + onClickAttributes?: (index: number) => void; +} + +const propsAreEqual = (prev: IProps, next: IProps): boolean => { + return ( + prev.isDarkTheme === next.isDarkTheme && + prev.disabled === next.disabled && + isEqual(prev.data, next.data) && + isEqual(prev.values, next.values) && + prev.onChangeValue === next.onChangeValue && + prev.onFocusInput === next.onFocusInput && + prev.onClickAttributes === next.onClickAttributes + ); +}; + +const AttributesForm: React.FC = memo((props) => { + const { localeText } = useLocale(); + const { + isDarkTheme, + disabled, + data, + values, + onChangeValue, + onFocusInput, + onClickAttributes, + } = props; + + return ( +
+ {data.map((item, index) => ( + + {item.field} + + + +
+ + + onChangeImageDisplayOpts({ + ...displayOption, + brightness: value, + }) + } + min={0} + max={200} + /> +
+
+ + + onChangeImageDisplayOpts({ + ...displayOption, + contrast: value, + }) + } + min={0} + max={200} + /> +
+
+ + + onChangeImageDisplayOpts({ + ...displayOption, + saturate: value, + }) + } + min={0} + max={200} + /> +
+ + ); + }, [ + displayOption.brightness, + displayOption.contrast, + displayOption.saturate, + onChangeImageDisplayOpts, + onChangeAnnotsDisplayOpts, + ]); + + return ( + + + + + + ); + }, +); + +export default DisplaySettings; diff --git a/packages/components/src/Annotator/components/EditorStatus/index.less b/packages/components/src/Annotator/components/EditorStatus/index.less new file mode 100644 index 0000000..2abf72f --- /dev/null +++ b/packages/components/src/Annotator/components/EditorStatus/index.less @@ -0,0 +1,28 @@ +.dds-annotator-editor-status { + position: relative; + height: 30px; + padding: 0 8px; + display: flex; + align-items: center; + justify-content: center; + gap: 8px; + color: #fff; + font-size: 14px; + font-weight: 500; + border-radius: 5px; + + svg { + width: 20px; + height: 20px; + } +} + +.dds-annotator-editor-status-1 { + border: 1px solid #26a1f4; + background: rgba(38, 161, 244, 0.38); +} + +.dds-annotator-editor-status-2 { + border: 1px solid #ffd305; + background: rgba(255, 211, 5, 0.38); +} diff --git a/packages/components/src/Annotator/components/EditorStatus/index.tsx b/packages/components/src/Annotator/components/EditorStatus/index.tsx new file mode 100644 index 0000000..6c9d428 --- /dev/null +++ b/packages/components/src/Annotator/components/EditorStatus/index.tsx @@ -0,0 +1,40 @@ +import { memo } from 'react'; +import classNames from 'classnames'; +import { useLocale } from 'dds-utils/locale'; +import { EditorMode } from '../../type'; +import { ReactComponent as LabelIcon } from '../../assets/label.svg'; +import { ReactComponent as ReviewIcon } from '../../assets/review.svg'; +import './index.less'; + +interface IProps { + mode: EditorMode; +} + +const EditorStatus: React.FC = memo(({ mode }) => { + const { localeText } = useLocale(); + + if (mode === EditorMode.View) return null; + + return ( +
+ {mode === EditorMode.Edit ? ( + <> + + {localeText('DDSAnnotator.status.labeling')} + + ) : ( + <> + + {localeText('DDSAnnotator.status.reviewing')} + + )} +
+ ); +}); + +export default EditorStatus; diff --git a/packages/components/src/Annotator/components/LabelSelector/index.less b/packages/components/src/Annotator/components/LabelSelector/index.less new file mode 100644 index 0000000..c8fe833 --- /dev/null +++ b/packages/components/src/Annotator/components/LabelSelector/index.less @@ -0,0 +1,77 @@ +.dds-annotator-label-selector { + width: 220px; + margin-left: -5px; + + .ant-select { + width: 100%; + + .ant-select-selector { + background-color: transparent !important; + color: #fff; + } + + .ant-select-selection-item { + display: flex; + align-items: center; + + svg { + width: 16px; + height: 16px; + margin-right: 10px; + } + } + + .ant-select-arrow { + color: rgba(255, 255, 255, 0.5); + } + } + + .ant-select-open .ant-select-selection-item { + color: rgba(255, 255, 255, 0.5); + } + + &-option { + &-color { + width: 12px; + height: 12px; + margin-right: 10px; + background-color: #fff; + } + + .ant-select-item-option-content { + display: flex; + align-items: center; + } + + svg { + width: 16px; + height: 16px; + margin-right: 10px; + } + } +} + +.dds-annotator-editor-light { + .dds-annotator-label-selector { + .ant-select { + .ant-select-selector { + color: #000; + border: 1px solid #acacac; + } + + .ant-select-arrow { + color: rgba(0, 0, 0, 0.5); + } + } + + .ant-select-open .ant-select-selection-item { + color: rgba(0, 0, 0, 0.5); + } + + &-option { + &-color { + background-color: #000; + } + } + } +} diff --git a/packages/components/src/Annotator/components/LabelSelector/index.tsx b/packages/components/src/Annotator/components/LabelSelector/index.tsx new file mode 100644 index 0000000..8cebe23 --- /dev/null +++ b/packages/components/src/Annotator/components/LabelSelector/index.tsx @@ -0,0 +1,103 @@ +import { Select } from 'antd'; +import { useLocale } from 'dds-utils/locale'; +import { memo, useMemo } from 'react'; +import { Category, DrawData } from '../../type'; +import CategoryCreator from '../CategoryCreator'; +import { + EBasicToolItem, + EBasicToolTypeMap, + LABEL_TOOL_MAP, + OBJECT_ICON, +} from '../../constants'; +import './index.less'; + +interface IProps { + drawData: DrawData; + latestLabelId: string; + isSeperate?: boolean; + labelOptions: Category[]; + labelColors?: Record; + onChangeObjectLabel: (labelId: string) => void; + onCreateCategory: (name: string) => void; +} + +const LabelSelector: React.FC = memo( + ({ + drawData, + latestLabelId, + isSeperate, + labelOptions, + labelColors, + onChangeObjectLabel, + onCreateCategory, + }) => { + const { localeText } = useLocale(); + const TypeIcon = useMemo(() => { + if (labelOptions.length > 0) { + const labelType = labelOptions[0]?.labelType; + // @ts-ignore + const toolType = labelType && LABEL_TOOL_MAP[labelType]; + const objectType = + EBasicToolTypeMap[toolType as unknown as EBasicToolItem]; + if (objectType) { + return OBJECT_ICON[objectType]; + } + } + return undefined; + }, [labelOptions]); + + return ( +
+ +
+ ); + }, +); + +export default LabelSelector; diff --git a/packages/components/src/Annotator/components/MainToolBar/index.less b/packages/components/src/Annotator/components/MainToolBar/index.less deleted file mode 100644 index a8bdefb..0000000 --- a/packages/components/src/Annotator/components/MainToolBar/index.less +++ /dev/null @@ -1,83 +0,0 @@ -.dds-annotator-maintoolbar { - position: absolute; - left: 1rem; - top: 50%; - transform: translateY(-50%); - display: flex; - flex-direction: column; - justify-content: center; - align-items: center; - z-index: 99; - background-color: #212121; - border-radius: 10px; - padding: 0.5rem; - width: 50px; - pointer-events: auto; - font-weight: 600; - - .maintoolbar-btn { - width: 32px; - height: 32px; - margin: 0.25rem 0; - border: 0; - background-color: transparent; - border-radius: 5px; - - svg { - scale: 1.2; - } - - &:hover { - color: @colorPrimary; - background-color: @colorPrimary; - transform: scale(1.2); - } - } - - .maintoolbar-btn-active { - color: @colorPrimary; - background-color: @colorPrimary; - } - - .maintoolbar-divider { - width: 100%; - margin: 8px 6px; - border-bottom: 1px solid #bbb; - } -} - -.dds-annotator-maintoolbar-popover { - display: flex; - flex-direction: column; - justify-content: center; - align-items: flex-start; - - .popover-title { - font-weight: 600; - font-size: 14px; - margin-right: 10px; - } - - .popover-key { - min-width: 30px; - justify-content: center; - border-radius: 2px; - padding: 2px 5px; - color: rgba(0, 0, 0, 0.8); - box-shadow: 0 1px 3px rgba(0, 0, 0, 0.3), 0 1px 2px rgba(0, 0, 0, 0.1); - font-size: 12px; - font-weight: 600; - } - - .popover-divider { - width: 100%; - margin: 10px 0; - border-bottom: 1px solid rgba(0, 0, 0, 0.1); - } - - .popover-description { - max-width: 220px; - font-size: 13px; - color: rgba(0, 0, 0, 0.8); - } -} diff --git a/packages/components/src/Annotator/components/MainToolBar/index.tsx b/packages/components/src/Annotator/components/MainToolBar/index.tsx deleted file mode 100644 index 473d5e2..0000000 --- a/packages/components/src/Annotator/components/MainToolBar/index.tsx +++ /dev/null @@ -1,304 +0,0 @@ -import { Button, Popover } from 'antd'; -import Icon from '@ant-design/icons'; -import classNames from 'classnames'; -import { - EBasicToolItem, - EObjectType, - EActionToolItem, - EToolType, - OBJECT_ICON, - EDITOR_TOOL_ICON, -} from '../../constants'; -import { FloatWrapper } from '../FloatWrapper'; -import { ReactComponent as DragToolIcon } from '../../assets/drag.svg'; -import { useKeyPress } from 'ahooks'; -import { - EDITOR_SHORTCUTS, - EShortcuts, - TShortcutItem, -} from '../../constants/shortcuts'; -import { memo, useMemo } from 'react'; -import { getIconFromShortcut } from '../ShortcutsInfo'; -import { useLocale } from 'dds-utils/locale'; -import './index.less'; - -type TToolItem = { - key: T; - name: string; - shortcut: TShortcutItem; - icon: JSX.Element; - description?: string; -}; - -interface IProps { - selectedTool: EToolType; - isAIAnnotationActive: boolean; - onChangeSelectedTool: (type: EToolType) => void; - onActiveAIAnnotation: (active: boolean) => void; - undo: () => void; - redo: () => void; - repeatPrevious: () => void; - deleteAll: () => void; -} - -export const MainToolBar: React.FC = memo( - ({ - selectedTool, - isAIAnnotationActive, - onChangeSelectedTool, - onActiveAIAnnotation, - undo, - redo, - repeatPrevious, - deleteAll, - }) => { - const { localeText } = useLocale(); - - const basicTools: TToolItem[] = [ - { - key: EBasicToolItem.Drag, - name: localeText('DDSAnnotator.toolbar.drag'), - shortcut: EDITOR_SHORTCUTS[EShortcuts.DragTool], - icon: , - description: localeText('DDSAnnotator.toolbar.drag.desc'), - }, - { - key: EBasicToolItem.Rectangle, - name: localeText('DDSAnnotator.toolbar.rectangle'), - shortcut: EDITOR_SHORTCUTS[EShortcuts.RectangleTool], - icon: , - description: localeText('DDSAnnotator.toolbar.rectangle.desc'), - }, - { - key: EBasicToolItem.Polygon, - name: localeText('DDSAnnotator.toolbar.polygon'), - shortcut: EDITOR_SHORTCUTS[EShortcuts.PolygonTool], - icon: , - description: localeText('DDSAnnotator.toolbar.polygon.desc'), - }, - { - key: EBasicToolItem.Skeleton, - name: localeText('DDSAnnotator.toolbar.skeleton'), - shortcut: EDITOR_SHORTCUTS[EShortcuts.SkeletonTool], - icon: , - description: localeText('DDSAnnotator.toolbar.skeleton.desc'), - }, - { - key: EBasicToolItem.Mask, - name: localeText('DDSAnnotator.toolbar.mask'), - shortcut: EDITOR_SHORTCUTS[EShortcuts.MaskTool], - icon: , - description: localeText('DDSAnnotator.toolbar.mask.desc'), - }, - ]; - - const smartTools: TToolItem[] = [ - { - key: EActionToolItem.SmartAnnotation, - name: localeText('DDSAnnotator.toolbar.aiAnno'), - shortcut: EDITOR_SHORTCUTS[EShortcuts.SmartAnnotation], - icon: ( - - ), - description: localeText('DDSAnnotator.toolbar.aiAnno.desc'), - }, - ]; - - const actionTools = [ - { - key: EActionToolItem.Undo, - name: localeText('DDSAnnotator.toolbar.undo'), - icon: , - shortcut: EDITOR_SHORTCUTS[EShortcuts.Undo], - handler: undo, - description: localeText('DDSAnnotator.toolbar.undo.desc'), - }, - { - key: EActionToolItem.Redo, - name: localeText('DDSAnnotator.toolbar.redo'), - icon: , - shortcut: EDITOR_SHORTCUTS[EShortcuts.Redo], - handler: redo, - description: localeText('DDSAnnotator.toolbar.redo.desc'), - }, - { - key: EActionToolItem.RepeatPrevious, - name: localeText('DDSAnnotator.toolbar.repeatPrevious'), - icon: ( - - ), - shortcut: EDITOR_SHORTCUTS[EShortcuts.RepeatPrevious], - handler: repeatPrevious, - description: localeText('DDSAnnotator.toolbar.repeatPrevious.desc'), - }, - { - key: EActionToolItem.DeleteAll, - name: localeText('DDSAnnotator.toolbar.deleteAll'), - icon: , - shortcut: EDITOR_SHORTCUTS[EShortcuts.DeleteAll], - handler: deleteAll, - description: localeText('DDSAnnotator.toolbar.deleteAll.desc'), - }, - ]; - - const basicToolKeys: string[] = useMemo(() => { - return basicTools.reduce((keys: string[], tool) => { - return keys.concat(tool.shortcut.shortcut); - }, []); - }, [basicTools]); - - const smartToolKeys: string[] = useMemo(() => { - return smartTools.reduce((keys: string[], tool) => { - return keys.concat(tool.shortcut.shortcut); - }, []); - }, [actionTools]); - - /** Active Basic Tool */ - useKeyPress( - basicToolKeys, - (event) => { - const activeTool = basicTools.find((tool) => { - return tool.shortcut.shortcut.includes(event.key); - }); - if (activeTool) { - onChangeSelectedTool(activeTool.key); - } - }, - { - exactMatch: true, - }, - ); - - /** Active AI Annotation */ - useKeyPress( - smartToolKeys, - (event) => { - const smartTool = smartTools.find((tool) => { - return tool.shortcut.shortcut.includes(event.key); - }); - if (smartTool) { - onActiveAIAnnotation(!isAIAnnotationActive); - } - }, - { - exactMatch: true, - }, - ); - - /** Undo */ - useKeyPress( - EDITOR_SHORTCUTS[EShortcuts.Undo].shortcut, - (event: KeyboardEvent) => { - event.preventDefault(); - undo(); - }, - { - exactMatch: true, - }, - ); - - /** Redo */ - useKeyPress( - EDITOR_SHORTCUTS[EShortcuts.Redo].shortcut, - (event: KeyboardEvent) => { - event.preventDefault(); - redo(); - }, - { - exactMatch: true, - }, - ); - - /** Repeat Previous */ - useKeyPress( - EDITOR_SHORTCUTS[EShortcuts.RepeatPrevious].shortcut, - (event: KeyboardEvent) => { - event.preventDefault(); - repeatPrevious(); - }, - { - exactMatch: true, - }, - ); - - /** Delete All */ - useKeyPress( - EDITOR_SHORTCUTS[EShortcuts.DeleteAll].shortcut, - (event: KeyboardEvent) => { - event.preventDefault(); - deleteAll(); - }, - { - exactMatch: true, - }, - ); - - const popoverContent = ( - item: TToolItem, - ) => { - const icon = getIconFromShortcut(item.shortcut.shortcut, false); - return ( -
-
- {item.name} - {icon} -
-
-
{item.description}
-
- ); - }; - - return ( - -
- {basicTools.map((item) => ( - -
- - ); - }, -); diff --git a/packages/components/src/Annotator/components/ModelSelectModal/index.less b/packages/components/src/Annotator/components/ModelSelectModal/index.less new file mode 100644 index 0000000..76dba70 --- /dev/null +++ b/packages/components/src/Annotator/components/ModelSelectModal/index.less @@ -0,0 +1,61 @@ +.dds-annotator-model-selector-modal { + display: flex; + gap: 30px; + + &-option { + position: relative; + display: flex; + flex-direction: column; + align-items: center; + justify-content: space-between; + padding-top: 30px; + padding-block-end: 20px; + margin-block: 25px; + width: 220px; + height: 180px; + border: 0.5px solid #d6d6d6; + + &-icon { + svg { + width: 55px; + height: 55px; + } + } + + &-name { + color: #000; + font-size: 18px; + font-weight: 500; + user-select: none; + } + + &-description { + text-align: center; + width: 80%; + color: rgba(0, 0, 0, 0.4); + font-size: 12px; + font-weight: 400; + text-overflow: ellipsis; + user-select: none; + } + + &-tag { + position: absolute; + top: 10px; + right: 10px; + margin: 0; + } + + &:hover { + border: 2px solid #165cff; + background: rgba(185, 206, 255, 0.11); + box-shadow: 0 4px 4px 0 rgba(0, 0, 0, 0.25); + } + } + + &-option-hightlight { + border: 2px solid #165cff5f; + box-shadow: 0 1px 2px 0 rgba(0, 0, 0, 0.25); + background: rgba(185, 206, 255, 0.3); + } +} diff --git a/packages/components/src/Annotator/components/ModelSelectModal/index.tsx b/packages/components/src/Annotator/components/ModelSelectModal/index.tsx new file mode 100644 index 0000000..8c240a0 --- /dev/null +++ b/packages/components/src/Annotator/components/ModelSelectModal/index.tsx @@ -0,0 +1,96 @@ +import { + EBasicToolItem, + EnumModelType, + MODEL_INTRO_MAP, + TOOL_MODELS_MAP, +} from '../../constants'; +import Icon from '@ant-design/icons'; +import { Modal, Tag } from 'antd'; +import { memo, useMemo } from 'react'; +import './index.less'; +import { useLocale } from 'dds-utils'; +import classNames from 'classnames'; + +interface IProps { + selectedTool: EBasicToolItem; + AIAnnotation: boolean; + selectedModel?: EnumModelType; + onSelectModel: (type: EnumModelType) => void; + onCloseModal: () => void; +} + +const ModelSelectModal: React.FC = memo( + ({ + selectedTool, + AIAnnotation, + selectedModel, + onSelectModel, + onCloseModal, + }) => { + const { localeText } = useLocale(); + + const autoOpen = useMemo(() => { + if ( + AIAnnotation && + TOOL_MODELS_MAP[selectedTool] && + TOOL_MODELS_MAP[selectedTool]!.length > 1 && + !selectedModel + ) { + return true; + } + return false; + }, [AIAnnotation, selectedTool, selectedModel]); + + return ( + +
+ {TOOL_MODELS_MAP[selectedTool]?.map((model, index) => { + const intro = MODEL_INTRO_MAP[model]; + if (!intro) return <>; + return ( +
onSelectModel(model)} + key={index} + > + +
+ {intro.name} +
+
+ {localeText(intro.description)} +
+ {intro.hightlight && ( + + {'New'} + + )} +
+ ); + })} +
+ + ); + }, +); + +export default ModelSelectModal; diff --git a/packages/components/src/Annotator/components/ModelSelector/index.less b/packages/components/src/Annotator/components/ModelSelector/index.less new file mode 100644 index 0000000..ce81e89 --- /dev/null +++ b/packages/components/src/Annotator/components/ModelSelector/index.less @@ -0,0 +1,77 @@ +.dds-annotator-model-selector { + width: 220px; + margin-left: -5px; + + .ant-select { + width: 100%; + + .ant-select-selector { + background-color: transparent !important; + color: #fff; + } + + .ant-select-selection-item { + display: flex; + align-items: center; + + svg { + width: 16px; + height: 16px; + margin-right: 10px; + } + } + + .ant-select-arrow { + color: rgba(255, 255, 255, 0.5); + } + } + + .ant-select-open .ant-select-selection-item { + color: rgba(255, 255, 255, 0.5); + } + + &-option { + &-color { + width: 12px; + height: 12px; + margin-right: 10px; + background-color: #fff; + } + + .ant-select-item-option-content { + display: flex; + align-items: center; + } + + svg { + width: 16px; + height: 16px; + margin-right: 10px; + } + } +} + +.dds-annotator-editor-light { + .dds-annotator-model-selector { + .ant-select { + .ant-select-selector { + color: #000; + border: 1px solid #acacac; + } + + .ant-select-arrow { + color: rgba(0, 0, 0, 0.5); + } + } + + .ant-select-open .ant-select-selection-item { + color: rgba(0, 0, 0, 0.5); + } + + &-option { + &-color { + background-color: #000; + } + } + } +} diff --git a/packages/components/src/Annotator/components/ModelSelector/index.tsx b/packages/components/src/Annotator/components/ModelSelector/index.tsx new file mode 100644 index 0000000..759c87f --- /dev/null +++ b/packages/components/src/Annotator/components/ModelSelector/index.tsx @@ -0,0 +1,56 @@ +import { Select } from 'antd'; +import { useLocale } from 'dds-utils/locale'; +import { memo } from 'react'; +import { DrawData } from '../../type'; +import { + EnumModelType, + EObjectType, + MODEL_INTRO_MAP, + OBJECT_AI_ICON, +} from '../../constants'; +import './index.less'; +import Icon from '@ant-design/icons'; + +interface IProps { + drawData: DrawData; + modelOptions: EnumModelType[]; + onSelectModel: (type: EnumModelType) => void; +} + +const ModelSelector: React.FC = memo( + ({ drawData, modelOptions, onSelectModel }) => { + const { localeText } = useLocale(); + + return ( +
+ +
+ ); + }, +); + +export default ModelSelector; diff --git a/packages/components/src/Annotator/components/ObjectList/index.less b/packages/components/src/Annotator/components/ObjectList/index.less index b814b4e..5643f3e 100644 --- a/packages/components/src/Annotator/components/ObjectList/index.less +++ b/packages/components/src/Annotator/components/ObjectList/index.less @@ -21,10 +21,15 @@ .ant-tabs-tab { padding: 12px; margin: 0; + font-size: 16px; } - .ant-tabs-tab-active { - border-bottom: 2px solid @colorPrimary; + // .ant-tabs-tab-active { + // border-bottom: 2px solid @colorPrimary; + // } + + .ant-tabs-ink-bar { + background: transparent; } .ant-tabs-nav { @@ -33,6 +38,10 @@ top: 0; z-index: 1; background: #000; + + &::before { + display: none; + } } .ant-collapse-item { @@ -98,7 +107,31 @@ right: 5%; top: 50%; transform: translateY(-50%); - border: 0; + + button { + border: 0; + } + + &-color-btn { + width: 28px; + height: 28px; + margin: 0 0.5rem; + border: 0; + background-color: transparent; + color: #fff; + border-radius: 5px; + box-shadow: unset; + + &:hover { + background-color: @colorPrimary; + transform: scale(1.2); + } + } + + &-color-btn-active { + color: @colorPrimary; + background-color: @colorPrimary; + } } .tab-collapse { @@ -202,13 +235,13 @@ .label-icon { margin-left: 15px; + margin-right: 12px; width: 15px; height: 15px; } .label { flex: 1; - max-width: 100px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; @@ -216,11 +249,45 @@ .label-actions { margin-right: 5%; + display: flex; + align-items: center; + justify-content: center; } .label-btn { border: 0; } + + .attr-btn { + border: 0; + + svg { + width: 22px; + height: 22px; + fill: @colorPrimary; + } + } + + .attr-btn-warn { + svg { + fill: #f53f3f; + } + } + + .frame-count { + display: flex; + align-items: center; + justify-content: center; + gap: 8px; + margin: 0 8px; + font-size: 14px; + + svg { + width: 14px; + height: 14px; + fill: #fff; + } + } } .collapse-item:hover { diff --git a/packages/components/src/Annotator/components/ObjectList/index.tsx b/packages/components/src/Annotator/components/ObjectList/index.tsx index 872a090..e6bd0f7 100644 --- a/packages/components/src/Annotator/components/ObjectList/index.tsx +++ b/packages/components/src/Annotator/components/ObjectList/index.tsx @@ -1,14 +1,29 @@ -import React, { memo, useEffect, useMemo, useRef, useState } from 'react'; +import React, { + memo, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; import { Button, Collapse, List, Tabs, Tooltip } from 'antd'; import { OBJECT_ICON } from '../../constants'; import { ReactComponent as DownArrorIcon } from '../../assets/downArror.svg'; +import { ReactComponent as Palette } from '../../assets/palette.svg'; +import { ReactComponent as Attribute } from '../../assets/attribute.svg'; +import { ReactComponent as Layer } from '../../assets/layer.svg'; import classNames from 'classnames'; import Icon, { DeleteOutlined, EyeInvisibleOutlined, EyeOutlined, } from '@ant-design/icons'; -import { IAnnotationObject } from '../../type'; +import { + Category, + DrawData, + IAnnotationObject, + IAnnotsDisplayOptions, +} from '../../type'; import { useKeyPress } from 'ahooks'; import { EDITOR_SHORTCUTS, EShortcuts } from '../../constants/shortcuts'; import { useLocale } from 'dds-utils/locale'; @@ -16,10 +31,11 @@ import VirtualList, { ListRef } from 'rc-virtual-list'; import { useWindowResize } from 'dds-hooks'; import { isEqual } from 'lodash'; import './index.less'; +import { Updater } from 'use-immer'; export interface IProps { objects: IAnnotationObject[]; - labelColors: Record; + framesObjects?: IAnnotationObject[][]; activeObjectIndex: number; className?: string; supportEdit?: boolean; @@ -30,6 +46,10 @@ export interface IProps { onChangeCategoryHidden: (category: string, hidden: boolean) => void; onDeleteObject: (index: number) => void; onChangeActiveClassName: (className: string) => void; + categories: Category[]; + setDrawDataWithHistory: Updater; + colorByCategory: boolean; + onChangeAnnotsDisplayOpts: (options: IAnnotsDisplayOptions) => void; } enum ETab { @@ -46,23 +66,27 @@ type TObjectItem = IAnnotationObject & { const propsAreEqual = (prev: IProps, next: IProps): boolean => { return ( isEqual(prev.objects, next.objects) && + isEqual(prev.framesObjects, next.framesObjects) && prev.activeObjectIndex === next.activeObjectIndex && prev.supportEdit === next.supportEdit && prev.activeClassName === next.activeClassName && prev.className === next.className && - isEqual(prev.labelColors, next.labelColors) && prev.onChangeActiveClassName === next.onChangeActiveClassName && prev.onFocusObject === next.onFocusObject && prev.onDeleteObject === next.onDeleteObject && prev.onChangeObjectHidden === next.onChangeObjectHidden && - prev.onChangeCategoryHidden === next.onChangeCategoryHidden + prev.onChangeCategoryHidden === next.onChangeCategoryHidden && + prev.setDrawDataWithHistory === next.setDrawDataWithHistory && + isEqual(prev.categories, next.categories) && + prev.colorByCategory === next.colorByCategory && + prev.onChangeAnnotsDisplayOpts === next.onChangeAnnotsDisplayOpts ); }; export const ObjectList: React.FC = memo((props) => { const { objects, - labelColors, + framesObjects, activeObjectIndex, className, supportEdit, @@ -73,8 +97,11 @@ export const ObjectList: React.FC = memo((props) => { onDeleteObject, onChangeCategoryHidden, onChangeActiveClassName, + categories, + setDrawDataWithHistory, + colorByCategory, + onChangeAnnotsDisplayOpts, } = props; - const { localeText } = useLocale(); const DEFAULT_CLASS_NAME = localeText( @@ -103,6 +130,27 @@ export const ObjectList: React.FC = memo((props) => { }); }; + const switchColorMode = () => { + onChangeAnnotsDisplayOpts({ + colorByCategory: !colorByCategory, + }); + }; + + const showEditingAttributes = useCallback( + (object: IAnnotationObject, label: Category, index: number) => { + onActiveObject(index); + setDrawDataWithHistory((s) => { + s.editingAttribute = { + index, + labelId: object.labelId, + attributes: label.attributes || [], + values: object.attributes || [], + }; + }); + }, + [onActiveObject], + ); + /** Hide All Objects */ useKeyPress( EDITOR_SHORTCUTS[EShortcuts.HideAll].shortcut, @@ -123,11 +171,13 @@ export const ObjectList: React.FC = memo((props) => { obj: IAnnotationObject, index: number, ) => { - const label = obj.label || DEFAULT_CLASS_NAME; - if (!acc[label]) { - acc[label] = []; + const labelName = + categories.find((c) => c.id === obj.labelId)?.name || + DEFAULT_CLASS_NAME; + if (!acc[labelName]) { + acc[labelName] = []; } - acc[label].push({ ...obj, originIndex: index }); + acc[labelName].push({ ...obj, originIndex: index }); return acc; }, {}, @@ -167,35 +217,36 @@ export const ObjectList: React.FC = memo((props) => { {objects.length > 0 && Object.keys(objectMapByClass) .sort() - .map((label) => { - const subObjects = objectMapByClass[label]; + .map((labelName) => { + const subObjects = objectMapByClass[labelName]; const isHidden = subObjects.every((item) => item.hidden); + const firstColor = subObjects[0]?.color; return ( { onChangeActiveClassName( - label === activeClassName ? '' : label, + labelName === activeClassName ? '' : labelName, ); }} > - {activeClassName === label && ( + {activeClassName === labelName && (
)} -
{label}
+
{labelName}
{subObjects.length} {supportEdit && ( @@ -219,7 +270,7 @@ export const ObjectList: React.FC = memo((props) => { shape={'circle'} onClick={(event) => { event.stopPropagation(); - onChangeCategoryHidden(label, !isHidden); + onChangeCategoryHidden(labelName, !isHidden); }} /> @@ -234,7 +285,7 @@ export const ObjectList: React.FC = memo((props) => {
} > - {activeClassName === label && ( + {activeClassName === labelName && ( = memo((props) => { itemKey={'originIndex'} ref={virtualListRef} > - {(object: TObjectItem, objIndex: number) => ( - { - onFocusObject(object.originIndex); - }} - onClick={(event) => { - event.stopPropagation(); - onActiveObject(object.originIndex); - }} - > - {activeObjectIndex === object.originIndex && ( -
- )} - -
{object.label}
-
- -
- - )} + )} + +
+ + ); + }} )} @@ -344,33 +443,41 @@ export const ObjectList: React.FC = memo((props) => { items={[ { key: ETab.Class, - label: localeText('DDSAnnotator.annotsList.categories'), + label: localeText('DDSAnnotator.annotsList.labels'), children: classTab, }, - // { - // key: ETab.Object, - // label: localeText('DDSAnnotator.annotsList.objects'), - // children: objectTab, - // }, ]} tabBarExtraContent={ - objects.length > 0 && ( - +
+ - ) + {objects.length > 0 && ( + +
} /> diff --git a/packages/components/src/Annotator/components/PointItem/PointItem.tsx b/packages/components/src/Annotator/components/PointItem/index.tsx similarity index 97% rename from packages/components/src/Annotator/components/PointItem/PointItem.tsx rename to packages/components/src/Annotator/components/PointItem/index.tsx index c10d771..c6ed230 100644 --- a/packages/components/src/Annotator/components/PointItem/PointItem.tsx +++ b/packages/components/src/Annotator/components/PointItem/index.tsx @@ -40,9 +40,7 @@ const PointItem: React.FC = ({ }} /> )} -
+
{point.name ? `#${index + 1} ${point.name}` : `${index + 1} `}
diff --git a/packages/components/src/Annotator/components/PointsEditModal/index.less b/packages/components/src/Annotator/components/PointsEditModal/index.less new file mode 100644 index 0000000..4ded0a1 --- /dev/null +++ b/packages/components/src/Annotator/components/PointsEditModal/index.less @@ -0,0 +1,65 @@ +.dds-annotator-points-editor { + position: absolute; + right: 1rem; + top: 1rem; + box-shadow: 2px 2px 12px 3px rgba(0, 0, 0, 0.6); + opacity: 0; + transition: opacity 0.15s ease; + pointer-events: none; + z-index: 99; + border-radius: 6px; + overflow: hidden; + + .ant-card-head { + background-color: @colorPrimary; + color: #fff; + font-size: 15px; + padding: 0; + min-height: auto; + } + + .ant-card-body { + padding: 0; + } + + .btn { + border: 0; + } + + .title { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + padding: 0 12px; + min-height: 40px; + cursor: pointer; + + .extra-btn { + cursor: pointer; + + &:hover { + transform: scale(1.05); + } + } + } + + .content { + display: flex; + flex-direction: column; + gap: 5px; + width: 280px; + height: 140px; + overflow-y: scroll; + padding: 10px 0 12px 10px; + } + + &:hover { + box-shadow: 2px 2px 12px 3px rgba(0, 0, 0, 0.6); + } +} + +.dds-annotator-points-editor-visible { + opacity: 1; + pointer-events: all; +} diff --git a/packages/components/src/Annotator/components/PointsEditModal/index.tsx b/packages/components/src/Annotator/components/PointsEditModal/index.tsx new file mode 100644 index 0000000..dd78595 --- /dev/null +++ b/packages/components/src/Annotator/components/PointsEditModal/index.tsx @@ -0,0 +1,114 @@ +import { Card } from 'antd'; +import classNames from 'classnames'; +import { FloatWrapper } from '../FloatWrapper'; +import { memo, useMemo, useState } from 'react'; +import { useLocale } from 'dds-utils/locale'; +import { EditState, EditorMode, IAnnotationObject } from '../../type'; +import { + EElementType, + EObjectType, + KEYPOINTS_VISIBLE_TYPE, +} from '../../constants'; +import './index.less'; +import PointItem from '../PointItem'; +import { DownCircleOutlined, UpCircleOutlined } from '@ant-design/icons'; +import { Updater } from 'use-immer'; + +interface IProps { + mode: EditorMode; + isAiAnnotation: boolean; + currObject: IAnnotationObject | undefined; + currObjectIndex: number; + focusObjectIndex: number; + focusEleType: EElementType; + focusEleIndex: number; + onChangePointVisible: ( + pointIndex: number, + visible: KEYPOINTS_VISIBLE_TYPE, + ) => void; + setEditState: Updater; +} + +const PointsEditModal: React.FC = memo( + ({ + mode, + isAiAnnotation, + currObject, + currObjectIndex, + focusObjectIndex, + focusEleType, + focusEleIndex, + onChangePointVisible, + setEditState, + }) => { + const { localeText } = useLocale(); + const [collapsed, setCollapsed] = useState(true); + + const show = useMemo(() => { + if ( + currObjectIndex > -1 && + currObject?.type === EObjectType.Skeleton && + !isAiAnnotation + ) { + return true; + } + return false; + }, [mode, currObject, currObjectIndex, isAiAnnotation]); + + const onFocusEleIndex = (index: number) => { + setEditState((s) => { + s.focusObjectIndex = currObjectIndex; + s.focusEleIndex = index; + s.focusEleType = EElementType.Circle; + }); + }; + + return ( + + setCollapsed((s) => !s)}> + {localeText('DDSAnnotator.points.editor')} +
+ {collapsed ? : } +
+
+ } + > + {!collapsed && ( +
{ + event.stopPropagation(); + }} + > + {currObject && + currObject.keypoints && + currObject.keypoints.points.map((ele, eleIndex) => ( + onFocusEleIndex(eleIndex)} + onVisibleChange={(visible) => { + onChangePointVisible(eleIndex, visible); + }} + /> + ))} +
+ )} + + + ); + }, +); + +export default PointsEditModal; diff --git a/packages/components/src/Annotator/components/ScaleToolBar/index.less b/packages/components/src/Annotator/components/ScaleToolBar/index.less deleted file mode 100644 index 6e2f5f8..0000000 --- a/packages/components/src/Annotator/components/ScaleToolBar/index.less +++ /dev/null @@ -1,110 +0,0 @@ -.dds-annotator-scaletoolbar { - position: absolute; - bottom: 1rem; - left: 1rem; - display: flex; - flex-direction: row; - justify-content: center; - align-items: center; - z-index: 99; - background-color: #212121; - border-radius: 10px; - padding: 0.4rem 0.5rem; - - &-btn { - width: 32px; - height: 32px; - margin: 0 0.25rem; - border: 0; - background-color: transparent; - color: #fff; - border-radius: 5px; - box-shadow: unset; - - &:hover { - background-color: @colorPrimary; - transform: scale(1.2); - } - } - - &-btn-active { - color: @colorPrimary; - background-color: @colorPrimary; - } - - &-btn-disabled { - color: rgba(255, 255, 255, 0.25); - pointer-events: none; - - svg { - fill: rgba(255, 255, 255, 0.25); - } - } - - &-scale-text { - color: rgba(255, 255, 255, 0.8); - margin: 0 8px; - user-select: none; - } - - &-divider { - height: 24px; - margin: 0 8px; - border-left: 1px solid #bbb; - } - - &-popover { - border-radius: 10px; - - .ant-popover-inner { - padding: 0; - } - } -} - -.dds-annotator-scaletoolbar-pop-container { - border-radius: 10px; - color: #fff; - padding-bottom: 8px; - - &-header { - display: flex; - justify-content: space-between; - align-items: center; - gap: 24px; - border-bottom: 1px solid rgba(107, 114, 128); - padding: 8px 8px 8px 16px; - margin-bottom: 8px; - } - - &-btn { - width: 24px; - height: 24px; - margin: 0 0.25rem; - border: 0; - background-color: transparent; - color: #fff; - box-shadow: unset; - font-size: 12px; - } - - &-btn:hover { - background-color: @colorPrimary; - transform: scale(1.2); - - svg { - fill: #000; - } - } - - &-option { - display: flex; - flex-flow: column nowrap; - padding: 4px 16px; - width: 240px; - - .ant-slider { - margin: 5px 8px; - } - } -} diff --git a/packages/components/src/Annotator/components/ScaleToolBar/index.tsx b/packages/components/src/Annotator/components/ScaleToolBar/index.tsx deleted file mode 100644 index e3c0239..0000000 --- a/packages/components/src/Annotator/components/ScaleToolBar/index.tsx +++ /dev/null @@ -1,209 +0,0 @@ -import { Button, Popover, Slider } from 'antd'; -import Icon, { ZoomInOutlined, ZoomOutOutlined } from '@ant-design/icons'; -import classNames from 'classnames'; -import { useKeyPress } from 'ahooks'; -import { MAX_SCALE, MIN_SCALE } from '../../constants'; -import { EDITOR_SHORTCUTS, EShortcuts } from '../../constants/shortcuts'; -import { useLocale } from 'dds-utils/locale'; -import { FloatWrapper } from '../FloatWrapper'; -import { ReactComponent as ImgSetting } from '../../assets/imgSetting.svg'; -import { ReactComponent as Palette } from '../../assets/palette.svg'; -import { ReactComponent as DisplayReset } from '../../assets/displayReset.svg'; -import { ReactComponent as ZoomResize } from '../../assets/zoomResize.svg'; -import { memo, useMemo } from 'react'; -import { - DEFAULT_IMG_DISPLAY_OPTIONS, - IAnnotsDisplayOptions, - IImageDisplayOptions, -} from '../../type'; -import './index.less'; - -interface IProps { - scale: number; - displayOption: IImageDisplayOptions; - colorByCategory: boolean; - onZoomIn: () => void; - onZoomOut: () => void; - onReset: () => void; - onChangeImageDisplayOpts: (options: IImageDisplayOptions) => void; - onChangeAnnotsDisplayOpts: (options: IAnnotsDisplayOptions) => void; -} - -export const ScaleToolBar: React.FC = memo( - ({ - scale, - displayOption, - colorByCategory, - onZoomIn, - onZoomOut, - onReset, - onChangeImageDisplayOpts, - onChangeAnnotsDisplayOpts, - }) => { - const { localeText } = useLocale(); - - const disabledZoomIn = scale >= MAX_SCALE; - const disabledZoomOut = scale <= MIN_SCALE; - - useKeyPress(EDITOR_SHORTCUTS[EShortcuts.ZoomIn].shortcut, () => { - if (disabledZoomIn) return; - onZoomIn(); - }); - - useKeyPress(EDITOR_SHORTCUTS[EShortcuts.ZoomOut].shortcut, () => { - if (disabledZoomOut) return; - onZoomOut(); - }); - - useKeyPress(EDITOR_SHORTCUTS[EShortcuts.Reset].shortcut, () => { - onReset(); - }); - - const popoverContent = useMemo(() => { - return ( -
-
-
{localeText('DDSAnnotator.imgDisplayTool.title')}
- -
-
- - - onChangeImageDisplayOpts({ - ...displayOption, - brightness: value, - }) - } - min={0} - max={200} - /> -
-
- - - onChangeImageDisplayOpts({ - ...displayOption, - contrast: value, - }) - } - min={0} - max={200} - /> -
-
- - - onChangeImageDisplayOpts({ - ...displayOption, - saturate: value, - }) - } - min={0} - max={200} - /> -
-
- ); - }, [ - displayOption.brightness, - displayOption.contrast, - displayOption.saturate, - onChangeImageDisplayOpts, - onChangeAnnotsDisplayOpts, - ]); - - const mouseEventHandler = (event: React.MouseEvent) => { - // enable mouseup propagate only for sliders - if (event.type === 'mouseup') { - return; - } else { - event.stopPropagation(); - } - }; - - const switchColorMode = () => { - onChangeAnnotsDisplayOpts({ - colorByCategory: !colorByCategory, - }); - }; - - return ( - -
- - - - - - {localeText('DDSAnnotator.colorMode')} - - } - trigger="hover" - color={'#212121'} - > - - -
- - ); - }, -); diff --git a/packages/components/src/Annotator/components/AnnotationEditor/index.less b/packages/components/src/Annotator/components/SegConfirmModal/index.less similarity index 78% rename from packages/components/src/Annotator/components/AnnotationEditor/index.less rename to packages/components/src/Annotator/components/SegConfirmModal/index.less index 6cfa178..dbfe7bd 100644 --- a/packages/components/src/Annotator/components/AnnotationEditor/index.less +++ b/packages/components/src/Annotator/components/SegConfirmModal/index.less @@ -1,4 +1,4 @@ -.dds-annotator-anno-editor { +.dds-annotator-seg-confirm { position: absolute; right: 1rem; top: 1rem; @@ -33,9 +33,8 @@ .content { display: flex; - flex-direction: column; align-items: center; - justify-content: flex-end; + justify-content: space-between; gap: 12px; .item { @@ -46,15 +45,6 @@ width: 100%; } - .list { - display: flex; - flex-direction: column; - gap: 5px; - width: 100%; - height: 150px; - overflow-y: scroll; - } - .selector { width: 100%; } @@ -71,7 +61,7 @@ } } -.dds-annotator-anno-editor-visible { +.dds-annotator-seg-confirm-visible { opacity: 1; pointer-events: all; } diff --git a/packages/components/src/Annotator/components/SegConfirmModal/index.tsx b/packages/components/src/Annotator/components/SegConfirmModal/index.tsx new file mode 100644 index 0000000..e39492e --- /dev/null +++ b/packages/components/src/Annotator/components/SegConfirmModal/index.tsx @@ -0,0 +1,76 @@ +import { Button, Card } from 'antd'; +import classNames from 'classnames'; +import { FloatWrapper } from '../FloatWrapper'; +import { memo, useMemo } from 'react'; +import { useKeyPress } from 'ahooks'; +import { EDITOR_SHORTCUTS, EShortcuts } from '../../constants/shortcuts'; +import { useLocale } from 'dds-utils/locale'; +import { EditorMode, IAnnotationObject } from '../../type'; +import { EObjectType } from '../../constants'; +import './index.less'; + +interface IProps { + mode: EditorMode; + isAiAnnotation: boolean; + latestLabelId: string; + currObject: IAnnotationObject | undefined; + onFinishCurrCreate: (labelId: string) => void; +} + +const SegConfirmModal: React.FC = memo( + ({ mode, isAiAnnotation, latestLabelId, currObject, onFinishCurrCreate }) => { + const { localeText } = useLocale(); + + const show = useMemo(() => { + if (mode !== EditorMode.Edit) return false; + if ( + currObject?.type === EObjectType.Mask || + (currObject?.type === EObjectType.Polygon && isAiAnnotation) + ) { + return true; + } + return false; + }, [mode, currObject, isAiAnnotation]); + + useKeyPress( + EDITOR_SHORTCUTS[EShortcuts.SaveCurrObject].shortcut, + (event: KeyboardEvent) => { + if (currObject) { + event.preventDefault(); + onFinishCurrCreate(latestLabelId); + } + }, + { + exactMatch: true, + }, + ); + + return ( + + {localeText('DDSAnnotator.seg.tool')}
+ } + > +
+
{localeText('DDSAnnotator.seg.tool.content')}
+ +
+ + + ); + }, +); + +export default SegConfirmModal; diff --git a/packages/components/src/Annotator/components/ShortcutsInfo/index.less b/packages/components/src/Annotator/components/ShortcutsInfo/index.less index a67fc9f..cec2e1b 100644 --- a/packages/components/src/Annotator/components/ShortcutsInfo/index.less +++ b/packages/components/src/Annotator/components/ShortcutsInfo/index.less @@ -50,3 +50,16 @@ color: rgba(255, 255, 255, 0.9); } } + +.dds-annotator-shortcutsinfo-icon { + svg { + margin-left: 2px; + margin-right: 12px; + fill: #fff; + cursor: pointer; + + &:hover { + fill: @colorPrimary; + } + } +} diff --git a/packages/components/src/Annotator/components/ShortcutsInfo/index.tsx b/packages/components/src/Annotator/components/ShortcutsInfo/index.tsx index c6ca6b1..e5ae72b 100644 --- a/packages/components/src/Annotator/components/ShortcutsInfo/index.tsx +++ b/packages/components/src/Annotator/components/ShortcutsInfo/index.tsx @@ -1,5 +1,5 @@ import { Dropdown, Menu, MenuProps, Tooltip } from 'antd'; -import { ReactComponent as KeyboardIcon } from '../../assets/keyboard.svg'; +import { ReactComponent as KeyboardIcon } from '../../assets/keyboard-down.svg'; import Icon from '@ant-design/icons'; import { memo, useMemo } from 'react'; import { @@ -12,9 +12,11 @@ import { import { useLocale } from 'dds-utils/locale'; import './index.less'; import classNames from 'classnames'; +import { EditorMode } from '../../type'; interface IProps { - viewOnly: boolean; + mode: EditorMode; + // viewOnly: boolean; } export const getIconFromShortcut = (keys: string[], withStyle = true) => { @@ -30,9 +32,12 @@ export const getIconFromShortcut = (keys: string[], withStyle = true) => { const combineKeys = key.split('.'); combineKeys.forEach((key, idx) => { const letter = ( - + {convertAliasToSymbol(key)} ); @@ -41,7 +46,7 @@ export const getIconFromShortcut = (keys: string[], withStyle = true) => { icons.push( @@ -55,7 +60,7 @@ export const getIconFromShortcut = (keys: string[], withStyle = true) => { const letter = ( @@ -68,7 +73,7 @@ export const getIconFromShortcut = (keys: string[], withStyle = true) => { icons.push( @@ -81,7 +86,7 @@ export const getIconFromShortcut = (keys: string[], withStyle = true) => { return {icons}; }; -export const ShortcutsInfo: React.FC = memo(({ viewOnly }) => { +export const ShortcutsInfo: React.FC = memo(({ mode }) => { const { localeText } = useLocale(); const convertShortcutsToMenuProps = ( @@ -91,11 +96,26 @@ export const ShortcutsInfo: React.FC = memo(({ viewOnly }) => { for (const key in shortcuts) { if (shortcuts.hasOwnProperty(key)) { // @ts-ignore - const { type, descTextKey, shortcut } = shortcuts[key]; + const { name, type, descTextKey, shortcut } = shortcuts[key]; const description = localeText(descTextKey); - if (viewOnly && type !== EShortcutType.ViewAction) { + if (mode === EditorMode.View && type !== EShortcutType.ViewAction) { continue; } + if (mode === EditorMode.Review) { + if ( + [EShortcutType.AnnotationAction, EShortcutType.Tool].includes(type) + ) { + continue; + } + if ( + [EShortcutType.GeneralAction].includes(type) && + name !== 'Accept' && + name !== 'Reject' + ) { + continue; + } + } + if (categories[type]) { categories[type].children.push({ key, @@ -123,7 +143,7 @@ export const ShortcutsInfo: React.FC = memo(({ viewOnly }) => { const items = useMemo(() => { return convertShortcutsToMenuProps(EDITOR_SHORTCUTS) || []; - }, [viewOnly]); + }, [mode]); return ( = memo(({ viewOnly }) => { > diff --git a/packages/components/src/Annotator/components/SliderToolBar/index.less b/packages/components/src/Annotator/components/SliderToolBar/index.less new file mode 100644 index 0000000..5ab1a0b --- /dev/null +++ b/packages/components/src/Annotator/components/SliderToolBar/index.less @@ -0,0 +1,200 @@ +.dds-annotator-slidertoolbar { + position: relative; + height: 100%; + background: #212121; + border-radius: 0; + display: flex; + flex-direction: column; + justify-content: space-between; + align-items: center; + gap: 12px; + width: 50px; + pointer-events: auto; + font-weight: 600; + padding: 1rem 0.5rem 2rem; + z-index: 99; + overflow-y: scroll; + + /* Hide scrollbar */ + scrollbar-width: none; /* firefox */ + -ms-overflow-style: none; /* IE 10+ */ + &::-webkit-scrollbar { + display: none; /* Chrome Safari */ + } + + &-content { + display: flex; + flex-direction: column; + justify-content: center; + align-items: center; + gap: 12px; + } + + .slidertoolbar-btn { + width: 32px; + height: 32px; + border: 0; + background-color: transparent; + border-radius: 5px; + + svg { + color: #fff; + fill: #fff; + scale: 1.2; + } + + &:hover { + background-color: @colorPrimary; + transform: scale(1.2); + } + } + + .slidertoolbar-btn-active { + color: @colorPrimary; + background-color: @colorPrimary; + } + + // .slidertoolbar-tool-btn-active { + // svg { + // color: @colorPrimary; + // fill: @colorPrimary; + // } + // } + + .slidertoolbar-btn-disabled { + color: rgba(255, 255, 255, 0.25); + pointer-events: none; + + svg { + fill: rgba(255, 255, 255, 0.25); + } + } + + .slidertoolbar-annotool-active-wrap { + display: flex; + flex-direction: column; + align-items: center; + gap: 12px; + width: 50px; + padding: 10px 0; + background: #484848; + border-radius: 12px; + } + + .slidertoolbar-scale-text { + font-size: 12px; + font-weight: normal; + color: rgba(255, 255, 255, 0.8); + margin: 12px 0; + user-select: none; + } + + .slidertoolbar-divider { + width: 100%; + margin: 8px 6px; + border-bottom: 1px solid #bbb; + } +} + +.dds-annotator-slidertoolbar-popover { + display: flex; + flex-direction: column; + justify-content: center; + align-items: flex-start; + + .popover-title { + font-weight: 600; + font-size: 14px; + margin-right: 10px; + } + + .popover-key { + min-width: 30px; + justify-content: center; + border-radius: 2px; + padding: 2px 5px; + color: rgba(0, 0, 0, 0.8); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.3), 0 1px 2px rgba(0, 0, 0, 0.1); + font-size: 12px; + font-weight: 600; + } + + .popover-divider { + width: 100%; + margin: 10px 0; + border-bottom: 1px solid rgba(0, 0, 0, 0.1); + } + + .popover-description { + max-width: 220px; + font-size: 13px; + color: rgba(0, 0, 0, 0.8); + } +} + +.dds-annotator-editor-light { + .dds-annotator-slidertoolbar { + background: #fff; + + .slidertoolbar-btn { + box-shadow: none; + + svg { + color: #000; + fill: #000; + scale: 1.2; + } + + &:hover { + svg { + color: #fff; + fill: #fff; + } + } + } + + .slidertoolbar-btn-active { + svg { + color: #fff; + fill: #fff; + scale: 1.2; + } + } + + .slidertoolbar-btn-disabled { + color: rgba(0, 0, 0, 0.25); + + svg { + fill: rgba(0, 0, 0, 0.25); + } + } + + .slidertoolbar-annotool-active-wrap { + background: #484848; + } + + .slidertoolbar-scale-text { + color: rgba(0, 0, 0, 0.8); + } + + .slidertoolbar-divider { + border-bottom: 1px solid #bbb; + } + } + + .dds-annotator-slidertoolbar-popover { + .popover-key { + color: rgba(255, 255, 255, 0.8); + box-shadow: 0 1px 3px rgba(255, 255, 255, 0.3), + 0 1px 2px rgba(255, 255, 255, 0.1); + } + + .popover-divider { + border-bottom: 1px solid rgba(255, 255, 255, 0.1); + } + + .popover-description { + color: rgba(255, 255, 255, 0.8); + } + } +} diff --git a/packages/components/src/Annotator/components/SliderToolBar/index.tsx b/packages/components/src/Annotator/components/SliderToolBar/index.tsx new file mode 100644 index 0000000..81340c3 --- /dev/null +++ b/packages/components/src/Annotator/components/SliderToolBar/index.tsx @@ -0,0 +1,422 @@ +import { Button, Popover } from 'antd'; +import Icon, { ZoomInOutlined, ZoomOutOutlined } from '@ant-design/icons'; +import classNames from 'classnames'; +import { + EBasicToolItem, + EObjectType, + EActionToolItem, + EToolType, + OBJECT_ICON, + EDITOR_TOOL_ICON, + MAX_SCALE, + MIN_SCALE, + OBJECT_AI_ICON, + TOOL_MODELS_MAP, + EnumModelType, +} from '../../constants'; +import { ReactComponent as DragToolIcon } from '../../assets/drag.svg'; +import { useKeyPress } from 'ahooks'; +import { + EDITOR_SHORTCUTS, + EShortcuts, + TShortcutItem, +} from '../../constants/shortcuts'; +import { memo, useMemo } from 'react'; +import { getIconFromShortcut } from '../ShortcutsInfo'; +import { useLocale } from 'dds-utils/locale'; +import { ReactComponent as ZoomResize } from '../../assets/zoomResize.svg'; +import './index.less'; + +type TToolItem = { + key: T; + name: string; + shortcut: TShortcutItem; + icon: JSX.Element; + aiIcon?: JSX.Element; + aiModels?: EnumModelType[]; + description?: string; +}; + +interface IProps { + selectedTool: EToolType; + manualMode?: boolean; + limitToolTypes?: EBasicToolItem[]; + supportRepeat?: boolean; + isAIAnnotationActive: boolean; + onChangeSelectedTool: (type: EToolType) => void; + onActiveAIAnnotation: (active: boolean) => void; + undo: () => void; + redo: () => void; + repeatPrevious?: () => void; + deleteAll: () => void; + scale: number; + onZoomIn: () => void; + onZoomOut: () => void; + onZoomReset: () => void; + onlySupportZoom: boolean; + hideUndoRedoActions?: boolean; +} + +const SliderToolBar: React.FC = memo( + ({ + selectedTool, + manualMode, + supportRepeat, + limitToolTypes, + isAIAnnotationActive, + onChangeSelectedTool, + onActiveAIAnnotation, + undo, + redo, + repeatPrevious, + deleteAll, + scale, + onZoomIn, + onZoomOut, + onZoomReset, + onlySupportZoom, + hideUndoRedoActions, + }) => { + const { localeText } = useLocale(); + + const dragTools: TToolItem[] = useMemo(() => { + return [ + { + key: EBasicToolItem.Drag, + name: localeText('DDSAnnotator.toolbar.drag'), + shortcut: EDITOR_SHORTCUTS[EShortcuts.DragTool], + icon: , + description: localeText('DDSAnnotator.toolbar.drag.desc'), + }, + ]; + }, []); + + const annoTools: TToolItem[] = useMemo(() => { + const typeTools = [ + { + key: EBasicToolItem.Rectangle, + name: localeText('DDSAnnotator.toolbar.rectangle'), + shortcut: EDITOR_SHORTCUTS[EShortcuts.RectangleTool], + icon: , + aiIcon: , + aiModels: TOOL_MODELS_MAP[EBasicToolItem.Rectangle], + description: localeText('DDSAnnotator.toolbar.rectangle.desc'), + }, + { + key: EBasicToolItem.Polygon, + name: localeText('DDSAnnotator.toolbar.polygon'), + shortcut: EDITOR_SHORTCUTS[EShortcuts.PolygonTool], + icon: , + aiIcon: , + description: localeText('DDSAnnotator.toolbar.polygon.desc'), + }, + { + key: EBasicToolItem.Skeleton, + name: localeText('DDSAnnotator.toolbar.skeleton'), + shortcut: EDITOR_SHORTCUTS[EShortcuts.SkeletonTool], + icon: , + aiIcon: , + description: localeText('DDSAnnotator.toolbar.skeleton.desc'), + }, + { + key: EBasicToolItem.Mask, + name: localeText('DDSAnnotator.toolbar.mask'), + shortcut: EDITOR_SHORTCUTS[EShortcuts.MaskTool], + icon: , + aiIcon: , + description: localeText('DDSAnnotator.toolbar.mask.desc'), + }, + ]; + if (limitToolTypes) { + return typeTools.filter((item) => limitToolTypes.includes(item.key)); + } + return typeTools; + }, [limitToolTypes]); + + const smartTool: TToolItem = { + key: EActionToolItem.SmartAnnotation, + name: localeText('DDSAnnotator.toolbar.aiAnno'), + shortcut: EDITOR_SHORTCUTS[EShortcuts.SmartAnnotation], + icon: ( + + ), + description: localeText('DDSAnnotator.toolbar.aiAnno.desc'), + }; + + const actionTools = [ + ...(!hideUndoRedoActions + ? [ + { + key: EActionToolItem.Undo, + name: localeText('DDSAnnotator.toolbar.undo'), + icon: , + shortcut: EDITOR_SHORTCUTS[EShortcuts.Undo], + handler: undo, + description: localeText('DDSAnnotator.toolbar.undo.desc'), + }, + { + key: EActionToolItem.Redo, + name: localeText('DDSAnnotator.toolbar.redo'), + icon: , + shortcut: EDITOR_SHORTCUTS[EShortcuts.Redo], + handler: redo, + description: localeText('DDSAnnotator.toolbar.redo.desc'), + }, + ] + : []), + ...(supportRepeat + ? [ + { + key: EActionToolItem.RepeatPrevious, + name: localeText('DDSAnnotator.toolbar.repeatPrevious'), + icon: ( + + ), + shortcut: EDITOR_SHORTCUTS[EShortcuts.RepeatPrevious], + handler: repeatPrevious, + description: localeText( + 'DDSAnnotator.toolbar.repeatPrevious.desc', + ), + }, + ] + : []), + { + key: EActionToolItem.DeleteAll, + name: localeText('DDSAnnotator.toolbar.deleteAll'), + icon: , + shortcut: EDITOR_SHORTCUTS[EShortcuts.DeleteAll], + handler: deleteAll, + description: localeText('DDSAnnotator.toolbar.deleteAll.desc'), + }, + ]; + + const basicToolKeys: string[] = useMemo(() => { + return [...dragTools, ...annoTools].reduce((keys: string[], tool) => { + return keys.concat(tool.shortcut.shortcut); + }, []); + }, [dragTools, annoTools]); + + /** Active Basic Tool */ + useKeyPress( + basicToolKeys, + (event) => { + const activeTool = [...dragTools, ...annoTools].find((tool) => { + return tool.shortcut.shortcut.includes(event.key); + }); + if (activeTool) { + onChangeSelectedTool(activeTool.key); + } + }, + { + exactMatch: true, + }, + ); + + /** Active AI Annotation */ + useKeyPress( + EDITOR_SHORTCUTS[EShortcuts.SmartAnnotation].shortcut, + () => { + if (selectedTool !== EBasicToolItem.Drag) { + onActiveAIAnnotation(!isAIAnnotationActive); + } + }, + { + exactMatch: true, + }, + ); + + /** Undo */ + useKeyPress( + EDITOR_SHORTCUTS[EShortcuts.Undo].shortcut, + (event: KeyboardEvent) => { + event.preventDefault(); + undo(); + }, + { + exactMatch: true, + }, + ); + + /** Redo */ + useKeyPress( + EDITOR_SHORTCUTS[EShortcuts.Redo].shortcut, + (event: KeyboardEvent) => { + event.preventDefault(); + redo(); + }, + { + exactMatch: true, + }, + ); + + /** Repeat Previous */ + useKeyPress( + EDITOR_SHORTCUTS[EShortcuts.RepeatPrevious].shortcut, + (event: KeyboardEvent) => { + event.preventDefault(); + repeatPrevious?.(); + }, + { + exactMatch: true, + }, + ); + + /** Delete All */ + useKeyPress( + EDITOR_SHORTCUTS[EShortcuts.DeleteAll].shortcut, + (event: KeyboardEvent) => { + event.preventDefault(); + deleteAll(); + }, + { + exactMatch: true, + }, + ); + + const disabledZoomIn = scale >= MAX_SCALE; + const disabledZoomOut = scale <= MIN_SCALE; + + useKeyPress(EDITOR_SHORTCUTS[EShortcuts.ZoomIn].shortcut, () => { + if (disabledZoomIn) return; + onZoomIn(); + }); + + useKeyPress(EDITOR_SHORTCUTS[EShortcuts.ZoomOut].shortcut, () => { + if (disabledZoomOut) return; + onZoomOut(); + }); + + useKeyPress(EDITOR_SHORTCUTS[EShortcuts.Reset].shortcut, () => { + onZoomReset(); + }); + + const popoverContent = ( + item: TToolItem, + ) => { + const icon = getIconFromShortcut(item.shortcut.shortcut, false); + return ( +
+
+ {item.name} + {icon} +
+
+
{item.description}
+
+ ); + }; + + return ( +
{ + event.stopPropagation(); + }} + > + {!onlySupportZoom ? ( +
+ {dragTools.map((item) => ( + +
+ ))} +
+ {actionTools.map((item) => ( + +
+ ) : ( +
+ )} +
+
+
+ ); + }, +); + +export default SliderToolBar; diff --git a/packages/components/src/Annotator/components/SmartAnnotationControl/index.less b/packages/components/src/Annotator/components/SmartAnnotationControl/index.less index a8be35f..746d7f1 100644 --- a/packages/components/src/Annotator/components/SmartAnnotationControl/index.less +++ b/packages/components/src/Annotator/components/SmartAnnotationControl/index.less @@ -23,6 +23,14 @@ &-btn { border: 0; + + &:hover { + background-color: rgba(255, 255, 255, 0.2) !important; + + svg { + color: #fff; + } + } } &-title { diff --git a/packages/components/src/Annotator/components/SmartAnnotationControl/index.tsx b/packages/components/src/Annotator/components/SmartAnnotationControl/index.tsx index de90669..b22572a 100644 --- a/packages/components/src/Annotator/components/SmartAnnotationControl/index.tsx +++ b/packages/components/src/Annotator/components/SmartAnnotationControl/index.tsx @@ -6,15 +6,15 @@ import { EActionToolItem, ESubToolItem, EToolType, + EnumModelType, } from '../../constants'; import { CloseOutlined } from '@ant-design/icons'; import Icon from '@ant-design/icons/lib/components/Icon'; import { Button, Card, Select, Slider, Space } from 'antd'; import classNames from 'classnames'; -import { useMemo, memo } from 'react'; +import { useMemo, memo, useState } from 'react'; import { FloatWrapper } from '../FloatWrapper'; import { useLocale } from 'dds-utils/locale'; -import CategoryCreator from '../CategoryCreator'; import { OnAiAnnotationFunc } from '../../hooks/useActions'; import { useImmer } from 'use-immer'; import { ReactComponent as DragToolIcon } from '../../assets/drag.svg'; @@ -26,21 +26,19 @@ import './index.less'; interface IProps { selectedTool: EToolType; selectedSubTool: ESubToolItem; + selectedModel?: EnumModelType; AIAnnotation: boolean; hasPolygonPreds: boolean; isBatchEditing: boolean; isCtrlPressed: boolean; naturalSize: ISize; - aiLabels: string[]; + aiLabels?: string; limitConf: number; categories: Category[]; - setAiLabels: (labels: string[]) => void; + setAiLabels: (labels?: string) => void; forceChangeTool: (tool: EBasicToolItem, subtool: ESubToolItem) => void; - onCreateCategory: (name: string) => void; onExitAIAnnotation: () => void; onAiAnnotation: OnAiAnnotationFunc; - onSaveAIPolygon: () => void; - onCancelAIPolygon: () => void; onChangeConfidenceRange: (range: [number, number]) => void; onChangeLimitConf: (value: number) => void; onAcceptValidObjects: () => void; @@ -51,8 +49,8 @@ const SmartAnnotationControl: React.FC = memo( ({ selectedTool, selectedSubTool, + selectedModel, AIAnnotation, - hasPolygonPreds, isBatchEditing, isCtrlPressed, aiLabels, @@ -60,11 +58,8 @@ const SmartAnnotationControl: React.FC = memo( naturalSize, limitConf, setAiLabels, - onCreateCategory, onExitAIAnnotation, onAiAnnotation, - onSaveAIPolygon, - onCancelAIPolygon, onChangeConfidenceRange, onChangeLimitConf, onAcceptValidObjects, @@ -72,6 +67,7 @@ const SmartAnnotationControl: React.FC = memo( forceChangeTool, }) => { const { localeText } = useLocale(); + const [inputText, setInputText] = useState(''); /** Parameters for requesting segmemt everything API */ const [samParams, setSamParams] = useImmer({ @@ -86,7 +82,10 @@ const SmartAnnotationControl: React.FC = memo( icon: DragToolIcon, }, [EBasicToolItem.Rectangle]: { - name: localeText('DDSAnnotator.smart.detection.name'), + name: + selectedModel === EnumModelType.Detection + ? localeText('DDSAnnotator.smart.detection.name') + : localeText('DDSAnnotator.smart.ivp.name'), icon: OBJECT_ICON[EObjectType.Rectangle], }, [EBasicToolItem.Polygon]: { @@ -105,9 +104,14 @@ const SmartAnnotationControl: React.FC = memo( const labelOptions = useMemo(() => { if (selectedTool === EBasicToolItem.Rectangle) { - return categories?.map((category) => ( - - {category.name} + let options = categories?.map((c) => c.name); + options = + inputText && !options.includes(inputText) + ? [inputText, ...options] + : options; + return options.map((text) => ( + + {text} )); } else if (selectedTool === EBasicToolItem.Polygon) { @@ -119,7 +123,7 @@ const SmartAnnotationControl: React.FC = memo( )); } - }, [selectedTool, categories]); + }, [selectedTool, categories, inputText]); const mouseEventHandler = (event: React.MouseEvent) => { if ( @@ -140,22 +144,27 @@ const SmartAnnotationControl: React.FC = memo( if (!AIAnnotation || selectedTool === EBasicToolItem.Drag) return false; if ( - selectedTool === EBasicToolItem.Mask && - selectedSubTool !== ESubToolItem.AutoSegmentEverything + (selectedTool === EBasicToolItem.Mask && + selectedSubTool !== ESubToolItem.AutoSegmentEverything) || + selectedTool === EBasicToolItem.Polygon ) return false; - if ( - selectedTool === EBasicToolItem.Rectangle && - isBatchEditing && - isCtrlPressed - ) - return false; + if (selectedTool === EBasicToolItem.Rectangle) { + if (selectedModel === EnumModelType.Detection) { + return !(isBatchEditing && isCtrlPressed); + } else if (selectedModel === EnumModelType.IVP) { + return isBatchEditing; + } else { + return false; + } + } return true; }, [ selectedTool, selectedSubTool, + selectedModel, AIAnnotation, isBatchEditing, isCtrlPressed, @@ -167,7 +176,12 @@ const SmartAnnotationControl: React.FC = memo( }; const aiDetectionTip = useMemo(() => { - if (isBatchEditing && isCtrlPressed) { + if ( + selectedTool === EBasicToolItem.Rectangle && + selectedModel === EnumModelType.Detection && + isBatchEditing && + isCtrlPressed + ) { return [ { text: localeText('DDSAnnotator.smart.tip.recover'), @@ -180,7 +194,7 @@ const SmartAnnotationControl: React.FC = memo( ]; } return []; - }, [isBatchEditing, isCtrlPressed]); + }, [isBatchEditing, isCtrlPressed, selectedModel]); const imageArea = useMemo(() => { return naturalSize.width * naturalSize.height; @@ -227,6 +241,7 @@ const SmartAnnotationControl: React.FC = memo( >
{selectedTool === EBasicToolItem.Rectangle && + selectedModel === EnumModelType.Detection && (isBatchEditing ? (
@@ -273,13 +288,10 @@ const SmartAnnotationControl: React.FC = memo( placeholder={localeText( 'DDSAnnotator.smart.detection.input', )} - showArrow={true} + showSearch value={aiLabels} - onChange={(values) => - Array.isArray(values) - ? setAiLabels(values) - : setAiLabels([values]) - } + onChange={(value) => setAiLabels(value)} + onSearch={(value) => setInputText(value)} onInputKeyDown={(e) => { if (e.code !== 'Enter') { e.stopPropagation(); @@ -289,20 +301,6 @@ const SmartAnnotationControl: React.FC = memo( getPopupContainer={() => document.getElementById('smart-annotation-editor') } - mode={'multiple'} - dropdownRender={(menu) => ( - <> - {menu} - { - { - onCreateCategory(value); - setAiLabels([...aiLabels, value]); - }} - /> - } - - )} > {labelOptions} @@ -314,6 +312,26 @@ const SmartAnnotationControl: React.FC = memo(
))} + {selectedTool === EBasicToolItem.Rectangle && + selectedModel === EnumModelType.IVP && ( +
+
+ {localeText('DDSAnnotator.smart.tip')}: + {localeText('DDSAnnotator.smart.tip.visualPrompt')} +
+
+ + +
+
+ )} {selectedTool === EBasicToolItem.Skeleton && (isBatchEditing ? ( <> @@ -363,13 +381,10 @@ const SmartAnnotationControl: React.FC = memo( placeholder={localeText( 'DDSAnnotator.smart.pose.input', )} - showArrow={true} + showSearch value={aiLabels} - onChange={(values) => - Array.isArray(values) - ? setAiLabels(values) - : setAiLabels([values]) - } + onChange={(value) => setAiLabels(value)} + onSearch={(value) => setInputText(value)} onInputKeyDown={(e) => { if (e.code !== 'Enter') { e.stopPropagation(); @@ -393,25 +408,6 @@ const SmartAnnotationControl: React.FC = memo( ))} - {selectedTool === EBasicToolItem.Polygon && ( - <> -
- {hasPolygonPreds - ? localeText('DDSAnnotator.smart.segmentation.tipsNext') - : localeText('DDSAnnotator.smart.segmentation.tipsInitial')} -
- {hasPolygonPreds && ( -
- - -
- )} - - )} {selectedTool === EBasicToolItem.Mask && selectedSubTool === ESubToolItem.AutoSegmentEverything && ( <> diff --git a/packages/components/src/Annotator/components/SubToolBar/index.less b/packages/components/src/Annotator/components/SubToolBar/index.less index c2447ce..f9d6e59 100644 --- a/packages/components/src/Annotator/components/SubToolBar/index.less +++ b/packages/components/src/Annotator/components/SubToolBar/index.less @@ -1,7 +1,4 @@ .dds-annotator-subtoolbar { - position: absolute; - left: 1rem; - top: 1rem; display: flex; flex-direction: row; justify-content: center; @@ -10,6 +7,7 @@ background-color: #212121; border-radius: 10px; padding: 0.5rem; + padding-left: 0; height: 50px; pointer-events: auto; font-weight: 600; @@ -42,11 +40,16 @@ } &-divider { - height: 100%; + height: 65%; margin: 10px 8px; border-left: 1px solid #fff; } + &-title { + margin: 0 0.25rem; + color: #fff; + } + &-slider { width: 100px; margin: 0 0.25rem; diff --git a/packages/components/src/Annotator/components/SubToolBar/index.tsx b/packages/components/src/Annotator/components/SubToolBar/index.tsx index c1f44be..bcf315d 100644 --- a/packages/components/src/Annotator/components/SubToolBar/index.tsx +++ b/packages/components/src/Annotator/components/SubToolBar/index.tsx @@ -1,129 +1,34 @@ import { Button, Popover, Slider } from 'antd'; -import Icon from '@ant-design/icons'; import classNames from 'classnames'; import { ESubToolItem } from '../../constants'; import { FloatWrapper } from '../FloatWrapper'; -import { TShortcutItem } from '../../constants/shortcuts'; -import { ReactComponent as PenAddIcon } from '../../assets/pen-add.svg'; -import { ReactComponent as PenEraseIcon } from '../../assets/pen-erase.svg'; -import { ReactComponent as BrushAddIcon } from '../../assets/brush-add.svg'; -import { ReactComponent as BrushEraseIcon } from '../../assets/brush-erase.svg'; -import { ReactComponent as MagicBoxIcon } from '../../assets/magic-box.svg'; -import { ReactComponent as ClickIcon } from '../../assets/magic-click.svg'; -import { ReactComponent as EdgeStitchIcon } from '../../assets/edge-stitch.svg'; -import { ReactComponent as SegmentEverythingIcon } from '../../assets/segment-everything.svg'; -import { ReactComponent as StrokeIcon } from '../../assets/magic-brush.svg'; -import { useLocale } from 'dds-utils/locale'; import { memo, useMemo } from 'react'; import { useKeyPress } from 'ahooks'; +import { TSubtoolOptions, TToolItem } from '@/Annotator/hooks/useSubtools'; import './index.less'; -type TToolItem = { - key: T; - name: string; - shortcut?: TShortcutItem; - icon: JSX.Element; - description?: string; - available: boolean; -}; interface IProps { + toolOptions: TSubtoolOptions; selectedSubTool: ESubToolItem; isAIAnnotationActive: boolean; - isSegEverythingAvailable: boolean; - isManualAvailable: boolean; brushSize: number; onChangeSubTool: (type: ESubToolItem) => void; onActiveAIAnnotation: (active: boolean) => void; onChangeBrushSize: (size: number) => void; } -export const SubToolBar: React.FC = memo( +const SubToolBar: React.FC = memo( ({ + toolOptions, selectedSubTool, isAIAnnotationActive, - isSegEverythingAvailable, - isManualAvailable, brushSize, onChangeSubTool, onChangeBrushSize, }) => { - const { localeText } = useLocale(); - - const basicMaskTools: TToolItem[] = [ - { - key: ESubToolItem.PenAdd, - name: localeText('DDSAnnotator.subtoolbar.mask.penAdd'), - icon: , - available: isManualAvailable, - }, - { - key: ESubToolItem.PenErase, - name: localeText('DDSAnnotator.subtoolbar.mask.penErase'), - icon: , - available: isManualAvailable, - }, - { - key: ESubToolItem.BrushAdd, - name: localeText('DDSAnnotator.subtoolbar.mask.brushAdd'), - icon: , - available: isManualAvailable, - }, - { - key: ESubToolItem.BrushErase, - name: localeText('DDSAnnotator.subtoolbar.mask.brushErase'), - icon: , - available: isManualAvailable, - }, - ]; - - const smartMaskTools: TToolItem[] = useMemo(() => { - return [ - { - key: ESubToolItem.AutoSegmentByBox, - name: localeText('DDSAnnotator.subtoolbar.mask.box'), - icon: , - available: true, - }, - { - key: ESubToolItem.AutoSegmentByStroke, - name: localeText('DDSAnnotator.subtoolbar.mask.stroke'), - icon: , - available: true, - }, - { - key: ESubToolItem.AutoSegmentByClick, - name: localeText('DDSAnnotator.subtoolbar.mask.click'), - icon: , - available: true, - }, - { - key: ESubToolItem.AutoEdgeStitching, - name: localeText('DDSAnnotator.subtoolbar.mask.edgeStitch'), - icon: , - available: true, - }, - { - key: ESubToolItem.AutoSegmentEverything, - name: localeText('DDSAnnotator.subtoolbar.mask.sam'), - icon: , - available: isSegEverythingAvailable, - description: isSegEverythingAvailable - ? localeText('DDSAnnotator.subtoolbar.mask.sam.desc') - : localeText('DDSAnnotator.subtoolbar.mask.sam.notAllow'), - }, - ]; - }, [isSegEverythingAvailable]); - - const toolsWithBrushSize = [ - ESubToolItem.BrushAdd, - ESubToolItem.BrushErase, - ESubToolItem.AutoSegmentByStroke, - ESubToolItem.AutoEdgeStitching, - ]; - const allSubTools = useMemo(() => { - return [...basicMaskTools, ...smartMaskTools]; - }, [basicMaskTools, smartMaskTools]); + return [...toolOptions.basicTools, ...toolOptions.smartTools]; + }, [toolOptions.basicTools, toolOptions.smartTools]); const shortcuts = useMemo(() => { const keys: string[] = []; @@ -141,7 +46,7 @@ export const SubToolBar: React.FC = memo( }); if (tool && tool.available) { if ( - smartMaskTools.find((item) => tool.key === item.key) && + toolOptions.smartTools.find((item) => tool.key === item.key) && !isAIAnnotationActive ) return; @@ -154,10 +59,10 @@ export const SubToolBar: React.FC = memo( ); const mouseEventHandler = (event: React.MouseEvent) => { - // enable mouseup propagate only for brush + const tool = allSubTools.find((item) => item.key === selectedSubTool); if ( - toolsWithBrushSize.includes(selectedSubTool) && - event.type === 'mouseup' + event.type === 'mouseup' && + (tool?.withSize || tool?.withCustomElement) ) { return; } else { @@ -221,16 +126,28 @@ export const SubToolBar: React.FC = memo( return (
- {basicMaskTools.map((item) => ToolItemBtn(item))} + {toolOptions.basicTools.map((item) => ToolItemBtn(item))} {isAIAnnotationActive && ( + <> + {toolOptions.basicTools.length > 0 && ( +
+ )} + {toolOptions.smartTools.map((item) => ToolItemBtn(item))} + + )} + {toolOptions.customElement && ( <>
- {smartMaskTools.map((item) => ToolItemBtn(item))} + {toolOptions.customElement} )} - {toolsWithBrushSize.includes(selectedSubTool) && ( + {!!allSubTools.find((item) => item.key === selectedSubTool) + ?.withSize && ( <>
+
+ {'Brush Size'} +
= memo( ); }, ); + +export default SubToolBar; diff --git a/packages/components/src/Annotator/components/TopPagination/index.tsx b/packages/components/src/Annotator/components/TopPagination/index.tsx index 0ee149d..3038f87 100644 --- a/packages/components/src/Annotator/components/TopPagination/index.tsx +++ b/packages/components/src/Annotator/components/TopPagination/index.tsx @@ -1,7 +1,7 @@ import { Button, Tooltip } from 'antd'; import { LeftOutlined, RightOutlined } from '@ant-design/icons'; import classNames from 'classnames'; -import { DrawImageData } from '../../type'; +import { AnnoItem } from '../../type'; import { memo, useState } from 'react'; import { useKeyPress } from 'ahooks'; import { EDITOR_SHORTCUTS, EShortcuts } from '../../constants/shortcuts'; @@ -9,7 +9,7 @@ import { useLocale } from 'dds-utils/locale'; import './index.less'; interface IProps { - list: DrawImageData[]; + list: AnnoItem[]; current: number; total: number; customText?: React.ReactElement; diff --git a/packages/components/src/Annotator/components/TopTools/index.less b/packages/components/src/Annotator/components/TopTools/index.less index ceec049..f74493a 100644 --- a/packages/components/src/Annotator/components/TopTools/index.less +++ b/packages/components/src/Annotator/components/TopTools/index.less @@ -3,15 +3,24 @@ display: flex; justify-content: space-between; align-items: center; + gap: 12px; padding: 0 16px; width: 100%; height: 56px; color: #fff; background: #1f1f1f; - border-bottom: 2px solid #141414; + border-bottom: 1px solid #141414; pointer-events: auto; + overflow-x: scroll; z-index: 1; + /* Hide scrollbar */ + scrollbar-width: none; /* firefox */ + -ms-overflow-style: none; /* IE 10+ */ + &::-webkit-scrollbar { + display: none; /* Chrome Safari */ + } + &-row { display: flex; align-items: center; diff --git a/packages/components/src/Annotator/constants/index.ts b/packages/components/src/Annotator/constants/index.ts index b9ccad4..73436b0 100644 --- a/packages/components/src/Annotator/constants/index.ts +++ b/packages/components/src/Annotator/constants/index.ts @@ -1,22 +1,19 @@ import { ReactComponent as RectIcon } from '../assets/rectangle.svg'; -import { ReactComponent as SkeletonIcon } from '../assets/point.svg'; -import { ReactComponent as MagicIcon } from '../assets/magic.svg'; +import { ReactComponent as RectAiIcon } from '../assets/rectangle-ai.svg'; import { ReactComponent as PolygonIcon } from '../assets/polygon.svg'; +import { ReactComponent as PolygonAiIcon } from '../assets/polygon-ai.svg'; +import { ReactComponent as SkeletonIcon } from '../assets/skeleton.svg'; +import { ReactComponent as SkeletonAiIcon } from '../assets/skeleton-ai.svg'; +import { ReactComponent as MaskIcon } from '../assets/mask.svg'; +import { ReactComponent as MaskAiIcon } from '../assets/mask-ai.svg'; +import { ReactComponent as MagicIcon } from '../assets/magic.svg'; import { ReactComponent as CustomIcon } from '../assets/custom.svg'; -import { ReactComponent as MaskIcon } from '../assets/brush.svg'; import { ReactComponent as UndoIcon } from '../assets/undo.svg'; import { ReactComponent as RedoIcon } from '../assets/redo.svg'; import { ReactComponent as RepeatIcon } from '../assets/repeat.svg'; import { ReactComponent as DeleteAllIcon } from '../assets/delete_all.svg'; - -export enum AnnotationType { - Classification = 'Classification', - Detection = 'Detection', - Segmentation = 'Segmentation', - Matting = 'Matting', - KeyPoints = 'KeyPoints', - Mask = 'Mask', -} +import { ReactComponent as TextPromptIcon } from '../assets/text-prompt.svg'; +import { ReactComponent as VisualPromptIcon } from '../assets/visual-prompt.svg'; export enum DisplayOption { showAnnotations = 'showAnnotations', @@ -45,13 +42,23 @@ export const MAX_SCALE = 20; export const BUTTON_SCALE_STEP = 0.5; export const WHEEL_SCALE_STEP = 0.1; +export enum ELabelType { + Rectangle = 'rect', + Polygon = 'polygon', + Mask = 'mask', + Skeleton = 'coco_keypoints_17', + Classification = 'classification', +} + export enum EObjectType { Custom = 'Custom', + Classification = 'Classification', Rectangle = 'Rectangle', Polygon = 'Polygon', Skeleton = 'Skeleton', Mask = 'Mask', Matting = 'Matting', + Point = 'Point', } export enum EElementType { @@ -69,14 +76,6 @@ export enum EBasicToolItem { Mask = 'Mask', } -export const EBasicToolTypeMap = { - [EBasicToolItem.Drag]: EObjectType.Custom, - [EBasicToolItem.Rectangle]: EObjectType.Rectangle, - [EBasicToolItem.Polygon]: EObjectType.Polygon, - [EBasicToolItem.Skeleton]: EObjectType.Skeleton, - [EBasicToolItem.Mask]: EObjectType.Mask, -}; - export enum ESubToolItem { PenAdd = 'PenAdd', PenErase = 'PenErase', @@ -87,6 +86,8 @@ export enum ESubToolItem { AutoSegmentByStroke = 'AutoSegmentByStroke', AutoSegmentEverything = 'AutoSegmentEverything', AutoEdgeStitching = 'AutoEdgeStitching', + PositiveVisualPrompt = 'PositiveVisualPrompt', + NegativeVisualPrompt = 'NegativeVisualPrompt', } export enum EActionToolItem { @@ -99,6 +100,64 @@ export enum EActionToolItem { export type EToolType = EBasicToolItem; +export const EBasicToolTypeMap = { + [EBasicToolItem.Drag]: EObjectType.Custom, + [EBasicToolItem.Rectangle]: EObjectType.Rectangle, + [EBasicToolItem.Polygon]: EObjectType.Polygon, + [EBasicToolItem.Skeleton]: EObjectType.Skeleton, + [EBasicToolItem.Mask]: EObjectType.Mask, +}; + +export enum EnumModelType { + Detection = 'ai_detection', + IVP = 'ivp', + SegmentByPolygon = 'ai_polygon', + SegmentByMask = 'ai_segmentation_mask', + Pose = 'ai_pose', + MaskEdgeStitching = 'ai_mask_edge_stitching', + SegmentEverything = 'ai_segment_everything', +} + +export const TOOL_MODELS_MAP: Record = { + [EBasicToolItem.Drag]: [], + [EBasicToolItem.Rectangle]: [EnumModelType.Detection, EnumModelType.IVP], + [EBasicToolItem.Polygon]: [EnumModelType.SegmentByPolygon], + [EBasicToolItem.Mask]: [EnumModelType.SegmentByMask], + [EBasicToolItem.Skeleton]: [EnumModelType.Pose], +}; + +export const MODEL_INTRO_MAP: Partial< + Record< + EnumModelType, + { + name: string; + icon: React.FunctionComponent>; + description: string; + hightlight: boolean; + } + > +> = { + [EnumModelType.Detection]: { + name: 'Grounding-DINO', + icon: TextPromptIcon, + description: 'DDSAnnotator.smart.gdino.desc', + hightlight: false, + }, + [EnumModelType.IVP]: { + name: 'iVP', + icon: VisualPromptIcon, + description: 'DDSAnnotator.smart.ivp.desc', + hightlight: true, + }, +}; + +export const LABEL_TOOL_MAP = { + [ELabelType.Rectangle]: EBasicToolItem.Rectangle, + [ELabelType.Polygon]: EBasicToolItem.Polygon, + [ELabelType.Mask]: EBasicToolItem.Mask, + [ELabelType.Skeleton]: EBasicToolItem.Skeleton, +}; + export const OBJECT_ICON: Record< EObjectType, React.FunctionComponent> @@ -106,9 +165,18 @@ export const OBJECT_ICON: Record< [EObjectType.Rectangle]: RectIcon, [EObjectType.Skeleton]: SkeletonIcon, [EObjectType.Polygon]: PolygonIcon, - [EObjectType.Custom]: CustomIcon, [EObjectType.Mask]: MaskIcon, [EObjectType.Matting]: MaskIcon, + [EObjectType.Point]: CustomIcon, + [EObjectType.Custom]: CustomIcon, + [EObjectType.Classification]: CustomIcon, +}; + +export const OBJECT_AI_ICON = { + [EObjectType.Rectangle]: RectAiIcon, + [EObjectType.Skeleton]: SkeletonAiIcon, + [EObjectType.Polygon]: PolygonAiIcon, + [EObjectType.Mask]: MaskAiIcon, }; export const EDITOR_TOOL_ICON: Record< diff --git a/packages/components/src/Annotator/constants/render.ts b/packages/components/src/Annotator/constants/render.ts index 21cfe5f..3a98beb 100644 --- a/packages/components/src/Annotator/constants/render.ts +++ b/packages/components/src/Annotator/constants/render.ts @@ -19,8 +19,8 @@ export const ANNO_STROKE_ALPHA = { export const ANNO_MASK_ALPHA = { CREATING: 0.7, - FOCUS: 0.6, - DEFAULT: 0.4, + FOCUS: 0.7, + DEFAULT: 0.5, }; export const ANNO_STROKE_COLOR = { @@ -33,7 +33,12 @@ export const ANNO_FILL_COLOR = { CREATING_NEGATIVE: '#e91d00', }; +export const PROMPT_STROKE_COLOR = { + POSITIVE: 'rgba(1, 128, 0, 1)', + NEGATIVE: 'rgba(255, 3, 0, 1)', +}; + export const PROMPT_FILL_COLOR = { - POSITIVE: 'rgba(1, 128, 0, 0.7)', - NEGATIVE: 'rgba(255, 3, 0, 0.7)', + POSITIVE: 'rgba(1, 128, 0, 0.6)', + NEGATIVE: 'rgba(255, 3, 0, 0.6)', }; diff --git a/packages/components/src/Annotator/constants/shortcuts.ts b/packages/components/src/Annotator/constants/shortcuts.ts index 09120de..d8e0f4b 100644 --- a/packages/components/src/Annotator/constants/shortcuts.ts +++ b/packages/components/src/Annotator/constants/shortcuts.ts @@ -96,7 +96,7 @@ export const EDITOR_SHORTCUTS: Record = { [EShortcuts.RepeatPrevious]: { name: 'RepeatPrevious', type: EShortcutType.GeneralAction, - shortcut: ['r'], + shortcut: ['ctrl.r', 'meta.r'], descTextKey: 'DDSAnnotator.shortcuts.general.repeatPrevious', }, [EShortcuts.DeleteAll]: { diff --git a/packages/components/src/Annotator/editor.tsx b/packages/components/src/Annotator/editor.tsx index 7c73615..b78dd8e 100755 --- a/packages/components/src/Annotator/editor.tsx +++ b/packages/components/src/Annotator/editor.tsx @@ -1,43 +1,26 @@ -import React, { useCallback, useEffect, useMemo, useRef } from 'react'; -import { Button, Divider, Dropdown, Modal } from 'antd'; -import { - EObjectType, - EElementType, - EBasicToolItem, - ESubToolItem, -} from './constants'; +import React, { useEffect, useMemo, useRef } from 'react'; +import { Dropdown, Modal } from 'antd'; +import { EBasicToolItem } from './constants'; import { Updater, useImmer } from 'use-immer'; -import TopTools from './components/TopTools'; import useLabels from './hooks/useLabels'; import useActions from './hooks/useActions'; -import PopoverMenu from './components/PopoverMenu'; import { ObjectList } from './components/ObjectList'; -import { MainToolBar } from './components/MainToolBar'; import SmartAnnotationControl from './components/SmartAnnotationControl'; -import { ScaleToolBar } from './components/ScaleToolBar'; -import { ArrowLeftOutlined } from '@ant-design/icons'; import { TopPagination } from './components/TopPagination'; -import { AnnotationEditor } from './components/AnnotationEditor'; -import { ShortcutsInfo } from './components/ShortcutsInfo'; import useHistory from './hooks/useHistory'; import useObjects from './hooks/useObjects'; import useCanvasContainer from './hooks/useCanvasContainer'; -import usePreviousState from './hooks/usePreviousState'; import { cloneDeep } from 'lodash'; -import { useLocale } from 'dds-utils/locale'; -import { SubToolBar } from './components/SubToolBar'; import { BaseObject, Category, DEFAULT_DRAW_DATA, DEFAULT_EDIT_STATE, DrawData, - DrawImageData, + AnnoItem, EditState, EditorMode, - EObjectStatus, DrawObject, - EQaAction, } from './type'; import useMouseCursor from './hooks/useMouseCursor'; import useShortcuts from './hooks/useShortcuts'; @@ -45,17 +28,32 @@ import useToolActions from './hooks/useToolActions'; import useMouseEvents from './hooks/useMouseEvents'; import useCanvasRender from './hooks/useCanvasRender'; import useDataEffect from './hooks/useDataEffect'; +import useSubTools from './hooks/useSubtools'; import { useToolInstances } from './tools/base'; import useColor from './hooks/useColor'; import { ImageView } from './components/ImageView'; +import useTranslate from './hooks/useTranslate'; +import ClassificationPanel from './components/Classification'; +import AttributeEditor from './components/AttributeEditor'; +import SegConfirmModal from './components/SegConfirmModal'; +import useAttributes from './hooks/useAttributes'; +import SliderToolBar from './components/SliderToolBar'; +import useTopTools from './hooks/useTopTools'; import './index.less'; +import classNames from 'classnames'; +import ModelSelectModal from './components/ModelSelectModal'; +import PointsEditModal from './components/PointsEditModal'; export interface EditProps { - isSeperate: boolean; + isOldMode?: boolean; // is old dataset design mode + isSeperate?: boolean; // is quickmode single editor + theme?: 'light' | 'dark'; visible: boolean; mode: EditorMode; + enableReviewerModify?: boolean; + limitToolTypes?: EBasicToolItem[]; categories: Category[]; - list: DrawImageData[]; + list: AnnoItem[]; current: number; pagination?: { show: boolean; @@ -63,13 +61,38 @@ export interface EditProps { customText?: React.ReactElement; customDisableNext?: boolean; }; + titleElements?: React.ReactElement[]; actionElements?: React.ReactElement[]; + layoutOptions?: { + wrapHeight?: string; + hideRightList?: boolean; + hideTopBar?: boolean; + hideTopBarActions?: boolean; + hideUndoRedoActions?: boolean; + hideReferenceLine?: boolean; + minPadding?: { + top: number; + left: number; + }; + }; + manualMode?: boolean; + forceColorByObject?: boolean; + limitActiveObject?: boolean; + limitActiveObjectAfterCreate?: boolean; + customDefaultDrawData?: Partial; + customDefaultEditState?: EditState; + customDrawData?: DrawData; + customEditState?: EditState; + customObjects?: DrawObject[]; + customObjectsFilter?: (imageData: any) => BaseObject[]; objectsFilter?: (imageData: any) => BaseObject[]; - onCancel?: () => void; - onSave?: (imageId: string, annotations: BaseObject[]) => Promise; onAutoSave?: (annotations: BaseObject[], naturalSize: ISize) => void; - onReviewResult?: (imageId: string, action: EQaAction) => Promise; - onEnterEdit?: () => void; + onCancel?: () => void; + onSave?: (id: string, labels: any[]) => Promise; + onCommit?: (id: string, labels: any[]) => Promise; + onReviewModify?: (id: string, labels: any[]) => Promise; + onReviewAccept?: (id: string, labels: any[]) => Promise; + onReviewReject?: (id: string, labels: any[]) => Promise; onPrev?: () => Promise; onNext?: () => Promise; setCategories?: Updater; @@ -77,6 +100,8 @@ export interface EditProps { const Edit: React.FC = (props) => { const { + theme = 'dark', + isOldMode, isSeperate, visible, categories, @@ -84,19 +109,28 @@ const Edit: React.FC = (props) => { current, pagination, mode, + enableReviewerModify, + limitToolTypes, + titleElements, actionElements, + layoutOptions, + manualMode, + forceColorByObject, + limitActiveObject, + limitActiveObjectAfterCreate, + customDefaultDrawData, onPrev, onNext, onCancel, onSave, - onEnterEdit, - onReviewResult, + onCommit, + onReviewModify, + onReviewAccept, + onReviewReject, setCategories, onAutoSave, objectsFilter, } = props; - - const { localeText } = useLocale(); const [modal, contextHolder] = Modal.useModal(); const [annotations, setAnnotations] = useImmer([]); @@ -105,47 +139,25 @@ const Edit: React.FC = (props) => { cloneDeep(DEFAULT_EDIT_STATE), ); - const [drawData, setDrawData] = useImmer( - cloneDeep(DEFAULT_DRAW_DATA), - ); + const [drawData, setDrawData] = useImmer({ + ...cloneDeep(DEFAULT_DRAW_DATA), + ...customDefaultDrawData, + }); const canvasRef = useRef(null); const activeCanvasRef = useRef(null); const imgRef = useRef(null); - const isCustomCursorActive = useMemo(() => { - const isToolWithSize = [ - ESubToolItem.AutoEdgeStitching, - ESubToolItem.AutoSegmentByStroke, - ESubToolItem.BrushAdd, - ESubToolItem.BrushErase, - ].includes(drawData.selectedSubTool); - - if ( - drawData.creatingObject && - drawData.activeObjectIndex > -1 && - drawData.creatingObject.type === EObjectType.Mask - ) { - return isToolWithSize; - } - if ( - drawData.selectedTool !== EBasicToolItem.Drag && - !drawData.isBatchEditing - ) { - return drawData.selectedTool === EBasicToolItem.Mask && isToolWithSize; - } - return false; - }, [drawData.selectedTool, drawData.selectedSubTool]); + const currAnnoItem = useMemo(() => { + return list[current]; + }, [list, current]); - const showReferenceLine = useMemo(() => { - return ( - drawData.selectedTool !== EBasicToolItem.Drag && !isCustomCursorActive - ); - }, [drawData.selectedTool, isCustomCursorActive]); + const currImageItem = currAnnoItem; - const { labelColors, getAnnotColor } = useColor({ + const { getAnnotColor, labelColors } = useColor({ categories, editState, + forceColorByObject, }); const { @@ -163,19 +175,24 @@ const Edit: React.FC = (props) => { isMousePress, } = useCanvasContainer({ visible, + drawData, allowMove: editState.allowMove, isRequiring: editState.isRequiring, - showReferenceLine, - minPadding: { + minPadding: layoutOptions?.minPadding || { top: 30, left: 80, }, - isCustomCursorActive, cursorSize: drawData.brushSize, + hideReferenceLine: !!layoutOptions?.hideReferenceLine, }); - const [preClientSize, clearPreClientSize] = - usePreviousState(clientSize); + const { translateObject, translateToObject } = useTranslate({ + isOldMode, + clientSize, + naturalSize, + categories, + getAnnotColor, + }); const { undo, @@ -189,8 +206,15 @@ const Edit: React.FC = (props) => { naturalSize, setDrawData, onAutoSave, + translateObject, }); + const { judgeEditingAttribute, onConfirmAttibuteEdit, onCancelAttibuteEdit } = + useAttributes({ + setDrawDataWithHistory, + categories, + }); + const { addObject, removeObject, @@ -200,29 +224,34 @@ const Edit: React.FC = (props) => { updateObject, updateObjectWithoutHistory, updateAllObjectWithoutHistory, + commitedObjects, + currObject, } = useObjects({ annotations, setAnnotations, - clientSize, - naturalSize, drawData, setDrawData, setDrawDataWithHistory, - editState, setEditState, mode, + translateToObject, + judgeEditingAttribute, + limitActiveObjectAfterCreate, + updateHistory, }); const { + labelOptions, + classificationOptions, aiLabels, setAiLabels, onChangeObjectHidden, onChangeCategoryHidden, onChangeActiveClass, onCreateCategory, - onChangePointVisible + onChangePointVisible, } = useLabels({ - visible, + isOldMode, mode, categories, setCategories, @@ -236,13 +265,14 @@ const Edit: React.FC = (props) => { const { onAiAnnotation, onSaveAnnotations, + onCommitAnnotations, onCancelAnnotations, - onReject, - onAccept, + onRejectAnnotations, + onAcceptAnnotations, + onModifyAnnotations, } = useActions({ mode, - list, - current, + currImageItem, modal, drawData, setDrawData, @@ -253,12 +283,18 @@ const Edit: React.FC = (props) => { clientSize, imagePos, containerMouse, - onCancel, - onSave, updateAllObject, hadChangeRecord, - latestLabel: editState.latestLabel, getAnnotColor, + categories, + translateObject, + onCancel, + onSave, + onCommit, + onReviewModify, + onReviewAccept, + onReviewReject, + classificationOptions, }); const { updateMouseCursor } = useMouseCursor({ @@ -268,9 +304,8 @@ const Edit: React.FC = (props) => { }); const { - onDeleteCurrObject, + onChangeObjectLabel, onFinishCurrCreate, - onCloseAnnotationEditor, onAcceptValidObjects, onAbortBatchObjects, selectTool, @@ -279,15 +314,16 @@ const Edit: React.FC = (props) => { onExitAIAnnotation, setBrushSize, activeAIAnnotation, - onSaveAIPolygon, - onCancelAIPolygon, onChangeSkeletonConf, onChangeLimitConf, onChangeAnnotsDisplayOpts, onChangeImageDisplayOpts, onChangeColorMode, + onChangePointResolution, + onSelectModel, } = useToolActions({ mode, + manualMode: !!manualMode, drawData, setDrawData, setDrawDataWithHistory, @@ -298,9 +334,14 @@ const Edit: React.FC = (props) => { clientSize, naturalSize, addObject, - removeObject, updateObject, updateAllObject, + onAiAnnotation, + }); + + const { showSubTools, currSubTools } = useSubTools({ + drawData, + onChangePointResolution, }); const { objectHooksMap } = useToolInstances({ @@ -324,9 +365,10 @@ const Edit: React.FC = (props) => { aiLabels, onAiAnnotation, getAnnotColor, + categories, }); - const { updateRender } = useCanvasRender({ + const { updateRender, renderPopoverMenu } = useCanvasRender({ visible, drawData, editState, @@ -359,18 +401,20 @@ const Edit: React.FC = (props) => { imagePos, containerMouse, getAnnotColor, + limitActiveObject, }); useShortcuts({ visible, mode, drawData, + categories, isMousePress, setDrawData, setEditState, onSaveAnnotations, - onAccept, - onReject, + onAcceptAnnotations, + onRejectAnnotations, onChangeObjectHidden, onChangeCategoryHidden, removeObject, @@ -380,12 +424,9 @@ const Edit: React.FC = (props) => { const { resetDataWithImageData } = useDataEffect({ imagePos, clientSize, - preClientSize, - clearPreClientSize, naturalSize, annotations, setAnnotations, - labelColors, drawData, setDrawData, editState, @@ -394,15 +435,10 @@ const Edit: React.FC = (props) => { updateRender, clearHistory, objectsFilter, + labelOptions, + customDefaultDrawData, }); - /** Copy annots from previous image */ - const repeatPrevious = useCallback(() => { - if (current > 0 && current < list.length) { - resetDataWithImageData(list[current - 1], visible, false); - } - }, [resetDataWithImageData, list, current, visible]); - // ================================================================================================================= // Effects // ================================================================================================================= @@ -410,12 +446,15 @@ const Edit: React.FC = (props) => { /** Limit bottom layer body scroll */ useEffect(() => { document.body.style.overflow = visible ? 'hidden' : 'overlay'; + return () => { + document.body.style.overflow = 'overlay'; + }; }, [visible]); /** Reset data when hiding the editor or switching images */ useEffect(() => { - resetDataWithImageData(list[current], visible); - }, [visible, mode, current, objectsFilter]); + resetDataWithImageData(currImageItem, visible); + }, [visible, mode, current, currImageItem?.id, objectsFilter]); useEffect(() => { onChangeColorMode(); @@ -426,156 +465,98 @@ const Edit: React.FC = (props) => { // ================================================================================================================= const fileName = useMemo(() => { - if ( - list[current]?.urlFullRes && - list[current]?.urlFullRes.indexOf('http') === 0 - ) { - const url = decodeURIComponent(list[current]?.urlFullRes); + if (currAnnoItem?.name) return currAnnoItem?.name; + if (currAnnoItem?.url && currAnnoItem?.url.indexOf('http') === 0) { + const url = decodeURIComponent(currAnnoItem?.url); return url.replace(/\?.*$/, '').split('/').pop() || ''; } return ''; - }, [list, current]); + }, [currAnnoItem]); + + const topBarCenterElement = + pagination && pagination.show ? ( + + ) : null; + + const { topToolsBar } = useTopTools({ + isOldMode, + isSeperate, + mode, + hideTopBarActions: layoutOptions?.hideTopBarActions, + fileName, + drawData, + editState, + titleElements, + actionElements, + enableReviewerModify, + labelOptions, + showSubTools, + currSubTools, + topBarCenterElement, + labelColors, + selectSubTool, + setBrushSize, + activeAIAnnotation, + onChangeImageDisplayOpts, + onChangeAnnotsDisplayOpts, + onChangeObjectLabel, + onCreateCategory, + onSaveAnnotations, + onCommitAnnotations, + onRejectAnnotations, + onAcceptAnnotations, + onModifyAnnotations, + onCancelAnnotations, + onSelectModel, + }); - const supportActions = useMemo(() => { - const actions = actionElements - ? actionElements.map((item) => ({ customElement: item })) - : []; - if (mode === EditorMode.Review && onReviewResult) { - actions.push( - ...[ - { - customElement: ( - - ), - }, - { - customElement: ( - - ), - }, - ], - ); - } - if (mode === EditorMode.Edit && !isSeperate) { - actions.push( - ...[ - { - customElement: ( - - ), - }, - ], - ); - } - actions.unshift({ - customElement: ( - <> - - - - ), - }); - return actions; - }, [mode, onReviewResult, onEnterEdit, onSaveAnnotations, list[current]]); - - const renderPopoverMenu = () => { - if ( - editState.focusObjectIndex > -1 && - drawData.objectList[editState.focusObjectIndex] && - !drawData.objectList[editState.focusObjectIndex].hidden && - editState.focusEleIndex > -1 && - editState.focusEleType === EElementType.Circle - ) { - const target = - drawData.objectList[editState.focusObjectIndex].keypoints?.points?.[ - editState.focusEleIndex - ]; - if (target) { - return ( - - ); - } - } - return <>; - }; + if (!visible) { + return null; + } - const isAnnotEditorVisible = - mode === EditorMode.Edit && - !( - drawData.isBatchEditing && - drawData.selectedTool === EBasicToolItem.Skeleton - ) && - !( - drawData.selectedTool === EBasicToolItem.Polygon && - drawData.AIAnnotation && - drawData.activeObjectIndex === -1 - ); - - const showSubTools = - drawData.selectedTool === EBasicToolItem.Mask || - (drawData.creatingObject && - drawData.creatingObject.type === EObjectType.Mask); - - const commitedObjects = useMemo(() => { - return drawData.objectList.filter((obj) => { - return obj.status === EObjectStatus.Commited; - }); - }, [drawData.isBatchEditing, drawData.objectList]); - - if (visible) { - return ( -
- , - onClick: () => onCancelAnnotations(), - }, - ]), - { - customElement: fileName, - }, - ]} - rightTools={supportActions} - > - {pagination && pagination.show && ( - - )} - -
-
-
+ return ( +
+ {!layoutOptions?.hideTopBar && topToolsBar} +
+ +
+ {currImageItem && ( = (props) => { children: ( <> { + // Possibly size not changed but image changed + updateRender(); + onLoadImg(event); + }} /> {renderPopoverMenu()} ), })} - {isAnnotEditorVisible && ( - + + + + setDrawData((s) => { + s.AIAnnotation = false; + }) + } + /> + {drawData.editingAttribute && ( + + )} +
+ {!layoutOptions?.hideRightList && ( +
+ {classificationOptions.length > 0 && ( + )} - - - {mode === EditorMode.Edit && ( - <> - - {showSubTools && ( - 0 - ) && - !drawData.isBatchEditing - } - brushSize={drawData.brushSize} - onChangeSubTool={selectSubTool} - onChangeBrushSize={setBrushSize} - onActiveAIAnnotation={activeAIAnnotation} - /> - )} - - )}
- -
-
{ - e.stopPropagation(); - }} - > - {contextHolder} -
+ )}
- ); - } else { - return <>; - } +
{ + e.stopPropagation(); + }} + > + {contextHolder} +
+
+ ); }; export default Edit; diff --git a/packages/components/src/Annotator/hooks/useActions.ts b/packages/components/src/Annotator/hooks/useActions.tsx similarity index 55% rename from packages/components/src/Annotator/hooks/useActions.ts rename to packages/components/src/Annotator/hooks/useActions.tsx index cd4dcdc..1179796 100644 --- a/packages/components/src/Annotator/hooks/useActions.ts +++ b/packages/components/src/Annotator/hooks/useActions.tsx @@ -1,47 +1,61 @@ import { getVisibleAreaForImage, translateBoundingBoxToRect, - translateObjectsToAnnotations, translatePointsToPointObjs, translatePointZoom, translateRectToAbsBbox, getCanvasPoint, getNaturalPoint, + translateRectToBoundingBox, + translatePointObjsToPointAttrs, + convertFrameObjectsIntoFramesObjects, + translateRectZoom, + translateAbsBBoxToRect, } from '../utils/compute'; -import { message } from 'antd'; +import { Modal, message } from 'antd'; import { Updater } from 'use-immer'; import { BODY_TEMPLATE, EBasicToolItem, EBasicToolTypeMap, + EnumModelType, EObjectType, ESubToolItem, } from '../constants'; -import { getImageBase64, isBase64 } from '../utils/base64'; +import { + getImageBase64, + isBase64, + isBlobUrl, + isHttpsUrl, +} from '../utils/base64'; import { useLocale } from 'dds-utils/locale'; import { useModel } from '@umijs/max'; import { - BaseObject, DrawData, - DrawImageData, + AnnoItem, EditState, EditorMode, IAnnotationObject, - MaskPromptItem, + PromptItem, EObjectStatus, - EQaAction, + Category, + VideoFramesData, } from '../type'; import { objectToRle, rleToCanvas } from '../tools/useMask'; import { CursorState } from 'ahooks/lib/useMouse'; import { ModalStaticFunctions } from 'antd/es/modal/confirm'; import { useCallback } from 'react'; -import { NsApiAnnotator, fetchModelResults } from '../sevices'; +import { + NsApiAnnotator, + fetchModelResults, + getOssUrlByBlobUrl, +} from '../sevices'; interface IProps { mode: EditorMode; - list: DrawImageData[]; - current: number; + currImageItem?: AnnoItem; modal: Omit; + framesData?: VideoFramesData; drawData: DrawData; setDrawData: Updater; setDrawDataWithHistory: Updater; @@ -53,11 +67,16 @@ interface IProps { imagePos: React.MutableRefObject; updateAllObject: (objectList: IAnnotationObject[]) => void; hadChangeRecord: boolean; - latestLabel: string; getAnnotColor: (category: string, forceColorByCategory?: boolean) => string; + categories: Category[]; + translateObject?: (object: any) => any; onCancel?: () => void; - onSave?: (imageId: string, annotations: BaseObject[]) => Promise; - onReviewResult?: (imageId: string, action: EQaAction) => Promise; + onSave?: (id: string, labels: any[]) => Promise; + onCommit?: (id: string, labels: any[]) => Promise; + onReviewModify?: (id: string, labels: any[]) => Promise; + onReviewAccept?: (id: string, labels: any[]) => Promise; + onReviewReject?: (id: string, labels: any[]) => Promise; + classificationOptions?: Category[]; } export type OnAiAnnotationFunc = ({ @@ -65,15 +84,15 @@ export type OnAiAnnotationFunc = ({ drawData, aiLabels, bbox, - maskPrompts, + promptsQueue, segmentationClicks, segmentEverythingParams, }: { type?: EObjectType; drawData?: DrawData; - aiLabels?: string[]; + aiLabels?: string; bbox?: IBoundingBox; - maskPrompts?: MaskPromptItem[]; + promptsQueue?: PromptItem[]; segmentationClicks?: { point: IPoint; isPositive: boolean; @@ -83,9 +102,9 @@ export type OnAiAnnotationFunc = ({ const useActions = ({ mode, - list, - current, + currImageItem, modal, + framesData, drawData: editorDrawData, setDrawData, setDrawDataWithHistory, @@ -97,11 +116,16 @@ const useActions = ({ containerMouse, updateAllObject, hadChangeRecord, - latestLabel, + categories, getAnnotColor, + translateObject, onCancel, onSave, - onReviewResult, + onCommit, + onReviewModify, + onReviewAccept, + onReviewReject, + classificationOptions, }: IProps) => { const { localeText } = useLocale(); const { setLoading } = useModel('global'); @@ -111,17 +135,16 @@ const useActions = ({ s.isRequiring = requiring; }); - const requestAiDetection = async (source: string, aiLabels: string[]) => { + const requestAiDetection = async (source: string, aiLabels: string) => { try { setLoading(true); - const result = - await fetchModelResults( - NsApiAnnotator.EnumModelType.Detection, - { - image: source, - text: aiLabels.join(','), - }, - ); + const result = await fetchModelResults( + EnumModelType.Detection, + { + image: source, + text: aiLabels, + }, + ); if (result) { const { objects, suggestThreshold } = result; @@ -134,7 +157,7 @@ const useActions = ({ }; return { rect: { ...rect, visible: true }, - label: item.categoryName, + labelId: editState.latestLabelId, type: EObjectType.Rectangle, hidden: false, status: @@ -142,7 +165,7 @@ const useActions = ({ ? EObjectStatus.Checked : EObjectStatus.Unchecked, conf: item.normalizedScore, - color: getAnnotColor(item.categoryName, true), + color: getAnnotColor(editState.latestLabelId, true), }; }) .reverse(); @@ -150,7 +173,7 @@ const useActions = ({ s.isBatchEditing = true; s.limitConf = limitConf; const commitedObjects = s.objectList.filter( - (obj) => obj.status === EObjectStatus.Commited, + (obj) => obj?.status === EObjectStatus.Commited, ); s.objectList = [...commitedObjects, ...newObjects]; if (s.creatingObject && s.objectList[s.activeObjectIndex]) { @@ -166,134 +189,19 @@ const useActions = ({ } }; - const requestAiSegmentByPolygon = async ( - drawData: DrawData, - source: string, - bbox?: IBoundingBox, - segmentationClicks?: { - point: IPoint; - isPositive: boolean; - }[], - ) => { - const existPolygons = - drawData.creatingObject?.polygon?.group.map((polygon) => { - return polygon.reduce((acc: number[], point) => { - const { x, y } = getNaturalPoint( - [point.x, point.y], - naturalSize, - clientSize, - ); - return acc.concat([x, y]); - }, []); - }) || []; - - const clicks = - segmentationClicks?.map((click) => { - const { x, y } = getNaturalPoint( - [click.point.x, click.point.y], - naturalSize, - clientSize, - ); - return { - isPositive: click.isPositive, - position: [x, y], - }; - }) || []; - - const reqParams = { - image: source, - mask: drawData.prompt.segmentationMask || '', - polygons: existPolygons, - clicks: clicks, - }; - - if (bbox) { - const { xmin, ymin, xmax, ymax } = bbox; - const topleftPoint = getNaturalPoint( - [xmin, ymin], - naturalSize, - clientSize, - ); - const bottomRightPoint = getNaturalPoint( - [xmax, ymax], - naturalSize, - clientSize, - ); - Object.assign(reqParams, { - rect: [ - topleftPoint.x, - topleftPoint.y, - bottomRightPoint.x, - bottomRightPoint.y, - ], - }); - } - - try { - setLoading(true); - const result = - await fetchModelResults( - NsApiAnnotator.EnumModelType.SegmentByPolygon, - reqParams, - ); - if (result) { - const { polygon, mask } = result; - - if (polygon && polygon.length > 0) { - const predictPolygons = polygon.map((item) => { - const result: IPolygon = []; - for (let i = 0; i < item.length; i += 2) { - const x = item[i]; - const y = item[i + 1]; - const canvasPoint = getCanvasPoint( - [x, y], - naturalSize, - clientSize, - ); - result.push(canvasPoint); - } - return result; - }); - - const creatingObj = { - type: EObjectType.Polygon, - hidden: false, - label: latestLabel, - color: getAnnotColor(latestLabel), - currIndex: -1, - polygon: { - visible: true, - group: predictPolygons, - }, - status: EObjectStatus.Checked, - }; - - setDrawDataWithHistory((s) => { - s.creatingObject = creatingObj; - s.prompt.segmentationMask = mask; - }); - } - - message.success(localeText('DDSAnnotator.smart.msg.success')); - } - } catch (error: any) { - message.error(localeText('DDSAnnotator.smart.msg.error')); - } finally { - setLoading(false); - } - }; - const convertPromptFormat = ( - prompt: MaskPromptItem[], + prompt: PromptItem[], ): { type: string; isPositive: boolean; point?: number[]; rect?: number[]; stroke?: number[]; + radius?: number; + polygons?: number[][]; }[] => { const newPromptArr = prompt.map((item) => { - const { type, isPositive, point, rect, stroke, radius } = item; + const { type, isPositive, point, rect, stroke, radius, polygons } = item; const newItem = { type, isPositive }; @@ -342,31 +250,125 @@ const useActions = ({ }); } + if (polygons) { + const transformedPolygons = polygons.map((polygon) => { + const res = []; + for (let i = 0; i < polygon.length; i += 2) { + const transformedPoint = getNaturalPoint( + [polygon[i], polygon[i + 1]], + naturalSize, + clientSize, + ); + res.push(transformedPoint.x, transformedPoint.y); + } + return res; + }); + Object.assign(newItem, { + polygons: transformedPolygons, + }); + } + return newItem; }); return newPromptArr; }; - const requestAiSegmentByMask = async ( - drawData: DrawData, - source: string, - maskPrompts?: MaskPromptItem[], + const requestIvpDetection = async ( + base64Img: string, + promptsQueue?: PromptItem[], ) => { - if (!maskPrompts) return; + if (!promptsQueue || !currImageItem) return; - const currMask = - drawData.creatingObject?.maskCanvasElement || - drawData.creatingObject?.tempMaskSteps - ? objectToRle( - clientSize, - naturalSize, - drawData.creatingObject?.tempMaskSteps || [], - drawData.creatingObject?.maskCanvasElement, - ) - : []; + if (promptsQueue.every((prompt) => !prompt.isPositive)) { + message.error(localeText('DDSAnnotator.smart.msg.positivePrompt')); + setDrawDataWithHistory((s) => { + s.prompt.creatingPrompt = undefined; + }); + return; + } + + try { + setLoading(true); + + let url = base64Img; + if (isHttpsUrl(currImageItem.url)) { + url = currImageItem.url; + } else if (isBlobUrl(currImageItem.url)) { + url = await getOssUrlByBlobUrl( + currImageItem.fileName || 'image', + currImageItem.url, + ); + } + + const reqParams = { + promptImage: url, + inferImage: url, + prompts: convertPromptFormat(promptsQueue || []), + labelTypes: ['bbox'], + }; + + const result = await fetchModelResults( + EnumModelType.IVP, + reqParams, + ); + + if (result) { + const { objects } = result; + const limitConf = 0.3; + const newObjects: IAnnotationObject[] = objects + .filter((item) => { + return item.bbox; + }) + .map((item) => { + const [xmin, ymin, xmax, ymax] = item.bbox!; + const rect = translateRectZoom( + translateAbsBBoxToRect({ xmin, ymin, xmax, ymax }), + naturalSize, + clientSize, + ); + return { + rect: { ...rect, visible: true }, + labelId: editState.latestLabelId, + type: EObjectType.Rectangle, + hidden: false, + status: + item.score >= limitConf + ? EObjectStatus.Checked + : EObjectStatus.Unchecked, + conf: item.score, + color: getAnnotColor(editState.latestLabelId, true), + }; + }) + .reverse(); + + setDrawDataWithHistory((s) => { + s.isBatchEditing = true; + s.limitConf = limitConf; + const commitedObjects = s.objectList.filter( + (obj) => obj.status === EObjectStatus.Commited, + ); + s.objectList = [...commitedObjects, ...newObjects]; + if (s.creatingObject && s.objectList[s.activeObjectIndex]) { + s.creatingObject = { ...s.objectList[s.activeObjectIndex] }; + } + s.prompt.promptsQueue = promptsQueue; + s.prompt.creatingPrompt = undefined; + }); + message.success(localeText('DDSAnnotator.smart.msg.success')); + } + } catch (error: any) { + message.error(localeText('DDSAnnotator.smart.msg.error')); + setDrawDataWithHistory((s) => { + s.prompt.creatingPrompt = undefined; + }); + } finally { + setLoading(false); + } + }; - // record visible area currently for model + const getCurrVisibleBbox = () => { + // record visible area currently for model prediction const { xmin, ymin, xmax, ymax } = getVisibleAreaForImage( imagePos.current, clientSize, @@ -392,12 +394,117 @@ const useActions = ({ ); area = [Math.round(x1), Math.round(y1), Math.round(x2), Math.round(y2)]; } + return area; + }; + + const requestAiSegmentByPolygon = async ( + drawData: DrawData, + source: string, + promptsQueue?: PromptItem[], + ) => { + if (!promptsQueue) return; + + const reqParams = { + image: editState.imageCacheIdForPolygon + ? `image_id://${editState.imageCacheIdForPolygon}` + : source, + density: drawData.pointResolution, + area: getCurrVisibleBbox(), + prompts: convertPromptFormat(promptsQueue || []), + }; + + if (drawData.prompt.sessionId) { + Object.assign(reqParams, { sessionId: drawData.prompt.sessionId }); + } + + try { + setLoading(true); + const result = await fetchModelResults( + EnumModelType.SegmentByPolygon, + reqParams, + ); + if (result) { + const { image, polygons, sessionId } = result; + + if (polygons && polygons.length > 0) { + const predictPolygons = polygons + .filter((item) => { + return item.length >= 6; + }) + .map((item) => { + const result: IPolygon = []; + for (let i = 0; i < item.length; i += 2) { + const x = item[i]; + const y = item[i + 1]; + const canvasPoint = getCanvasPoint( + [x, y], + naturalSize, + clientSize, + ); + result.push(canvasPoint); + } + return result; + }); + + const creatingObj = { + type: EObjectType.Polygon, + hidden: false, + labelId: editState.latestLabelId, + color: + drawData.creatingObject?.color || + getAnnotColor(editState.latestLabelId), + currIndex: -1, + polygon: { + visible: true, + group: predictPolygons, + }, + status: EObjectStatus.Checked, + }; + + setDrawDataWithHistory((s) => { + s.creatingObject = creatingObj; + s.prompt.promptsQueue = promptsQueue; + s.prompt.sessionId = sessionId; + s.prompt.creatingPrompt = undefined; + }); + setEditState((s) => { + s.imageCacheIdForPolygon = image.replace(/^image_id:\/\//, ''); + }); + message.success(localeText('DDSAnnotator.smart.msg.success')); + } + } + } catch (error: any) { + message.error(localeText('DDSAnnotator.smart.msg.error')); + setDrawDataWithHistory((s) => { + s.prompt.creatingPrompt = undefined; + }); + } finally { + setLoading(false); + } + }; + + const requestAiSegmentByMask = async ( + drawData: DrawData, + source: string, + promptsQueue?: PromptItem[], + ) => { + if (!promptsQueue) return; + const currMask = + drawData.creatingObject?.maskCanvasElement || + drawData.creatingObject?.tempMaskSteps + ? objectToRle( + clientSize, + naturalSize, + drawData.creatingObject?.tempMaskSteps || [], + drawData.creatingObject?.maskCanvasElement, + ) + : []; const reqParams: NsApiAnnotator.FetchAIMaskSegmentReq = { maskRle: currMask || [], - maskId: drawData.prompt.segmentationMask || '', - prompt: convertPromptFormat(maskPrompts || []), - area, + maskId: drawData.prompt.sessionId || '', + prompt: convertPromptFormat(promptsQueue || []), + area: getCurrVisibleBbox(), }; if (editState.imageCacheId) { @@ -408,19 +515,19 @@ const useActions = ({ try { setLoading(true); - const result = - await fetchModelResults( - NsApiAnnotator.EnumModelType.SegmentByMask, - reqParams, - ); + const result = await fetchModelResults( + EnumModelType.SegmentByMask, + reqParams, + ); if (result) { const { maskId, maskRle, imageId } = result; const color = - drawData.creatingObject?.color || getAnnotColor(latestLabel); + drawData.creatingObject?.color || + getAnnotColor(editState.latestLabelId); const creatingObj = { type: EObjectType.Mask, hidden: false, - label: latestLabel, + labelId: editState.latestLabelId, currIndex: -1, maskCanvasElement: rleToCanvas(maskRle, naturalSize, color), maskRle, @@ -429,9 +536,9 @@ const useActions = ({ }; setDrawDataWithHistory((s) => { s.creatingObject = creatingObj; - s.prompt.maskPrompts = maskPrompts; - s.prompt.segmentationMask = maskId; - s.prompt.creatingMask = undefined; + s.prompt.promptsQueue = promptsQueue; + s.prompt.sessionId = maskId; + s.prompt.creatingPrompt = undefined; }); setEditState((s) => { s.imageCacheId = imageId; @@ -441,7 +548,7 @@ const useActions = ({ } catch (error: any) { message.error(localeText('DDSAnnotator.smart.msg.error')); setDrawDataWithHistory((s) => { - s.prompt.creatingMask = undefined; + s.prompt.creatingPrompt = undefined; }); } finally { setLoading(false); @@ -451,13 +558,13 @@ const useActions = ({ const requestAiPoseEstimation = async ( drawData: DrawData, source: string, - aiLabels: string[], + aiLabels: string, ) => { // TODO: Integrate custom templates const { lines, pointNames, pointColors } = BODY_TEMPLATE; const reqParams = { image: source, - targets: aiLabels.join(','), + targets: aiLabels, template: { lines, pointNames, @@ -484,16 +591,19 @@ const useActions = ({ obj.status === EObjectStatus.Checked, ); if (skeletonObjs.length > 0) { - const annotations = translateObjectsToAnnotations( - skeletonObjs, - naturalSize, - clientSize, - ); - const objects = annotations.map((item) => { + const objects = skeletonObjs.map((item) => { return { - categoryName: item.categoryName, - points: item.points, - boundingBox: item.boundingBox, + categoryName: aiLabels, + points: item.keypoints + ? translatePointObjsToPointAttrs( + item.keypoints.points, + naturalSize, + clientSize, + ).points + : undefined, + boundingBox: item.rect + ? translateRectToBoundingBox(item.rect, clientSize) + : undefined, }; }); Object.assign(reqParams, { objects }); @@ -502,8 +612,8 @@ const useActions = ({ try { setLoading(true); - const result = await fetchModelResults( - NsApiAnnotator.EnumModelType.Pose, + const result = await fetchModelResults( + EnumModelType.Pose, reqParams, ); @@ -512,10 +622,10 @@ const useActions = ({ if (objects && objects.length > 0) { const skeletonObjs = objects.map((obj) => { - let { categoryName, boundingBox, points, conf } = obj; + let { boundingBox, points, conf } = obj; const newObj: IAnnotationObject = { - label: categoryName, - color: getAnnotColor(categoryName), + labelId: editState.latestLabelId, + color: getAnnotColor(editState.latestLabelId), type: EObjectType.Skeleton, hidden: false, conf, @@ -571,12 +681,12 @@ const useActions = ({ source: string, ) => { if ( - !drawData.prompt.creatingMask?.stroke || - !drawData.prompt.creatingMask?.radius + !drawData.prompt.creatingPrompt?.stroke || + !drawData.prompt.creatingPrompt?.radius ) return; - const { stroke, radius } = drawData.prompt.creatingMask; + const { stroke, radius } = drawData.prompt.creatingPrompt; const maskObjects = drawData.objectList.filter( (item) => item.type === EObjectType.Mask, @@ -587,7 +697,7 @@ const useActions = ({ 'To ensure valid results when using intelligent edge stitching, make sure to use at least 2 mask objects.', ); setDrawData((s) => { - s.prompt.creatingMask = undefined; + s.prompt.creatingPrompt = undefined; }); return; } @@ -595,7 +705,9 @@ const useActions = ({ const rleList = maskObjects.map((item) => { const maskRle = objectToRle(clientSize, naturalSize, [], item.maskCanvasElement) || []; - return { maskRle, categoryName: item.label }; + const categoryName = + categories.find((c) => c.id === item.labelId)?.name || ''; + return { maskRle, categoryName }; }); const points = stroke.reduce((acc: number[], point: IPoint) => { @@ -620,18 +732,19 @@ const useActions = ({ try { setLoading(true); - const result = - await fetchModelResults( - NsApiAnnotator.EnumModelType.MaskEdgeStitching, - reqParams, - ); + const result = await fetchModelResults( + EnumModelType.MaskEdgeStitching, + reqParams, + ); if (result && result.rleList?.length > 0) { const maskObjects = result.rleList.map((item) => { - const color = getAnnotColor(item.categoryName); + const labelId = + categories.find((c) => c.name === item.categoryName)?.id || ''; + const color = getAnnotColor(labelId); return { type: EObjectType.Mask, hidden: false, - label: item.categoryName, + labelId: labelId, maskRle: item.maskRle, maskCanvasElement: rleToCanvas(item.maskRle, naturalSize, color), conf: 1, @@ -655,7 +768,7 @@ const useActions = ({ } finally { setLoading(false); setDrawData((s) => { - s.prompt.creatingMask = undefined; + s.prompt.creatingPrompt = undefined; }); } }; @@ -676,22 +789,21 @@ const useActions = ({ try { setLoading(true); - const result = - await fetchModelResults( - NsApiAnnotator.EnumModelType.SegmentEverything, - reqParams, - ); + const result = await fetchModelResults( + EnumModelType.SegmentEverything, + reqParams, + ); if (result && result.rleList?.length > 0) { // change to display different color setEditState((s) => { s.annotsDisplayOptions.colorByCategory = false; }); const maskObjects: IAnnotationObject[] = result.rleList.map((item) => { - const color = getAnnotColor(latestLabel); + const color = getAnnotColor(editState.latestLabelId); return { type: EObjectType.Mask, hidden: false, - label: latestLabel, + labelId: editState.latestLabelId, maskRle: item.maskRle, maskCanvasElement: rleToCanvas(item.maskRle, naturalSize, color), conf: 1, @@ -716,10 +828,8 @@ const useActions = ({ async ({ type, drawData: propsDrawData, - aiLabels = [], - bbox, - maskPrompts, - segmentationClicks, + aiLabels, + promptsQueue, segmentEverythingParams, }) => { if (isRequiring) return; @@ -727,10 +837,10 @@ const useActions = ({ const drawData = propsDrawData || editorDrawData; if ( - !aiLabels.length && - [EBasicToolItem.Rectangle, EBasicToolItem.Skeleton].includes( - drawData.selectedTool, - ) + !aiLabels && + (drawData.selectedTool === EBasicToolItem.Skeleton || + (drawData.selectedTool === EBasicToolItem.Rectangle && + drawData.selectedModel === EnumModelType.Detection)) ) { message.warning(localeText('DDSAnnotator.smart.msg.labelRequired')); return; @@ -740,7 +850,7 @@ const useActions = ({ localeText('DDSAnnotator.smart.msg.loading'), 100000, ); - let imgSrc = `${list[current].urlFullRes}`; + let imgSrc = `${currImageItem?.url}`; try { setIsRequiring(true); @@ -756,20 +866,19 @@ const useActions = ({ const aiType = type || EBasicToolTypeMap[drawData.selectedTool]; switch (aiType) { case EObjectType.Rectangle: { - await requestAiDetection(imgSrc, aiLabels); + if (drawData.selectedModel === EnumModelType.Detection) { + await requestAiDetection(imgSrc, aiLabels || ''); + } else { + await requestIvpDetection(imgSrc, promptsQueue); + } break; } case EObjectType.Skeleton: { - await requestAiPoseEstimation(drawData, imgSrc, aiLabels); + await requestAiPoseEstimation(drawData, imgSrc, aiLabels || ''); break; } case EObjectType.Polygon: { - await requestAiSegmentByPolygon( - drawData, - imgSrc, - bbox, - segmentationClicks, - ); + await requestAiSegmentByPolygon(drawData, imgSrc, promptsQueue); break; } case EObjectType.Mask: { @@ -780,7 +889,7 @@ const useActions = ({ ) { await requestSegmentEverything(imgSrc, segmentEverythingParams); } else { - await requestAiSegmentByMask(drawData, imgSrc, maskPrompts); + await requestAiSegmentByMask(drawData, imgSrc, promptsQueue); } break; } @@ -801,31 +910,155 @@ const useActions = ({ [editorDrawData], ); - const onSaveAnnotations = async (drawData: DrawData) => { + const translateDrawData = useCallback( + (drawData: DrawData): [string, any[]] => { + let objectList = []; + if (framesData) { + objectList = convertFrameObjectsIntoFramesObjects( + drawData.objectList, + framesData.objects, + framesData.list.length, + framesData.activeIndex, + ).map((objs) => { + const availObjs: any = {}; + objs.forEach((obj, frameIndex) => { + if (obj && !obj.frameEmpty) { + // TODO: adapt for old format + const { labelId, attributes, labelValue } = + translateObject?.(obj); + availObjs.labelId = labelId; + availObjs.attributes = attributes; + if (!availObjs.labelValue) availObjs.labelValue = {}; + availObjs.labelValue[String(frameIndex)] = labelValue; + } + }); + return availObjs; + }); + } else { + objectList = drawData.objectList.map((obj) => translateObject?.(obj)); + } + return [ + framesData?.id || currImageItem?.id || '', + [ + ...drawData.classifications.map((item) => { + const label = categories.find((c) => c.id === item.labelId); + return { + ...item, + attributes: + item.attributes || label?.attributes?.map(() => null) || [], + }; + }), + ...objectList, + ], + ]; + }, + [currImageItem, translateObject, framesData], + ); + + const judgeLimitCommit = (labels: any[]) => { + const errorList: string[] = []; + // check classification + classificationOptions?.forEach((item, idx) => { + const value = labels.find((label) => label.labelId === item.id); + if (!value || [undefined, null, ''].includes(value.labelValue)) { + errorList.push( + localeText('DDSAnnotator.save.check.classification', { + idx: idx + 1, + }), + ); + } + }); + // check label + labels.forEach((item, idx) => { + const label = categories.find((label) => label.id === item.labelId); + if ( + label?.attributes?.find( + (attribute, index) => + attribute.required && + [undefined, null, ''].includes(item.attributes?.[index]), + ) + ) { + errorList.push( + localeText('DDSAnnotator.save.check.label', { + idx: idx + 1, + labelName: label.labelName, + }), + ); + } + }); + + if (errorList.length > 0) { + Modal.warning({ + width: 480, + title: localeText('DDSAnnotator.save.check.error'), + content: ( +
+ {errorList.map((item, index) => ( + + {item} +
+ + ))} + {localeText('DDSAnnotator.save.check.tip')} +
+ ), + }); + return true; + } + + return false; + }; + + const onSaveAnnotations = async () => { if (isRequiring || !onSave) return; - if (drawData.objectList.find((item) => !item.label)) { - message.warning( - 'There are annotations without a category. Please check.', - ); - return; + const [id, labels] = translateDrawData(editorDrawData); + console.log('>>> save', id, labels); + if (judgeLimitCommit(labels)) return; + + setIsRequiring(true); + try { + await onSave(id, labels); + } catch (error) { + console.error(error); } + setIsRequiring(false); + }; + + const onCommitAnnotations = async () => { + if (isRequiring || !onCommit) return; + + const [id, labels] = translateDrawData(editorDrawData); + if (judgeLimitCommit(labels)) return; setIsRequiring(true); try { - const annotations = translateObjectsToAnnotations( - drawData.objectList, - naturalSize, - clientSize, - ); - await onSave(list[current].id, annotations); + await onCommit(id, labels); } catch (error) { console.error(error); } setIsRequiring(false); }; - const onCancelAnnotations = () => { + const onRejectAnnotations = async () => { + if (mode === EditorMode.Review && onReviewReject) { + onReviewReject(...translateDrawData(editorDrawData)); + } + }; + + const onAcceptAnnotations = async () => { + if (mode === EditorMode.Review && onReviewAccept) { + onReviewAccept(...translateDrawData(editorDrawData)); + } + }; + + const onModifyAnnotations = async () => { + if (mode === EditorMode.Review && onReviewModify) { + onReviewModify(...translateDrawData(editorDrawData)); + } + }; + + const onCancelAnnotations = async () => { if (mode === EditorMode.Edit && hadChangeRecord) { modal.confirm({ getContainer: () => document.body, @@ -842,24 +1075,14 @@ const useActions = ({ if (onCancel) onCancel(); }; - const onReject = () => { - if (mode === EditorMode.Review && onReviewResult) { - onReviewResult(list[current]?.id || '', EQaAction.Reject); - } - }; - - const onAccept = () => { - if (mode === EditorMode.Review && onReviewResult) { - onReviewResult(list[current]?.id || '', EQaAction.Accept); - } - }; - return { onAiAnnotation, onSaveAnnotations, + onCommitAnnotations, onCancelAnnotations, - onReject, - onAccept, + onRejectAnnotations, + onAcceptAnnotations, + onModifyAnnotations, }; }; diff --git a/packages/components/src/Annotator/hooks/useAttributes.ts b/packages/components/src/Annotator/hooks/useAttributes.ts new file mode 100644 index 0000000..fa15f12 --- /dev/null +++ b/packages/components/src/Annotator/hooks/useAttributes.ts @@ -0,0 +1,72 @@ +import { useCallback } from 'react'; +import { + Category, + DrawData, + IAnnotationObject, + IAttributeValue, +} from '../type'; +import { Updater } from 'use-immer'; + +interface IProps { + setDrawDataWithHistory: Updater; + categories: Category[]; +} + +export default function useAttributes({ + setDrawDataWithHistory, + categories, +}: IProps) { + const judgeEditingAttribute = useCallback( + (object: IAnnotationObject, index: number) => { + const label = categories.find((item) => item.id === object.labelId); + if (label?.attributes && label.attributes.length > 0) { + return { + index, + labelId: object.labelId, + attributes: label.attributes, + values: object.attributes || [], + }; + } + return undefined; + }, + [categories], + ); + + const onConfirmAttibuteEdit = useCallback((values: IAttributeValue[]) => { + setDrawDataWithHistory((s) => { + if (s.editingAttribute) { + if (s.objectList[s.editingAttribute.index]) { + // object attributes + s.objectList[s.editingAttribute.index].attributes = values; + } else { + // classification attributes + const i = s.classifications.findIndex( + (item) => item.labelId === s.editingAttribute?.labelId, + ); + if (i > -1) { + s.classifications[i].attributes = values; + } else { + s.classifications.push({ + labelId: s.editingAttribute?.labelId, + labelValue: null, + attributes: values, + }); + } + } + s.editingAttribute = undefined; + } + }); + }, []); + + const onCancelAttibuteEdit = () => { + setDrawDataWithHistory((s) => { + s.editingAttribute = undefined; + }); + }; + + return { + judgeEditingAttribute, + onConfirmAttibuteEdit, + onCancelAttibuteEdit, + }; +} diff --git a/packages/components/src/Annotator/hooks/useCanvasContainer.tsx b/packages/components/src/Annotator/hooks/useCanvasContainer.tsx index 93b5a6f..92c7daa 100644 --- a/packages/components/src/Annotator/hooks/useCanvasContainer.tsx +++ b/packages/components/src/Annotator/hooks/useCanvasContainer.tsx @@ -13,8 +13,12 @@ import { MAX_SCALE, BUTTON_SCALE_STEP, WHEEL_SCALE_STEP, + ESubToolItem, + EObjectType, + EBasicToolItem, } from '../constants'; import { fixedFloatNum } from 'dds-utils/digit'; +import { DrawData } from '../type'; interface IProps { isRequiring: boolean; @@ -24,10 +28,10 @@ interface IProps { left: number; }; allowMove: boolean; - isCustomCursorActive: boolean; cursorSize: number; - showReferenceLine?: boolean; + drawData: DrawData; onClickMaskBg?: React.MouseEventHandler; + hideReferenceLine?: boolean; } export default function useCanvasContainer({ @@ -35,10 +39,10 @@ export default function useCanvasContainer({ visible, minPadding = { top: 0, left: 0 }, allowMove, - showReferenceLine, - isCustomCursorActive, + drawData, cursorSize, onClickMaskBg, + hideReferenceLine, }: IProps) { const containerRef = useRef(null); const containerSize = useSize(() => containerRef.current); @@ -87,8 +91,8 @@ export default function useCanvasContainer({ const [movingImgAnchor, setMovingImgAnchor] = useImmer(null); - const initClientSizeToFit = (naturalSize: ISize) => { - if (naturalSize && containerSize) { + const initClientSizeToFit = (naturalSize: ISize, containerSize: ISize) => { + if (naturalSize?.width && containerSize?.height) { const containerWidth = containerSize.width; const containerHeight = containerSize.height; const [width, height, scale] = zoomImgSize( @@ -112,8 +116,10 @@ export default function useCanvasContainer({ /** Initial position to fit container */ useEffect(() => { - initClientSizeToFit(naturalSize); - }, [naturalSize, containerSize]); + if (naturalSize && containerSize) { + initClientSizeToFit(naturalSize, containerSize); + } + }, [containerSize]); const adaptImagePosWhileZoom = () => { if (!containerSize) return; @@ -199,8 +205,15 @@ export default function useCanvasContainer({ const onReset = useCallback(() => { lastScalePosRef.current = undefined; - initClientSizeToFit(naturalSize); - }, [naturalSize.width, naturalSize.height]); + if (containerSize && naturalSize) { + initClientSizeToFit(naturalSize, containerSize); + } + }, [ + naturalSize.width, + naturalSize.height, + containerSize?.width, + containerSize?.height, + ]); // Reset data when hidden. useEffect(() => { @@ -219,8 +232,9 @@ export default function useCanvasContainer({ const [isMousePress, setMousePress] = useState(false); useEventListener('mousedown', () => { + if (!visible || !containerRef.current || !isInCanvas(containerMouse)) + return; setMousePress(true); - if (!visible || !containerRef.current) return; setMovingImgAnchor({ x: contentMouse.elementX, y: contentMouse.elementY, @@ -261,11 +275,16 @@ export default function useCanvasContainer({ } }, [allowMove]); - const onLoadImg = (e: React.UIEvent) => { + const onLoadImg = ( + e: React.UIEvent, + withoutInitClientSize?: boolean, + ) => { const img = e.target as HTMLImageElement; const naturalSize = { width: img.naturalWidth, height: img.naturalHeight }; setNaturalSize(naturalSize); - initClientSizeToFit(naturalSize); + if (containerSize && naturalSize && !withoutInitClientSize) { + initClientSizeToFit(naturalSize, containerSize); + } }; const onClickBg = (event: React.MouseEvent) => { @@ -274,6 +293,44 @@ export default function useCanvasContainer({ } }; + const isCustomCursorActive = useMemo(() => { + const isToolWithSize = [ + ESubToolItem.AutoEdgeStitching, + ESubToolItem.AutoSegmentByStroke, + ESubToolItem.BrushAdd, + ESubToolItem.BrushErase, + ].includes(drawData.selectedSubTool); + + if ( + drawData.creatingObject && + drawData.activeObjectIndex > -1 && + [EObjectType.Mask, EObjectType.Polygon].includes( + drawData.creatingObject.type, + ) + ) { + return isToolWithSize; + } + if ( + drawData.selectedTool !== EBasicToolItem.Drag && + !drawData.isBatchEditing + ) { + return ( + [EBasicToolItem.Mask, EBasicToolItem.Polygon].includes( + drawData.selectedTool, + ) && isToolWithSize + ); + } + return false; + }, [drawData.selectedTool, drawData.selectedSubTool]); + + const showReferenceLine = useMemo(() => { + return ( + drawData.selectedTool !== EBasicToolItem.Drag && + !isCustomCursorActive && + !hideReferenceLine + ); + }, [drawData.selectedTool, isCustomCursorActive, hideReferenceLine]); + /** Container render function */ const CanvasContainer = ({ children, @@ -296,57 +353,49 @@ export default function useCanvasContainer({ {/* leftLine */}
{/* rightLine */}
{/* upLine */}
{/* downLine */}
diff --git a/packages/components/src/Annotator/hooks/useCanvasRender.ts b/packages/components/src/Annotator/hooks/useCanvasRender.tsx similarity index 76% rename from packages/components/src/Annotator/hooks/useCanvasRender.ts rename to packages/components/src/Annotator/hooks/useCanvasRender.tsx index 1fa8d10..18fee88 100644 --- a/packages/components/src/Annotator/hooks/useCanvasRender.ts +++ b/packages/components/src/Annotator/hooks/useCanvasRender.tsx @@ -7,7 +7,12 @@ import { ICreatingObject, } from '../type'; import { translateAnnotCoord } from '../utils/compute'; -import { EObjectType } from '../constants'; +import { + EBasicToolItem, + EElementType, + EnumModelType, + EObjectType, +} from '../constants'; import { addFilter, clearCanvas, @@ -23,8 +28,9 @@ import { ANNO_STROKE_ALPHA, ANNO_STROKE_COLOR, } from '../constants/render'; -import { RenderStyles, ToolInstanceHookReturn } from '../tools/base'; +import { ToolInstanceHookReturn } from '../tools/base'; import { hexToRgba } from '../utils/color'; +import PopoverMenu from '../components/PopoverMenu'; interface IProps { visible: boolean; @@ -37,10 +43,6 @@ interface IProps { activeCanvasRef: React.RefObject; imgRef: React.RefObject; objectHooksMap: Record; - getCustomObjectStyles?: ( - object: IAnnotationObject, - color: string, - ) => Partial; } const useCanvasRender = ({ @@ -54,7 +56,6 @@ const useCanvasRender = ({ activeCanvasRef, imgRef, objectHooksMap, - getCustomObjectStyles, }: IProps) => { // ================================================================================================================= // Render @@ -84,7 +85,6 @@ const useCanvasRender = ({ fillColor = ANNO_FILL_COLOR.CREATING; } - const customStyles = getCustomObjectStyles?.(object, color) || {}; return { strokeColor, fillColor, @@ -92,7 +92,7 @@ const useCanvasRender = ({ strokeDash: [0], thickness: 2, pointAplha: 1, - ...customStyles, + ...(object.customStyles || {}), }; }; @@ -139,17 +139,32 @@ const useCanvasRender = ({ const { prompt } = theDrawData; if ( - prompt.maskPrompts || - prompt.creatingMask || + prompt.creatingPrompt || + prompt.promptsQueue || prompt.activeRectWhileLoading ) { - objectHooksMap[EObjectType.Mask].renderPrompt({ - prompt, - }); - } else if (prompt.segmentationClicks) { - objectHooksMap[EObjectType.Polygon].renderPrompt({ - prompt, - }); + if ( + theDrawData.selectedTool === EBasicToolItem.Mask || + theDrawData.creatingObject?.type === EObjectType.Mask + ) { + objectHooksMap[EObjectType.Mask].renderPrompt({ + prompt, + }); + } else if ( + theDrawData.selectedTool === EBasicToolItem.Polygon || + theDrawData.creatingObject?.type === EObjectType.Polygon + ) { + objectHooksMap[EObjectType.Polygon].renderPrompt({ + prompt, + }); + } else if ( + theDrawData.selectedTool === EBasicToolItem.Rectangle && + theDrawData.selectedModel === EnumModelType.IVP + ) { + objectHooksMap[EObjectType.Rectangle].renderPrompt({ + prompt, + }); + } } return; }; @@ -209,16 +224,23 @@ const useCanvasRender = ({ if ( obj.hidden || index === activeObjectIndex || - index === editState.focusObjectIndex + index === editState.focusObjectIndex || + obj.frameEmpty ) { return; } - renderObject(obj, false); + renderObject(obj, drawData.editingAttribute?.index === index); }); }; const updateRender = (updateDrawData?: DrawData) => { - if (!visible || !canvasRef.current || !imgRef.current) return; + if ( + !visible || + !canvasRef.current || + !imgRef.current || + !imgRef.current.complete + ) + return; resizeSmoothCanvas(canvasRef.current, { width: containerMouse.elementW, @@ -258,14 +280,41 @@ const useCanvasRender = ({ editState.focusObjectIndex > -1 && editState.focusObjectIndex !== drawData.activeObjectIndex && theDrawData.objectList[editState.focusObjectIndex] && - !theDrawData.objectList[editState.focusObjectIndex].hidden + !theDrawData.objectList[editState.focusObjectIndex].hidden && + !theDrawData.objectList[editState.focusObjectIndex].frameEmpty ) { renderObject(theDrawData.objectList[editState.focusObjectIndex], true); } }; + const renderPopoverMenu = () => { + if ( + editState.focusObjectIndex > -1 && + drawData.objectList[editState.focusObjectIndex] && + !drawData.objectList[editState.focusObjectIndex].hidden && + editState.focusEleIndex > -1 && + editState.focusEleType === EElementType.Circle + ) { + const target = + drawData.objectList[editState.focusObjectIndex].keypoints?.points?.[ + editState.focusEleIndex + ]; + if (target) { + return ( + + ); + } + } + return <>; + }; + return { updateRender, + renderPopoverMenu, }; }; diff --git a/packages/components/src/Annotator/hooks/useColor.ts b/packages/components/src/Annotator/hooks/useColor.ts index 8cd65b0..a6cdcc0 100644 --- a/packages/components/src/Annotator/hooks/useColor.ts +++ b/packages/components/src/Annotator/hooks/useColor.ts @@ -5,11 +5,16 @@ import { Category, EditState } from '../type'; interface IProps { categories: Category[]; editState: EditState; + forceColorByObject?: boolean; } -export default function useColor({ categories, editState }: IProps) { +export default function useColor({ + categories, + editState, + forceColorByObject, +}: IProps) { const labelColors = useMemo(() => { - return getCategoryColors(categories.map((item) => item.name)); + return getCategoryColors(categories.map((item) => item.id)); }, [categories]); const colorSeedRef = useRef(0); @@ -31,12 +36,13 @@ export default function useColor({ categories, editState }: IProps) { }, [editState.annotsDisplayOptions.colorByCategory]); const getAnnotColor = useCallback( - (category: string, forceColorByCategory?: boolean) => { + (categoryId: string, forceColorByCategory?: boolean) => { if ( - editState.annotsDisplayOptions.colorByCategory || - forceColorByCategory + !forceColorByObject && + (editState.annotsDisplayOptions.colorByCategory || forceColorByCategory) ) { - return labelColors[category] || '#fff'; + const catagory = categories.find((item) => item.id === categoryId); + return catagory?.renderColor || labelColors[categoryId] || '#fff'; } else { return getUniformHexColor(colorSeedRef.current); } @@ -46,6 +52,7 @@ export default function useColor({ categories, editState }: IProps) { labelColors, getUniformHexColor, colorSeedRef.current, + forceColorByObject, ], ); diff --git a/packages/components/src/Annotator/hooks/useDataEffect.ts b/packages/components/src/Annotator/hooks/useDataEffect.ts index 3ae8367..389ed8f 100644 --- a/packages/components/src/Annotator/hooks/useDataEffect.ts +++ b/packages/components/src/Annotator/hooks/useDataEffect.ts @@ -2,62 +2,68 @@ import { useCallback, useEffect } from 'react'; import { cloneDeep } from 'lodash'; import { BaseObject, + Category, DEFAULT_DRAW_DATA, DEFAULT_EDIT_STATE, DrawData, - DrawImageData, + AnnoItem, DrawObject, EditState, + VideoFramesData, } from '../type'; -import { scaleDrawData } from '../utils/compute'; +import { scaleDrawData, scaleFramesObjects } from '../utils/compute'; import { Updater } from 'use-immer'; +import usePreviousState from './usePreviousState'; interface IProps { imagePos: React.MutableRefObject; clientSize: ISize; - preClientSize?: ISize; - clearPreClientSize: () => void; naturalSize: ISize; annotations: DrawObject[]; setAnnotations: Updater; - labelColors: Record; drawData: DrawData; setDrawData: Updater; + setFramesData?: Updater; editState: EditState; setEditState: Updater; - initObjectList: ( - annotations: DrawObject[], - labelColors: Record, - ) => void; + initObjectList: (annotations: DrawObject[]) => void; updateRender: (updateDrawData?: DrawData) => void; clearHistory: () => void; objectsFilter?: (imageData: any) => BaseObject[]; + labelOptions: Category[]; + customDefaultDrawData?: Partial; } const useDataEffect = ({ imagePos, clientSize, - preClientSize, - clearPreClientSize, naturalSize, annotations, setAnnotations, - labelColors, drawData, setDrawData, + setFramesData, editState, setEditState, initObjectList, updateRender, clearHistory, objectsFilter, + labelOptions, + customDefaultDrawData, }: IProps) => { + const [preClientSize, clearPreClientSize] = + usePreviousState(clientSize); + /** * Rebuilds the draw data for the annotation tool. * @param {boolean} isUpdateDrawData - Optional parameter that specifies whether to update draw data. * @return {void} */ - const rebuildDrawData = (isForce?: boolean) => { + const rebuildDrawData = ( + isForce?: boolean, + theAnnotations?: DrawObject[], + ) => { if ( !clientSize.width || !clientSize.height || @@ -65,15 +71,15 @@ const useDataEffect = ({ !naturalSize.height ) return; - if (!drawData.initialized || isForce) { - // Initialization - setDrawData((s) => { - s.initialized = true; - }); - initObjectList(annotations, labelColors); + initObjectList(theAnnotations || annotations); } else if (drawData.initialized && preClientSize) { // scale change + if (setFramesData) { + setFramesData?.((s) => { + s.objects = scaleFramesObjects(s.objects, preClientSize, clientSize); + }); + } const updateDrawData = scaleDrawData(drawData, preClientSize, clientSize); setDrawData(updateDrawData); updateRender(updateDrawData); @@ -87,10 +93,13 @@ const useDataEffect = ({ brushSize: drawData.brushSize, selectedTool: drawData.selectedTool, selectedSubTool: drawData.selectedSubTool, + selectedModel: drawData.selectedModel, AIAnnotation: drawData.AIAnnotation, + ...customDefaultDrawData, }); }, [ DEFAULT_DRAW_DATA, + customDefaultDrawData, drawData.brushSize, drawData.selectedSubTool, drawData.selectedTool, @@ -100,31 +109,37 @@ const useDataEffect = ({ const resetEditData = useCallback(() => { setEditState({ ...cloneDeep(DEFAULT_EDIT_STATE), + latestLabelId: labelOptions?.[0]?.id || '', imageDisplayOptions: editState.imageDisplayOptions, annotsDisplayOptions: editState.annotsDisplayOptions, }); }, [ DEFAULT_EDIT_STATE, + labelOptions, editState.imageDisplayOptions, editState.annotsDisplayOptions, ]); const applyImageAnnots = useCallback( - (imageData: DrawImageData) => { + (imageData: AnnoItem) => { const annotations = imageData?.objects ? [...imageData?.objects] : []; const currAnnotations = - imageData && objectsFilter ? objectsFilter(imageData) : annotations; + imageData && objectsFilter + ? objectsFilter(imageData) || [] + : annotations; setAnnotations(currAnnotations); + rebuildDrawData(true, currAnnotations); }, - [objectsFilter], + [objectsFilter, rebuildDrawData], ); const resetDataWithImageData = useCallback( ( - imageData: DrawImageData, + imageData: AnnoItem, visible: boolean, clearHistoryQueue: boolean = true, ) => { + setAnnotations([]); resetDrawData(); resetEditData(); if (clearHistoryQueue) clearHistory(); @@ -148,7 +163,19 @@ const useDataEffect = ({ /** Annotations / naturalSize changed */ useEffect(() => { rebuildDrawData(true); - }, [annotations, naturalSize.width, naturalSize.height]); + }, [naturalSize.width, naturalSize.height]); + + useEffect(() => { + if (!labelOptions?.length) return; + setEditState((s) => { + if ( + !s.latestLabelId || + !labelOptions.find((item) => item.id === s.latestLabelId) + ) { + s.latestLabelId = labelOptions[0]?.id; + } + }); + }, [labelOptions]); return { rebuildDrawData, diff --git a/packages/components/src/Annotator/hooks/useHistory.ts b/packages/components/src/Annotator/hooks/useHistory.ts index 2c6cb5e..6d7c85f 100644 --- a/packages/components/src/Annotator/hooks/useHistory.ts +++ b/packages/components/src/Annotator/hooks/useHistory.ts @@ -1,19 +1,23 @@ import { useCallback, useState } from 'react'; import { DraftFunction, Updater, useImmer } from 'use-immer'; import { cloneDeep, isEqual } from 'lodash'; -import { scaleDrawData, translateObjectsToAnnotations } from '../utils/compute'; -import { BaseObject, DrawData } from '../type'; +import { scaleDrawData, scaleFramesObjects } from '../utils/compute'; +import { BaseObject, DrawData, VideoFramesData } from '../type'; export interface HistoryItem { drawData: DrawData; + framesData?: VideoFramesData; clientSize: ISize; } interface IProps { clientSize: ISize; naturalSize: ISize; + framesData?: VideoFramesData; + setFramesData?: Updater; setDrawData: Updater; onAutoSave?: (annotations: BaseObject[], naturalSize: ISize) => void; + translateObject?: (object: any) => any; } const useHistory = ({ @@ -21,28 +25,35 @@ const useHistory = ({ naturalSize, onAutoSave, setDrawData, + translateObject, + framesData, + setFramesData, }: IProps) => { const [historyQueue, setHistoryQueue] = useImmer([]); const [currentIndex, setCurrIndex] = useState(0); const maxCacheSize = 20; const autoSave = (item: HistoryItem) => { - const annotations = translateObjectsToAnnotations( - item.drawData.objectList, - naturalSize, - item.clientSize, - true, - ); - if (onAutoSave) onAutoSave(annotations, naturalSize); + if (onAutoSave) { + const annotations = item.drawData.objectList.map( + (obj) => translateObject?.(obj) || {}, + ); + onAutoSave(annotations, naturalSize); + } }; - /** - * Undo the last action - */ - const undo = useCallback(() => { - if (currentIndex > 0) { - setCurrIndex((prevIndex) => prevIndex - 1); - const record = historyQueue[currentIndex - 1]; + const updateCurrentRecord = useCallback( + (record: HistoryItem) => { + if (record.framesData) { + setFramesData?.({ + ...record.framesData, + objects: scaleFramesObjects( + record.framesData.objects, + record.clientSize, + clientSize, + ), + }); + } const updateDrawData = scaleDrawData( record.drawData, record.clientSize, @@ -50,8 +61,19 @@ const useHistory = ({ ); setDrawData(updateDrawData); autoSave(record); + }, + [clientSize.width, clientSize.height], + ); + + /** + * Undo the last action + */ + const undo = useCallback(() => { + if (currentIndex > 0) { + setCurrIndex((prevIndex) => prevIndex - 1); + updateCurrentRecord(historyQueue[currentIndex - 1]); } - }, [currentIndex, historyQueue, clientSize.width, clientSize.height]); + }, [currentIndex, historyQueue, updateCurrentRecord]); /** * Redo the last undone action @@ -59,21 +81,22 @@ const useHistory = ({ const redo = useCallback(() => { if (currentIndex < historyQueue.length - 1) { setCurrIndex((prevIndex) => prevIndex + 1); - const record = historyQueue[currentIndex + 1]; - const updateDrawData = scaleDrawData( - record.drawData, - record.clientSize, - clientSize, - ); - setDrawData(updateDrawData); - autoSave(record); + updateCurrentRecord(historyQueue[currentIndex + 1]); } - }, [currentIndex, historyQueue, clientSize.width, clientSize.height]); + }, [currentIndex, historyQueue, updateCurrentRecord]); /** * Update the history queue with the new objects */ - const updateHistory = (item: HistoryItem) => { + const updateHistory = ( + drawData: DrawData, + theframesData?: VideoFramesData, + ) => { + const item = { + drawData, + clientSize, + framesData: theframesData || framesData, + }; setHistoryQueue((queue) => { if (queue[currentIndex] && isEqual(item, queue[currentIndex])) { return queue; @@ -85,6 +108,7 @@ const useHistory = ({ // fix to change image current render return queue; } + // console.log('>>> updata history', item.drawData, framesData); queue.splice(currentIndex + 1); queue.push(item); if (queue.length > maxCacheSize) { @@ -105,21 +129,11 @@ const useHistory = ({ if (typeof updater === 'function') { setDrawData((s) => { updater(s); - updateHistory( - cloneDeep({ - drawData: s, - clientSize, - }), - ); + updateHistory(cloneDeep(s)); }); } else { setDrawData(updater); - updateHistory( - cloneDeep({ - drawData: updater, - clientSize, - }), - ); + updateHistory(cloneDeep(updater)); } }; diff --git a/packages/components/src/Annotator/hooks/useLabels.ts b/packages/components/src/Annotator/hooks/useLabels.ts index 1f46655..a44a89f 100644 --- a/packages/components/src/Annotator/hooks/useLabels.ts +++ b/packages/components/src/Annotator/hooks/useLabels.ts @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useState } from 'react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; import { Updater } from 'use-immer'; import { Category, @@ -7,11 +7,18 @@ import { EditorMode, IAnnotationObject, } from '../type'; -import { EElementType, KEYPOINTS_VISIBLE_TYPE } from '../constants'; +import { + EBasicToolItem, + EBasicToolTypeMap, + EElementType, + ELabelType, + KEYPOINTS_VISIBLE_TYPE, + LABEL_TOOL_MAP, +} from '../constants'; import { cloneDeep } from 'lodash'; interface IProps { - visible: boolean; + isOldMode?: boolean; mode: EditorMode; categories: Category[]; setCategories?: Updater; @@ -26,7 +33,7 @@ interface IProps { } export default function useLabels({ - visible, + isOldMode, categories, setCategories, drawData, @@ -35,7 +42,45 @@ export default function useLabels({ updateObjectWithoutHistory, updateAllObjectWithoutHistory, }: IProps) { - const [aiLabels, setAiLabels] = useState([]); + const [aiLabels, setAiLabels] = useState(undefined); + const curObjects = drawData.objectList; + + const labelOptions: Category[] = useMemo(() => { + if (isOldMode) return categories; + + if ( + drawData.objectList[drawData.activeObjectIndex] || + drawData.selectedTool !== EBasicToolItem.Drag + ) { + const toolType = drawData.objectList[drawData.activeObjectIndex] + ? Object.keys(EBasicToolTypeMap).find( + (key) => + drawData.objectList[drawData.activeObjectIndex].type === + EBasicToolTypeMap[key as unknown as EBasicToolItem], + ) + : drawData.selectedTool; + const labelType = Object.keys(LABEL_TOOL_MAP).find( + // @ts-ignore + (key) => toolType === LABEL_TOOL_MAP[key], + ); + return categories.filter((category) => category.labelType === labelType); + } + + return []; + }, [ + categories, + drawData.objectList, + drawData.activeObjectIndex, + drawData.selectedTool, + ]); + + const classificationOptions: Category[] = useMemo(() => { + return ( + categories?.filter( + (category) => category.labelType === ELabelType.Classification, + ) || [] + ); + }, [categories]); const onCreateCategory = useCallback( (name: string) => { @@ -52,20 +97,6 @@ export default function useLabels({ [categories], ); - useEffect(() => { - const allLabels = categories.map((item) => item.name); - const commonLabels = aiLabels.filter((item) => allLabels.includes(item)); - setAiLabels(commonLabels); - }, [categories]); - - useEffect(() => { - if (!visible) { - setAiLabels([]); - } - }, [visible]); - - const curObjects = drawData.objectList; - const onChangeObjectHidden = useCallback( (index: number, hidden: boolean) => { const newObject = { ...drawData.objectList[index] }; @@ -76,10 +107,14 @@ export default function useLabels({ ); const onChangeCategoryHidden = useCallback( - (category: string, hidden: boolean) => { + (categoryName: string, hidden: boolean) => { const updatedObjects = drawData.objectList.map((item) => { const temp = { ...item }; - if (temp.label === category) temp.hidden = hidden; + if ( + categories.find((c) => c.id === item.labelId)?.name === categoryName + ) { + temp.hidden = hidden; + } return temp; }); updateAllObjectWithoutHistory(updatedObjects); @@ -113,16 +148,19 @@ export default function useLabels({ * * @param {KEYPOINTS_VISIBLE_TYPE} visible - The visibility value for the keypoint. */ - const onChangePointVisible = useCallback((pointIndex: number, visible: KEYPOINTS_VISIBLE_TYPE) => { - const newObject = cloneDeep( - drawData.objectList[drawData.activeObjectIndex], - ); - const point = newObject.keypoints?.points?.[pointIndex]; - if (point) { - point.visible = visible; - } - updateObjectWithoutHistory(newObject, drawData.activeObjectIndex); - }, [drawData.activeObjectIndex, drawData.objectList]); + const onChangePointVisible = useCallback( + (pointIndex: number, visible: KEYPOINTS_VISIBLE_TYPE) => { + const newObject = cloneDeep( + drawData.objectList[drawData.activeObjectIndex], + ); + const point = newObject.keypoints?.points?.[pointIndex]; + if (point) { + point.visible = visible; + } + updateObjectWithoutHistory(newObject, drawData.activeObjectIndex); + }, + [drawData.activeObjectIndex, drawData.objectList], + ); const onChangeActiveClass = useCallback((name: string) => { setDrawData((s) => { @@ -133,14 +171,19 @@ export default function useLabels({ useEffect(() => { if (drawData.activeObjectIndex < 0) return; - const activeItemLabel = - drawData.objectList[drawData.activeObjectIndex].label; - if (activeItemLabel !== drawData.activeClassName) { - onChangeActiveClass(activeItemLabel); + const activeItemLabelName = + categories.find( + (item) => + item.id === drawData.objectList[drawData.activeObjectIndex].labelId, + )?.name || ''; + if (activeItemLabelName !== drawData.activeClassName) { + onChangeActiveClass(activeItemLabelName); } }, [drawData.activeObjectIndex]); return { + labelOptions, + classificationOptions, aiLabels, setAiLabels, curObjects, diff --git a/packages/components/src/Annotator/hooks/useMouseEvents.tsx b/packages/components/src/Annotator/hooks/useMouseEvents.tsx index 996b108..458301e 100644 --- a/packages/components/src/Annotator/hooks/useMouseEvents.tsx +++ b/packages/components/src/Annotator/hooks/useMouseEvents.tsx @@ -19,6 +19,7 @@ import { EBasicToolItem, EBasicToolTypeMap, EElementType, + EnumModelType, EObjectType, } from '../constants'; import { Updater } from 'use-immer'; @@ -271,13 +272,17 @@ const useMouseEvents = ({ }); } else { s.activeObjectIndex = index; - s.creatingObject = { - ...drawData.objectList[index], - currIndex: undefined, - startPoint: undefined, - tempMaskSteps: [], - maskStep: undefined, - }; + if (!drawData.objectList[index].frameEmpty) { + s.creatingObject = { + ...drawData.objectList[index], + currIndex: undefined, + startPoint: undefined, + tempMaskSteps: [], + maskStep: undefined, + }; + } else { + s.creatingObject = undefined; + } if ( s.selectedTool !== EBasicToolItem.Drag && @@ -287,6 +292,9 @@ const useMouseEvents = ({ s.selectedTool = EBasicToolItem.Drag; } } + if (s.editingAttribute?.index !== index) { + s.editingAttribute = undefined; + } }); }, [clientSize.width, clientSize.height, contentMouse, drawData.objectList], @@ -333,7 +341,9 @@ const useMouseEvents = ({ backgroundColor: drawData.objectList[index]?.color || '#fff', }} /> - {drawData.objectList[index]?.label} + {categories.find( + (c) => c.id === drawData.objectList[index]?.labelId, + )?.name || ''} {drawData.isBatchEditing && ` (${fixedFloatNum(drawData.objectList[index]?.conf || 0)})`}
@@ -347,7 +357,8 @@ const useMouseEvents = ({ !visible || editState.allowMove || editState.isRequiring || - !isInCanvas(contentMouse) + !isInCanvas(contentMouse) || + !isInCanvas(containerMouse) ) return; @@ -371,8 +382,11 @@ const useMouseEvents = ({ // 2. Create object if ( drawData.selectedTool !== EBasicToolItem.Drag && - !drawData.isBatchEditing + (!drawData.isBatchEditing || drawData.selectedModel === EnumModelType.IVP) ) { + setDrawData((s) => { + s.editingAttribute = undefined; + }); const objectType = EBasicToolTypeMap[drawData.selectedTool]; if ( mode === EditorMode.Edit && @@ -385,9 +399,9 @@ const useMouseEvents = ({ }, basic: { hidden: false, - label: editState.latestLabel || categories[0].name, + labelId: editState.latestLabelId || categories[0].id, status: EObjectStatus.Commited, - color: getAnnotColor(editState.latestLabel || categories[0].name), + color: getAnnotColor(editState.latestLabelId || categories[0].name), }, }) ) { diff --git a/packages/components/src/Annotator/hooks/useObjects.ts b/packages/components/src/Annotator/hooks/useObjects.ts index 365c448..17af253 100644 --- a/packages/components/src/Annotator/hooks/useObjects.ts +++ b/packages/components/src/Annotator/hooks/useObjects.ts @@ -1,10 +1,4 @@ -import { AnnotationType, EElementType, EObjectType } from '../constants'; -import { - getObjectType, - translateBoundingBoxToRect, - translatePointsToPointObjs, - getSegmentationPoints, -} from '../utils/compute'; +import { EElementType, EObjectType } from '../constants'; import { Updater } from 'use-immer'; import { BaseObject, @@ -12,12 +6,13 @@ import { EditState, EditorMode, IAnnotationObject, - EObjectStatus, DrawObject, + IEditingAttribute, + EObjectStatus, + VideoFramesData, } from '../type'; -import { rleToCanvas } from '../tools/useMask'; -import { useCallback } from 'react'; -import { generateUniformHexColor } from '../utils/color'; +import { useCallback, useMemo } from 'react'; +import { cloneDeep } from 'lodash'; interface IProps { mode: EditorMode; @@ -26,11 +21,16 @@ interface IProps { drawData: DrawData; setDrawData: Updater; setDrawDataWithHistory: Updater; - editState: EditState; + framesData?: VideoFramesData; + setFramesData?: Updater; setEditState: Updater; - clientSize: ISize; - naturalSize: ISize; - displayAnnotationType?: AnnotationType; + translateToObject?: (annotation: any, videoFrameCount?: number) => any; + judgeEditingAttribute?: ( + object: IAnnotationObject, + index: number, + ) => IEditingAttribute | undefined; + limitActiveObjectAfterCreate?: boolean; + updateHistory: (drawData: DrawData, theframesData?: VideoFramesData) => void; } const useObjects = ({ @@ -38,114 +38,51 @@ const useObjects = ({ drawData, setDrawData, setDrawDataWithHistory, + framesData, + setFramesData, setEditState, - clientSize, - naturalSize, - editState, - displayAnnotationType, + translateToObject, + judgeEditingAttribute, + limitActiveObjectAfterCreate, + updateHistory, }: IProps) => { - const translateAnnotationToObject = ( - annotation: DrawObject, - labelColors: Record, - ): IAnnotationObject => { - let { - categoryName, - boundingBox, - points, - lines, - pointNames, - pointColors, - segmentation, - mask, - alpha, - } = annotation; - - const color = editState.annotsDisplayOptions.colorByCategory - ? labelColors[categoryName || ''] || '#ffffff' - : generateUniformHexColor(); - - const newObj: IAnnotationObject = { - label: categoryName || '', - type: EObjectType.Rectangle, - hidden: false, - conf: annotation.conf || 1, - labelId: annotation.labelId, - compareResult: annotation.compareResult, - status: EObjectStatus.Commited, - color, - }; - - if (boundingBox) { - const rect = translateBoundingBoxToRect(boundingBox, clientSize); - Object.assign(newObj, { rect: { visible: true, ...rect } }); - } - - if ( - points && - points.length > 0 && - lines && - lines.length > 0 && - pointNames && - pointColors - ) { - const pointObjs: IElement[] = translatePointsToPointObjs( - points, - pointNames, - pointColors, - naturalSize, - clientSize, - ); - Object.assign(newObj, { - keypoints: { - points: pointObjs, - lines, - }, - }); - } - if (segmentation) { - const group = getSegmentationPoints( - segmentation, - naturalSize, - clientSize, - ); - const polygon: IElement = { - group, - visible: true, - }; - Object.assign(newObj, { polygon }); - } - - if (mask && mask.length) { - Object.assign(newObj, { - maskRle: mask, - maskCanvasElement: rleToCanvas(mask, naturalSize, color), - }); - } - - if (alpha) { - const alphaImageElement = new Image(); - alphaImageElement.src = alpha; - // alphaImageElement.crossOrigin = 'anonymous'; - Object.assign(newObj, { - alpha, - alphaImageElement, - }); - } - - newObj.type = getObjectType(newObj, displayAnnotationType); - return newObj; - }; - - const initObjectList = ( - annotations: DrawObject[], - labelColors: Record, - ) => { - setDrawDataWithHistory((s) => { - s.objectList = annotations - .map((annotation) => { - return translateAnnotationToObject(annotation, labelColors); - }) - .filter((annotation) => annotation.type !== EObjectType.Custom); + const initObjectList = (annotations: DrawObject[]) => { + setDrawData((s) => { + const newDrawData = cloneDeep(s); + const newFramesData = cloneDeep(framesData); + newDrawData.initialized = true; + if (newFramesData) { + // video + const objects = annotations.map( + (annotation) => + translateToObject?.(annotation, newFramesData.list.length) || {}, + ); + newFramesData.objects = objects + .filter((item) => !!item.objects) + .map((item) => item.objects); + newDrawData.classifications = objects + .filter((item) => !!item.classification) + .map((item) => item.classification); + newDrawData.objectList = newFramesData.objects.map( + (item) => item[newFramesData.activeIndex], + ); + setFramesData?.(newFramesData); + } else { + // image + const objects = annotations.map( + (annotation) => translateToObject?.(annotation) || {}, + ); + newDrawData.classifications = objects.filter( + (item) => item.type === EObjectType.Classification, + ); + newDrawData.objectList = objects.filter( + (item) => + item.type !== EObjectType.Custom && + item.type !== EObjectType.Classification, + ); + } + updateHistory(cloneDeep(newDrawData), cloneDeep(newFramesData)); + return newDrawData; }); }; @@ -153,47 +90,82 @@ const useObjects = ({ if (mode !== EditorMode.Edit) return; setDrawDataWithHistory((s) => { s.objectList.push(object); - s.creatingObject = { ...object }; - s.activeObjectIndex = notActive ? -1 : s.objectList.length - 1; + + if (limitActiveObjectAfterCreate) { + s.creatingObject = undefined; + s.activeObjectIndex = -1; + } else { + s.creatingObject = { ...object }; + s.activeObjectIndex = notActive ? -1 : s.objectList.length - 1; + + // Show attribut editor + if (judgeEditingAttribute) { + s.editingAttribute = judgeEditingAttribute( + object, + s.objectList.length - 1, + ); + } + } }); }; - const removeObject = useCallback( - (index: number) => { - if (mode !== EditorMode.Edit || !drawData.objectList[index]) return; - setDrawDataWithHistory((s) => { - if (s.objectList[index]) { - s.objectList.splice(index, 1); - s.activeObjectIndex = -1; - s.creatingObject = undefined; - } - }); - setEditState((s) => { - s.focusObjectIndex = -1; - s.focusEleIndex = -1; - s.focusEleType = EElementType.Rect; - }); - }, - [mode, drawData.objectList], - ); + const removeObject = (index: number) => { + if (mode !== EditorMode.Edit || !drawData.objectList[index]) return; + setEditState((s) => { + s.focusObjectIndex = -1; + s.focusEleIndex = -1; + s.focusEleType = EElementType.Rect; + }); + + const newFramesData = cloneDeep(framesData); + const newDrawData = cloneDeep(drawData); + if (newFramesData && newFramesData.objects[index]) { + newFramesData.objects.splice(index, 1); + setFramesData?.(newFramesData); + } + if (newDrawData.objectList[index]) { + newDrawData.objectList.splice(index, 1); + newDrawData.activeObjectIndex = -1; + newDrawData.creatingObject = undefined; + newDrawData.editingAttribute = undefined; + } + setDrawData(newDrawData); + updateHistory(cloneDeep(newDrawData), cloneDeep(newFramesData)); + }; const removeAllObjects = useCallback(() => { if (mode !== EditorMode.Edit) return; - setDrawDataWithHistory((s) => { - s.objectList = []; - s.creatingObject = undefined; - s.prompt = {}; - }); setEditState((s) => { s.focusObjectIndex = -1; s.focusEleIndex = -1; s.focusEleType = EElementType.Rect; }); + + const newFramesData = cloneDeep(framesData); + const newDrawData = cloneDeep(drawData); + if (newFramesData) { + newFramesData.objects = []; + setFramesData?.(newFramesData); + } + newDrawData.objectList = []; + newDrawData.activeObjectIndex = -1; + newDrawData.creatingObject = undefined; + newDrawData.editingAttribute = undefined; + setDrawData(newDrawData); + updateHistory(cloneDeep(newDrawData), cloneDeep(newFramesData)); }, [mode]); const updateObject = (object: IAnnotationObject, index: number) => { if (mode !== EditorMode.Edit || !drawData.objectList[index]) return; setDrawDataWithHistory((s) => { + // Change label & Show attribut editor + if ( + object.labelId !== s.objectList[index].labelId && + judgeEditingAttribute + ) { + s.editingAttribute = judgeEditingAttribute(object, index); + } + s.objectList[index] = object; if (s.creatingObject && s.activeObjectIndex === index) { s.creatingObject = { ...object }; @@ -232,6 +204,22 @@ const useObjects = ({ }); }; + const commitedObjects = useMemo(() => { + return drawData.objectList.filter((obj) => { + return obj.status === EObjectStatus.Commited; + }); + }, [drawData.isBatchEditing, drawData.objectList]); + + const currObject = useMemo(() => { + return ( + drawData.objectList[drawData.activeObjectIndex] || drawData.creatingObject + ); + }, [ + drawData.objectList, + drawData.activeObjectIndex, + drawData.creatingObject, + ]); + return { initObjectList, addObject, @@ -241,6 +229,8 @@ const useObjects = ({ updateAllObject, updateObjectWithoutHistory, updateAllObjectWithoutHistory, + commitedObjects, + currObject, }; }; diff --git a/packages/components/src/Annotator/hooks/useShortcuts.ts b/packages/components/src/Annotator/hooks/useShortcuts.ts index 097834f..b40ce9a 100644 --- a/packages/components/src/Annotator/hooks/useShortcuts.ts +++ b/packages/components/src/Annotator/hooks/useShortcuts.ts @@ -2,18 +2,25 @@ import { Updater } from 'use-immer'; import { useKeyPress } from 'ahooks'; import { EObjectType } from '../constants'; import { EDITOR_SHORTCUTS, EShortcuts } from '../constants/shortcuts'; -import { DrawData, EditState, EditorMode, IAnnotationObject } from '../type'; +import { + Category, + DrawData, + EditState, + EditorMode, + IAnnotationObject, +} from '../type'; interface IProps { visible: boolean; mode: EditorMode; drawData: DrawData; + categories: Category[]; isMousePress: boolean; setDrawData: Updater; setEditState: Updater; - onSaveAnnotations: (drawData: DrawData) => Promise; - onAccept: () => void; - onReject: () => void; + onSaveAnnotations?: () => void; + onAcceptAnnotations?: () => void; + onRejectAnnotations?: () => void; onChangeObjectHidden: (index: number, hidden: boolean) => void; onChangeCategoryHidden: (category: string, hidden: boolean) => void; removeObject: (index: number) => void; @@ -24,12 +31,13 @@ const useShortcuts = ({ visible, mode, drawData, + categories, isMousePress, setDrawData, setEditState, onSaveAnnotations, - onAccept, - onReject, + onAcceptAnnotations, + onRejectAnnotations, onChangeObjectHidden, onChangeCategoryHidden, removeObject, @@ -41,7 +49,7 @@ const useShortcuts = ({ (event: KeyboardEvent) => { event.preventDefault(); if (mode === EditorMode.Edit) { - onSaveAnnotations(drawData); + onSaveAnnotations?.(); } }, { @@ -54,7 +62,7 @@ const useShortcuts = ({ EDITOR_SHORTCUTS[EShortcuts.Accept].shortcut, (event: KeyboardEvent) => { event.preventDefault(); - onAccept(); + onAcceptAnnotations?.(); }, { exactMatch: true, @@ -66,7 +74,7 @@ const useShortcuts = ({ EDITOR_SHORTCUTS[EShortcuts.Reject].shortcut, (event: KeyboardEvent) => { event.preventDefault(); - onReject(); + onRejectAnnotations?.(); }, { exactMatch: true, @@ -149,8 +157,10 @@ const useShortcuts = ({ (event) => { if (drawData.activeObjectIndex < 0) return; event.preventDefault(); - const { label, hidden } = drawData.objectList[drawData.activeObjectIndex]; - onChangeCategoryHidden(label, !hidden); + const { labelId, hidden } = + drawData.objectList[drawData.activeObjectIndex]; + const labelName = categories.find((c) => c.id === labelId)?.name || ''; + onChangeCategoryHidden(labelName, !hidden); }, { exactMatch: true, @@ -209,14 +219,14 @@ const useShortcuts = ({ drawData.creatingObject && drawData.creatingObject.type === EObjectType.Polygon ) { - const { polygon, type, hidden, label, status, color } = + const { polygon, type, hidden, labelId, status, color } = drawData.creatingObject!; if (polygon && polygon.group && polygon.group[0].length > 2) { const newObject: IAnnotationObject = { polygon, type, hidden, - label, + labelId, status, color, }; diff --git a/packages/components/src/Annotator/hooks/useSubtools.tsx b/packages/components/src/Annotator/hooks/useSubtools.tsx new file mode 100644 index 0000000..cb5a92b --- /dev/null +++ b/packages/components/src/Annotator/hooks/useSubtools.tsx @@ -0,0 +1,281 @@ +import { TShortcutItem } from '../constants/shortcuts'; +import { useLocale } from 'dds-utils/locale'; +import { + EBasicToolItem, + EnumModelType, + EObjectType, + ESubToolItem, +} from '../constants'; +import Icon from '@ant-design/icons'; +import { ReactComponent as PenAddIcon } from '../assets/pen-add.svg'; +import { ReactComponent as PenEraseIcon } from '../assets/pen-erase.svg'; +import { ReactComponent as BrushAddIcon } from '../assets/brush-add.svg'; +import { ReactComponent as BrushEraseIcon } from '../assets/brush-erase.svg'; +import { ReactComponent as MagicBoxIcon } from '../assets/magic-box.svg'; +import { ReactComponent as ClickIcon } from '../assets/magic-click.svg'; +import { ReactComponent as EdgeStitchIcon } from '../assets/edge-stitch.svg'; +import { ReactComponent as SegmentEverythingIcon } from '../assets/segment-everything.svg'; +import { ReactComponent as StrokeIcon } from '../assets/magic-brush.svg'; +import { ReactComponent as AddPromptIcon } from '../assets/add-prompt.svg'; +import { ReactComponent as RemovePromptIcon } from '../assets/remove-prompt.svg'; +import { useMemo } from 'react'; +import { DrawData } from '../type'; +import { Slider } from 'antd'; + +export type TToolItem = { + key: T; + name: string; + shortcut?: TShortcutItem; + icon: JSX.Element; + available: boolean; + description?: string; + withSize?: boolean; + withCustomElement?: boolean; +}; + +export type TSubtoolOptions = { + basicTools: TToolItem[]; + smartTools: TToolItem[]; + customElement?: React.ReactNode; +}; + +interface IProps { + drawData: DrawData; + onChangePointResolution: (value: number, update?: boolean) => void; +} + +const useSubTools = ({ drawData, onChangePointResolution }: IProps) => { + const { localeText } = useLocale(); + + const isSegEverythingAvailable = useMemo(() => { + return ( + (drawData.objectList.length === 0 && !drawData.creatingObject) || + drawData.isBatchEditing + ); + }, [drawData.objectList, drawData.creatingObject, drawData.isBatchEditing]); + + const isManualAvailable = useMemo(() => { + return ( + !drawData.prompt.sessionId && + !( + drawData.prompt.promptsQueue && drawData.prompt.promptsQueue.length > 0 + ) && + !drawData.isBatchEditing + ); + }, [drawData.prompt, drawData.isBatchEditing]); + + const basicMaskTools: TToolItem[] = useMemo( + () => [ + { + key: ESubToolItem.PenAdd, + name: localeText('DDSAnnotator.subtoolbar.mask.penAdd'), + icon: , + available: isManualAvailable, + }, + { + key: ESubToolItem.PenErase, + name: localeText('DDSAnnotator.subtoolbar.mask.penErase'), + icon: , + available: isManualAvailable && !!drawData.creatingObject, + }, + { + key: ESubToolItem.BrushAdd, + name: localeText('DDSAnnotator.subtoolbar.mask.brushAdd'), + icon: , + available: isManualAvailable, + withSize: true, + }, + { + key: ESubToolItem.BrushErase, + name: localeText('DDSAnnotator.subtoolbar.mask.brushErase'), + icon: , + available: isManualAvailable && !!drawData.creatingObject, + withSize: true, + }, + ], + [isManualAvailable, drawData.creatingObject], + ); + + const smartMaskTools: TToolItem[] = useMemo(() => { + return [ + { + key: ESubToolItem.AutoSegmentByBox, + name: localeText('DDSAnnotator.subtoolbar.mask.box'), + icon: , + available: true, + }, + { + key: ESubToolItem.AutoSegmentByStroke, + name: localeText('DDSAnnotator.subtoolbar.mask.stroke'), + icon: , + available: true, + withSize: true, + }, + { + key: ESubToolItem.AutoSegmentByClick, + name: localeText('DDSAnnotator.subtoolbar.mask.click'), + icon: , + available: true, + }, + { + key: ESubToolItem.AutoEdgeStitching, + name: localeText('DDSAnnotator.subtoolbar.mask.edgeStitch'), + icon: , + available: true, + withSize: true, + }, + { + key: ESubToolItem.AutoSegmentEverything, + name: localeText('DDSAnnotator.subtoolbar.mask.sam'), + icon: , + available: isSegEverythingAvailable, + description: isSegEverythingAvailable + ? localeText('DDSAnnotator.subtoolbar.mask.sam.desc') + : localeText('DDSAnnotator.subtoolbar.mask.sam.notAllow'), + }, + ]; + }, [isSegEverythingAvailable]); + + const smartPolygonTools: TToolItem[] = useMemo(() => { + return [ + { + key: ESubToolItem.AutoSegmentByBox, + name: localeText('DDSAnnotator.subtoolbar.mask.box'), + icon: , + available: true, + withCustomElement: true, + }, + { + key: ESubToolItem.AutoSegmentByStroke, + name: localeText('DDSAnnotator.subtoolbar.mask.stroke'), + icon: , + available: true, + withSize: true, + withCustomElement: true, + }, + { + key: ESubToolItem.AutoSegmentByClick, + name: localeText('DDSAnnotator.subtoolbar.mask.click'), + icon: , + available: true, + withCustomElement: true, + }, + ]; + }, []); + + const ivpTools: TToolItem[] = useMemo(() => { + return [ + { + key: ESubToolItem.PositiveVisualPrompt, + name: localeText('DDSAnnotator.subtoolbar.visualprompt.positive'), + icon: , + available: true, + }, + { + key: ESubToolItem.NegativeVisualPrompt, + name: localeText('DDSAnnotator.subtoolbar.visualprompt.negative'), + icon: , + available: true, + }, + ]; + }, []); + + const showSubTools = useMemo(() => { + if (drawData.selectedTool === EBasicToolItem.Mask) return true; + + if ( + drawData.selectedTool === EBasicToolItem.Polygon && + drawData.AIAnnotation + ) + return true; + + if ( + drawData.selectedTool === EBasicToolItem.Rectangle && + drawData.AIAnnotation && + drawData.selectedModel === EnumModelType.IVP + ) + return true; + + if (drawData.creatingObject?.type === EObjectType.Mask) return true; + + if ( + drawData.creatingObject?.type === EObjectType.Polygon && + drawData.AIAnnotation + ) + return true; + + return false; + }, [ + drawData.selectedTool, + drawData.creatingObject, + drawData.AIAnnotation, + drawData.selectedModel, + ]); + + const currSubTools: TSubtoolOptions = useMemo(() => { + if ( + drawData.selectedTool === EBasicToolItem.Mask || + drawData.creatingObject?.type === EObjectType.Mask + ) { + return { + basicTools: basicMaskTools, + smartTools: smartMaskTools, + }; + } else if ( + drawData.selectedTool === EBasicToolItem.Polygon || + drawData.creatingObject?.type === EObjectType.Polygon + ) { + return { + basicTools: [], + smartTools: smartPolygonTools, + customElement: ( + <> +
+ {localeText('DDSAnnotator.subtoolbar.polygon.pointResolution')} +
+
+ onChangePointResolution(value, true)} + /> +
+ + ), + }; + } else if ( + drawData.selectedTool === EBasicToolItem.Rectangle && + drawData.AIAnnotation && + drawData.selectedModel === EnumModelType.IVP + ) { + return { + basicTools: [], + smartTools: ivpTools, + }; + } + return { + basicTools: [], + smartTools: [], + }; + }, [ + drawData.selectedTool, + drawData.creatingObject, + drawData.AIAnnotation, + drawData.selectedModel, + basicMaskTools, + smartMaskTools, + smartPolygonTools, + ivpTools, + drawData.pointResolution, + ]); + + return { + showSubTools, + currSubTools, + }; +}; + +export default useSubTools; diff --git a/packages/components/src/Annotator/hooks/useToolActions.ts b/packages/components/src/Annotator/hooks/useToolActions.ts index 639f46c..a53914e 100644 --- a/packages/components/src/Annotator/hooks/useToolActions.ts +++ b/packages/components/src/Annotator/hooks/useToolActions.ts @@ -1,7 +1,12 @@ import { useCallback } from 'react'; import { Updater } from 'use-immer'; import { Modal, message } from 'antd'; -import { EBasicToolItem, EObjectType, ESubToolItem } from '../constants'; +import { + EBasicToolItem, + EnumModelType, + EObjectType, + ESubToolItem, +} from '../constants'; import { DrawData, EditState, @@ -14,13 +19,15 @@ import { import { objectToRle, rleToCanvas } from '../tools/useMask'; import { useLocale } from 'dds-utils/locale'; import { cloneDeep } from 'lodash'; +import { OnAiAnnotationFunc } from './useActions'; interface IProps { mode: EditorMode; drawData: DrawData; + manualMode?: boolean; setDrawData: Updater; setDrawDataWithHistory: Updater; - setAiLabels: (labels: string[]) => void; + setAiLabels: (labels?: string) => void; editState: EditState; setEditState: Updater; getAnnotColor: (category: string) => string; @@ -30,13 +37,14 @@ interface IProps { object: IAnnotationObject, notActive?: boolean | undefined, ) => void; - removeObject: (index: number) => void; updateObject: (object: IAnnotationObject, index: number) => void; updateAllObject: (objectList: IAnnotationObject[]) => void; + onAiAnnotation: OnAiAnnotationFunc; } const useToolActions = ({ mode, + manualMode, drawData, setDrawData, setDrawDataWithHistory, @@ -46,52 +54,23 @@ const useToolActions = ({ clientSize, naturalSize, addObject, - removeObject, updateObject, updateAllObject, getAnnotColor, + onAiAnnotation, }: IProps) => { const { localeText } = useLocale(); - const onDeleteCurrObject = useCallback(() => { - if ( - drawData.isBatchEditing && - drawData.objectList[drawData.activeObjectIndex]?.status !== - EObjectStatus.Commited - ) { - setDrawData((s) => { - s.objectList[s.activeObjectIndex].status = EObjectStatus.Unchecked; - s.creatingObject = undefined; - s.prompt = {}; - s.activeObjectIndex = -1; - }); - return; - } - - if (drawData.activeObjectIndex > -1) { - removeObject(drawData.activeObjectIndex); - } - setDrawData((s) => { - s.creatingObject = undefined; - s.prompt = {}; - s.activeObjectIndex = -1; - }); - }, [ - drawData.isBatchEditing, - drawData.objectList, - drawData.activeObjectIndex, - ]); - // TODO const getColorForMaskObj = useCallback( - (label: string) => { + (labelId: string) => { if (editState.annotsDisplayOptions.colorByCategory) { - return getAnnotColor(label); + return getAnnotColor(labelId); } if (drawData.activeObjectIndex > -1) { return drawData.objectList[drawData.activeObjectIndex].color; } - return drawData.creatingObject?.color || getAnnotColor(label); + return drawData.creatingObject?.color || getAnnotColor(labelId); }, [ editState.annotsDisplayOptions.colorByCategory, @@ -102,8 +81,37 @@ const useToolActions = ({ ], ); + const onChangeObjectLabel = (labelId: string) => { + const editObject = drawData.objectList[drawData.activeObjectIndex]; + if (editObject) { + const newObject = { + ...drawData.objectList[drawData.activeObjectIndex], + attributes: undefined, + }; + newObject.labelId = labelId; + if (editState.annotsDisplayOptions.colorByCategory) { + newObject.color = getAnnotColor(labelId); + } + if (newObject.type === EObjectType.Mask && newObject.maskRle) { + newObject.maskCanvasElement = rleToCanvas( + newObject.maskRle, + naturalSize, + newObject.color, + ); + } + // batch editing set conf to 1 + if (drawData.isBatchEditing) { + newObject.conf = 1; + } + updateObject(newObject, drawData.activeObjectIndex); + } + setEditState((s) => { + s.latestLabelId = labelId; + }); + }; + const onFinishCurrCreate = useCallback( - (label: string) => { + (labelId: string) => { if (drawData.creatingObject?.type === EObjectType.Mask) { const maskRle = objectToRle( clientSize, @@ -112,10 +120,11 @@ const useToolActions = ({ drawData.creatingObject?.maskCanvasElement, ); if (maskRle && maskRle.length > 0) { - const color = getColorForMaskObj(label); + const color = getColorForMaskObj(labelId); const newObject = { + ...drawData.objectList[drawData.activeObjectIndex], type: EObjectType.Mask, - label, + labelId, hidden: false, maskRle, maskCanvasElement: rleToCanvas(maskRle, naturalSize, color), @@ -139,13 +148,32 @@ const useToolActions = ({ localeText('DDSAnnotator.anno.mask.translateToRleError'), ); } + } else if (drawData.creatingObject?.type === EObjectType.Polygon) { + const color = getAnnotColor(labelId); + const newObject = { + ...drawData.objectList[drawData.activeObjectIndex], + type: EObjectType.Polygon, + labelId, + hidden: false, + polygon: drawData.creatingObject?.polygon, + conf: 1, + status: EObjectStatus.Commited, + color, + }; + if (drawData.activeObjectIndex > -1) { + // edit existing polygon + updateObject(newObject, drawData.activeObjectIndex); + } else { + // add new polygon + addObject(newObject, true); + } } else { const newObject = { ...drawData.objectList[drawData.activeObjectIndex], }; - newObject.label = label; + newObject.labelId = labelId; if (editState.annotsDisplayOptions.colorByCategory) { - newObject.color = getAnnotColor(label); + newObject.color = getAnnotColor(labelId); } // batch editing set conf to 1 if (drawData.isBatchEditing) { @@ -157,12 +185,19 @@ const useToolActions = ({ s.creatingObject = undefined; s.prompt = {}; s.activeObjectIndex = -1; + if ( + [ESubToolItem.PenErase, ESubToolItem.BrushErase].includes( + s.selectedSubTool, + ) + ) { + s.selectedSubTool = ESubToolItem.PenAdd; + } }); setEditState((s) => { - s.latestLabel = label; + s.latestLabelId = labelId; }); }, - [drawData.creatingObject], + [drawData.creatingObject, drawData.activeObjectIndex, drawData.objectList], ); const onCloseAnnotationEditor = useCallback(() => { @@ -181,7 +216,7 @@ const useToolActions = ({ .map((obj) => { obj.status = EObjectStatus.Commited; if (obj.type !== EObjectType.Mask) { - obj.color = getAnnotColor(obj.label); + obj.color = getAnnotColor(obj.labelId); } return obj; }); @@ -189,8 +224,9 @@ const useToolActions = ({ s.isBatchEditing = false; s.activeObjectIndex = -1; s.creatingObject = undefined; + s.prompt = {}; }); - setAiLabels([]); + setAiLabels(undefined); }, [drawData.objectList]); const onAbortBatchObjects = useCallback(() => { @@ -202,6 +238,7 @@ const useToolActions = ({ s.isBatchEditing = false; s.activeObjectIndex = -1; s.creatingObject = undefined; + s.prompt = {}; }); }, [drawData.objectList]); @@ -209,7 +246,7 @@ const useToolActions = ({ (tool: EBasicToolItem) => { if ( mode !== EditorMode.Edit || - tool === drawData.selectedTool || + (tool === drawData.selectedTool && drawData.AIAnnotation) || drawData.isBatchEditing ) return; @@ -219,12 +256,27 @@ const useToolActions = ({ s.selectedSubTool = s.AIAnnotation ? ESubToolItem.AutoSegmentByBox : ESubToolItem.PenAdd; + } else if (tool === EBasicToolItem.Polygon) { + s.selectedSubTool = ESubToolItem.AutoSegmentByBox; + } else if ( + tool === EBasicToolItem.Rectangle && + s.selectedModel === EnumModelType.IVP + ) { + s.selectedSubTool = ESubToolItem.PositiveVisualPrompt; } + s.AIAnnotation = false; s.activeObjectIndex = -1; s.creatingObject = undefined; + s.editingAttribute = undefined; + s.prompt = {}; }); }, - [mode, drawData.selectedTool, drawData.isBatchEditing], + [ + mode, + drawData.selectedTool, + drawData.isBatchEditing, + drawData.selectedModel, + ], ); const selectSubTool = useCallback( @@ -232,9 +284,11 @@ const useToolActions = ({ if ( mode !== EditorMode.Edit || tool === drawData.selectedSubTool || - drawData.isBatchEditing + (drawData.selectedTool === EBasicToolItem.Mask && + drawData.isBatchEditing) ) return; + setDrawData((s) => { s.selectedSubTool = tool; }); @@ -242,7 +296,7 @@ const useToolActions = ({ // save unfinished mask object if (tool === ESubToolItem.AutoEdgeStitching && drawData.creatingObject) { onFinishCurrCreate( - drawData.creatingObject.label || editState.latestLabel || '', + drawData.creatingObject.labelId || editState.latestLabelId || '', ); } }, @@ -260,7 +314,7 @@ const useToolActions = ({ ); const onExitAIAnnotation = useCallback(() => { - setDrawData((s) => { + setDrawDataWithHistory((s) => { s.objectList = s.objectList.filter( (obj) => obj.status === EObjectStatus.Commited, ); @@ -281,6 +335,40 @@ const useToolActions = ({ [mode], ); + const setPointResolution = useCallback( + (value: number) => { + if (mode !== EditorMode.Edit) return; + setDrawData((s) => { + s.pointResolution = value; + }); + }, + [mode], + ); + + const onChangePointResolution = useCallback( + (value: number, update?: boolean) => { + setPointResolution(value); + if ( + update && + drawData.creatingObject && + drawData.creatingObject.type === EObjectType.Polygon && + drawData.prompt.promptsQueue && + drawData.prompt.promptsQueue.length > 0 + ) { + const updateDrawData: DrawData = { + ...drawData, + pointResolution: value, + }; + onAiAnnotation({ + type: EObjectType.Polygon, + drawData: updateDrawData, + promptsQueue: drawData.prompt.promptsQueue, + }); + } + }, + [drawData.creatingObject, drawData.prompt], + ); + const displayAIModeUnavailableModal = () => { Modal.info({ centered: true, @@ -300,7 +388,8 @@ const useToolActions = ({ displayAIModeUnavailableModal(); return; } - if (mode !== EditorMode.Edit || drawData.isBatchEditing) return; + if (mode !== EditorMode.Edit || drawData.isBatchEditing || manualMode) + return; setDrawData((s) => { s.AIAnnotation = active; }); @@ -308,31 +397,6 @@ const useToolActions = ({ [mode, drawData.isBatchEditing], ); - const onSaveAIPolygon = useCallback(() => { - const label = drawData.creatingObject?.label || ''; - const color = getAnnotColor(label); - addObject({ - type: EObjectType.Polygon, - polygon: drawData.creatingObject?.polygon, - label, - color, - hidden: false, - status: EObjectStatus.Commited, - }); - setDrawData((s) => { - s.activeObjectIndex = s.objectList.length - 1; - s.prompt = {}; - }); - }, [drawData.creatingObject]); - - const onCancelAIPolygon = useCallback(() => { - setDrawData((s) => { - s.creatingObject = undefined; - s.activeObjectIndex = -1; - s.prompt = {}; - }); - }, []); - const onChangeSkeletonConf = useCallback( (range: [number, number]) => { setDrawDataWithHistory((s) => { @@ -404,7 +468,7 @@ const useToolActions = ({ const onChangeColorMode = useCallback(() => { if (!drawData.objectList || !drawData.objectList.length) return; const newObjectList = cloneDeep(drawData.objectList).map((item) => { - const color = getAnnotColor(item.label); + const color = getAnnotColor(item.labelId); if ( item.type === EObjectType.Mask && item.maskRle && @@ -421,8 +485,20 @@ const useToolActions = ({ updateAllObject(newObjectList); }, [drawData.objectList, getAnnotColor]); + const onSelectModel = useCallback((type: EnumModelType) => { + setDrawData((s) => { + s.selectedModel = type; + if (type === EnumModelType.IVP) { + s.selectedSubTool = ESubToolItem.PositiveVisualPrompt; + } else { + // TODO + s.selectedSubTool = ESubToolItem.PenAdd; + } + }); + }, []); + return { - onDeleteCurrObject, + onChangeObjectLabel, onFinishCurrCreate, onCloseAnnotationEditor, onAcceptValidObjects, @@ -434,13 +510,13 @@ const useToolActions = ({ setBrushSize, activeAIAnnotation, displayAIModeUnavailableModal, - onSaveAIPolygon, - onCancelAIPolygon, onChangeSkeletonConf, onChangeLimitConf, onChangeImageDisplayOpts, onChangeAnnotsDisplayOpts, onChangeColorMode, + onChangePointResolution, + onSelectModel, }; }; diff --git a/packages/components/src/Annotator/hooks/useTopTools.tsx b/packages/components/src/Annotator/hooks/useTopTools.tsx new file mode 100644 index 0000000..e907a86 --- /dev/null +++ b/packages/components/src/Annotator/hooks/useTopTools.tsx @@ -0,0 +1,279 @@ +import { useMemo } from 'react'; +import { Button, Tooltip } from 'antd'; +import Icon, { ArrowLeftOutlined } from '@ant-design/icons'; +import { ReactComponent as LogoIcon } from '../assets/logo.svg'; +import { ReactComponent as DocsIcon } from '../assets/docs.svg'; +import { + EBasicToolItem, + EnumModelType, + ESubToolItem, + TOOL_MODELS_MAP, +} from '../constants'; +import { + DrawData, + EditState, + EditorMode, + IImageDisplayOptions, + IAnnotsDisplayOptions, + Category, +} from '../type'; +import { useLocale } from 'dds-utils/locale'; +import DisplaySettings from '../components/DisplaySettings'; +import { ShortcutsInfo } from '../components/ShortcutsInfo'; +import EditorStatus from '../components/EditorStatus'; +import TopTools from '../components/TopTools'; +import LabelSelector from '../components/LabelSelector'; +import ModelSelector from '../components/ModelSelector'; +import SubToolBar from '../components/SubToolBar'; +import { TSubtoolOptions } from './useSubtools'; + +interface IProps { + isOldMode?: boolean; + isSeperate?: boolean; + mode: EditorMode; + fileName?: string; + drawData: DrawData; + editState: EditState; + hideTopBarActions?: boolean; + titleElements?: React.ReactElement[]; + actionElements?: React.ReactElement[]; + enableReviewerModify?: boolean; + labelOptions: Category[]; + showSubTools: boolean; + currSubTools: TSubtoolOptions; + topBarCenterElement?: React.ReactElement | null; + labelColors?: Record; + selectSubTool: (tool: ESubToolItem) => void; + onSelectModel: (type: EnumModelType) => void; + setBrushSize: (size: number) => void; + activeAIAnnotation: (active: boolean) => void; + onChangeImageDisplayOpts: (value: IImageDisplayOptions) => void; + onChangeAnnotsDisplayOpts: (value: IAnnotsDisplayOptions) => void; + onChangeObjectLabel: (labelId: string) => void; + onCreateCategory: (name: string) => void; + onSaveAnnotations: () => Promise; + onCommitAnnotations: () => Promise; + onRejectAnnotations: () => Promise; + onAcceptAnnotations: () => Promise; + onModifyAnnotations: () => Promise; + onCancelAnnotations: () => Promise; +} + +const useTopTools = ({ + isOldMode, + isSeperate, + mode, + fileName, + drawData, + editState, + hideTopBarActions, + titleElements, + actionElements, + enableReviewerModify, + labelOptions, + labelColors, + showSubTools, + currSubTools, + topBarCenterElement, + selectSubTool, + setBrushSize, + activeAIAnnotation, + onChangeImageDisplayOpts, + onChangeAnnotsDisplayOpts, + onChangeObjectLabel, + onCreateCategory, + onSaveAnnotations, + onCommitAnnotations, + onRejectAnnotations, + onAcceptAnnotations, + onModifyAnnotations, + onCancelAnnotations, + onSelectModel, +}: IProps) => { + const { localeText } = useLocale(); + const jumpDocs = () => { + window.open('https://docs.deepdataspace.com'); + }; + + const supportActions = useMemo(() => { + const actions = actionElements + ? actionElements.map((item) => ({ customElement: item })) + : []; + if (hideTopBarActions) return actions; + if (mode === EditorMode.Review) { + actions.push( + ...[ + { + customElement: ( + + ), + }, + ...(isOldMode || !enableReviewerModify + ? [] + : [ + { + customElement: ( + + ), + }, + ]), + { + customElement: ( + + ), + }, + ], + ); + } + if (mode === EditorMode.Edit && !isSeperate) { + actions.push({ + customElement: ( + + ), + }); + if (!isOldMode) { + actions.push({ + customElement: ( + + ), + }); + } + } + actions.unshift({ + customElement: ( + <> + {mode === EditorMode.Edit && ( +
+ + + + +
+ )} + + + + ), + }); + return actions; + }, [ + mode, + isOldMode, + enableReviewerModify, + hideTopBarActions, + onSaveAnnotations, + onCommitAnnotations, + onCancelAnnotations, + onRejectAnnotations, + onAcceptAnnotations, + onModifyAnnotations, + ]); + + const leftTools = () => { + const actions = []; + if (titleElements) { + actions.push(...titleElements.map((item) => ({ customElement: item }))); + } else { + if (isSeperate || mode === EditorMode.Edit) { + actions.push({ + customElement: ( + + + + ), + }); + } else { + actions.push({ + title: localeText('DDSAnnotator.exit'), + icon: , + onClick: () => onCancelAnnotations(), + }); + } + if (mode !== EditorMode.Edit && fileName) { + actions.push({ customElement: fileName }); + } + } + if ( + mode === EditorMode.Edit && + TOOL_MODELS_MAP[drawData.selectedTool] && + TOOL_MODELS_MAP[drawData.selectedTool].length > 1 && + drawData.AIAnnotation && + drawData.selectedModel + ) { + actions.push({ + customElement: ( + + ), + }); + } + if ( + mode === EditorMode.Edit && + (drawData.objectList[drawData.activeObjectIndex] || + drawData.selectedTool !== EBasicToolItem.Drag) + ) { + actions.push({ + customElement: ( + + ), + }); + } + if (mode === EditorMode.Edit && showSubTools) { + actions.push({ + customElement: ( + + ), + }); + } + return actions; + }; + + const topToolsBar = ( + + {topBarCenterElement} + + ); + + return { + topToolsBar, + }; +}; + +export default useTopTools; diff --git a/packages/components/src/Annotator/hooks/useTranslate.ts b/packages/components/src/Annotator/hooks/useTranslate.ts new file mode 100644 index 0000000..caecb9f --- /dev/null +++ b/packages/components/src/Annotator/hooks/useTranslate.ts @@ -0,0 +1,403 @@ +import { + BODY_TEMPLATE, + ELabelType, + EObjectType, + KEYPOINTS_VISIBLE_TYPE, +} from '../constants'; +import { + getObjectType, + translateBoundingBoxToRect, + translatePointsToPointObjs, + translatePointObjsToPointAttrs, + getSegmentationPoints, + translateRectToBoundingBox, + translatePolygonsToSegmentation, + translatePointsToRect, + translatePointGroupsToPoints, + translateRectToPointsArray, + translatePolygonsToPointsArrayGroup, + newTranslatePointsToPointObjs, + newTranslatePointObjsToPointAttrs, + getCanvasPoint, + getNaturalPoint, +} from '../utils/compute'; +import { + IAnnotationObject, + EObjectStatus, + DrawObject, + Category, + BaseObject, +} from '../type'; +import { rleToCanvas } from '../tools/useMask'; +import { cloneDeep } from 'lodash'; + +interface IProps { + isOldMode?: boolean; + clientSize: ISize; + naturalSize: ISize; + categories: Category[]; + getAnnotColor: (category: string) => string; +} + +const useTranslate = ({ + isOldMode, + clientSize, + naturalSize, + categories, + getAnnotColor, +}: IProps) => { + /** + * Use for annotator & old project + * @param annotation + * @returns + */ + const translateAnnotationToObject = ( + annotation: DrawObject, + ): IAnnotationObject => { + let { + categoryId, + boundingBox, + points, + lines, + pointNames, + pointColors, + segmentation, + mask, + alpha, + point, + } = annotation; + + const color = getAnnotColor(categoryId || ''); + const newObj: IAnnotationObject = { + labelId: categoryId || '', + type: EObjectType.Rectangle, + hidden: false, + conf: annotation.conf || 1, + customStyles: annotation.customStyles, + status: EObjectStatus.Commited, + color, + }; + + if (boundingBox) { + const rect = translateBoundingBoxToRect(boundingBox, clientSize); + Object.assign(newObj, { rect: { visible: true, ...rect } }); + } + + if ( + points && + points.length > 0 && + lines && + lines.length > 0 && + pointNames && + pointColors + ) { + const pointObjs: IElement[] = translatePointsToPointObjs( + points, + pointNames, + pointColors, + naturalSize, + clientSize, + ); + Object.assign(newObj, { + keypoints: { + points: pointObjs, + lines, + }, + }); + } + if (segmentation) { + const group = getSegmentationPoints( + segmentation, + naturalSize, + clientSize, + ); + const polygon: IElement = { + group, + visible: true, + }; + Object.assign(newObj, { polygon }); + } + + if (mask && mask.length) { + Object.assign(newObj, { + maskRle: mask, + maskCanvasElement: rleToCanvas(mask, naturalSize, color), + }); + } + + if (alpha) { + const alphaImageElement = new Image(); + alphaImageElement.src = alpha; + // alphaImageElement.crossOrigin = 'anonymous'; + Object.assign(newObj, { + alpha, + alphaImageElement, + }); + } + + if (point) { + Object.assign(newObj, { + point: { + ...getCanvasPoint(point, naturalSize, clientSize), + visible: KEYPOINTS_VISIBLE_TYPE.labeledVisible, + }, + }); + } + + newObj.type = getObjectType(newObj); + return newObj; + }; + + /** + * Use for annotator & old project + * @param annotation + * @returns + */ + const translateObjectToAnnotation = (obj: IAnnotationObject): BaseObject => { + const { labelId, rect, keypoints, polygon, maskRle, point } = obj; + const labelName = + categories.find((item) => item.id === labelId)?.name || ''; + const annoObj = { + categoryId: labelId, + categoryName: labelName, + }; + if (rect) { + Object.assign(annoObj, { + boundingBox: translateRectToBoundingBox(rect, clientSize), + }); + } + if (keypoints) { + Object.assign(annoObj, { + lines: keypoints.lines, + ...translatePointObjsToPointAttrs( + keypoints.points, + naturalSize, + clientSize, + ), + }); + } + if (polygon) { + const segmentation = translatePolygonsToSegmentation( + polygon, + naturalSize, + clientSize, + ); + Object.assign(annoObj, { + segmentation, + }); + } + if (maskRle) { + Object.assign(annoObj, { + mask: maskRle, + }); + } + if (point) { + const { x, y } = getNaturalPoint( + [point.x, point.y], + naturalSize, + clientSize, + ); + Object.assign(annoObj, { + point: [x, y], + }); + } + return annoObj; + }; + + /** + * Use for new project + * @param label + * @returns + */ + const translateLabelToObject = ( + originLabel: { + labelId: string; + labelValue: any; + attributes?: (string | number | number[])[]; + }, + videoFrameCount?: number, + ) => { + const { labelId, labelValue } = originLabel; + const color = getAnnotColor(labelId); + const label = categories.find((item) => item.id === labelId); + // confirm format correct + const attributes = + label?.attributes?.map( + (_, index) => originLabel.attributes?.[index] || null, + ) || undefined; + const newObj: IAnnotationObject = { + labelId, + type: EObjectType.Custom, + hidden: false, + status: EObjectStatus.Commited, + color, + attributes, + }; + + const convertLabelValue = (newObj: IAnnotationObject, labelValue: any) => { + switch (label?.labelType) { + case ELabelType.Rectangle: { + const rect = translatePointsToRect( + labelValue, + naturalSize, + clientSize, + ); + Object.assign(newObj, { + rect: { visible: true, ...rect }, + type: EObjectType.Rectangle, + }); + break; + } + case ELabelType.Polygon: { + const group = translatePointGroupsToPoints( + labelValue, + naturalSize, + clientSize, + ); + const polygon: IElement = { + group, + visible: true, + }; + Object.assign(newObj, { + polygon, + type: EObjectType.Polygon, + }); + break; + } + case ELabelType.Skeleton: { + const pointObjs: IElement[] = newTranslatePointsToPointObjs( + labelValue, + BODY_TEMPLATE.pointNames, + BODY_TEMPLATE.pointColors, + naturalSize, + clientSize, + ); + Object.assign(newObj, { + keypoints: { + points: pointObjs, + lines: BODY_TEMPLATE.lines, + }, + type: EObjectType.Skeleton, + }); + break; + } + case ELabelType.Mask: { + Object.assign(newObj, { + maskRle: labelValue, + maskCanvasElement: rleToCanvas(labelValue, naturalSize, color), + type: EObjectType.Mask, + }); + break; + } + case ELabelType.Classification: { + Object.assign(newObj, { + labelValue, + type: EObjectType.Classification, + }); + break; + } + } + return newObj; + }; + + if (videoFrameCount && videoFrameCount > 0) { + if (label?.labelType === ELabelType.Classification) { + return { + classification: convertLabelValue(newObj, labelValue), + }; + } else { + const objects: any[] = new Array(videoFrameCount).fill(undefined); + let tempObj: any; + Object.keys(labelValue).forEach((key: string) => { + tempObj = convertLabelValue(cloneDeep(newObj), labelValue[key]); + objects[Number(key)] = { + ...tempObj, + frameEmpty: false, + }; + }); + return { + objects: objects.map( + (item) => + item || { + ...cloneDeep(tempObj), + frameEmpty: true, + }, + ), + }; + } + } + { + return convertLabelValue(newObj, labelValue); + } + }; + + /** + * Use for new project + * @param obj + * @returns + */ + const translateObjectToLabel = (obj: IAnnotationObject) => { + const { labelId, rect, keypoints, polygon, maskRle, attributes } = obj; + const label = categories.find((item) => item.id === labelId); + + const annoObj: any = { + labelId: labelId, + attributes: attributes || label?.attributes?.map(() => null) || [], + }; + switch (label?.labelType) { + case ELabelType.Rectangle: { + if (rect) { + annoObj.labelValue = translateRectToPointsArray( + rect, + clientSize, + naturalSize, + ); + } + break; + } + case ELabelType.Polygon: { + if (polygon) { + annoObj.labelValue = translatePolygonsToPointsArrayGroup( + polygon, + naturalSize, + clientSize, + ); + } + break; + } + case ELabelType.Skeleton: { + if (keypoints) { + const { points } = newTranslatePointObjsToPointAttrs( + keypoints.points, + naturalSize, + clientSize, + ); + annoObj.labelValue = points; + } + break; + } + case ELabelType.Mask: { + if (maskRle) { + annoObj.labelValue = maskRle; + } + break; + } + } + return annoObj; + }; + + return { + translateAnnotationToObject, + translateObjectToAnnotation, + translateLabelToObject, + translateObjectToLabel, + translateObject: isOldMode + ? translateObjectToAnnotation + : translateObjectToLabel, + translateToObject: isOldMode + ? translateAnnotationToObject + : translateLabelToObject, + }; +}; + +export default useTranslate; diff --git a/packages/components/src/Annotator/index.less b/packages/components/src/Annotator/index.less index 29876dd..5f1e56d 100644 --- a/packages/components/src/Annotator/index.less +++ b/packages/components/src/Annotator/index.less @@ -1,4 +1,30 @@ .dds-annotator { + &-loading { + position: fixed; + top: 0; + left: 0; + bottom: 0; + right: 0; + display: flex; + align-items: center; + justify-content: center; + background: rgba(255, 255, 255, 0.2); + z-index: 10001; + } + + &-logo { + margin-left: -5px; + width: 33px; + height: 33px; + cursor: pointer; + } + + &-logo-replace { + margin-left: -5px; + width: 33px; + height: 33px; + } + .edit-wrap { position: absolute; inset: 0; @@ -17,6 +43,22 @@ } } + &-qk-actions { + display: flex; + align-items: center; + justify-content: center; + gap: 18px; + + svg { + cursor: pointer; + fill: #fff; + + &:hover { + fill: @colorPrimary; + } + } + } + &-dropdown-options { display: flex; flex-direction: column; @@ -51,10 +93,10 @@ } .dds-annotator-editor { + position: relative; top: 0; left: 0; z-index: 100; - position: relative; width: 100%; height: 100vh; background-color: #000; @@ -69,11 +111,11 @@ .left-slider { position: relative; - width: 0; height: 100%; - background: #262626; - border-right: 2px solid #141414; - backdrop-filter: blur(12px); + background: #212121; + padding: 0; + border-radius: 0; + border-left: 1px solid black; overflow-y: scroll; overflow-x: hidden; z-index: 1; @@ -83,18 +125,60 @@ position: relative; flex: 1; height: 100%; + display: flex; + flex-direction: column; + overflow: hidden; + + .draw-area { + position: relative; + flex: 1; + width: 100%; + } } .right-slider { position: relative; width: 256px; + min-width: 256px; height: 100%; background: #262626; border-left: 2px solid #141414; backdrop-filter: blur(12px); - overflow-y: scroll; - overflow-x: hidden; + display: flex; + flex-direction: column; z-index: 1; + + .classifications { + max-height: 50%; + overflow-x: hidden; + overflow-y: scroll; + } + + .object-list { + flex: 1; + width: 100%; + overflow-y: scroll; + overflow-x: hidden; + } + } + } +} + +.dds-annotator-editor-light { + background-color: #f7f7f7; + + .dds-annnotator-toptools { + background-color: #f1f2f4; + border-bottom: 1px solid #f7f7f7; + + &-row { + &-icon { + color: #000; + + svg { + fill: #fff; + } + } } } } diff --git a/packages/components/src/Annotator/preview.tsx b/packages/components/src/Annotator/preview.tsx index 23dc36e..e26e049 100755 --- a/packages/components/src/Annotator/preview.tsx +++ b/packages/components/src/Annotator/preview.tsx @@ -1,11 +1,5 @@ import React, { useCallback, useEffect, useRef, useState } from 'react'; -import { - AnnotationType, - DisplayOption, - EElementType, - MAX_SCALE, - MIN_SCALE, -} from './constants'; +import { DisplayOption, EElementType, MAX_SCALE, MIN_SCALE } from './constants'; import { useImmer } from 'use-immer'; import TopTools from './components/TopTools'; import PopoverMenu from './components/PopoverMenu'; @@ -19,7 +13,6 @@ import { import useHistory from './hooks/useHistory'; import useObjects from './hooks/useObjects'; import useCanvasContainer from './hooks/useCanvasContainer'; -import usePreviousState from './hooks/usePreviousState'; import { cloneDeep, isEmpty } from 'lodash'; import { BaseObject, @@ -27,18 +20,17 @@ import { DEFAULT_DRAW_DATA, DEFAULT_EDIT_STATE, DrawData, - DrawImageData, + AnnoItem, DrawObject, EditState, EditorMode, - IAnnotationObject, } from './type'; import useColor from './hooks/useColor'; import useMouseCursor from './hooks/useMouseCursor'; import useMouseEvents from './hooks/useMouseEvents'; import useCanvasRender from './hooks/useCanvasRender'; import useDataEffect from './hooks/useDataEffect'; -import { RenderStyles, useToolInstances } from './tools/base'; +import { useToolInstances } from './tools/base'; import classNames from 'classnames'; import { ReactComponent as DoubleRightIcon } from './assets/doubleRight.svg'; import { ReactComponent as DownloadIcon } from './assets/download.svg'; @@ -47,26 +39,24 @@ import { EDITOR_SHORTCUTS, EShortcuts } from './constants/shortcuts'; import { message } from 'antd'; import { ImageView } from './components/ImageView'; import './index.less'; +import useTranslate from './hooks/useTranslate'; export interface PreviewProps { + isOldMode?: boolean; // is old dataset design mode visible: boolean; categories: Category[]; - list: DrawImageData[]; + list: AnnoItem[]; current: number; objectsFilter?: (imageData: any) => BaseObject[]; - getCustomObjectStyles?: ( - object: IAnnotationObject, - color: string, - ) => Partial; onCancel?: () => void; onPrev?: () => Promise; onNext?: () => Promise; - displayAnnotationType?: AnnotationType; displayOptionsResult: { [key in DisplayOption]?: boolean }; } const Preview: React.FC = (props) => { const { + isOldMode, visible, categories, list, @@ -75,8 +65,6 @@ const Preview: React.FC = (props) => { onNext, onCancel, objectsFilter, - getCustomObjectStyles, - displayAnnotationType, displayOptionsResult, } = props; @@ -107,6 +95,7 @@ const Preview: React.FC = (props) => { CanvasContainer, } = useCanvasContainer({ visible, + drawData, allowMove: editState.allowMove, isRequiring: editState.isRequiring, minPadding: { @@ -114,13 +103,27 @@ const Preview: React.FC = (props) => { left: 300, }, cursorSize: drawData.brushSize, - showReferenceLine: false, - isCustomCursorActive: false, onClickMaskBg: onCancel, }); - const [preClientSize, clearPreClientSize] = - usePreviousState(clientSize); + const { getAnnotColor } = useColor({ + categories, + editState, + }); + + const { updateMouseCursor } = useMouseCursor({ + topCanvas: activeCanvasRef.current, + editState, + drawData, + }); + + const { translateToObject } = useTranslate({ + isOldMode, + clientSize, + naturalSize, + categories, + getAnnotColor, + }); const { clearHistory, updateHistory, setDrawDataWithHistory } = useHistory({ clientSize, @@ -131,26 +134,13 @@ const Preview: React.FC = (props) => { const { addObject, initObjectList, updateObject } = useObjects({ annotations, setAnnotations, - clientSize, - naturalSize, drawData, setDrawData, setDrawDataWithHistory, - editState, setEditState, mode: EditorMode.View, - displayAnnotationType, - }); - - const { labelColors, getAnnotColor } = useColor({ - categories, - editState, - }); - - const { updateMouseCursor } = useMouseCursor({ - topCanvas: activeCanvasRef.current, - editState, - drawData, + translateToObject, + updateHistory, }); const { objectHooksMap } = useToolInstances({ @@ -173,6 +163,7 @@ const Preview: React.FC = (props) => { updateMouseCursor, displayOptionsResult, getAnnotColor, + categories, }); const { updateRender } = useCanvasRender({ @@ -186,7 +177,6 @@ const Preview: React.FC = (props) => { activeCanvasRef, imgRef, objectHooksMap, - getCustomObjectStyles, }); useMouseEvents({ @@ -217,15 +207,12 @@ const Preview: React.FC = (props) => { document.body.style.overflow = visible ? 'hidden' : 'overlay'; }, [visible]); - const { resetDataWithImageData, rebuildDrawData } = useDataEffect({ + const { resetDataWithImageData } = useDataEffect({ imagePos, clientSize, - preClientSize, - clearPreClientSize, naturalSize, annotations, setAnnotations, - labelColors, drawData, setDrawData, editState, @@ -234,6 +221,7 @@ const Preview: React.FC = (props) => { updateRender, clearHistory, objectsFilter, + labelOptions: categories, }); /** Reset data when hiding the editor or switching images */ @@ -243,8 +231,8 @@ const Preview: React.FC = (props) => { /** Custom options changed */ useEffect(() => { - rebuildDrawData(true); - }, [displayAnnotationType, displayOptionsResult, getCustomObjectStyles]); + updateRender(); + }, [displayOptionsResult]); // ================================================================================================================= // Preview @@ -323,12 +311,12 @@ const Preview: React.FC = (props) => { if ( editState.focusObjectIndex > -1 && drawData.objectList[editState.focusObjectIndex] && - !drawData.objectList[editState.focusObjectIndex].hidden && + !drawData.objectList[editState.focusObjectIndex]?.hidden && editState.focusEleIndex > -1 && editState.focusEleType === EElementType.Circle ) { const target = - drawData.objectList[editState.focusObjectIndex].keypoints?.points?.[ + drawData.objectList[editState.focusObjectIndex]?.keypoints?.points?.[ editState.focusEleIndex ]; if (target) { @@ -386,7 +374,7 @@ const Preview: React.FC = (props) => { children: ( <> = (props) => { : metadata[key]}
))} - { - list[current]?.caption ? ( -
- {'caption'} -
- {list[current].caption} -
- ) : null - }
diff --git a/packages/components/src/Annotator/sevices/index.ts b/packages/components/src/Annotator/sevices/index.ts index 897d43c..4a32855 100644 --- a/packages/components/src/Annotator/sevices/index.ts +++ b/packages/components/src/Annotator/sevices/index.ts @@ -2,21 +2,14 @@ import { request } from '@umijs/max'; import { Modal } from 'antd'; import { globalLocaleText } from 'dds-utils/locale'; -import { EnumTaskStatus } from '../constants'; +import { EnumModelType, EnumTaskStatus } from '../constants'; export namespace NsApiAnnotator { - export enum EnumModelType { - Detection = 'ai_detection', - SegmentByPolygon = 'ai_segmentation', - SegmentByMask = 'ai_segmentation_mask', - Pose = 'ai_pose', - MaskEdgeStitching = 'ai_mask_edge_stitching', - SegmentEverything = 'ai_segment_everything', - } - export type ModelParam = T extends EnumModelType.Detection ? FetchAIDetectionReq + : T extends EnumModelType.IVP + ? FetchIVPReq : T extends EnumModelType.SegmentByPolygon ? FetchAIPolygonSegmentReq : T extends EnumModelType.SegmentByMask @@ -32,6 +25,8 @@ export namespace NsApiAnnotator { export type ModelResult = T extends EnumModelType.Detection ? FetchAIDetectionRsp + : T extends EnumModelType.IVP + ? FetchIVPRsp : T extends EnumModelType.SegmentByPolygon ? FetchAIPolygonSegmentRsp : T extends EnumModelType.SegmentByMask @@ -49,15 +44,32 @@ export namespace NsApiAnnotator { text: string; } - export interface FetchAIPolygonSegmentReq { - image: string; - mask: string; - polygons: number[][]; - clicks: { + export interface FetchIVPReq { + promptImage: string; + inferImage: string; + prompts: { + type: string; // 'rect' | 'point' isPositive: boolean; - position: number[]; + rect?: number[]; // [xmin, ymin, xmax, ymax]; + point?: number[]; // [x, y] + }[]; + labelTypes: string[]; // ["bbox", "mask"] + } + + export interface FetchAIPolygonSegmentReq { + image: string; // image_id:// | base64:// | http:// | https:// + density: number; // (0, 1) default 0.2 + area: number[]; // [xmin, ymin, xmax, ymax]; + prompts: { + type: string; // 'rect' | 'point' | 'stroke' | 'modify'; + isPositive: boolean; // + rect?: number[]; // [xmin, ymin, xmax, ymax]; + point?: number[]; // [x, y] + stroke?: number[]; // [x1, y1, x2, y2, ...]; + radius?: number; // brush size while using stroke prompt + polygons?: number[][]; // [[x1, y1, x2, y2, ...], [xn, yn, xn+1, yn+1, ...], ....]; }[]; - rect?: number[]; + sessionId?: string; } export interface FetchAIMaskSegmentReq { @@ -123,9 +135,18 @@ export namespace NsApiAnnotator { suggestThreshold: number; } + export interface FetchIVPRsp { + objects: Array<{ + bbox?: number[]; + mask?: number[]; + score: number; + }>; + } + export interface FetchAIPolygonSegmentRsp { - polygon: number[][]; - mask: string; + image: string; // image_id:// + sessionId: string; + polygons: number[][]; // [[x1, y1, x2, y2, ...], [xn, yn, xn+1, yn+1, ...], ....] } export interface FetchAIMaskSegmentRsp { @@ -164,10 +185,22 @@ export namespace NsApiAnnotator { uuid: string; result: ModelResult; } + export interface FetchUploadSignatureRsp { + downloadUrl: string; + uploadUrl: string; + } + + export interface FetchBetchUploadSignatureRsp { + fileUrls: { + fileName: string; + downloadUrl: string; + uploadUrl: string; + }[]; + } } async function fetchTaskUuid( - type: NsApiAnnotator.EnumModelType, + type: EnumModelType, params: any, options?: { [key: string]: any }, ) { @@ -185,7 +218,7 @@ async function fetchTaskUuid( ); } -function fetchTaskResults( +function fetchTaskResults( taskUuid: string, options?: { [key: string]: any }, ) { @@ -198,7 +231,7 @@ function fetchTaskResults( ); } -function fetchMaskTaskResults( +function fetchMaskTaskResults( taskUuid: string, options?: { [key: string]: any }, ) { @@ -211,8 +244,8 @@ function fetchMaskTaskResults( ); } -export async function pollTaskResults( - type: NsApiAnnotator.EnumModelType, +export async function pollTaskResults( + type: EnumModelType, taskUuid: string, maxAttempts = 5000, interval = 1000, @@ -221,9 +254,9 @@ export async function pollTaskResults( while (attempts < maxAttempts) { const fetchTaskResultsRequest = [ - NsApiAnnotator.EnumModelType.SegmentByMask, - NsApiAnnotator.EnumModelType.MaskEdgeStitching, - NsApiAnnotator.EnumModelType.SegmentEverything, + EnumModelType.SegmentByMask, + EnumModelType.MaskEdgeStitching, + EnumModelType.SegmentEverything, ].includes(type) ? fetchMaskTaskResults : fetchTaskResults; @@ -246,8 +279,8 @@ export async function pollTaskResults( throw new Error('Max attempts exceeded'); } -export async function fetchModelResults( - type: NsApiAnnotator.EnumModelType, +export async function fetchModelResults( + type: EnumModelType, params: NsApiAnnotator.ModelParam, ) { try { @@ -269,3 +302,86 @@ export async function fetchModelResults( } } } + +export async function fetchUploadSignature( + params: { + fileName: string; + }, + options?: { [key: string]: any }, +) { + return request( + `${process.env.MODEL_API_PATH}/upload_signature`, + { + method: 'POST', + data: { + ...params, + }, + ...(options || {}), + }, + ); +} + +export async function fetchBatchUploadSignature( + params: { + fileNames: string[]; + }, + options?: { [key: string]: any }, +) { + return request( + `${process.env.MODEL_API_PATH}/batch_upload_signatures`, + { + method: 'POST', + data: { + ...params, + }, + ...(options || {}), + }, + ); +} + +export const putFile = async ( + uploadUrl: string, + file?: File, + contentType?: string, +): Promise => { + return new Promise((resolve, reject) => { + if (!file) reject(null); + fetch(uploadUrl, { + method: 'PUT', + headers: { + 'Content-Type': contentType || '', + }, + body: file, + }) + .then((response) => { + if (response.status === 200) { + resolve(response); + } else { + console.error('Upload file error: ', uploadUrl, response); + reject(null); + } + }) + .catch((error) => { + console.error('Upload file error: ', uploadUrl, error); + reject(null); + }); + }); +}; + +export async function getOssUrlByBlobUrl( + fileName: string, + blobUrl: string, +): Promise { + try { + const { downloadUrl, uploadUrl } = await fetchUploadSignature({ fileName }); + const response = await fetch(blobUrl); + if (!response.ok) { + throw new Error('Failed to fetch file'); + } + const blobData = await response.blob(); + await putFile(uploadUrl, blobData as File); + return downloadUrl; + } catch (error: any) { + throw new Error('Failed to get oss url', error.message); + } +} diff --git a/packages/components/src/Annotator/tools/base.ts b/packages/components/src/Annotator/tools/base.ts index 9345171..a5a28bb 100644 --- a/packages/components/src/Annotator/tools/base.ts +++ b/packages/components/src/Annotator/tools/base.ts @@ -15,6 +15,7 @@ import { setRectBetweenPixels, } from '../utils/compute'; import { + Category, DrawData, EditState, EObjectStatus, @@ -24,13 +25,13 @@ import { } from '../type'; import { CursorState } from 'ahooks/lib/useMouse'; import { Updater } from 'use-immer'; -import { HistoryItem } from '../hooks/useHistory'; import { OnAiAnnotationFunc } from '../hooks/useActions'; import useRectangle from './useRectangle'; import usePolygon from './usePolygon'; import useSkeleton from './useSkeleton'; import useMask from './useMask'; import useMatting from './useMatting'; +import usePoint from './usePoint'; export type RenderStyles = { strokeColor: string; @@ -70,7 +71,7 @@ export namespace ToolHooksFunc { point: { x: number; y: number }; basic: { hidden: boolean; - label: string; + labelId: string; status: EObjectStatus; color: string; }; @@ -121,7 +122,7 @@ export interface ToolInstanceHookProps { drawData: DrawData; setDrawData: Updater; setDrawDataWithHistory: Updater; - updateHistory: (item: HistoryItem) => void; + updateHistory: (drawData: DrawData) => void; updateObject: (object: IAnnotationObject, index: number) => void; addObject: (object: IAnnotationObject, notActive?: boolean) => void; clientSize: ISize; @@ -133,9 +134,10 @@ export interface ToolInstanceHookProps { activeCanvasRef: React.RefObject; updateMouseCursor: (value: string, position?: Direction) => void; getAnnotColor: (category: string) => string; - aiLabels?: string[]; + aiLabels?: string; onAiAnnotation?: OnAiAnnotationFunc; displayOptionsResult?: { [key in DisplayOption]?: boolean }; + categories: Category[]; } export type ToolInstanceHook = ( @@ -148,6 +150,7 @@ export const useToolInstances = (props: ToolInstanceHookProps) => { const skeletonHooks = useSkeleton(props); const maskHooks = useMask(props); const mattingHooks = useMatting(props); + const pointHooks = usePoint(props); const objectHooksMap: Record = { [EObjectType.Rectangle]: rectangleHooks, @@ -155,6 +158,7 @@ export const useToolInstances = (props: ToolInstanceHookProps) => { [EObjectType.Skeleton]: skeletonHooks, [EObjectType.Mask]: maskHooks, [EObjectType.Matting]: mattingHooks, + [EObjectType.Point]: pointHooks, [EObjectType.Custom]: rectangleHooks, // todo }; diff --git a/packages/components/src/Annotator/tools/useMask.ts b/packages/components/src/Annotator/tools/useMask.ts index a9b4a78..9957e3f 100644 --- a/packages/components/src/Annotator/tools/useMask.ts +++ b/packages/components/src/Annotator/tools/useMask.ts @@ -29,10 +29,10 @@ import { PROMPT_FILL_COLOR, } from '../constants/render'; import { - EMaskPromptType, + EPromptType, ICreatingMaskStep, ICreatingObject, - MaskPromptItem, + PromptItem, } from '../type'; import { hexToRgbArray, hexToRgba } from '../utils/color'; import { cloneDeep } from 'lodash'; @@ -467,12 +467,12 @@ const useMask: ToolInstanceHook = ({ const renderPrompt: ToolHooksFunc.RenderPrompt = ({ prompt }) => { // draw creating prompt - if (prompt.creatingMask) { + if (prompt.creatingPrompt) { const strokeColor = ANNO_STROKE_COLOR.CREATING; const fillColor = ANNO_FILL_COLOR.CREATING; - switch (prompt.creatingMask.type) { - case EMaskPromptType.Rect: { - const { startPoint } = prompt.creatingMask; + switch (prompt.creatingPrompt.type) { + case EPromptType.Rect: { + const { startPoint } = prompt.creatingPrompt; const rect = getRectFromPoints( startPoint!, { @@ -498,10 +498,10 @@ const useMask: ToolInstanceHook = ({ ); break; } - case EMaskPromptType.Point: { - if (!prompt.creatingMask.point) break; + case EPromptType.Point: { + if (!prompt.creatingPrompt.point) break; const canvasCoordPoint = translatePointCoord( - prompt.creatingMask.point, + prompt.creatingPrompt.point, { x: -imagePos.current.x, y: -imagePos.current.y, @@ -511,29 +511,31 @@ const useMask: ToolInstanceHook = ({ activeCanvasRef.current!, canvasCoordPoint, 4, - prompt.creatingMask.isPositive + prompt.creatingPrompt.isPositive ? PROMPT_FILL_COLOR.POSITIVE : PROMPT_FILL_COLOR.NEGATIVE, 2, '#fff', ); } - case EMaskPromptType.EdgeStitch: - case EMaskPromptType.Stroke: { - if (!prompt.creatingMask.stroke || !prompt.creatingMask.radius) break; + case EPromptType.EdgeStitch: + case EPromptType.Stroke: { + if (!prompt.creatingPrompt.stroke || !prompt.creatingPrompt.radius) + break; const canvasCoordStroke = translatePolygonCoord( - prompt.creatingMask.stroke, + prompt.creatingPrompt.stroke, { x: -imagePos.current.x, y: -imagePos.current.y, }, ); const radius = - (prompt.creatingMask.radius * clientSize.width) / naturalSize.width; + (prompt.creatingPrompt.radius * clientSize.width) / + naturalSize.width; const color = - prompt.creatingMask.type === EMaskPromptType.EdgeStitch + prompt.creatingPrompt.type === EPromptType.EdgeStitch ? hexToRgba(strokeColor, ANNO_MASK_ALPHA.CREATING) - : prompt.creatingMask.isPositive + : prompt.creatingPrompt.isPositive ? PROMPT_FILL_COLOR.POSITIVE : PROMPT_FILL_COLOR.NEGATIVE; drawQuadraticPath( @@ -562,9 +564,9 @@ const useMask: ToolInstanceHook = ({ } // draw existing prompts - if (prompt.maskPrompts) { - prompt.maskPrompts.forEach((item) => { - if (item.type === EMaskPromptType.Point) { + if (prompt.promptsQueue) { + prompt.promptsQueue.forEach((item) => { + if (item.type === EPromptType.Point) { const canvasCoordPoint = translatePointCoord(item.point!, { x: -imagePos.current.x, y: -imagePos.current.y, @@ -629,34 +631,29 @@ const useMask: ToolInstanceHook = ({ ) ) { // Brush tool need not push history when mousedown - updateHistory( - cloneDeep({ - drawData: s, - clientSize, - }), - ); + updateHistory(cloneDeep(s)); } } - s.prompt.segmentationMask = undefined; + s.prompt.sessionId = undefined; break; case ESubToolItem.AutoSegmentByBox: - s.prompt.creatingMask = { - type: EMaskPromptType.Rect, + s.prompt.creatingPrompt = { + type: EPromptType.Rect, startPoint: mouse, isPositive: true, }; break; case ESubToolItem.AutoSegmentByClick: - s.prompt.creatingMask = { - type: EMaskPromptType.Point, + s.prompt.creatingPrompt = { + type: EPromptType.Point, startPoint: mouse, point: mouse, isPositive: getPromptBoolean(event), }; break; case ESubToolItem.AutoSegmentByStroke: - s.prompt.creatingMask = { - type: EMaskPromptType.Stroke, + s.prompt.creatingPrompt = { + type: EPromptType.Stroke, startPoint: mouse, stroke: [mouse], radius: s.brushSize, @@ -664,8 +661,8 @@ const useMask: ToolInstanceHook = ({ }; break; case ESubToolItem.AutoEdgeStitching: - s.prompt.creatingMask = { - type: EMaskPromptType.EdgeStitch, + s.prompt.creatingPrompt = { + type: EPromptType.EdgeStitch, startPoint: mouse, stroke: [mouse], radius: s.brushSize, @@ -708,26 +705,26 @@ const useMask: ToolInstanceHook = ({ }, tempMaskSteps: [], }; - s.prompt.segmentationMask = undefined; + s.prompt.sessionId = undefined; break; case ESubToolItem.AutoSegmentByBox: - s.prompt.creatingMask = { - type: EMaskPromptType.Rect, + s.prompt.creatingPrompt = { + type: EPromptType.Rect, startPoint: point, isPositive: true, }; break; case ESubToolItem.AutoSegmentByClick: - s.prompt.creatingMask = { - type: EMaskPromptType.Point, + s.prompt.creatingPrompt = { + type: EPromptType.Point, startPoint: point, point: point, isPositive: getPromptBoolean(event), }; break; case ESubToolItem.AutoSegmentByStroke: - s.prompt.creatingMask = { - type: EMaskPromptType.Stroke, + s.prompt.creatingPrompt = { + type: EPromptType.Stroke, startPoint: point, stroke: [point], radius: s.brushSize, @@ -735,8 +732,8 @@ const useMask: ToolInstanceHook = ({ }; break; case ESubToolItem.AutoEdgeStitching: - s.prompt.creatingMask = { - type: EMaskPromptType.EdgeStitch, + s.prompt.creatingPrompt = { + type: EPromptType.EdgeStitch, startPoint: point, stroke: [point], radius: s.brushSize, @@ -757,7 +754,7 @@ const useMask: ToolInstanceHook = ({ event, object, }) => { - if (object || drawData.prompt.creatingMask) { + if (object || drawData.prompt.creatingPrompt) { updateMouseCursor('crosshair'); const allowRecordMousePath = [ ESubToolItem.BrushAdd, @@ -782,7 +779,7 @@ const useMask: ToolInstanceHook = ({ ].includes(drawData.selectedSubTool); setDrawData((s) => { if (isCreatingPrompt) { - s.prompt.creatingMask?.stroke?.push(mouse); + s.prompt.creatingPrompt?.stroke?.push(mouse); } else { s.creatingObject?.maskStep?.points.push(mouse); } @@ -810,7 +807,7 @@ const useMask: ToolInstanceHook = ({ }; const finishMaskWhenMouseUp = () => { - if (!drawData.creatingObject && !drawData.prompt.creatingMask) return; + if (!drawData.creatingObject && !drawData.prompt.creatingPrompt) return; const mouse = { x: contentMouse.elementX, y: contentMouse.elementY, @@ -843,75 +840,75 @@ const useMask: ToolInstanceHook = ({ s.creatingObject.maskStep = undefined; } } - s.prompt.segmentationMask = undefined; + s.prompt.sessionId = undefined; }); break; } case ESubToolItem.AutoSegmentByBox: { - if (!drawData.prompt.creatingMask?.startPoint) break; + if (!drawData.prompt.creatingPrompt?.startPoint) break; if ( - mouse.x === drawData.prompt.creatingMask.startPoint?.x || - mouse.y === drawData.prompt.creatingMask.startPoint?.y + mouse.x === drawData.prompt.creatingPrompt.startPoint?.x || + mouse.y === drawData.prompt.creatingPrompt.startPoint?.y ) { - setDrawData((s) => (s.prompt.creatingMask = undefined)); + setDrawData((s) => (s.prompt.creatingPrompt = undefined)); break; } const rect = getRectFromPoints( - drawData.prompt.creatingMask.startPoint as IPoint, + drawData.prompt.creatingPrompt.startPoint as IPoint, mouse, { width: contentMouse.elementW, height: contentMouse.elementH, }, ); - const promptItem: MaskPromptItem = { - type: EMaskPromptType.Rect, + const promptItem: PromptItem = { + type: EPromptType.Rect, isPositive: true, rect, }; setDrawDataWithHistory((s) => { s.prompt.activeRectWhileLoading = rect; }); - const maskPrompts = drawData.prompt.maskPrompts - ? [...drawData.prompt.maskPrompts, promptItem] + const promptsQueue = drawData.prompt.promptsQueue + ? [...drawData.prompt.promptsQueue, promptItem] : [promptItem]; - onAiAnnotation?.({ type: EObjectType.Mask, drawData, maskPrompts }); + onAiAnnotation?.({ type: EObjectType.Mask, drawData, promptsQueue }); break; } case ESubToolItem.AutoSegmentByClick: { if ( !isInCanvas(contentMouse) || !isInCanvas(containerMouse) || - !drawData.prompt.creatingMask?.point + !drawData.prompt.creatingPrompt?.point ) break; - const promptItem: MaskPromptItem = { - type: EMaskPromptType.Point, - isPositive: drawData.prompt.creatingMask.isPositive, - point: drawData.prompt.creatingMask.point, + const promptItem: PromptItem = { + type: EPromptType.Point, + isPositive: drawData.prompt.creatingPrompt.isPositive, + point: drawData.prompt.creatingPrompt.point, }; - const maskPrompts = drawData.prompt.maskPrompts - ? [...drawData.prompt.maskPrompts, promptItem] + const promptsQueue = drawData.prompt.promptsQueue + ? [...drawData.prompt.promptsQueue, promptItem] : [promptItem]; - onAiAnnotation?.({ type: EObjectType.Mask, drawData, maskPrompts }); + onAiAnnotation?.({ type: EObjectType.Mask, drawData, promptsQueue }); break; } case ESubToolItem.AutoSegmentByStroke: { - if (!drawData.prompt.creatingMask?.stroke) break; - const promptItem: MaskPromptItem = { - type: EMaskPromptType.Stroke, - isPositive: drawData.prompt.creatingMask.isPositive, - stroke: drawData.prompt.creatingMask.stroke, + if (!drawData.prompt.creatingPrompt?.stroke) break; + const promptItem: PromptItem = { + type: EPromptType.Stroke, + isPositive: drawData.prompt.creatingPrompt.isPositive, + stroke: drawData.prompt.creatingPrompt.stroke, radius: drawData.brushSize, }; - const maskPrompts = drawData.prompt.maskPrompts - ? [...drawData.prompt.maskPrompts, promptItem] + const promptsQueue = drawData.prompt.promptsQueue + ? [...drawData.prompt.promptsQueue, promptItem] : [promptItem]; - onAiAnnotation?.({ type: EObjectType.Mask, drawData, maskPrompts }); + onAiAnnotation?.({ type: EObjectType.Mask, drawData, promptsQueue }); break; } case ESubToolItem.AutoEdgeStitching: { - if (!drawData.prompt.creatingMask?.stroke) break; + if (!drawData.prompt.creatingPrompt?.stroke) break; onAiAnnotation?.({ type: EObjectType.Mask, drawData }); break; } diff --git a/packages/components/src/Annotator/tools/usePoint.ts b/packages/components/src/Annotator/tools/usePoint.ts new file mode 100644 index 0000000..af131a8 --- /dev/null +++ b/packages/components/src/Annotator/tools/usePoint.ts @@ -0,0 +1,77 @@ +import { drawCircleWithFill } from '../utils/draw'; +import { ToolInstanceHook, ToolHooksFunc } from './base'; + +const usePoint: ToolInstanceHook = ({ canvasRef }) => { + const renderObject: ToolHooksFunc.RenderObject = ({ object, styles }) => { + const { point } = object; + if (point && point.visible) { + const { x, y } = point; + const { strokeColor, fillColor } = styles; + drawCircleWithFill( + canvasRef.current!, + { x, y }, + 4, + fillColor, + 2, + strokeColor, + ); + } + }; + + const renderCreatingObject: ToolHooksFunc.RenderCreatingObject = () => { + // todo + }; + + const renderEditingObject: ToolHooksFunc.RenderEditingObject = () => { + // to do + }; + + const renderPrompt: ToolHooksFunc.RenderPrompt = () => { + // nothing in rect + }; + + const startEditingWhenMouseDown: ToolHooksFunc.StartEditingWhenMouseDown = + () => { + return false; + }; + + const startCreatingWhenMouseDown: ToolHooksFunc.StartCreatingWhenMouseDown = + () => { + return false; + }; + + const updateEditingWhenMouseMove: ToolHooksFunc.UpdateEditingWhenMouseMove = + () => { + return false; + }; + + const updateCreatingWhenMouseMove: ToolHooksFunc.UpdateCreatingWhenMouseMove = + () => { + return false; + }; + + const finishEditingWhenMouseUp: ToolHooksFunc.FinishEditingWhenMouseUp = + () => { + return false; + }; + + const finishCreatingWhenMouseUp: ToolHooksFunc.FinishCreatingWhenMouseUp = + () => { + return false; + }; + + return { + renderObject, + renderCreatingObject, + renderEditingObject, + renderPrompt, + startEditingWhenMouseDown, + startCreatingWhenMouseDown, + updateEditingWhenMouseMove, + updateCreatingWhenMouseMove, + finishEditingWhenMouseUp, + finishCreatingWhenMouseUp, + }; +}; + +export default usePoint; diff --git a/packages/components/src/Annotator/tools/usePolygon.ts b/packages/components/src/Annotator/tools/usePolygon.ts index 2bde58b..bcc9a12 100644 --- a/packages/components/src/Annotator/tools/usePolygon.ts +++ b/packages/components/src/Annotator/tools/usePolygon.ts @@ -2,20 +2,23 @@ import { drawCircleWithFill, drawLine, drawPolygonWithFill, + drawQuadraticPath, + drawRect, + shadeEverythingButRect, } from '../utils/draw'; -import { EElementType, EObjectType } from '../constants'; +import { EElementType, EObjectType, ESubToolItem } from '../constants'; import { getClosestPointOnLineSegment, - getInnerPolygonIndexFromGroup, getLinesFromPolygon, getRectFromPoints, - getReferencePointsFromRect, isInCanvas, isPointOnPoint, movePoint, movePolygon, translateAnnotCoord, translatePointCoord, + translatePolygonCoord, + translateRectCoord, } from '../utils/compute'; import { ToolInstanceHook, @@ -26,14 +29,18 @@ import { import { hexToRgba } from '../utils/color'; import { ANNO_FILL_ALPHA, + ANNO_FILL_COLOR, ANNO_STROKE_ALPHA, + ANNO_STROKE_COLOR, PROMPT_FILL_COLOR, } from '../constants/render'; import { cloneDeep } from 'lodash'; +import { EPromptType, PromptItem } from '../type'; const usePolygon: ToolInstanceHook = ({ editState, clientSize, + naturalSize, imagePos, containerMouse, canvasRef, @@ -42,6 +49,7 @@ const usePolygon: ToolInstanceHook = ({ setEditState, drawData, setDrawData, + setDrawDataWithHistory, updateHistory, updateMouseCursor, updateObject, @@ -95,7 +103,6 @@ const usePolygon: ToolInstanceHook = ({ }); const { polygon } = annotObject; if (polygon && polygon.visible) { - const innerPolygonIdx = getInnerPolygonIndexFromGroup(polygon.group); // draw creating polygon polygon.group.forEach((polygon, polygonIdx) => { if (currIndex === polygonIdx) { @@ -134,28 +141,29 @@ const usePolygon: ToolInstanceHook = ({ } }); } else { - if (!innerPolygonIdx.includes(polygonIdx)) { - drawPolygonWithFill( - activeCanvasRef.current, - polygon, - hexToRgba('#1f4dd8', 0.5), + // draw polygon + drawPolygonWithFill( + activeCanvasRef.current, + polygon, + hexToRgba('#1f4dd8', 0.5), + '#1f4dd8', + 2, + [0], + ); + + // draw points + polygon.forEach((point) => { + drawCircleWithFill( + activeCanvasRef.current!, + point, + 4, + styles.strokeColor, + 3, '#1f4dd8', - 2, - [0], ); - } + }); } }); - innerPolygonIdx.forEach((index) => { - drawPolygonWithFill( - activeCanvasRef.current, - polygon.group[index], - 'rgba(255, 255, 255, 0.8)', - '#1f4dd8', - 2, - [0], - ); - }); } }; @@ -167,35 +175,18 @@ const usePolygon: ToolInstanceHook = ({ }) => { const { polygon } = object; if (polygon && polygon.visible) { - const innerPolygonIdx = getInnerPolygonIndexFromGroup(polygon.group); const isFocusOnPolygon = isFocus && editState.focusEleType === EElementType.Polygon && editState.focusEleIndex === 0; - polygon.group.forEach((polygon, index) => { - if (!innerPolygonIdx.includes(index)) { - const fillColor = isFocusOnPolygon - ? hexToRgba(color, 0.2) - : 'transparent'; - drawPolygonWithFill( - activeCanvasRef.current, - polygon, - fillColor, - styles.strokeColor, - styles.thickness, - styles.strokeDash, - ); - } - }); - - innerPolygonIdx.forEach((index) => { + polygon.group.forEach((polygon) => { const fillColor = isFocusOnPolygon - ? 'rgba(255, 255, 255, 0.8)' + ? hexToRgba(color, 0.2) : 'transparent'; drawPolygonWithFill( activeCanvasRef.current, - polygon.group[index], + polygon, fillColor, styles.strokeColor, styles.thickness, @@ -259,31 +250,168 @@ const usePolygon: ToolInstanceHook = ({ }; const renderPrompt: ToolHooksFunc.RenderPrompt = ({ prompt }) => { - // draw segmentation reference points - if (prompt.segmentationClicks) { - prompt.segmentationClicks.forEach((click) => { - const canvasCoordPoint = translatePointCoord(click.point, { - x: -imagePos.current.x, - y: -imagePos.current.y, - }); - drawCircleWithFill( - activeCanvasRef.current!, - canvasCoordPoint, - 4, - click.isPositive + // draw creating prompt + if (prompt.creatingPrompt) { + const strokeColor = ANNO_STROKE_COLOR.CREATING; + const fillColor = ANNO_FILL_COLOR.CREATING; + switch (prompt.creatingPrompt.type) { + case EPromptType.Rect: { + const { startPoint } = prompt.creatingPrompt; + const rect = getRectFromPoints( + startPoint!, + { + x: contentMouse.elementX, + y: contentMouse.elementY, + }, + { + width: contentMouse.elementW, + height: contentMouse.elementH, + }, + ); + const canvasCoordRect = translateRectCoord(rect, { + x: -imagePos.current.x, + y: -imagePos.current.y, + }); + drawRect( + activeCanvasRef.current, + canvasCoordRect, + strokeColor, + 2, + [0], + fillColor, + ); + break; + } + case EPromptType.Point: { + if (!prompt.creatingPrompt.point) break; + const canvasCoordPoint = translatePointCoord( + prompt.creatingPrompt.point, + { + x: -imagePos.current.x, + y: -imagePos.current.y, + }, + ); + drawCircleWithFill( + activeCanvasRef.current!, + canvasCoordPoint, + 4, + prompt.creatingPrompt.isPositive + ? PROMPT_FILL_COLOR.POSITIVE + : PROMPT_FILL_COLOR.NEGATIVE, + 2, + '#fff', + ); + } + case EPromptType.Stroke: { + if (!prompt.creatingPrompt.stroke || !prompt.creatingPrompt.radius) + break; + const canvasCoordStroke = translatePolygonCoord( + prompt.creatingPrompt.stroke, + { + x: -imagePos.current.x, + y: -imagePos.current.y, + }, + ); + const radius = + (prompt.creatingPrompt.radius * clientSize.width) / + naturalSize.width; + const color = prompt.creatingPrompt.isPositive ? PROMPT_FILL_COLOR.POSITIVE - : PROMPT_FILL_COLOR.NEGATIVE, - 2, - '#fff', + : PROMPT_FILL_COLOR.NEGATIVE; + drawQuadraticPath( + activeCanvasRef.current!, + canvasCoordStroke, + color, + radius, + ); + break; + } + default: + break; + } + + // draw active area while loading ai annotations + if (editState.isRequiring && prompt.activeRectWhileLoading) { + const canvasCoordRect = translateRectCoord( + prompt.activeRectWhileLoading, + { + x: -imagePos.current.x, + y: -imagePos.current.y, + }, ); + shadeEverythingButRect(activeCanvasRef.current!, canvasCoordRect); + } + } + + // draw existing prompts + if (prompt.promptsQueue) { + prompt.promptsQueue.forEach((item) => { + if (item.type === EPromptType.Point) { + const canvasCoordPoint = translatePointCoord(item.point!, { + x: -imagePos.current.x, + y: -imagePos.current.y, + }); + drawCircleWithFill( + activeCanvasRef.current!, + canvasCoordPoint, + 4, + item.isPositive + ? PROMPT_FILL_COLOR.POSITIVE + : PROMPT_FILL_COLOR.NEGATIVE, + 2, + '#fff', + ); + } }); } }; + const updateAiPolygonWhenMouseDown = (event: MouseEvent) => { + const point = { + x: contentMouse.elementX, + y: contentMouse.elementY, + }; + setDrawData((s) => { + switch (s.selectedSubTool) { + case ESubToolItem.AutoSegmentByBox: + s.prompt.creatingPrompt = { + type: EPromptType.Rect, + startPoint: point, + isPositive: true, + }; + break; + case ESubToolItem.AutoSegmentByClick: + s.prompt.creatingPrompt = { + type: EPromptType.Point, + startPoint: point, + point: point, + isPositive: getPromptBoolean(event), + }; + break; + case ESubToolItem.AutoSegmentByStroke: { + s.prompt.creatingPrompt = { + type: EPromptType.Stroke, + startPoint: point, + stroke: [point], + radius: s.brushSize, + isPositive: getPromptBoolean(event), + }; + break; + } + default: { + } + } + }); + }; + const startEditingWhenMouseDown: ToolHooksFunc.StartEditingWhenMouseDown = ({ object, event, }) => { + if (drawData.AIAnnotation) { + updateAiPolygonWhenMouseDown(event); + return true; + } if (event?.button === 2) return false; if ( editBaseElementWhenMouseDown({ @@ -299,18 +427,38 @@ const usePolygon: ToolInstanceHook = ({ }; const startCreatingWhenMouseDown: ToolHooksFunc.StartCreatingWhenMouseDown = - ({ point, basic }) => { + ({ event, point, basic }) => { setDrawData((s) => { if (!s.creatingObject || s.activeObjectIndex > -1) { s.activeObjectIndex = -1; if (s.AIAnnotation) { - // by drawing rectangle under AI mode - s.creatingObject = { - type: EObjectType.Rectangle, - startPoint: point, - ...basic, - color: '#fff', - }; + switch (s.selectedSubTool) { + case ESubToolItem.AutoSegmentByBox: + s.prompt.creatingPrompt = { + type: EPromptType.Rect, + startPoint: point, + isPositive: true, + }; + break; + case ESubToolItem.AutoSegmentByClick: + s.prompt.creatingPrompt = { + type: EPromptType.Point, + startPoint: point, + point: point, + isPositive: getPromptBoolean(event), + }; + break; + case ESubToolItem.AutoSegmentByStroke: { + s.prompt.creatingPrompt = { + type: EPromptType.Stroke, + startPoint: point, + stroke: [point], + radius: s.brushSize, + isPositive: getPromptBoolean(event), + }; + break; + } + } } else { // create a new polygon manually s.creatingObject = { @@ -322,12 +470,7 @@ const usePolygon: ToolInstanceHook = ({ currIndex: 0, ...basic, }; - updateHistory( - cloneDeep({ - drawData: s, - clientSize, - }), - ); + updateHistory(cloneDeep(drawData)); } } else { if (!s.AIAnnotation) { @@ -340,31 +483,50 @@ const usePolygon: ToolInstanceHook = ({ s.creatingObject.currIndex = -1; } else if (s.creatingObject.polygon) { polygon.group[currIndex].push(point); - updateHistory( - cloneDeep({ - drawData: s, - clientSize, - }), - ); + updateHistory(cloneDeep(s)); } } else { polygon.group.push([point]); s.creatingObject.currIndex = polygon.group.length - 1; - updateHistory( - cloneDeep({ - drawData: s, - clientSize, - }), - ); + updateHistory(cloneDeep(s)); } + } else { + updateAiPolygonWhenMouseDown(event); } } }); return true; }; + const updatePolygonWhenMouseMove: ToolHooksFunc.UpdateCreatingWhenMouseMove = + ({ event }) => { + const allowRecordMousePath = + drawData.selectedSubTool === ESubToolItem.AutoSegmentByStroke; + // Left/Right button is pressed while mousemove + const isMousePress = event.buttons === 1 || event.buttons === 2; + if ( + drawData.prompt.creatingPrompt && + allowRecordMousePath && + isMousePress + ) { + const mouse = { + x: contentMouse.elementX, + y: contentMouse.elementY, + }; + setDrawData((s) => { + s.prompt.creatingPrompt?.stroke?.push(mouse); + }); + return true; + } + return false; + }; + const updateEditingWhenMouseMove: ToolHooksFunc.UpdateEditingWhenMouseMove = - () => { + ({ event }) => { + if (drawData.AIAnnotation) { + updateMouseCursor('crosshair'); + return updatePolygonWhenMouseMove({ event }); + } const { focusEleType, focusEleIndex, @@ -442,134 +604,174 @@ const usePolygon: ToolInstanceHook = ({ }; const updateCreatingWhenMouseMove: ToolHooksFunc.UpdateCreatingWhenMouseMove = - ({ object }) => { - return !!object; + ({ event }) => { + return updatePolygonWhenMouseMove({ event }); }; - const finishEditingWhenMouseUp: ToolHooksFunc.FinishEditingWhenMouseUp = ({ - object, - }) => { - const isResizingOrMoving = - editState.startRectResizeAnchor || editState.startElementMovePoint; + const getExistPolygonPrompts = (): PromptItem[] => { + if ( + drawData.prompt.promptsQueue && + drawData.prompt.promptsQueue.length > 0 + ) { + return drawData.prompt.promptsQueue; + } else { + // add exsit polygon as prompt item while editing instance by ai + const addExistPolygon = + !drawData.prompt.sessionId && drawData.creatingObject; - const isMouseStand = - editState.startElementMovePoint && - editState.startElementMovePoint.initPoint?.x === contentMouse.elementX && - editState.startElementMovePoint.initPoint?.y === contentMouse.elementY; + if (addExistPolygon) { + const existPolygons = + drawData.creatingObject?.polygon?.group.map((polygon) => { + return polygon.reduce((acc: number[], point) => { + return acc.concat([point.x, point.y]); + }, []); + }) || []; - const isRemovePolygonPoints = - isMouseStand && - editState.focusPolygonInfo.index > -1 && - editState.focusPolygonInfo.pointIndex > -1; + const modifyPromptItem: PromptItem = { + type: EPromptType.Modify, + isPositive: true, + polygons: existPolygons, + }; - if (isRemovePolygonPoints) { - const copyObject = cloneDeep(object); - const { index, pointIndex } = editState.focusPolygonInfo; - const polygon = copyObject.polygon?.group[index]; - if (polygon && index > -1 && pointIndex > -1 && polygon.length >= 3) { - polygon.splice(pointIndex, 1); + return [modifyPromptItem]; + } else { + return []; } - updateObject(copyObject, drawData.activeObjectIndex); - } else if (isResizingOrMoving) { - updateObject(object, drawData.activeObjectIndex); } - - setEditState((s) => { - s.startRectResizeAnchor = undefined; - s.startElementMovePoint = undefined; - }); - return true; }; - const finishCreatingWhenMouseUp: ToolHooksFunc.FinishCreatingWhenMouseUp = ({ - event, - object, - }) => { - if (!object) return false; - + const finishAiPolygonWhenMouseUp = () => { const mouse = { x: contentMouse.elementX, y: contentMouse.elementY, }; - if (drawData.AIAnnotation) { - if (object.type === EObjectType.Polygon) { - if (!isInCanvas(contentMouse) || !isInCanvas(containerMouse)) - return false; - // add reference points - const click = { - isPositive: getPromptBoolean(event), - point: mouse, + const existPrompts = getExistPolygonPrompts(); + switch (drawData.selectedSubTool) { + case ESubToolItem.AutoSegmentByBox: { + if (!drawData.prompt.creatingPrompt?.startPoint) break; + if ( + mouse.x === drawData.prompt.creatingPrompt.startPoint?.x || + mouse.y === drawData.prompt.creatingPrompt.startPoint?.y + ) { + setDrawData((s) => (s.prompt.creatingPrompt = undefined)); + break; + } + const rect = getRectFromPoints( + drawData.prompt.creatingPrompt.startPoint as IPoint, + mouse, + { + width: contentMouse.elementW, + height: contentMouse.elementH, + }, + ); + const promptItem: PromptItem = { + type: EPromptType.Rect, + isPositive: true, + rect, }; - const existClicks = drawData.prompt.segmentationClicks || []; - setDrawData((s) => { - s.prompt.segmentationClicks = [...existClicks, click]; + setDrawDataWithHistory((s) => { + s.prompt.activeRectWhileLoading = rect; }); + const promptsQueue = [...existPrompts, promptItem]; onAiAnnotation?.({ type: EObjectType.Polygon, drawData, - segmentationClicks: [...existClicks, click], - aiLabels: [object.label], + promptsQueue, }); - } else { - // first click + break; + } + case ESubToolItem.AutoSegmentByClick: { if ( - contentMouse.elementX === object.startPoint?.x && - contentMouse.elementY === object.startPoint?.y - ) { - if (!isInCanvas(contentMouse)) return false; - // draw point - const firstClick = { - isPositive: true, - point: mouse, - }; - setDrawData((s) => { - s.prompt.segmentationClicks = [firstClick]; - }); - onAiAnnotation?.({ - type: EObjectType.Polygon, - drawData, - segmentationClicks: [firstClick], - }); - } else { - // draw bbox - const rect = getRectFromPoints(object.startPoint as IPoint, mouse, { - width: contentMouse.elementW, - height: contentMouse.elementH, - }); - const points = getReferencePointsFromRect(rect); - const bbox = { - xmin: rect.x, - ymin: rect.y, - xmax: rect.x + rect.width, - ymax: rect.y + rect.height, - }; - const clicks = points.map((point, index) => { - return { - // Only the center point is positive - isPositive: index === points.length - 1 ? true : false, - point, - }; - }); - setDrawData((s) => { - s.prompt.segmentationClicks = [...clicks]; - }); - onAiAnnotation?.({ - type: EObjectType.Polygon, - drawData, - segmentationClicks: clicks, - bbox, - }); + !isInCanvas(contentMouse) || + !isInCanvas(containerMouse) || + !drawData.prompt.creatingPrompt?.point + ) + break; + const promptItem: PromptItem = { + type: EPromptType.Point, + isPositive: drawData.prompt.creatingPrompt.isPositive, + point: drawData.prompt.creatingPrompt.point, + }; + const promptsQueue = [...existPrompts, promptItem]; + onAiAnnotation?.({ + type: EObjectType.Polygon, + drawData, + promptsQueue, + }); + break; + } + case ESubToolItem.AutoSegmentByStroke: { + if (!drawData.prompt.creatingPrompt?.stroke) break; + const promptItem: PromptItem = { + type: EPromptType.Stroke, + isPositive: drawData.prompt.creatingPrompt.isPositive, + stroke: drawData.prompt.creatingPrompt.stroke, + radius: drawData.brushSize, + }; + const promptsQueue = [...existPrompts, promptItem]; + onAiAnnotation?.({ + type: EObjectType.Polygon, + drawData, + promptsQueue, + }); + break; + } + } + }; + + const finishEditingWhenMouseUp: ToolHooksFunc.FinishEditingWhenMouseUp = ({ + object, + }) => { + if (drawData.AIAnnotation) { + finishAiPolygonWhenMouseUp(); + } else { + const isResizingOrMoving = + editState.startRectResizeAnchor || editState.startElementMovePoint; + + const isMouseStand = + editState.startElementMovePoint && + editState.startElementMovePoint.initPoint?.x === + contentMouse.elementX && + editState.startElementMovePoint.initPoint?.y === contentMouse.elementY; + + const isRemovePolygonPoints = + isMouseStand && + editState.focusPolygonInfo.index > -1 && + editState.focusPolygonInfo.pointIndex > -1; + + if (isRemovePolygonPoints) { + const copyObject = cloneDeep(object); + const { index, pointIndex } = editState.focusPolygonInfo; + const polygon = copyObject.polygon?.group[index]; + if (polygon && index > -1 && pointIndex > -1 && polygon.length >= 3) { + polygon.splice(pointIndex, 1); } - setDrawData((s) => (s.creatingObject = undefined)); + updateObject(copyObject, drawData.activeObjectIndex); + } else if (isResizingOrMoving) { + updateObject(object, drawData.activeObjectIndex); } + + setEditState((s) => { + s.startRectResizeAnchor = undefined; + s.startElementMovePoint = undefined; + }); + } + return true; + }; + + const finishCreatingWhenMouseUp: ToolHooksFunc.FinishCreatingWhenMouseUp = ({ + object, + }) => { + if (drawData.AIAnnotation) { + finishAiPolygonWhenMouseUp(); } else { - if (object.currIndex === -1) { - const { polygon, type, hidden, label, status, color } = object; + if (object && object.currIndex === -1) { + const { polygon, type, hidden, labelId, status, color } = object; const newObject = { polygon, type, hidden, - label, + labelId, status, color, }; diff --git a/packages/components/src/Annotator/tools/useRectangle.ts b/packages/components/src/Annotator/tools/useRectangle.ts index 4424a1d..270ce43 100644 --- a/packages/components/src/Annotator/tools/useRectangle.ts +++ b/packages/components/src/Annotator/tools/useRectangle.ts @@ -1,6 +1,15 @@ -import { drawRect, drawText, shadeEverythingButRect } from '../utils/draw'; -import { EObjectType } from '../constants'; -import { getRectFromPoints, translateRectCoord } from '../utils/compute'; +import { + drawCircleWithFill, + drawRect, + drawText, + shadeEverythingButRect, +} from '../utils/draw'; +import { EnumModelType, EObjectType, ESubToolItem } from '../constants'; +import { + getRectFromPoints, + translatePointCoord, + translateRectCoord, +} from '../utils/compute'; import { ToolInstanceHook, ToolHooksFunc, @@ -8,9 +17,13 @@ import { editBaseElementWhenMouseDown, updateEditingRectWhenMouseMove, } from './base'; -import { EObjectStatus } from '../type'; +import { EObjectStatus, EPromptType, PromptItem } from '../type'; import { hexToRgba } from '../utils/color'; -import { ANNO_FILL_ALPHA } from '../constants/render'; +import { + ANNO_FILL_ALPHA, + PROMPT_FILL_COLOR, + PROMPT_STROKE_COLOR, +} from '../constants/render'; const useRectangle: ToolInstanceHook = ({ contentMouse, @@ -26,6 +39,8 @@ const useRectangle: ToolInstanceHook = ({ addObject, getAnnotColor, displayOptionsResult, + categories, + onAiAnnotation, }) => { const renderObject: ToolHooksFunc.RenderObject = ({ object, @@ -42,10 +57,14 @@ const useRectangle: ToolInstanceHook = ({ if (drawData.isBatchEditing) { if ( object.status === EObjectStatus.Unchecked && - !editState.isCtrlPressed + (!editState.isCtrlPressed || + drawData.selectedModel === EnumModelType.IVP) ) return; - if (editState.isCtrlPressed) { + if ( + editState.isCtrlPressed && + drawData.selectedModel === EnumModelType.Detection + ) { if (object.status !== EObjectStatus.Unchecked) { strokeColor = hexToRgba(color, 0.8); strokeDash = [2]; @@ -69,10 +88,12 @@ const useRectangle: ToolInstanceHook = ({ // draw text if (displayOptionsResult?.showBoxText) { + const labelName = + categories.find((c) => c.id === object.labelId)?.name || ''; const label = object?.conf && object.conf > 0 && object.conf < 1 - ? `${object.label}(${object.conf.toFixed(3)})` - : object.label; + ? `${labelName}(${object.conf.toFixed(3)})` + : labelName; drawText( canvasRef.current!, label || '', @@ -142,8 +163,112 @@ const useRectangle: ToolInstanceHook = ({ } }; - const renderPrompt: ToolHooksFunc.RenderPrompt = () => { - // nothing in rect + const renderPrompt: ToolHooksFunc.RenderPrompt = ({ prompt }) => { + // draw creating prompt + if (prompt.creatingPrompt) { + const strokeColor = prompt.creatingPrompt.isPositive + ? PROMPT_STROKE_COLOR.POSITIVE + : PROMPT_STROKE_COLOR.NEGATIVE; + const fillColor = prompt.creatingPrompt.isPositive + ? PROMPT_FILL_COLOR.POSITIVE + : PROMPT_FILL_COLOR.NEGATIVE; + + switch (prompt.creatingPrompt.type) { + case EPromptType.Rect: { + const { startPoint } = prompt.creatingPrompt; + const rect = getRectFromPoints( + startPoint!, + { + x: contentMouse.elementX, + y: contentMouse.elementY, + }, + { + width: contentMouse.elementW, + height: contentMouse.elementH, + }, + ); + const canvasCoordRect = translateRectCoord(rect, { + x: -imagePos.current.x, + y: -imagePos.current.y, + }); + drawRect( + activeCanvasRef.current, + canvasCoordRect, + strokeColor, + 2, + [0], + fillColor, + ); + break; + } + case EPromptType.Point: { + if (!prompt.creatingPrompt.point) break; + const canvasCoordPoint = translatePointCoord( + prompt.creatingPrompt.point, + { + x: -imagePos.current.x, + y: -imagePos.current.y, + }, + ); + drawCircleWithFill( + activeCanvasRef.current!, + canvasCoordPoint, + 4, + prompt.creatingPrompt.isPositive + ? PROMPT_FILL_COLOR.POSITIVE + : PROMPT_FILL_COLOR.NEGATIVE, + 2, + '#fff', + ); + } + default: + break; + } + } + + // draw existing prompts + if (prompt.promptsQueue) { + prompt.promptsQueue.forEach((item) => { + switch (item.type) { + case EPromptType.Rect: { + const canvasCoordRect = translateRectCoord(item.rect!, { + x: -imagePos.current.x, + y: -imagePos.current.y, + }); + drawRect( + activeCanvasRef.current, + canvasCoordRect, + item.isPositive + ? PROMPT_STROKE_COLOR.POSITIVE + : PROMPT_STROKE_COLOR.NEGATIVE, + 2, + [0], + item.isPositive + ? PROMPT_FILL_COLOR.POSITIVE + : PROMPT_FILL_COLOR.NEGATIVE, + ); + break; + } + case EPromptType.Point: { + const canvasCoordPoint = translatePointCoord(item.point!, { + x: -imagePos.current.x, + y: -imagePos.current.y, + }); + drawCircleWithFill( + activeCanvasRef.current!, + canvasCoordPoint, + 4, + item.isPositive + ? PROMPT_FILL_COLOR.POSITIVE + : PROMPT_FILL_COLOR.NEGATIVE, + 2, + '#fff', + ); + break; + } + } + }); + } }; const startEditingWhenMouseDown: ToolHooksFunc.StartEditingWhenMouseDown = ({ @@ -167,12 +292,21 @@ const useRectangle: ToolInstanceHook = ({ const startCreatingWhenMouseDown: ToolHooksFunc.StartCreatingWhenMouseDown = ({ point, basic }) => { setDrawData((s) => { - s.activeObjectIndex = -1; - s.creatingObject = { - type: EObjectType.Rectangle, - startPoint: point, - ...basic, - }; + if (s.AIAnnotation && s.selectedModel === EnumModelType.IVP) { + s.prompt.creatingPrompt = { + type: EPromptType.Rect, + startPoint: point, + point, + isPositive: s.selectedSubTool !== ESubToolItem.NegativeVisualPrompt, + }; + } else { + s.activeObjectIndex = -1; + s.creatingObject = { + type: EObjectType.Rectangle, + startPoint: point, + ...basic, + }; + } }); return true; }; @@ -212,6 +346,64 @@ const useRectangle: ToolInstanceHook = ({ const finishCreatingWhenMouseUp: ToolHooksFunc.FinishCreatingWhenMouseUp = ({ object, }) => { + const mouse = { + x: contentMouse.elementX, + y: contentMouse.elementY, + }; + if ( + drawData.AIAnnotation && + drawData.selectedModel === EnumModelType.IVP && + drawData.prompt.creatingPrompt?.startPoint + ) { + const { startPoint } = drawData.prompt.creatingPrompt; + if (mouse.x === startPoint.x || mouse.y === startPoint.y) { + setDrawData((s) => { + s.prompt.creatingPrompt = undefined; + }); + return true; + // TODO + // if (!isInCanvas(contentMouse)) return false; + // const promptItem: PromptItem = { + // type: EPromptType.Point, + // isPositive: drawData.prompt.creatingPrompt.isPositive, + // point: startPoint, + // }; + // const promptsQueue = [ + // ...(drawData.prompt.promptsQueue || []), + // promptItem, + // ]; + // onAiAnnotation?.({ + // type: EObjectType.Rectangle, + // drawData, + // promptsQueue, + // }); + // return true; + } else { + const rect = getRectFromPoints( + drawData.prompt.creatingPrompt.startPoint as IPoint, + mouse, + { + width: contentMouse.elementW, + height: contentMouse.elementH, + }, + ); + const promptItem: PromptItem = { + type: EPromptType.Rect, + isPositive: drawData.prompt.creatingPrompt.isPositive, + rect, + }; + const promptsQueue = [ + ...(drawData.prompt.promptsQueue || []), + promptItem, + ]; + onAiAnnotation?.({ + type: EObjectType.Rectangle, + drawData, + promptsQueue, + }); + } + return true; + } if (!object || !object.startPoint) return false; // Need to check if it can form a rectangle if ( @@ -233,12 +425,12 @@ const useRectangle: ToolInstanceHook = ({ ); const newObject = { type: EObjectType.Rectangle, - label: object.label, + labelId: object.labelId, hidden: false, rect: { visible: true, ...newRect }, conf: 1, status: EObjectStatus.Commited, - color: getAnnotColor(object.label), + color: getAnnotColor(object.labelId), }; addObject(newObject); return true; diff --git a/packages/components/src/Annotator/tools/useSkeleton.ts b/packages/components/src/Annotator/tools/useSkeleton.ts index 64b788a..e1cf86b 100644 --- a/packages/components/src/Annotator/tools/useSkeleton.ts +++ b/packages/components/src/Annotator/tools/useSkeleton.ts @@ -367,7 +367,7 @@ const useSkeleton: ToolInstanceHook = ({ const updatedObjs = getKeypointsFromRect(pointObjs, newRect); const newObject = { type: EObjectType.Skeleton, - label: object.label, + labelId: object.labelId, hidden: false, color: object.color, rect: { visible: true, ...newRect }, diff --git a/packages/components/src/Annotator/type.ts b/packages/components/src/Annotator/type.ts index 70b9678..a66dfaa 100644 --- a/packages/components/src/Annotator/type.ts +++ b/packages/components/src/Annotator/type.ts @@ -1,18 +1,43 @@ import { EBasicToolItem, EElementType, + ELabelType, + EnumModelType, EObjectType, ESubToolItem, EToolType, } from './constants'; import { RectAnchor } from './utils/compute'; +export enum EActionType { + Radio = 'radio', + Checkbox = 'checkbox', + Text = 'text', +} + +export interface IAttribute { + field: string; + type: EActionType; + required: boolean; + options?: { label: string }[]; +} + +export type IAttributeValue = string | number | number[] | null; + export interface Category { id: string; name: string; + labelName?: string; + labelType?: ELabelType; + renderColor?: string; + description?: string; + attributes?: IAttribute[]; + valueType?: EActionType; + valueOptions?: { label: string }[]; } export interface BaseObject { + id?: string; /** catagory */ categoryId?: string; categoryName?: string; @@ -22,7 +47,8 @@ export interface BaseObject { /** matting url */ alpha?: string; /** - * keypoints：[x, y, z, w, visible, conf, ...]. (Needs to be split manually.) + * keypoints: [x, y, visible, conf, ...] + * (old mode)keypoints：[x, y, z, w, visible, conf, ...]. (Needs to be split manually.) * visible 0: not labeled, v=1: labeled but not visible, and v=2: labeled and visible. */ points?: number[]; @@ -33,21 +59,19 @@ export interface BaseObject { lines?: number[]; /** mask */ mask?: number[]; + /** point */ + point?: number[]; } export interface DrawObject extends BaseObject { conf?: number; - labelId?: string; - compareResult?: string; + // custom styles + customStyles?: Record; } -export interface DrawImageData { +export interface AnnoItem extends Record { id: string; url: string; - urlFullRes: string; - objects: DrawObject[]; - metadata?: Record; - caption?: string; } export enum EObjectStatus { @@ -56,25 +80,38 @@ export enum EObjectStatus { Commited, } +export interface VideoFramesData { + id: string; + list: AnnoItem[]; + objects: IAnnotationObject[][]; // objects[objectIndex][frameIndex] + activeIndex: number; +} + export interface IAnnotationObject { type: EObjectType; - label: string; + labelId: string; hidden: boolean; - color: string; // hex + color: string; + customStyles?: Record; + attributes?: IAttributeValue[]; + status: EObjectStatus; + + // value rect?: IElement; polygon?: IElement; keypoints?: { points: IElement[]; lines: number[]; }; + point?: IElement; maskRle?: number[]; maskCanvasElement?: any; alpha?: string; alphaImageElement?: any; conf?: number; - labelId?: string; - compareResult?: string; - status: EObjectStatus; + + // for video frame attribute + frameEmpty?: boolean; } export interface ICreatingMaskStep { @@ -97,34 +134,43 @@ export interface ICreatingObject extends IAnnotationObject { tempMaskSteps?: ICreatingMaskStep[]; } -export enum EMaskPromptType { +export enum EPromptType { Rect = 'rect', Point = 'point', Stroke = 'stroke', EdgeStitch = 'edgeStitch', + Modify = 'modify', } -export type MaskPromptItem = { - type: EMaskPromptType; +export type PromptItem = { + type: EPromptType; isPositive: boolean; + /** Rect */ startPoint?: IPoint; rect?: IRect; + /** Point */ point?: IPoint; + /** Stroke / EdgeStitching */ stroke?: IPoint[]; radius?: number; + /** Modify */ + polygons?: number[][]; }; export interface IPrompt { - creatingMask?: MaskPromptItem; - maskPrompts?: MaskPromptItem[]; - segmentationClicks?: { - point: IPoint; - isPositive: boolean; - }[]; - segmentationMask?: string; + creatingPrompt?: PromptItem; + promptsQueue?: PromptItem[]; + sessionId?: string; activeRectWhileLoading?: IRect; } +export interface IEditingAttribute { + index: number; // Object Index || -1 + labelId: string; + attributes: IAttribute[]; + values?: IAttributeValue[]; +} + /** * Need to be saved in history */ @@ -135,16 +181,24 @@ export interface DrawData { selectedTool: EToolType; selectedSubTool: ESubToolItem; AIAnnotation: boolean; + selectedModel?: EnumModelType; brushSize: number; + pointResolution: number; /** drawed */ objectList: IAnnotationObject[]; + classifications: { + labelId: string; + labelValue: IAttributeValue; + attributes?: IAttributeValue[]; + }[]; /** drawing */ activeClassName: string; activeObjectIndex: number; creatingObject?: ICreatingObject; // - editing / creating isBatchEditing: boolean; // active while handle batch predictions by model + editingAttribute?: IEditingAttribute; limitConf: number; /** prompt actions */ @@ -166,7 +220,7 @@ export interface EditState { isLoadingError: boolean; isRequiring: boolean; allowMove: boolean; - latestLabel: string; + latestLabelId: string; startRectResizeAnchor?: RectAnchor; startElementMovePoint?: { topLeftPoint: IPoint; @@ -183,6 +237,8 @@ export interface EditState { lineIndex: number; }; imageCacheId?: string; + // TODO + imageCacheIdForPolygon?: string; isCtrlPressed: boolean; hideCreatingObject: boolean; imageDisplayOptions: IImageDisplayOptions; @@ -195,26 +251,24 @@ export const enum EditorMode { Review, } -export enum EQaAction { - Accept = 'accept', - Reject = 'reject', - ForceAccept = 'force_accept', -} - export const DEFAULT_DRAW_DATA: DrawData = { initialized: false, /** Selected tool */ selectedTool: EBasicToolItem.Drag, selectedSubTool: ESubToolItem.PenAdd, + selectedModel: undefined, AIAnnotation: false, /** drawed */ objectList: [], + classifications: [], activeObjectIndex: -1, activeClassName: '', creatingObject: undefined, + editingAttribute: undefined, brushSize: 20, + pointResolution: 0.5, prompt: {}, isBatchEditing: false, limitConf: 0, @@ -235,7 +289,7 @@ export const DEFAULT_EDIT_STATE: EditState = { isLoadingError: false, isRequiring: false, allowMove: false, - latestLabel: '', + latestLabelId: '', startRectResizeAnchor: undefined, startElementMovePoint: undefined, focusObjectIndex: -1, diff --git a/packages/components/src/Annotator/utils/base64.ts b/packages/components/src/Annotator/utils/base64.ts index 7dac5e3..22a3320 100644 --- a/packages/components/src/Annotator/utils/base64.ts +++ b/packages/components/src/Annotator/utils/base64.ts @@ -42,6 +42,11 @@ export const isBlobUrl = (str: string) => { return blobUrlRegex.test(str); }; +export const isHttpsUrl = (str: string) => { + const httpsRegex = /^https?:\/\//i; + return httpsRegex.test(str); +}; + export const getImgBase64ByBlob = (blobUrl: Blob) => { return new Promise((resolve, reject) => { const fileReader = new FileReader(); diff --git a/packages/components/src/Annotator/utils/color.ts b/packages/components/src/Annotator/utils/color.ts index a371ec5..eb0adde 100644 --- a/packages/components/src/Annotator/utils/color.ts +++ b/packages/components/src/Annotator/utils/color.ts @@ -83,19 +83,10 @@ export const createColorList = (count: number) => { return colors; }; -export const getCategoryColors = (list: string[], cur?: string) => { +export const getCategoryColors = (list: string[]) => { if (!list.length) return {}; const sortList = [...list]; - if (cur === 'All') { - sortList.shift(); - } else if (cur) { - // Move cur to the first position in the array. - const curIndex = sortList.findIndex((item) => item === cur); - sortList.splice(curIndex, 1); - sortList[0] = cur; - } - const colors = createColorList(sortList.length); const result: Record = {}; sortList.forEach((item, index) => { diff --git a/packages/components/src/Annotator/utils/compute.ts b/packages/components/src/Annotator/utils/compute.ts index 7ef7f63..1493390 100644 --- a/packages/components/src/Annotator/utils/compute.ts +++ b/packages/components/src/Annotator/utils/compute.ts @@ -1,18 +1,12 @@ import { - AnnotationType, EElementType, EObjectType, KEYPOINTS_VISIBLE_TYPE, } from '../constants'; -import { - BaseObject, - DrawData, - IAnnotationObject, - MaskPromptItem, -} from '../type'; +import { DrawData, IAnnotationObject, PromptItem } from '../type'; import { CursorState } from 'ahooks/lib/useMouse'; import { rgbArrayToRgba, rgbaToRgbArray } from './color'; -import { cloneDeep, isNumber } from 'lodash'; +import { cloneDeep, isEqual, isNumber } from 'lodash'; /** * Calculate the scaled width and height. @@ -116,6 +110,27 @@ export const getSegmentationPoints = ( return groups; }; +export const translatePointGroupsToPoints = ( + pointGroups: number[][], + naturalSize: ISize, + clientSize: ISize, +): IPoint[][] => { + const groups: IPoint[][] = []; + pointGroups.forEach((nums) => { + const points = []; + for (let i = 0; i < nums.length; i += 2) { + const point = getCanvasPoint( + [nums[i], nums[i + 1]], + naturalSize, + clientSize, + ); + points.push(point); + } + groups.push(points); + }); + return groups; +}; + /** * translate points to rect * @param startPoint @@ -270,6 +285,22 @@ export const translateRectZoom = ( height: (rect.height * toSize.height) / fromSize.height, }); +/** + * translate rect to points + * @param theRect + * @param fromSize + * @param toSize + * @returns + */ +export const translateRectToPointsArray = ( + theRect: IRect, + fromSize: ISize, + toSize: ISize, +): number[] => { + const rect = translateRectZoom(theRect, fromSize, toSize); + return [rect.x, rect.y, rect.x + rect.width, rect.y + rect.height]; +}; + /** * zoom point size * @param point @@ -285,6 +316,25 @@ export const translatePointZoom = ( y: (point.y * toSize.height) / formSize.height, }); +/** + * transtlate points to rect + * @param box + * @param size + * @returns + */ +export const translatePointsToRect = ( + points: [number, number, number, number], + formSize: ISize, + toSize: ISize, +): IRect => ({ + x: ((points[0] || 0) / formSize.width) * toSize.width, + y: ((points[1] || 0) / formSize.height) * toSize.height, + width: + (((points[2] || 0) - (points[0] || 0)) / formSize.width) * toSize.width, + height: + (((points[3] || 0) - (points[1] || 0)) / formSize.height) * toSize.height, +}); + /** * transtlate bounding box to rect * @param box @@ -310,6 +360,8 @@ export const translateAbsBBoxToRect = (box: IBoundingBox): IRect => ({ /** * format points + * keypoints: [x, y, z, w, visible, conf, ...] + * visible 0: not labeled, v=1: labeled but not visible, and v=2: labeled and visible. * @param box * @param size * @returns @@ -371,6 +423,71 @@ export const translatePointObjsToPointAttrs = ( }; }; +/** + * format points (new model) + * keypoints: [x, y, visible, conf, ...] + * visible 0: not labeled, v=1: labeled but not visible, and v=2: labeled and visible. + * @param box + * @param size + * @returns + */ +export const newTranslatePointsToPointObjs = ( + points: number[], + pointNames: string[], + pointColors: string[], + naturalSize: ISize, + clientSize: ISize, +): IElement[] => { + const pointList = []; + for (let i = 0; i * 4 < points.length; i++) { + const { x, y } = getCanvasPoint( + [points[i * 4], points[i * 4 + 1]], + naturalSize, + clientSize, + ); + const color = rgbArrayToRgba(pointColors.slice(i * 3, i * 3 + 3), 1); + const point = { + x, + y, + visible: points[i * 4 + 2], + color, + name: pointNames[i], + }; + pointList.push(point); + } + return pointList; +}; + +export const newTranslatePointObjsToPointAttrs = ( + pointObjs: IElement[], + naturalSize: ISize, + clientSize: ISize, +): { + points: number[]; + pointNames: string[]; + pointColors: string[]; +} => { + const points = []; + const pointNames = []; + const pointColors = []; + + for (let i = 0; i < pointObjs.length; i++) { + const point = pointObjs[i]; + const { x, y } = point; + const rgb = rgbaToRgbArray(point.color!); + const naturalPoint = getNaturalPoint([x, y], naturalSize, clientSize); + points.push(naturalPoint.x, naturalPoint.y, point.visible, 1); + pointNames.push(point.name!); + pointColors.push(rgb[0] || '255', rgb[1] || '255', rgb[2] || '255'); + } + + return { + points, + pointNames, + pointColors, + }; +}; + /** * Determine if two rects are the same.（Only compare the decimal places after the second digit） * @param aRect @@ -541,7 +658,7 @@ export const judgeFocusOnSingleObject = ( object: IAnnotationObject, clientSize?: ISize, ): boolean => { - if (object.hidden) { + if (object.hidden || object.frameEmpty) { return false; } @@ -1074,43 +1191,33 @@ export const isValidRect = (rect: IRect) => { }; // TODO: How to confirm ObjectType -export const getObjectType = ( - obj: IAnnotationObject, - displayType?: AnnotationType, -): EObjectType => { - if (obj.maskRle && (!displayType || displayType === AnnotationType.Mask)) { +export const getObjectType = (obj: IAnnotationObject): EObjectType => { + if (obj.maskRle) { return EObjectType.Mask; } - if (obj.alpha && (!displayType || displayType === AnnotationType.Matting)) { + if (obj.alpha) { return EObjectType.Matting; } - if ( - obj.keypoints && - (!displayType || displayType === AnnotationType.KeyPoints) - ) { + if (obj.keypoints) { return EObjectType.Skeleton; } - if ( - obj.polygon && - (!displayType || displayType === AnnotationType.Segmentation) - ) { + if (obj.polygon) { return EObjectType.Polygon; } - if ( - obj.rect && - isValidRect(obj.rect) && - (!displayType || displayType === AnnotationType.Detection) - ) { + if (obj.point) { + return EObjectType.Point; + } + if (obj.rect && isValidRect(obj.rect)) { return EObjectType.Rectangle; } return EObjectType.Custom; }; -export const translatePolygonsToSegmentation = ( +export const translatePolygonsToPointsArrayGroup = ( polygons: IElement, naturalSize: ISize, clientSize: ISize, -): string => { +): number[][] => { const arr = polygons.group.map((polygon) => { return polygon.reduce((acc: number[], point: IPoint) => { const { x, y } = point; @@ -1118,7 +1225,19 @@ export const translatePolygonsToSegmentation = ( return acc.concat([naturalPoint.x, naturalPoint.y]); }, []); }); + return arr; +}; +export const translatePolygonsToSegmentation = ( + polygons: IElement, + naturalSize: ISize, + clientSize: ISize, +): string => { + const arr = translatePolygonsToPointsArrayGroup( + polygons, + naturalSize, + clientSize, + ); const res = arr .map((polygon) => { @@ -1129,55 +1248,6 @@ export const translatePolygonsToSegmentation = ( return res; }; -export const translateObjectsToAnnotations = ( - objectList: IAnnotationObject[], - naturalSize: ISize, - clientSize: ISize, - needNormalizeBbox: boolean = true, -): BaseObject[] => { - const annotations = objectList.map((obj) => { - const { label, rect, keypoints, polygon, maskRle } = obj; - const annoObj = { - categoryName: label, - }; - if (rect) { - Object.assign(annoObj, { - boundingBox: needNormalizeBbox - ? translateRectToBoundingBox(rect, clientSize) - : translateRectToAbsBbox(rect), - }); - } - if (keypoints) { - Object.assign(annoObj, { - lines: keypoints.lines, - ...translatePointObjsToPointAttrs( - keypoints.points, - naturalSize, - clientSize, - ), - }); - } - if (polygon) { - const segmentation = translatePolygonsToSegmentation( - polygon, - naturalSize, - clientSize, - ); - Object.assign(annoObj, { - segmentation, - }); - } - if (maskRle) { - Object.assign(annoObj, { - mask: maskRle, - }); - } - return annoObj; - }); - - return annotations; -}; - export const getClosestPointOnLineSegment = ( point: IPoint, lineStart: IPoint, @@ -1346,7 +1416,7 @@ export const translateAnnotCoord = ( annoObj: IAnnotationObject, newCoordOrigin: IPoint, ): IAnnotationObject => { - const { rect, polygon, keypoints } = annoObj; + const { rect, polygon, keypoints, point } = annoObj; const newAnnoObj = { ...annoObj }; if (rect) { @@ -1379,6 +1449,13 @@ export const translateAnnotCoord = ( }; } + if (point) { + newAnnoObj.point = { + ...point, + ...translatePointCoord(point, newCoordOrigin), + }; + } + return newAnnoObj; }; @@ -1416,15 +1493,18 @@ export const scaleObject = ( }); newObj.polygon = { ...newObj.polygon, group: newGroups }; } + if (newObj.point) { + const newPoint = translatePointZoom(newObj.point, preSize, curSize); + newObj.point = { ...newObj.point, ...newPoint }; + } return newObj; }; - const scalePromptItem = ( - promptItem: MaskPromptItem, + promptItem: PromptItem, preSize: ISize, curSize: ISize, -): MaskPromptItem => { - const { point, startPoint, rect, stroke } = promptItem; +): PromptItem => { + const { point, startPoint, rect, stroke, polygons } = promptItem; const scaledPromptItem = { ...promptItem }; if (point) { Object.assign(scaledPromptItem, { @@ -1448,9 +1528,43 @@ const scalePromptItem = ( }), }); } + if (polygons) { + Object.assign(scaledPromptItem, { + polygons: polygons.map((polygon) => { + const res = []; + for (let i = 0; i < polygon.length; i += 2) { + const point = { x: polygon[i], y: polygon[i + 1] }; + const scaledPoint = translatePointZoom(point, preSize, curSize); + res.push(scaledPoint.x, scaledPoint.y); + } + return res; + }), + }); + } return scaledPromptItem; }; +/** + * Scale frames objects + * @param preSize + * @param curSize + */ +export const scaleFramesObjects = ( + framesObjects: IAnnotationObject[][], + preSize: ISize, + curSize: ISize, +) => { + const updateFramesObjects = cloneDeep(framesObjects); + return updateFramesObjects.map((objs) => { + if (objs) { + return objs.map((obj) => { + return obj ? scaleObject(obj, preSize, curSize) : obj; + }); + } + return objs; + }); +}; + /** * Scale draw data * @param preSize @@ -1511,34 +1625,19 @@ export const scaleDrawData = ( } } - if (updateDrawData.prompt.segmentationClicks) { - updateDrawData.prompt.segmentationClicks = - updateDrawData.prompt.segmentationClicks.map((click) => { - if (click.point) { - const newPoint = translatePointZoom(click.point, preSize, curSize); - return { - ...click, - point: newPoint, - }; - } - return click; - }); - } - - if (updateDrawData.prompt.creatingMask) { - updateDrawData.prompt.creatingMask = scalePromptItem( - updateDrawData.prompt.creatingMask, + if (updateDrawData.prompt.creatingPrompt) { + updateDrawData.prompt.creatingPrompt = scalePromptItem( + updateDrawData.prompt.creatingPrompt, preSize, curSize, ); } - if (updateDrawData.prompt.maskPrompts) { - updateDrawData.prompt.maskPrompts = updateDrawData.prompt.maskPrompts?.map( - (item) => { + if (updateDrawData.prompt.promptsQueue) { + updateDrawData.prompt.promptsQueue = + updateDrawData.prompt.promptsQueue?.map((item) => { return scalePromptItem(item, preSize, curSize); - }, - ); + }); } if (updateDrawData.prompt.activeRectWhileLoading) { @@ -1552,6 +1651,41 @@ export const scaleDrawData = ( return updateDrawData; }; +export const convertFrameObjectsIntoFramesObjects = ( + currFrameObjects: IAnnotationObject[], + framesObjects: IAnnotationObject[][], + frameCount: number, + activeIndex: number, +) => { + const tempObjects = [...framesObjects]; + currFrameObjects.forEach((item, objectIdx) => { + const objectframes = + tempObjects[objectIdx] || new Array(frameCount).fill(undefined); + tempObjects[objectIdx] = objectframes.map((obj, frameIdx) => { + if (frameIdx === activeIndex) { + return item; + } + let resultObject = obj; + if (frameIdx > activeIndex) { + // frame change to after active frame + resultObject = isEqual(obj, objectframes[activeIndex]) ? item : obj; + } + return { + ...resultObject, + type: item.type, + labelId: item.labelId, + hidden: item.hidden, + color: item.color, + customStyles: item.customStyles, + attributes: item.attributes, + status: item.status, + frameEmpty: obj?.frameEmpty || Boolean(!obj), + }; + }); + }); + return tempObjects; +}; + export const getVisibleAreaForImage = ( imagePos: IPoint, clientSize: ISize, diff --git a/packages/components/src/Annotator/view.tsx b/packages/components/src/Annotator/view.tsx index 7719520..9942556 100755 --- a/packages/components/src/Annotator/view.tsx +++ b/packages/components/src/Annotator/view.tsx @@ -1,10 +1,9 @@ import React, { useEffect, useMemo, useRef, useState } from 'react'; -import { AnnotationType, DisplayOption } from './constants'; +import { DisplayOption } from './constants'; import { useImmer } from 'use-immer'; import { cloneDeep } from 'lodash'; import useHistory from './hooks/useHistory'; import useObjects from './hooks/useObjects'; -import usePreviousState from './hooks/usePreviousState'; import { BaseObject, Category, @@ -13,38 +12,35 @@ import { DrawData, EditState, EditorMode, - IAnnotationObject, - DrawImageData, + AnnoItem, DrawObject, } from './type'; import useColor from './hooks/useColor'; import useMouseCursor from './hooks/useMouseCursor'; import useCanvasRender from './hooks/useCanvasRender'; import useDataEffect from './hooks/useDataEffect'; -import { RenderStyles, useToolInstances } from './tools/base'; +import { useToolInstances } from './tools/base'; import { zoomImgSize } from './utils/compute'; import { CursorState } from 'ahooks/lib/useMouse'; import { ImageView } from './components/ImageView'; import './index.less'; +import useTranslate from './hooks/useTranslate'; export interface ViewProps { + isOldMode?: boolean; // is old dataset design mode categories: Category[]; - data: DrawImageData; + data: AnnoItem; objectsFilter?: (imageData: any) => BaseObject[]; - getCustomObjectStyles?: ( - object: IAnnotationObject, - color: string, - ) => Partial; currentSize?: ISize; wrapWidth?: number; wrapHeight?: number; minHeight?: number; - displayAnnotationType?: AnnotationType; displayOptionsResult?: { [key in DisplayOption]?: boolean }; } const View: React.FC = (props) => { const { + isOldMode, categories, data, currentSize, @@ -52,8 +48,6 @@ const View: React.FC = (props) => { wrapHeight, minHeight, objectsFilter, - getCustomObjectStyles, - displayAnnotationType, displayOptionsResult, } = props; @@ -116,14 +110,19 @@ const View: React.FC = (props) => { return [mouse, mouse]; }, [clientSize]); - const [preClientSize, clearPreClientSize] = - usePreviousState(clientSize); - - const { labelColors, getAnnotColor } = useColor({ + const { getAnnotColor } = useColor({ categories, editState, }); + const { translateToObject } = useTranslate({ + isOldMode, + clientSize, + naturalSize, + categories, + getAnnotColor, + }); + const { clearHistory, updateHistory, setDrawDataWithHistory } = useHistory({ clientSize, naturalSize, @@ -133,15 +132,13 @@ const View: React.FC = (props) => { const { addObject, initObjectList, updateObject } = useObjects({ annotations, setAnnotations, - clientSize, - naturalSize, drawData, setDrawData, setDrawDataWithHistory, - editState, setEditState, mode: EditorMode.View, - displayAnnotationType, + translateToObject, + updateHistory, }); const { updateMouseCursor } = useMouseCursor({ @@ -170,6 +167,7 @@ const View: React.FC = (props) => { updateMouseCursor, displayOptionsResult, getAnnotColor, + categories, }); const { updateRender } = useCanvasRender({ @@ -183,22 +181,18 @@ const View: React.FC = (props) => { activeCanvasRef, imgRef, objectHooksMap, - getCustomObjectStyles, }); // ================================================================================================================= // Effects // ================================================================================================================= - const { resetDataWithImageData, rebuildDrawData } = useDataEffect({ + const { resetDataWithImageData } = useDataEffect({ imagePos, clientSize, - preClientSize, - clearPreClientSize, naturalSize, annotations, setAnnotations, - labelColors, drawData, setDrawData, editState, @@ -207,6 +201,7 @@ const View: React.FC = (props) => { updateRender, clearHistory, objectsFilter, + labelOptions: categories, }); /** Reset data when hiding the editor or switching images */ @@ -216,8 +211,8 @@ const View: React.FC = (props) => { /** Custom options changed */ useEffect(() => { - rebuildDrawData(true); - }, [displayAnnotationType, displayOptionsResult, getCustomObjectStyles]); + updateRender(); + }, [displayOptionsResult]); const onLoadImg = (e: React.UIEvent) => { // Set natural size. diff --git a/packages/components/src/locales/en-US.ts b/packages/components/src/locales/en-US.ts index e3d09f0..462c9d3 100644 --- a/packages/components/src/locales/en-US.ts +++ b/packages/components/src/locales/en-US.ts @@ -16,16 +16,21 @@ export default { /** DDSAnnotator */ 'DDSAnnotator.save': 'Save', + 'DDSAnnotator.commit': 'Commit', 'DDSAnnotator.cancel': 'Cancel', 'DDSAnnotator.delete': 'Delete', + 'DDSAnnotator.modify': 'Modify', 'DDSAnnotator.reject': 'Reject', 'DDSAnnotator.approve': 'Approve', 'DDSAnnotator.prev': 'Previous Image', 'DDSAnnotator.next': 'Next Image', 'DDSAnnotator.exit': 'Exit', + 'DDSAnnotator.docs': 'Docs', 'DDSAnnotator.shortcuts': 'Shortcuts', 'DDSAnnotator.confidence': 'Confidence', 'DDSAnnotator.annotsList.categories': 'Categories', + 'DDSAnnotator.annotsList.labels': 'Labels', + 'DDSAnnotator.annotsList.classification': 'Classification', 'DDSAnnotator.annotsList.objects': 'Objects', 'DDSAnnotator.annotsList.hideAll': 'Hide All', 'DDSAnnotator.annotsList.showAll': 'Show All', @@ -60,6 +65,9 @@ export default { 'DDSAnnotator.subtoolbar.mask.sam.notAllow': 'Unavailable when any instance exists', 'DDSAnnotator.subtoolbar.mask.edgeStitch': 'Edge Stitching Brush', + 'DDSAnnotator.subtoolbar.visualprompt.positive': 'Positive Visual Prompt', + 'DDSAnnotator.subtoolbar.visualprompt.negative': 'Negative Visual Prompt', + 'DDSAnnotator.subtoolbar.polygon.pointResolution': 'Point Resolution', 'DDSAnnotator.zoomTool.reset': 'Reset Zoom', 'DDSAnnotator.zoomIn': 'Zoom In', 'DDSAnnotator.zoomOut': 'Zoom Out', @@ -148,11 +156,14 @@ export default { 'DDSAnnotator.smart.infoModal.action': 'Visit Our Website', 'DDSAnnotator.smart.detection.name': 'Intelligent Object Detection', 'DDSAnnotator.smart.detection.input': 'Select or enter categories', + 'DDSAnnotator.smart.ivp.name': 'Interactive Visual Prompt (iVP)', 'DDSAnnotator.smart.segmentation.name': 'Intelligent Segmentation (Polygon)', 'DDSAnnotator.smart.pose.name': 'Intelligent Pose Estimation', 'DDSAnnotator.smart.mask.name': 'Intelligent Panoramic Segmentation', 'DDSAnnotator.smart.pose.input': 'Select template', 'DDSAnnotator.smart.pose.apply': 'Apply Results', + 'DDSAnnotator.smart.ivp.desc': 'Detect the objects with visual prompt', + 'DDSAnnotator.smart.gdino.desc': 'Detect the objects with text prompt', 'DDSAnnotator.smart.annotate': 'Auto-Annotate', 'DDSAnnotator.smart.retry': 'Retry', 'DDSAnnotator.smart.modelTyle': 'Model Type', @@ -169,6 +180,8 @@ export default { 'DDSAnnotator.smart.msg.confResults': '{count} matching annotations shown', 'DDSAnnotator.smart.msg.applyConf': '{count} annotations have been retained, with the others removed.', + 'DDSAnnotator.smart.msg.positivePrompt': + 'At least one positive visual prompt is required.', 'DDSAnnotator.smart.rateLimit.title': 'Tips', 'DDSAnnotator.smart.rateLimit.content': 'Sorry, our public server is currently under low capacity and unable to process your request. Please try again later.', @@ -181,4 +194,35 @@ export default { 'DDSAnnotator.smart.tip.recover': 'Recover unselected annotations', 'DDSAnnotator.smart.tip.overlayobject': 'View overlapping annotation objects', 'DDSAnnotator.smart.tip.annotationApplied': '{count} annotations applied.', + 'DDSAnnotator.smart.tip.visualPrompt': + 'Add more visual prompts or accept current objects', + 'DDSAnnotator.seg.tool': 'Segmentation tool', + 'DDSAnnotator.seg.tool.content': 'Accept the segmentation result.', + 'DDSAnnotator.confirm': 'Confirm', + 'DDSAnnotator.points.editor': 'Points Attributes', + 'DDSAnnotator.attribute.add': 'Add label attributes', + 'DDSAnnotator.attribute.edit': 'Edit label attributes', + 'DDSAnnotator.attribute.input': 'Please input', + 'DDSAnnotator.attribute.required': 'Please fill in all required fields.', + 'DDSAnnotator.attribute.newOperation.limit': + 'Please make sure to add the required label attribute before proceeding with other operations.', + 'DDSAnnotator.classification.required': + 'You have not filled in all classification questions.', + 'DDSAnnotator.label.attributes.required': + 'You have not filled in all required label attributes.', + 'DDSAnnotator.label.select': 'Select a label', + 'DDSAnnotator.model.select': 'Select a model', + 'DDSAnnotator.status.labeling': 'Labeling', + 'DDSAnnotator.status.reviewing': 'Reviewing', + 'DDSAnnotator.save.check.error': 'Pre Check Error', + 'DDSAnnotator.save.check.classification': + 'Classification #{idx} is required to have answer.', + 'DDSAnnotator.save.check.label': + 'Label ({labelName}) #{idx} is required to have manual attributes.', + 'DDSAnnotator.save.check.tip': 'Please modify first.', + + 'DDSAnnotator.video.track': 'Tracking', + 'DDSAnnotator.video.track.setting': 'Tracking settings', + 'DDSAnnotator.video.frame': 'Frames', + 'DDSAnnotator.video.track.backward': 'Backward inference frames', }; diff --git a/packages/components/src/locales/zh-CN.ts b/packages/components/src/locales/zh-CN.ts index 4b8fc9f..7e38ae9 100644 --- a/packages/components/src/locales/zh-CN.ts +++ b/packages/components/src/locales/zh-CN.ts @@ -15,16 +15,21 @@ export default { /** Annotator */ 'DDSAnnotator.save': '保存', + 'DDSAnnotator.commit': '提交', 'DDSAnnotator.cancel': '取消', 'DDSAnnotator.delete': '删除', + 'DDSAnnotator.modify': '修改', 'DDSAnnotator.reject': '拒绝', 'DDSAnnotator.approve': '通过', 'DDSAnnotator.prev': '上一张', 'DDSAnnotator.next': '下一张', 'DDSAnnotator.exit': '退出', + 'DDSAnnotator.docs': '文档', 'DDSAnnotator.shortcuts': '快捷键', 'DDSAnnotator.confidence': '置信区间', 'DDSAnnotator.annotsList.categories': '分类', + 'DDSAnnotator.annotsList.labels': '标注', + 'DDSAnnotator.annotsList.classification': '分类筛选', 'DDSAnnotator.annotsList.objects': '实例', 'DDSAnnotator.annotsList.hideAll': '隐藏全部', 'DDSAnnotator.annotsList.showAll': '显示全部', @@ -80,6 +85,9 @@ export default { 'DDSAnnotator.subtoolbar.mask.sam.notAllow': '当图中存在任意实例时, 该功能不可用', 'DDSAnnotator.subtoolbar.mask.edgeStitch': '智能边缘缝合', + 'DDSAnnotator.subtoolbar.visualprompt.positive': '正例视觉提示', + 'DDSAnnotator.subtoolbar.visualprompt.negative': '反例视觉提示', + 'DDSAnnotator.subtoolbar.polygon.pointResolution': '点密度', 'DDSAnnotator.annotsEditor.title': '修改标注实例', 'DDSAnnotator.annotsEditor.delete': '删除', 'DDSAnnotator.annotsEditor.finish': '完成', @@ -134,6 +142,7 @@ export default { '抱歉, DeepDataSpace的本地版本暂时不支持智能标注功能, 您可以前往官网了解更多信息或联系我们（deepdataspace_dm@idea.edu.cn）获取智能标注的体验通道。', 'DDSAnnotator.smart.infoModal.action': '前往官网', 'DDSAnnotator.smart.detection.name': '智能目标检测', + 'DDSAnnotator.smart.ivp.name': '交互式视觉提示 (iVP)', 'DDSAnnotator.smart.segmentation.name': '智能图像分割(多边形)', 'DDSAnnotator.smart.pose.name': '智能姿态估计', 'DDSAnnotator.smart.mask.name': '智能全景分割', @@ -143,6 +152,8 @@ export default { 'DDSAnnotator.smart.detection.input': '选择或输入类别', 'DDSAnnotator.smart.pose.input': '选择模版', 'DDSAnnotator.smart.pose.apply': '保留当前结果', + 'DDSAnnotator.smart.ivp.desc': '根据视觉提示检测任意目标', + 'DDSAnnotator.smart.gdino.desc': '输入任意描述词检测目标', 'DDSAnnotator.smart.minArea': '最小分割面积', 'DDSAnnotator.smart.iouThres': 'IoU阈值', 'DDSAnnotator.smart.segmentation.tipsInitial': @@ -155,6 +166,7 @@ export default { 'DDSAnnotator.smart.msg.labelRequired': '请至少选择一个目标类别', 'DDSAnnotator.smart.msg.confResults': '共有{count}条标注符合目标置信区间', 'DDSAnnotator.smart.msg.applyConf': '已保留{count}条标注，其他标注已移除', + 'DDSAnnotator.smart.msg.positivePrompt': '请确保至少添加一个正视觉提示', 'DDSAnnotator.smart.rateLimit.title': '提示', 'DDSAnnotator.smart.rateLimit.content': '非常抱歉，我们的公共服务器暂时负载不足，请稍后再试。', @@ -166,4 +178,31 @@ export default { 'DDSAnnotator.smart.tip.recover': '回收未选标注', 'DDSAnnotator.smart.tip.overlayobject': '查看重叠的标注对象', 'DDSAnnotator.smart.tip.annotationApplied': '已添加{count}个标注对象', + 'DDSAnnotator.smart.tip.visualPrompt': '添加更多视觉提示或接受当前结果', + 'DDSAnnotator.seg.tool': '分割工具', + 'DDSAnnotator.seg.tool.content': '接受本次分割结果.', + 'DDSAnnotator.confirm': '确认', + 'DDSAnnotator.points.editor': '关键点属性', + 'DDSAnnotator.attribute.add': '添加标签属性', + 'DDSAnnotator.attribute.edit': '编辑标签属性', + 'DDSAnnotator.attribute.input': '请输入', + 'DDSAnnotator.attribute.required': '请填写所有必填项', + 'DDSAnnotator.attribute.newOperation.limit': + '请确认添加必填标签属性后再进行其他操作', + 'DDSAnnotator.classification.required': '你还未填写所有分类筛选问题', + 'DDSAnnotator.label.attributes.required': '你还未填写所有必填的标签属性', + 'DDSAnnotator.label.select': '选择标签', + 'DDSAnnotator.model.select': '选择模型', + 'DDSAnnotator.status.labeling': '标注中', + 'DDSAnnotator.status.reviewing': '审核中', + 'DDSAnnotator.save.check.error': '预检查错误', + 'DDSAnnotator.save.check.classification': '分类筛选 #{idx} 必须有答案.', + 'DDSAnnotator.save.check.label': + '标注 ({labelName}) #{idx} 有必须手动添加的标签属性', + 'DDSAnnotator.save.check.tip': '请修改后再进行操作', + + 'DDSAnnotator.video.track': '推理', + 'DDSAnnotator.video.track.setting': '推理设置', + 'DDSAnnotator.video.frame': '帧', + 'DDSAnnotator.video.track.backward': '向后推理帧数', }; diff --git a/packages/components/tsconfig.json b/packages/components/tsconfig.json index a2c69ba..bf2a152 100644 --- a/packages/components/tsconfig.json +++ b/packages/components/tsconfig.json @@ -5,6 +5,8 @@ "paths": { "@/*": ["src/*"], "@@/*": ["../../applications/app/src/.umi/*"], + "dds-components": ["../../packages/components/src"], + "dds-components/*": ["../../packages/components/src/*"], "dds-utils/*": ["../../packages/utils/src/*"], "dds-hooks": ["../../packages/hooks/src"] }