From 375c96d56df4f8e5e42c05695e7c0e712217f5e1 Mon Sep 17 00:00:00 2001 From: Julio Sgarbi Date: Wed, 13 Dec 2023 17:31:51 -0400 Subject: [PATCH 01/10] feat: LLM interactive labeling --- src/components/Assistant/Assistant.styl | 96 +++++++++++++++ src/components/Assistant/Assistant.tsx | 110 ++++++++++++++++++ .../SidePanels/DetailsPanel/DetailsPanel.tsx | 15 +++ src/components/SidePanels/TabPanels/utils.ts | 36 ++++-- 4 files changed, 246 insertions(+), 11 deletions(-) create mode 100644 src/components/Assistant/Assistant.styl create mode 100644 src/components/Assistant/Assistant.tsx diff --git a/src/components/Assistant/Assistant.styl b/src/components/Assistant/Assistant.styl new file mode 100644 index 000000000..55f9bbdc7 --- /dev/null +++ b/src/components/Assistant/Assistant.styl @@ -0,0 +1,96 @@ +.assistant + display flex + flex-wrap wrap + padding 16px + + &__history + display flex + flex-wrap wrap + width 100% + cursor pointer + margin-top 10px + + &-item + width 100% + margin 2px 0 + padding 1px 5px + display: flex; + justify-content: space-between; + align-items center + + &-delete + width 8px + height 8px + display flex + opacity .5 + + &:hover + opacity 1 + + &:hover + background-color #F5F5F5 + + + .assist-form + --primary-action-color #00A8FF + --primary-action-padding 4px + --primary-action-border-radius 4px + --primary-action-surface-color-hover rgba(9, 109, 217, 0.12) + --text-input-min-height 40px + --tooltip-text-color #F5222D + + display grid + grid-template-columns: auto var(--text-input-min-height); + align-items center + gap 4px + width 100% + + &__assist-text + display flex + align-items center + min-height 40px !important + border 1px solid #DFDFDF !important + border-radius 4px !important + background-color #FAFAFA !important + padding 8px 16px !important + font-size 16px !important + line-height 24px !important + letter-spacing 0.5px !important + + &__primary-action + height 100% + width 100% + display flex + justify-content center + align-items flex-start + flex-shrink 0 + flex-grow 0 + + button + margin-top 4px + display flex + justify-content center + align-items center + flex-shrink 0 + flex-grow 0 + appearance none + border none + background-color transparent + color var(--primary-action-color) + border-radius var(--primary-action-border-radius) + + &:hover + background-color var(--primary-action-surface-color-hover) + + ~/_inline & + padding 0 + height calc(var(--text-input-min-height) - 8px); + width calc(var(--text-input-min-height) - 8px); + + ~/_inline & + grid-row 1 / 2 + grid-column 2 / -1 + + &__tooltipMessage + color var(--tooltip-text-color) + font-size 0.9em diff --git a/src/components/Assistant/Assistant.tsx b/src/components/Assistant/Assistant.tsx new file mode 100644 index 000000000..740c95a29 --- /dev/null +++ b/src/components/Assistant/Assistant.tsx @@ -0,0 +1,110 @@ +import { ChangeEvent, FC, useCallback, useEffect, useState } from 'react'; +import { observer } from 'mobx-react'; +import { Block, Elem } from '../../utils/bem'; +import { ReactComponent as IconSend } from '../../assets/icons/send.svg'; +import { IconCross } from '../../assets/icons'; +import Input from '../../common/Input/Input'; + +import './Assistant.styl'; + + +export const Assistant: FC<{ }> = observer(() => { + const [historyValue, setHistoryValue] = useState([]); + const [value, setValue] = useState(''); + + useEffect(() => { + const _history = JSON.parse(window.localStorage.getItem('llm_assistant') || '[]'); + + if (_history.length) { + setHistoryValue(_history); + setValue(historyValue[0]); + } + }, []); + + useEffect(() => { + window.localStorage.setItem('llm_assistant', JSON.stringify(historyValue)); + }, [historyValue]); + + const setHistory = useCallback((text: string) => { + const _history = [...historyValue]; + + _history.forEach((item: string, index: number) => { + if (item === text) { + _history.splice(index, 1); + } + }); + + _history.unshift(text); + + if (_history.length > 5) { + _history.pop(); + } + + setHistoryValue(_history); + }, [historyValue]); + + + const onSubmit = useCallback((e) => { + e?.preventDefault?.(); + + if (!value.trim()) return; + + setHistory(value); + }, [value]); + + const setValueFromHistory = useCallback((item: string) => { + setValue(item); + setHistory(item); + }, [historyValue]); + + const deleteValueFromHistory = useCallback((deleteItem: string) => { + const _history = [...historyValue]; + + _history.forEach((item: string, index: number) => { + if (item === deleteItem) { + _history.splice(index, 1); + } + }); + + setHistoryValue(_history); + }, [historyValue]); + + const renderHistory = () => { + return historyValue.map((item: string, index: number) => { + return ( + setValueFromHistory(item)}> + {item} + { + e.stopPropagation(); + deleteValueFromHistory(item); + }}> + + + + ); + }); + }; + + return ( + + + ) => setValue(e.target.value)} + onSubmit={onSubmit} + /> + + + + + + {historyValue.length > 0 && renderHistory()} + + + ); +}); diff --git a/src/components/SidePanels/DetailsPanel/DetailsPanel.tsx b/src/components/SidePanels/DetailsPanel/DetailsPanel.tsx index 21a2ed51c..fde4550d7 100644 --- a/src/components/SidePanels/DetailsPanel/DetailsPanel.tsx +++ b/src/components/SidePanels/DetailsPanel/DetailsPanel.tsx @@ -12,6 +12,8 @@ import { Relations as RelationsComponent } from './Relations'; // eslint-disable-next-line // @ts-ignore import { DraftPanel } from '../../DraftPanel/DraftPanel'; +import { Assistant } from '../../Assistant/Assistant'; + interface DetailsPanelProps extends PanelProps { regions: any; selection: any; @@ -70,6 +72,18 @@ const CommentsTab: FC = inject('store')(observer(({ store }) => { ); })); +const AssistTab: FC = inject('store')(observer(({ store }) => { + return ( + + + + + + + + ); +})); + const RelationsTab: FC = inject('store')(observer(({ currentEntity }) => { const { relationStore } = currentEntity; @@ -207,6 +221,7 @@ const SelectedRegion: FC<{region: any}> = observer(({ ); }); +export const Assist = observer(AssistTab); export const Comments = observer(CommentsTab); export const History = observer(HistoryTab); export const Relations = observer(RelationsTab); diff --git a/src/components/SidePanels/TabPanels/utils.ts b/src/components/SidePanels/TabPanels/utils.ts index 012ae8d28..01edab257 100644 --- a/src/components/SidePanels/TabPanels/utils.ts +++ b/src/components/SidePanels/TabPanels/utils.ts @@ -1,20 +1,27 @@ import { FC, MutableRefObject, ReactNode } from 'react'; import { clamp } from '../../../utils/utilities'; -import { DEFAULT_PANEL_HEIGHT, DEFAULT_PANEL_MAX_HEIGHT, DEFAULT_PANEL_MAX_WIDTH, DEFAULT_PANEL_MIN_HEIGHT, DEFAULT_PANEL_WIDTH, PANEL_HEADER_HEIGHT } from '../constants'; -import { Comments, History, Info, Relations } from '../DetailsPanel/DetailsPanel'; +import { + DEFAULT_PANEL_HEIGHT, + DEFAULT_PANEL_MAX_HEIGHT, + DEFAULT_PANEL_MAX_WIDTH, + DEFAULT_PANEL_MIN_HEIGHT, + DEFAULT_PANEL_WIDTH, + PANEL_HEADER_HEIGHT +} from '../constants'; +import { Assist, Comments, History, Info, Relations } from '../DetailsPanel/DetailsPanel'; import { OutlinerComponent } from '../OutlinerPanel/OutlinerPanel'; import { PanelProps } from '../PanelBase'; import { emptyPanel, JoinOrder, PanelBBox, PanelView, Side, ViewportSize } from './types'; export const determineLeftOrRight = (event: any, droppableElement?: ReactNode) => { - const element = droppableElement || event.target as HTMLElement; + const element = droppableElement || event.target as HTMLElement; const dropWidth = (element as HTMLElement).clientWidth as number; const x = event.pageX as number - (element as HTMLElement).getBoundingClientRect().left; const half = dropWidth / 2; - + return x > half ? Side.right : Side.left; }; - + export const determineDroppableArea = (droppingElement: HTMLElement) => droppingElement?.id?.includes('droppable'); export const stateRemovedTab = (state: Record, movingPanel: string, movingTab: number) => { @@ -49,7 +56,7 @@ export const setActiveDefaults = (state: Record) => { Object.keys(state).forEach((panelKey: string) => { const firstActiveTab = newState[panelKey].panelViews.findIndex((view) => view.active) ; - + newState[panelKey].panelViews[firstActiveTab > 0 ? firstActiveTab : 0].active = true; }); @@ -80,15 +87,15 @@ export const stateAddedTab = ( ) => { const newState = { ...state }; const panel = newState[receivingPanel]; - + panel.panelViews = newState[receivingPanel].panelViews.map((view) => { view.active = false; return view; }); - + let index = receivingTab + (dropSide === Side.right ? 1 : 0); - if (movingPanel === receivingPanel && index > 0) index -= 1; + if (movingPanel === receivingPanel && index > 0) index -= 1; panel.panelViews.splice(index, 0, movingTabData); return newState; }; @@ -107,6 +114,7 @@ export const panelComponents: {[key:string]: FC} = { 'history': History as FC, 'relations': Relations as FC, 'comments': Comments as FC, + 'assist': Assist as FC, 'info': Info as FC, }; @@ -142,6 +150,12 @@ const panelViews = [ component: panelComponents['comments'] as FC, active: false, }, + { + name: 'assist', + title: 'Assistant', + component: panelComponents['assist'] as FC, + active: false, + }, ]; export const enterprisePanelDefault: Record = { @@ -173,7 +187,7 @@ export const enterprisePanelDefault: Record = { detached: false, alignment: Side.right, maxHeight: DEFAULT_PANEL_MAX_HEIGHT, - panelViews: [panelViews[0], panelViews[2]], + panelViews: [panelViews[0], panelViews[2], panelViews[5]], }, }; @@ -191,7 +205,7 @@ export const openSourcePanelDefault: Record = { detached: false, alignment: Side.right, maxHeight: DEFAULT_PANEL_MAX_HEIGHT, - panelViews: [panelViews[3], panelViews[1]], + panelViews: [panelViews[3], panelViews[4], panelViews[1]], }, 'regions-relations': { order: 2, From 851c510108a7fcc7c3801ec259a3577c3a2432c7 Mon Sep 17 00:00:00 2001 From: Julio Sgarbi Date: Wed, 13 Dec 2023 18:06:37 -0400 Subject: [PATCH 02/10] change input to textarea --- src/components/Assistant/Assistant.styl | 26 +++++-------------------- src/components/Assistant/Assistant.tsx | 11 +++++------ 2 files changed, 10 insertions(+), 27 deletions(-) diff --git a/src/components/Assistant/Assistant.styl b/src/components/Assistant/Assistant.styl index 55f9bbdc7..88c94f54e 100644 --- a/src/components/Assistant/Assistant.styl +++ b/src/components/Assistant/Assistant.styl @@ -45,18 +45,6 @@ gap 4px width 100% - &__assist-text - display flex - align-items center - min-height 40px !important - border 1px solid #DFDFDF !important - border-radius 4px !important - background-color #FAFAFA !important - padding 8px 16px !important - font-size 16px !important - line-height 24px !important - letter-spacing 0.5px !important - &__primary-action height 100% width 100% @@ -65,6 +53,8 @@ align-items flex-start flex-shrink 0 flex-grow 0 + grid-row 1 / 2 + grid-column 2 / -1 button margin-top 4px @@ -78,19 +68,13 @@ background-color transparent color var(--primary-action-color) border-radius var(--primary-action-border-radius) + padding 0 + height calc(var(--text-input-min-height) - 8px); + width calc(var(--text-input-min-height) - 8px); &:hover background-color var(--primary-action-surface-color-hover) - ~/_inline & - padding 0 - height calc(var(--text-input-min-height) - 8px); - width calc(var(--text-input-min-height) - 8px); - - ~/_inline & - grid-row 1 / 2 - grid-column 2 / -1 - &__tooltipMessage color var(--tooltip-text-color) font-size 0.9em diff --git a/src/components/Assistant/Assistant.tsx b/src/components/Assistant/Assistant.tsx index 740c95a29..111cd979e 100644 --- a/src/components/Assistant/Assistant.tsx +++ b/src/components/Assistant/Assistant.tsx @@ -1,11 +1,11 @@ -import { ChangeEvent, FC, useCallback, useEffect, useState } from 'react'; +import { FC, useCallback, useEffect, useState } from 'react'; import { observer } from 'mobx-react'; import { Block, Elem } from '../../utils/bem'; import { ReactComponent as IconSend } from '../../assets/icons/send.svg'; import { IconCross } from '../../assets/icons'; -import Input from '../../common/Input/Input'; import './Assistant.styl'; +import { TextArea } from '../../common/TextArea/TextArea'; export const Assistant: FC<{ }> = observer(() => { @@ -39,7 +39,7 @@ export const Assistant: FC<{ }> = observer(() => { if (_history.length > 5) { _history.pop(); } - + setHistoryValue(_history); }, [historyValue]); @@ -88,12 +88,11 @@ export const Assistant: FC<{ }> = observer(() => { return ( - ) => setValue(e.target.value)} + onChange={setValue} onSubmit={onSubmit} /> From 7413f9e64fd2312ba45e17de7784633638663f39 Mon Sep 17 00:00:00 2001 From: hlomzik Date: Wed, 13 Dec 2023 18:08:55 -0400 Subject: [PATCH 03/10] Test assistant prompt --- src/env/development.js | 1 + src/env/production.js | 1 + src/stores/AppStore.js | 8 ++++++++ 3 files changed, 10 insertions(+) diff --git a/src/env/development.js b/src/env/development.js index 93060c9e3..808076b42 100644 --- a/src/env/development.js +++ b/src/env/development.js @@ -153,6 +153,7 @@ function configureApplication(params) { onSkipTask: params.onSkipTask ? params.onSkipTask : External.onSkipTask, onUnskipTask: params.onUnskipTask ? params.onUnskipTask : External.onUnskipTask, onPresignUrlForProject: params.onPresignUrlForProject, + onAssistantPrompt: params.onAssistantPrompt, onSubmitDraft: params.onSubmitDraft, onTaskLoad: params.onTaskLoad ? params.onTaskLoad : External.onTaskLoad, onLabelStudioLoad: params.onLabelStudioLoad ? params.onLabelStudioLoad : External.onLabelStudioLoad, diff --git a/src/env/production.js b/src/env/production.js index 03aa48f6e..2d6ee6825 100644 --- a/src/env/production.js +++ b/src/env/production.js @@ -66,6 +66,7 @@ function configureApplication(params) { onUnskipTask: params.onUnskipTask ? params.onUnskipTask : External.onUnskipTask, onSubmitDraft: params.onSubmitDraft, onPresignUrlForProject: params.onPresignUrlForProject, + onAssistantPrompt: params.onAssistantPrompt, onTaskLoad: params.onTaskLoad || External.onTaskLoad, onLabelStudioLoad: params.onLabelStudioLoad || External.onLabelStudioLoad, onEntityCreate: params.onEntityCreate || External.onEntityCreate, diff --git a/src/stores/AppStore.js b/src/stores/AppStore.js index 4f4f48c86..ac7c7e5c6 100644 --- a/src/stores/AppStore.js +++ b/src/stores/AppStore.js @@ -636,6 +636,12 @@ export default types return presignUrl; } + async function onAssistantPrompt(prompt) { + const result = await self.events.invoke('assistantPrompt', self, prompt); + + return result; + } + /** * Reset annotation store */ @@ -849,6 +855,8 @@ export default types setUsers, mergeUsers, + onAssistantPrompt, + showModal, toggleComments, toggleSettings, From 7588a045660e104f72468eb6c2b9c95591d1ace8 Mon Sep 17 00:00:00 2001 From: hlomzik Date: Wed, 13 Dec 2023 20:18:51 -0400 Subject: [PATCH 04/10] Connect Assistant component to backend Send actual request to ML-backend and show results from response. --- src/components/Assistant/Assistant.tsx | 5 +++-- src/components/SidePanels/DetailsPanel/DetailsPanel.tsx | 2 +- src/stores/AppStore.js | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/components/Assistant/Assistant.tsx b/src/components/Assistant/Assistant.tsx index 111cd979e..ed331adf4 100644 --- a/src/components/Assistant/Assistant.tsx +++ b/src/components/Assistant/Assistant.tsx @@ -7,8 +7,7 @@ import { IconCross } from '../../assets/icons'; import './Assistant.styl'; import { TextArea } from '../../common/TextArea/TextArea'; - -export const Assistant: FC<{ }> = observer(() => { +export const Assistant: FC<{ onPrompt: (prompt: string) => void }> = observer(({ onPrompt }) => { const [historyValue, setHistoryValue] = useState([]); const [value, setValue] = useState(''); @@ -49,6 +48,8 @@ export const Assistant: FC<{ }> = observer(() => { if (!value.trim()) return; + onPrompt(value); + setHistory(value); }, [value]); diff --git a/src/components/SidePanels/DetailsPanel/DetailsPanel.tsx b/src/components/SidePanels/DetailsPanel/DetailsPanel.tsx index fde4550d7..856f2ef37 100644 --- a/src/components/SidePanels/DetailsPanel/DetailsPanel.tsx +++ b/src/components/SidePanels/DetailsPanel/DetailsPanel.tsx @@ -77,7 +77,7 @@ const AssistTab: FC = inject('store')(observer(({ store }) => { - + diff --git a/src/stores/AppStore.js b/src/stores/AppStore.js index ac7c7e5c6..08e459bc8 100644 --- a/src/stores/AppStore.js +++ b/src/stores/AppStore.js @@ -639,6 +639,8 @@ export default types async function onAssistantPrompt(prompt) { const result = await self.events.invoke('assistantPrompt', self, prompt); + self.annotationStore.selected.deserializeResults(result[0]); + return result; } From 726d636f5543a27d6723819fd7bd2dc26f2cd16d Mon Sep 17 00:00:00 2001 From: Julio Sgarbi Date: Thu, 14 Dec 2023 11:15:54 -0400 Subject: [PATCH 05/10] fix loading new info --- src/components/Assistant/Assistant.tsx | 7 +++++-- .../SidePanels/DetailsPanel/DetailsPanel.tsx | 2 +- src/stores/AppStore.js | 13 +++---------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/components/Assistant/Assistant.tsx b/src/components/Assistant/Assistant.tsx index ed331adf4..a9cfaba50 100644 --- a/src/components/Assistant/Assistant.tsx +++ b/src/components/Assistant/Assistant.tsx @@ -7,7 +7,7 @@ import { IconCross } from '../../assets/icons'; import './Assistant.styl'; import { TextArea } from '../../common/TextArea/TextArea'; -export const Assistant: FC<{ onPrompt: (prompt: string) => void }> = observer(({ onPrompt }) => { +export const Assistant: FC<{ onPrompt: (prompt: string) => void, awaitingSuggestions: boolean }> = observer(({ onPrompt, awaitingSuggestions }) => { const [historyValue, setHistoryValue] = useState([]); const [value, setValue] = useState(''); @@ -55,6 +55,7 @@ export const Assistant: FC<{ onPrompt: (prompt: string) => void }> = observer(({ const setValueFromHistory = useCallback((item: string) => { setValue(item); + onPrompt(item); setHistory(item); }, [historyValue]); @@ -96,7 +97,9 @@ export const Assistant: FC<{ onPrompt: (prompt: string) => void }> = observer(({ onChange={setValue} onSubmit={onSubmit} /> - + diff --git a/src/components/SidePanels/DetailsPanel/DetailsPanel.tsx b/src/components/SidePanels/DetailsPanel/DetailsPanel.tsx index 856f2ef37..329bf78cd 100644 --- a/src/components/SidePanels/DetailsPanel/DetailsPanel.tsx +++ b/src/components/SidePanels/DetailsPanel/DetailsPanel.tsx @@ -77,7 +77,7 @@ const AssistTab: FC = inject('store')(observer(({ store }) => { - + diff --git a/src/stores/AppStore.js b/src/stores/AppStore.js index 08e459bc8..710dbecef 100644 --- a/src/stores/AppStore.js +++ b/src/stores/AppStore.js @@ -1,15 +1,6 @@ /* global LSF_VERSION */ -import { - destroy, - detach, - flow, - getEnv, getParent, - getSnapshot, - isRoot, - types, - walk -} from 'mobx-state-tree'; +import { destroy, detach, flow, getEnv, getParent, getSnapshot, isRoot, types, walk } from 'mobx-state-tree'; import uniqBy from 'lodash/uniqBy'; import InfoModal from '../components/Infomodal/Infomodal'; @@ -637,9 +628,11 @@ export default types } async function onAssistantPrompt(prompt) { + self.setFlags({ awaitingSuggestions: true }); const result = await self.events.invoke('assistantPrompt', self, prompt); self.annotationStore.selected.deserializeResults(result[0]); + self.setFlags({ awaitingSuggestions: false }); return result; } From 5c47b8e9b9cd3b3e1df271dd5551bf17273998e4 Mon Sep 17 00:00:00 2001 From: Julio Sgarbi Date: Thu, 14 Dec 2023 11:53:09 -0400 Subject: [PATCH 06/10] loading on assistant component --- src/components/Assistant/Assistant.styl | 43 +++++++++++++++++++++++++ src/components/Assistant/Assistant.tsx | 3 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/components/Assistant/Assistant.styl b/src/components/Assistant/Assistant.styl index 88c94f54e..af98932f4 100644 --- a/src/components/Assistant/Assistant.styl +++ b/src/components/Assistant/Assistant.styl @@ -45,6 +45,30 @@ gap 4px width 100% + &_disabled + opacity .5 !important + pointer-events none + + &_loading + --borderWidth: 3px; + background: #FFF; + position: relative; + border-radius 4px + pointer-events: none; + + &:after + content: ''; + position: absolute; + top: calc(-1 * var(--borderWidth)); + left: calc(-1 * var(--borderWidth)); + height: calc(100% + var(--borderWidth) * 2); + width: calc(100% + var(--borderWidth) * 2); + background: linear-gradient(60deg, #9254DE, #9254DE, #13C2C2, #13C2C2); + border-radius: calc(2 * var(--borderWidth)); + z-index: -1; + animation: animatedgradient 3s ease alternate infinite; + background-size: 300% 300%; + &__primary-action height 100% width 100% @@ -56,6 +80,10 @@ grid-row 1 / 2 grid-column 2 / -1 + &_loading + filter grayscale(1) + opacity .6 + button margin-top 4px display flex @@ -78,3 +106,18 @@ &__tooltipMessage color var(--tooltip-text-color) font-size 0.9em + + +@keyframes animatedgradient { + 0% { + background-position: 0% 50%; + } + 50% { + background-position: 100% 50%; + } + 100% { + background-position: 0% 50%; + } +} + + diff --git a/src/components/Assistant/Assistant.tsx b/src/components/Assistant/Assistant.tsx index a9cfaba50..d0975c7f0 100644 --- a/src/components/Assistant/Assistant.tsx +++ b/src/components/Assistant/Assistant.tsx @@ -89,8 +89,9 @@ export const Assistant: FC<{ onPrompt: (prompt: string) => void, awaitingSuggest return ( - +