diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index a3796cbeb..c88ba068b 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -201,6 +201,11 @@ export class RemoteBrowser { private networkRequestTimeout: NodeJS.Timeout | null = null; private pendingNetworkRequests: string[] = []; private readonly NETWORK_QUIET_PERIOD = 8000; + private readonly INITIAL_LOAD_QUIET_PERIOD = 3000; + private networkWaitStartTime: number = 0; + private progressInterval: NodeJS.Timeout | null = null; + private hasShownInitialLoader: boolean = false; + private isInitialLoadInProgress: boolean = false; /** * Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and @@ -432,17 +437,19 @@ export class RemoteBrowser { if (!this.currentPage) return; this.currentPage.on("domcontentloaded", async () => { - logger.info("DOM content loaded - triggering snapshot"); - await this.makeAndEmitDOMSnapshot(); + if (!this.isInitialLoadInProgress) { + logger.info("DOM content loaded - triggering snapshot"); + await this.makeAndEmitDOMSnapshot(); + } }); this.currentPage.on("response", async (response) => { const url = response.url(); - if ( - response.request().resourceType() === "document" || - url.includes("api/") || - url.includes("ajax") - ) { + const isDocumentRequest = response.request().resourceType() === "document"; + + if (!this.hasShownInitialLoader && isDocumentRequest && !url.includes("about:blank")) { + this.hasShownInitialLoader = true; + this.isInitialLoadInProgress = true; this.pendingNetworkRequests.push(url); if (this.networkRequestTimeout) { @@ -450,24 +457,54 @@ export class RemoteBrowser { this.networkRequestTimeout = null; } + if (this.progressInterval) { + clearInterval(this.progressInterval); + this.progressInterval = null; + } + + this.networkWaitStartTime = Date.now(); + this.progressInterval = setInterval(() => { + const elapsed = Date.now() - this.networkWaitStartTime; + const navigationProgress = Math.min((elapsed / this.INITIAL_LOAD_QUIET_PERIOD) * 40, 35); + const totalProgress = 60 + navigationProgress; + this.emitLoadingProgress(totalProgress, this.pendingNetworkRequests.length); + }, 500); + logger.debug( - `Network request received: ${url}. Total pending: ${this.pendingNetworkRequests.length}` + `Initial load network request received: ${url}. Using ${this.INITIAL_LOAD_QUIET_PERIOD}ms quiet period` ); this.networkRequestTimeout = setTimeout(async () => { logger.info( - `Network quiet period reached. Processing ${this.pendingNetworkRequests.length} requests` + `Initial load network quiet period reached (${this.INITIAL_LOAD_QUIET_PERIOD}ms)` ); + if (this.progressInterval) { + clearInterval(this.progressInterval); + this.progressInterval = null; + } + + this.emitLoadingProgress(100, this.pendingNetworkRequests.length); + this.pendingNetworkRequests = []; this.networkRequestTimeout = null; + this.isInitialLoadInProgress = false; await this.makeAndEmitDOMSnapshot(); - }, this.NETWORK_QUIET_PERIOD); + }, this.INITIAL_LOAD_QUIET_PERIOD); } }); } + private emitLoadingProgress(progress: number, pendingRequests: number): void { + this.socket.emit("domLoadingProgress", { + progress: Math.round(progress), + pendingRequests, + userId: this.userId, + timestamp: Date.now(), + }); + } + private async setupPageEventListeners(page: Page) { page.on('framenavigated', async (frame) => { if (frame === page.mainFrame()) { @@ -521,7 +558,13 @@ export class RemoteBrowser { const MAX_RETRIES = 3; let retryCount = 0; let success = false; - + + this.socket.emit("dom-snapshot-loading", { + userId: this.userId, + timestamp: Date.now(), + }); + this.emitLoadingProgress(0, 0); + while (!success && retryCount < MAX_RETRIES) { try { this.browser = (await chromium.launch({ @@ -545,7 +588,9 @@ export class RemoteBrowser { if (!this.browser || this.browser.isConnected() === false) { throw new Error('Browser failed to launch or is not connected'); } - + + this.emitLoadingProgress(20, 0); + const proxyConfig = await getDecryptedProxyConfig(userId); let proxyOptions: { server: string, username?: string, password?: string } = { server: '' }; @@ -623,6 +668,8 @@ export class RemoteBrowser { this.currentPage = await this.context.newPage(); + this.emitLoadingProgress(40, 0); + await this.setupPageEventListeners(this.currentPage); const viewportSize = await this.currentPage.viewportSize(); @@ -645,7 +692,9 @@ export class RemoteBrowser { // Still need to set up the CDP session even if blocker fails this.client = await this.currentPage.context().newCDPSession(this.currentPage); } - + + this.emitLoadingProgress(60, 0); + success = true; logger.log('debug', `Browser initialized successfully for user ${userId}`); } catch (error: any) { @@ -1521,9 +1570,6 @@ export class RemoteBrowser { this.isDOMStreamingActive = true; logger.info("DOM streaming started successfully"); - // Initial DOM snapshot - await this.makeAndEmitDOMSnapshot(); - this.setupScrollEventListener(); this.setupPageChangeListeners(); } catch (error) { diff --git a/src/components/browser/BrowserContent.tsx b/src/components/browser/BrowserContent.tsx index 46a8886d9..f592f846b 100644 --- a/src/components/browser/BrowserContent.tsx +++ b/src/components/browser/BrowserContent.tsx @@ -13,7 +13,7 @@ import { export const BrowserContent = () => { const { socket } = useSocketStore(); - const [tabs, setTabs] = useState(["current"]); + const [tabs, setTabs] = useState(["Loading..."]); const [tabIndex, setTabIndex] = React.useState(0); const [showOutputData, setShowOutputData] = useState(false); const { browserWidth } = useBrowserDimensionsStore(); @@ -125,7 +125,7 @@ export const BrowserContent = () => { useEffect(() => { getCurrentTabs() .then((response) => { - if (response) { + if (response && response.length > 0) { setTabs(response); } }) diff --git a/src/components/browser/BrowserWindow.tsx b/src/components/browser/BrowserWindow.tsx index 98642f8c5..207e8b56b 100644 --- a/src/components/browser/BrowserWindow.tsx +++ b/src/components/browser/BrowserWindow.tsx @@ -1,8 +1,6 @@ import React, { useCallback, useContext, useEffect, useRef, useState } from 'react'; import { useSocketStore } from '../../context/socket'; import { Button } from '@mui/material'; -import Canvas from "../recorder/Canvas"; -import { Highlighter } from "../recorder/Highlighter"; import { GenericModal } from '../ui/GenericModal'; import { useActionContext } from '../../context/browserActions'; import { useBrowserSteps, TextStep, ListStep } from '../../context/browserSteps'; @@ -38,12 +36,6 @@ interface AttributeOption { value: string; } -interface ScreencastData { - image: string; - userId: string; - viewport?: ViewportInfo | null; -} - interface ViewportInfo { width: number; height: number; @@ -146,8 +138,6 @@ const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null): export const BrowserWindow = () => { const { t } = useTranslation(); const { browserWidth, browserHeight } = useBrowserDimensionsStore(); - const [canvasRef, setCanvasReference] = useState | undefined>(undefined); - const [screenShot, setScreenShot] = useState(""); const [highlighterData, setHighlighterData] = useState<{ rect: DOMRect; selector: string; @@ -1303,17 +1293,6 @@ export const BrowserWindow = () => { }, []); const onMouseMove = (e: MouseEvent) => { - if (canvasRef && canvasRef.current && highlighterData) { - const canvasRect = canvasRef.current.getBoundingClientRect(); - if ( - e.pageX < canvasRect.left - || e.pageX > canvasRect.right - || e.pageY < canvasRect.top - || e.pageY > canvasRect.bottom - ) { - setHighlighterData(null); - } - } }; const resetListState = useCallback(() => { @@ -1331,35 +1310,15 @@ export const BrowserWindow = () => { } }, [getList, resetListState]); - const screencastHandler = useCallback((data: string | ScreencastData) => { - if (typeof data === 'string') { - setScreenShot(data); - } else if (data && typeof data === 'object' && 'image' in data) { - if (!data.userId || data.userId === user?.id) { - setScreenShot(data.image); - - if (data.viewport) { - setViewportInfo(data.viewport); - } - } - } - }, [user?.id]); - useEffect(() => { if (socket) { - socket.on("screencast", screencastHandler); socket.on("domcast", rrwebSnapshotHandler); socket.on("dom-mode-enabled", domModeHandler); socket.on("dom-mode-error", domModeErrorHandler); } - if (canvasRef?.current && !isDOMMode && screenShot) { - drawImage(screenShot, canvasRef.current); - } - return () => { if (socket) { - socket.off("screencast", screencastHandler); socket.off("domcast", rrwebSnapshotHandler); socket.off("dom-mode-enabled", domModeHandler); socket.off("dom-mode-error", domModeErrorHandler); @@ -1367,10 +1326,6 @@ export const BrowserWindow = () => { }; }, [ socket, - screenShot, - canvasRef, - isDOMMode, - screencastHandler, rrwebSnapshotHandler, domModeHandler, domModeErrorHandler, @@ -1847,24 +1802,7 @@ export const BrowserWindow = () => { const handleClick = (e: React.MouseEvent) => { if (highlighterData) { - let shouldProcessClick = false; - - if (!isDOMMode && canvasRef?.current) { - const canvasRect = canvasRef.current.getBoundingClientRect(); - const clickX = e.clientX - canvasRect.left; - const clickY = e.clientY - canvasRect.top; - const highlightRect = highlighterData.rect; - const mappedRect = - coordinateMapper.mapBrowserRectToCanvas(highlightRect); - - shouldProcessClick = - clickX >= mappedRect.left && - clickX <= mappedRect.right && - clickY >= mappedRect.top && - clickY <= mappedRect.bottom; - } else { - shouldProcessClick = true; - } + const shouldProcessClick = true; if (shouldProcessClick) { const options = getAttributeOptions( @@ -2209,17 +2147,7 @@ export const BrowserWindow = () => { !showAttributeModal && highlighterData?.rect != null && ( <> - {!isDOMMode && canvasRef?.current && ( - - )} - - {isDOMMode && highlighterData && ( + {highlighterData && (
{ borderRadius: "0px 0px 5px 5px", }} > - {isDOMMode ? ( + {currentSnapshot ? ( <> - {currentSnapshot ? ( - - ) : ( - - )} + {/* --- Loading overlay --- */} {isCachingChildSelectors && ( @@ -2492,11 +2416,7 @@ export const BrowserWindow = () => { )} ) : ( - + )}
@@ -2591,26 +2511,6 @@ const DOMLoadingIndicator: React.FC = () => { ); }; - -const drawImage = (image: string, canvas: HTMLCanvasElement): void => { - const ctx = canvas.getContext('2d'); - if (!ctx) return; - - const img = new Image(); - img.onload = () => { - requestAnimationFrame(() => { - ctx.drawImage(img, 0, 0, canvas.width, canvas.height); - }); - if (image.startsWith('blob:')) { - URL.revokeObjectURL(image); - } - }; - img.onerror = () => { - console.warn('Failed to load image'); - }; - img.src = image; -}; - const modalStyle = { top: '50%', left: '50%', diff --git a/src/pages/PageWrapper.tsx b/src/pages/PageWrapper.tsx index 1af536811..ea6763003 100644 --- a/src/pages/PageWrapper.tsx +++ b/src/pages/PageWrapper.tsx @@ -133,6 +133,10 @@ export const PageWrapper = () => { path="/register" element={} /> + } + /> } /> diff --git a/src/pages/RecordingPage.tsx b/src/pages/RecordingPage.tsx index 6e3624713..d53ce32d7 100644 --- a/src/pages/RecordingPage.tsx +++ b/src/pages/RecordingPage.tsx @@ -43,7 +43,7 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { const { setId, socket } = useSocketStore(); const { setWidth } = useBrowserDimensionsStore(); - const { browserId, setBrowserId, recordingId, recordingUrl, setRecordingUrl, setRecordingName, setRetrainRobotId } = useGlobalInfoStore(); + const { browserId, setBrowserId, recordingId, recordingUrl, setRecordingUrl, setRecordingName, setRetrainRobotId, setIsDOMMode } = useGlobalInfoStore(); const handleShowOutputData = useCallback(() => { setShowOutputData(true); @@ -77,6 +77,8 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { useEffect(() => { let isCancelled = false; const handleRecording = async () => { + setIsDOMMode(true); + const storedUrl = window.sessionStorage.getItem('recordingUrl'); if (storedUrl && !recordingUrl) { setRecordingUrl(storedUrl); @@ -137,9 +139,12 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => { if (browserId === 'new-recording') { socket?.emit('new-recording'); } + if (recordingUrl && socket) { + socket.emit('input:url', recordingUrl); + } setIsLoaded(true); } - }, [socket, browserId, recordingName, recordingId, isLoaded]); + }, [socket, browserId, recordingName, recordingId, recordingUrl, isLoaded]); useEffect(() => { socket?.on('loaded', handleLoaded); @@ -153,26 +158,20 @@ export const RecordingPage = ({ recordingName }: RecordingPageProps) => {
- {isLoaded ? ( - <> - - -
- - -
- - -
- - -
- - - - ) : ( - - )} + + +
+ + +
+ + +
+ + +
+ +