From 47b20b3611581b09b7d454e1b46233d9d73ab1cc Mon Sep 17 00:00:00 2001 From: Abu Bakkar Siddique Date: Fri, 18 Apr 2025 17:46:34 +0530 Subject: [PATCH 1/7] Added firecrawl with an input box to crawl website contents from url --- apps/studio/.env.example | 4 +- apps/studio/package.json | 1 + apps/studio/src/lib/projects/create.ts | 9 +- apps/studio/src/lib/services/crawler.ts | 58 ++++++++++ apps/studio/src/locales/en/translation.json | 4 + .../projects/PromptCreation/PromptingCard.tsx | 100 +++++++++++++++++- apps/web/client/messages/en.json | 4 + 7 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 apps/studio/src/lib/services/crawler.ts diff --git a/apps/studio/.env.example b/apps/studio/.env.example index 11d4b66d55..d7dc5668d5 100644 --- a/apps/studio/.env.example +++ b/apps/studio/.env.example @@ -4,4 +4,6 @@ VITE_SUPABASE_ANON_KEY= VITE_MIXPANEL_TOKEN= # Add your keys here to use Anthropic directly -VITE_ANTHROPIC_API_KEY= \ No newline at end of file +VITE_ANTHROPIC_API_KEY= +# Add your Firecrawl API key here +VITE_FIRECRAWL_API_KEY= \ No newline at end of file diff --git a/apps/studio/package.json b/apps/studio/package.json index 2286c72927..5070e7d23b 100644 --- a/apps/studio/package.json +++ b/apps/studio/package.json @@ -53,6 +53,7 @@ "@emotion/react": "^11.13.3", "@emotion/styled": "^11.13.0", "@fontsource-variable/inter": "^5.1.0", + "@mendable/firecrawl-js": "^1.24.0", "@onlook/foundation": "*", "@onlook/supabase": "*", "@onlook/ui": "*", diff --git a/apps/studio/src/lib/projects/create.ts b/apps/studio/src/lib/projects/create.ts index bfa99208bf..f72a99b3ae 100644 --- a/apps/studio/src/lib/projects/create.ts +++ b/apps/studio/src/lib/projects/create.ts @@ -89,9 +89,15 @@ export class CreateManager { } } - async sendPrompt(prompt: string, images: ImageMessageContext[], blank: boolean = false) { + async sendPrompt( + prompt: string, + images: ImageMessageContext[], + crawledContent?: string, + blank: boolean = false, + ) { sendAnalytics('prompt create project', { prompt, + crawledContent, blank, }); @@ -104,6 +110,7 @@ export class CreateManager { } else { result = await invokeMainChannel(MainChannels.CREATE_NEW_PROJECT_PROMPT, { prompt, + crawledContent, images, }); } diff --git a/apps/studio/src/lib/services/crawler.ts b/apps/studio/src/lib/services/crawler.ts new file mode 100644 index 0000000000..aecb6ecf30 --- /dev/null +++ b/apps/studio/src/lib/services/crawler.ts @@ -0,0 +1,58 @@ +import FirecrawlApp from '@mendable/firecrawl-js'; + +export interface CrawlOptions { + limit?: number; + scrapeOptions?: { + formats?: ( + | 'markdown' + | 'html' + | 'rawHtml' + | 'content' + | 'links' + | 'screenshot' + | 'screenshot@fullPage' + | 'extract' + | 'json' + | 'changeTracking' + )[]; + }; +} + +export class CrawlerService { + private static intance: CrawlerService; + + private app: FirecrawlApp; + + private constructor() { + this.app = new FirecrawlApp({ apiKey: process.env.VITE_FIRECRAWL_API_KEY }); + } + + static getInstance(): CrawlerService { + if (!this.intance) { + this.intance = new CrawlerService(); + } + return this.intance; + } + + async crawlUrl( + url: string, + options: CrawlOptions = { + limit: 100, + scrapeOptions: { + formats: ['markdown', 'html'], + }, + }, + ) { + try { + const response = await this.app.crawlUrl(url, options); + + if (!response.success) { + throw new Error(`Failed to crawl: ${response.error}`); + } + return response; + } catch (error) { + console.error('Error during crawling:', error); + throw error; + } + } +} diff --git a/apps/studio/src/locales/en/translation.json b/apps/studio/src/locales/en/translation.json index 0f153e79de..17b24fa8b7 100644 --- a/apps/studio/src/locales/en/translation.json +++ b/apps/studio/src/locales/en/translation.json @@ -73,6 +73,10 @@ "fileReference": "File Reference", "submit": "Start building your site" }, + "crawl": { + "title": "Duplicate a website", + "description": "Paste a link to a website that you want to duplicate" + }, "blankStart": "Start from a blank page" } }, diff --git a/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx b/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx index 402dc6533b..411ede6580 100644 --- a/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx +++ b/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx @@ -14,6 +14,8 @@ import { useEffect, useRef, useState } from 'react'; import useResizeObserver from 'use-resize-observer'; import { DraftImagePill } from '../../editor/EditPanel/ChatTab/ContextPills/DraftingImagePill'; import { useTranslation } from 'react-i18next'; +import { CrawlerService } from '@/lib/services/crawler'; +import { toast } from '@onlook/ui/use-toast'; export const PromptingCard = () => { const projectsManager = useProjectsManager(); @@ -28,6 +30,9 @@ export const PromptingCard = () => { const [isComposing, setIsComposing] = useState(false); const imageRef = useRef(null); const { t } = useTranslation(); + const [urlInput, setUrlInput] = useState(''); + const [isCrawling, setIsCrawling] = useState(false); + const [crawledValue, setCrawledValue] = useState(''); useEffect(() => { const handleEscapeKey = (e: KeyboardEvent) => { @@ -45,11 +50,11 @@ export const PromptingCard = () => { console.warn('Input is too short'); return; } - projectsManager.create.sendPrompt(inputValue, selectedImages, false); + projectsManager.create.sendPrompt(inputValue, selectedImages, crawledValue, false); }; const handleBlankSubmit = async () => { - projectsManager.create.sendPrompt('', [], true); + projectsManager.create.sendPrompt('', [], '', true); }; const handleDragOver = (e: React.DragEvent) => { @@ -179,6 +184,40 @@ export const PromptingCard = () => { } }; + const handleCrawlSubmit = async () => { + if (!urlInput.trim()) { + console.warn('URL input is empty'); + return; + } + setIsCrawling(true); + + try { + const crawler = CrawlerService.getInstance(); + + const response = await crawler.crawlUrl(urlInput); + + const responseData = response.data; + const html = responseData[0]?.html || ''; + const markdown = responseData[0]?.markdown || ''; + + setCrawledValue(`---MARKDOWN---\n${markdown}\n\n---HTML---\n${html}`); + + toast({ + title: 'URL Crawled', + description: `Data for ${urlInput} has been crawled successfully.`, + }); + } catch (error) { + console.error('Failed to crawl URL:', error); + toast({ + title: 'Failed to Crawl URL', + description: error instanceof Error ? error.message : 'An unknown error occurred', + variant: 'destructive', + }); + } finally { + setIsCrawling(false); + } + }; + return (
@@ -382,6 +421,63 @@ export const PromptingCard = () => { + + + + {t('projects.prompt.crawl.title')} + + + {t('projects.prompt.crawl.description')} + + + +
+
+ setUrlInput(e.target.value)} + placeholder="Enter URL to crawl..." + className={cn( + 'flex-1 h-9 px-3 rounded-md', + 'bg-background-secondary/80 backdrop-blur-sm', + 'border border-border', + 'text-sm text-foreground-primary', + 'placeholder:text-foreground-secondary', + 'focus:outline-none focus:ring-2 focus:ring-ring', + )} + /> + +
+
+ +
diff --git a/apps/web/client/messages/en.json b/apps/web/client/messages/en.json index 0063521289..8079f6ed70 100644 --- a/apps/web/client/messages/en.json +++ b/apps/web/client/messages/en.json @@ -75,7 +75,12 @@ }, "crawl": { "title": "Duplicate a website", - "description": "Paste a link to a website that you want to duplicate" + "description": "Paste a link to a website that you want to duplicate", + "input": { + "placeholder": "Paste a link of a website", + "crawling": "Getting Data", + "submit": "Get Data" + } }, "blankStart": "Start from a blank page" } From db20380e6ed0699c49eca3344b13e9789420855c Mon Sep 17 00:00:00 2001 From: Abu Bakkar Siddique Date: Fri, 18 Apr 2025 21:06:33 +0530 Subject: [PATCH 4/7] Added validation for firecrawl api key and trimmed the url using url constructor and fixed grammatical mistake --- apps/studio/src/lib/services/crawler.ts | 8 +++++++- apps/studio/src/locales/en/translation.json | 2 +- .../routes/projects/PromptCreation/PromptingCard.tsx | 12 +++++++----- apps/web/client/messages/en.json | 2 +- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/apps/studio/src/lib/services/crawler.ts b/apps/studio/src/lib/services/crawler.ts index 81c29aa436..68aefe3888 100644 --- a/apps/studio/src/lib/services/crawler.ts +++ b/apps/studio/src/lib/services/crawler.ts @@ -24,7 +24,13 @@ export class CrawlerService { private app: FirecrawlApp; private constructor() { - this.app = new FirecrawlApp({ apiKey: import.meta.env.VITE_FIRECRAWL_API_KEY }); + const apiKey = import.meta.env.VITE_FIRECRAWL_API_KEY; + if (!apiKey) { + throw new Error( + 'VITE_FIRECRAWL_API_KEY is not defined. Please provide a valid API key.', + ); + } + this.app = new FirecrawlApp({ apiKey }); } static getInstance(): CrawlerService { diff --git a/apps/studio/src/locales/en/translation.json b/apps/studio/src/locales/en/translation.json index c963c26ff3..005c2d5a70 100644 --- a/apps/studio/src/locales/en/translation.json +++ b/apps/studio/src/locales/en/translation.json @@ -77,7 +77,7 @@ "title": "Duplicate a website", "description": "Paste a link to a website that you want to duplicate", "input": { - "placeholder": "Paste a link of a website", + "placeholder": "Paste a link to a website", "crawling": "Getting Data", "submit": "Get Data" } diff --git a/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx b/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx index d488223fc1..3dd71d5d90 100644 --- a/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx +++ b/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx @@ -186,13 +186,14 @@ export const PromptingCard = () => { }; const handleCrawlSubmit = async () => { - if (!urlInput.trim()) { + const trimmedUrlInput = urlInput.trim(); + if (!trimmedUrlInput) { console.warn('URL input is empty'); return; } try { - const url = new URL(urlInput); + const url = new URL(trimmedUrlInput); if (!['http:', 'https:'].includes(url.protocol)) { console.warn('URL must start with http or https'); toast({ @@ -203,7 +204,7 @@ export const PromptingCard = () => { return; } } catch (error) { - console.warn('Invalid URL:', urlInput); + console.warn('Invalid URL:', trimmedUrlInput); toast({ title: 'Invalid URL', description: 'Please enter a valid URL format.', @@ -217,7 +218,7 @@ export const PromptingCard = () => { try { const crawler = CrawlerService.getInstance(); - const response = await crawler.crawlUrl(urlInput); + const response = await crawler.crawlUrl(trimmedUrlInput); const responseData = response.data; const html = responseData[0]?.html || ''; @@ -227,7 +228,7 @@ export const PromptingCard = () => { toast({ title: 'URL Crawled', - description: `Data for ${urlInput} has been crawled successfully.`, + description: `Data for ${trimmedUrlInput} has been crawled successfully.`, }); } catch (error) { console.error('Failed to crawl URL:', error); @@ -238,6 +239,7 @@ export const PromptingCard = () => { }); } finally { setIsCrawling(false); + setUrlInput(''); } }; diff --git a/apps/web/client/messages/en.json b/apps/web/client/messages/en.json index 8079f6ed70..d8b9f27dda 100644 --- a/apps/web/client/messages/en.json +++ b/apps/web/client/messages/en.json @@ -77,7 +77,7 @@ "title": "Duplicate a website", "description": "Paste a link to a website that you want to duplicate", "input": { - "placeholder": "Paste a link of a website", + "placeholder": "Paste a link to a website", "crawling": "Getting Data", "submit": "Get Data" } From 793342eb7923e506479599179ba0b1ea3a8ccf43 Mon Sep 17 00:00:00 2001 From: Abu Bakkar Siddique Date: Fri, 18 Apr 2025 23:10:57 +0530 Subject: [PATCH 5/7] make the url input clear on successful crawl --- apps/studio/src/locales/en/translation.json | 1 + .../src/routes/projects/PromptCreation/PromptingCard.tsx | 4 +++- apps/web/client/messages/en.json | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/studio/src/locales/en/translation.json b/apps/studio/src/locales/en/translation.json index 005c2d5a70..964e93d79c 100644 --- a/apps/studio/src/locales/en/translation.json +++ b/apps/studio/src/locales/en/translation.json @@ -78,6 +78,7 @@ "description": "Paste a link to a website that you want to duplicate", "input": { "placeholder": "Paste a link to a website", + "arialLabel": "URL input for web page crawling", "crawling": "Getting Data", "submit": "Get Data" } diff --git a/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx b/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx index 3dd71d5d90..2f42b9d617 100644 --- a/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx +++ b/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx @@ -230,6 +230,8 @@ export const PromptingCard = () => { title: 'URL Crawled', description: `Data for ${trimmedUrlInput} has been crawled successfully.`, }); + + setUrlInput(''); } catch (error) { console.error('Failed to crawl URL:', error); toast({ @@ -239,7 +241,6 @@ export const PromptingCard = () => { }); } finally { setIsCrawling(false); - setUrlInput(''); } }; @@ -476,6 +477,7 @@ export const PromptingCard = () => { type="url" value={urlInput} onChange={(e) => setUrlInput(e.target.value)} + aria-label={t('projects.prompt.crawl.input.ariaLabel')} placeholder={t('projects.prompt.crawl.input.placeholder')} className={cn( 'flex-1 h-9 px-3 rounded-md', diff --git a/apps/web/client/messages/en.json b/apps/web/client/messages/en.json index d8b9f27dda..dd561200e7 100644 --- a/apps/web/client/messages/en.json +++ b/apps/web/client/messages/en.json @@ -78,6 +78,7 @@ "description": "Paste a link to a website that you want to duplicate", "input": { "placeholder": "Paste a link to a website", + "arialLabel": "URL input for web page crawling", "crawling": "Getting Data", "submit": "Get Data" } From 3ee56d44951264528f0c0d8afa3594836d6212e4 Mon Sep 17 00:00:00 2001 From: Abu Bakkar Siddique Date: Sat, 19 Apr 2025 00:10:58 +0530 Subject: [PATCH 6/7] Added type-safe in the crawled response --- apps/studio/src/lib/projects/create.ts | 3 +- apps/studio/src/lib/services/crawler.ts | 41 +++++++++++++++++++ apps/studio/src/locales/en/translation.json | 2 +- .../projects/PromptCreation/PromptingCard.tsx | 28 ++++++++----- apps/web/client/messages/en.json | 2 +- 5 files changed, 63 insertions(+), 13 deletions(-) diff --git a/apps/studio/src/lib/projects/create.ts b/apps/studio/src/lib/projects/create.ts index f72a99b3ae..3b824b3f3a 100644 --- a/apps/studio/src/lib/projects/create.ts +++ b/apps/studio/src/lib/projects/create.ts @@ -4,6 +4,7 @@ import { MainChannels } from '@onlook/models/constants'; import { makeAutoObservable } from 'mobx'; import { ProjectTabs, type ProjectsManager } from '.'; import { invokeMainChannel, sendAnalytics } from '../utils'; +import type { CrawledContent } from '@/lib/services/crawler'; export enum CreateState { PROMPT = 'prompting', @@ -92,7 +93,7 @@ export class CreateManager { async sendPrompt( prompt: string, images: ImageMessageContext[], - crawledContent?: string, + crawledContent?: CrawledContent, blank: boolean = false, ) { sendAnalytics('prompt create project', { diff --git a/apps/studio/src/lib/services/crawler.ts b/apps/studio/src/lib/services/crawler.ts index 68aefe3888..ac9b693022 100644 --- a/apps/studio/src/lib/services/crawler.ts +++ b/apps/studio/src/lib/services/crawler.ts @@ -18,6 +18,47 @@ export interface CrawlOptions { }; } +export interface CrawlerResponse { + success: boolean; + error?: string; + data: Array<{ + html?: string; + markdown?: string; + }>; +} + +export interface CrawledContent { + markdown?: string; + html?: string; +} + +export function validateCrawlerResponse(response: unknown): response is CrawlerResponse { + if (!response || typeof response !== 'object') { + return false; + } + + if (!('success' in response) || typeof response.success !== 'boolean') { + return false; + } + + if (!('data' in response) || !Array.isArray(response.data)) { + return false; + } + + if (response.data.length === 0) { + return false; + } + + const firstItem = response.data[0]; + return ( + typeof firstItem === 'object' && + firstItem !== null && + ('html' in firstItem || 'markdown' in firstItem) && + (firstItem.html === undefined || typeof firstItem.html === 'string') && + (firstItem.markdown === undefined || typeof firstItem.markdown === 'string') + ); +} + export class CrawlerService { private static instance: CrawlerService; diff --git a/apps/studio/src/locales/en/translation.json b/apps/studio/src/locales/en/translation.json index 964e93d79c..e1f7dca5fe 100644 --- a/apps/studio/src/locales/en/translation.json +++ b/apps/studio/src/locales/en/translation.json @@ -78,7 +78,7 @@ "description": "Paste a link to a website that you want to duplicate", "input": { "placeholder": "Paste a link to a website", - "arialLabel": "URL input for web page crawling", + "ariaLabel": "URL input for web page crawling", "crawling": "Getting Data", "submit": "Get Data" } diff --git a/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx b/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx index 2f42b9d617..f2c4fcc3fb 100644 --- a/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx +++ b/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx @@ -14,7 +14,8 @@ import { useEffect, useRef, useState } from 'react'; import useResizeObserver from 'use-resize-observer'; import { DraftImagePill } from '../../editor/EditPanel/ChatTab/ContextPills/DraftingImagePill'; import { useTranslation } from 'react-i18next'; -import { CrawlerService } from '@/lib/services/crawler'; +import { CrawlerService, validateCrawlerResponse } from '@/lib/services/crawler'; +import type { CrawledContent } from '@/lib/services/crawler'; import { toast } from '@onlook/ui/use-toast'; export const PromptingCard = () => { @@ -32,7 +33,7 @@ export const PromptingCard = () => { const { t } = useTranslation(); const [urlInput, setUrlInput] = useState(''); const [isCrawling, setIsCrawling] = useState(false); - const [crawledValue, setCrawledValue] = useState(''); + const [crawledValue, setCrawledValue] = useState({ markdown: '', html: '' }); useEffect(() => { const handleEscapeKey = (e: KeyboardEvent) => { @@ -51,11 +52,11 @@ export const PromptingCard = () => { return; } projectsManager.create.sendPrompt(inputValue, selectedImages, crawledValue, false); - setCrawledValue(''); + setCrawledValue({ markdown: '', html: '' }); }; const handleBlankSubmit = async () => { - projectsManager.create.sendPrompt('', [], '', true); + projectsManager.create.sendPrompt('', [], { html: '', markdown: '' }, true); }; const handleDragOver = (e: React.DragEvent) => { @@ -188,7 +189,11 @@ export const PromptingCard = () => { const handleCrawlSubmit = async () => { const trimmedUrlInput = urlInput.trim(); if (!trimmedUrlInput) { - console.warn('URL input is empty'); + toast({ + title: 'URL Required', + description: 'Please enter a URL before submitting.', + variant: 'destructive', + }); return; } @@ -217,14 +222,17 @@ export const PromptingCard = () => { try { const crawler = CrawlerService.getInstance(); - const response = await crawler.crawlUrl(trimmedUrlInput); - const responseData = response.data; - const html = responseData[0]?.html || ''; - const markdown = responseData[0]?.markdown || ''; + if (!validateCrawlerResponse(response)) { + throw new Error('Invalid response format from crawler'); + } - setCrawledValue(`---MARKDOWN---\n${markdown}\n\n---HTML---\n${html}`); + const responseData = response.data; + setCrawledValue({ + html: responseData[0]?.html || '', + markdown: responseData[0]?.markdown || '', + }); toast({ title: 'URL Crawled', diff --git a/apps/web/client/messages/en.json b/apps/web/client/messages/en.json index dd561200e7..cb529b0f85 100644 --- a/apps/web/client/messages/en.json +++ b/apps/web/client/messages/en.json @@ -78,7 +78,7 @@ "description": "Paste a link to a website that you want to duplicate", "input": { "placeholder": "Paste a link to a website", - "arialLabel": "URL input for web page crawling", + "ariaLabel": "URL input for web page crawling", "crawling": "Getting Data", "submit": "Get Data" } From 3857b50a8ad1e85dbe0bce00ba24e9f1513e2655 Mon Sep 17 00:00:00 2001 From: Abu Bakkar Siddique Date: Sun, 20 Apr 2025 16:45:40 +0530 Subject: [PATCH 7/7] Removed UI and make it a toolcall --- apps/studio/electron/main/chat/index.ts | 42 ++++++ apps/studio/src/lib/projects/create.ts | 10 +- apps/studio/src/locales/en/translation.json | 10 -- .../projects/PromptCreation/PromptingCard.tsx | 140 +----------------- apps/web/client/messages/en.json | 10 -- .../ai/src/tools}/crawler.ts | 0 packages/ai/src/tools/helpers.ts | 37 +++++ packages/ai/src/tools/index.ts | 58 ++++++++ packages/models/src/chat/message/context.ts | 9 +- 9 files changed, 148 insertions(+), 168 deletions(-) rename {apps/studio/src/lib/services => packages/ai/src/tools}/crawler.ts (100%) diff --git a/apps/studio/electron/main/chat/index.ts b/apps/studio/electron/main/chat/index.ts index 66590255b9..a6de28cc93 100644 --- a/apps/studio/electron/main/chat/index.ts +++ b/apps/studio/electron/main/chat/index.ts @@ -25,6 +25,7 @@ import { z } from 'zod'; import { mainWindow } from '..'; import { PersistentStorage } from '../storage'; import { initModel } from './llmProvider'; +import { extractUrls } from '@onlook/ai/src/tools/helpers'; class LlmManager { private static instance: LlmManager; @@ -59,6 +60,43 @@ class LlmManager { return LlmManager.instance; } + private async processUrls(content: string): Promise { + const urls = extractUrls(content); + + if (urls.length === 0) { + return content; + } + + try { + const result = await streamText({ + model: await initModel(LLMProvider.ANTHROPIC, CLAUDE_MODELS.SONNET, { + requestType: StreamRequestType.SUGGESTIONS, + }), + messages: [ + { + role: 'user', + content: content, + }, + ], + tools: { crawl_urls: chatToolSet.crawl_urls }, + maxTokens: 4000, + }); + + const crawledContent = ` + Original request: + ${content} + + Referenced content from URLs: + ${JSON.stringify(result, null, 2)} + `; + + return crawledContent; + } catch (error) { + console.error('Error processing URLs:', error); + return content; + } + } + public async stream( messages: CoreMessage[], requestType: StreamRequestType, @@ -67,6 +105,10 @@ class LlmManager { skipSystemPrompt?: boolean; }, ): Promise { + const lastUserMessage = messages.findLast((m) => m.role === 'user'); + if (lastUserMessage && typeof lastUserMessage.content === 'string') { + lastUserMessage.content = await this.processUrls(lastUserMessage.content); + } const { abortController, skipSystemPrompt } = options || {}; this.abortController = abortController || new AbortController(); try { diff --git a/apps/studio/src/lib/projects/create.ts b/apps/studio/src/lib/projects/create.ts index 3b824b3f3a..bfa99208bf 100644 --- a/apps/studio/src/lib/projects/create.ts +++ b/apps/studio/src/lib/projects/create.ts @@ -4,7 +4,6 @@ import { MainChannels } from '@onlook/models/constants'; import { makeAutoObservable } from 'mobx'; import { ProjectTabs, type ProjectsManager } from '.'; import { invokeMainChannel, sendAnalytics } from '../utils'; -import type { CrawledContent } from '@/lib/services/crawler'; export enum CreateState { PROMPT = 'prompting', @@ -90,15 +89,9 @@ export class CreateManager { } } - async sendPrompt( - prompt: string, - images: ImageMessageContext[], - crawledContent?: CrawledContent, - blank: boolean = false, - ) { + async sendPrompt(prompt: string, images: ImageMessageContext[], blank: boolean = false) { sendAnalytics('prompt create project', { prompt, - crawledContent, blank, }); @@ -111,7 +104,6 @@ export class CreateManager { } else { result = await invokeMainChannel(MainChannels.CREATE_NEW_PROJECT_PROMPT, { prompt, - crawledContent, images, }); } diff --git a/apps/studio/src/locales/en/translation.json b/apps/studio/src/locales/en/translation.json index e1f7dca5fe..0f153e79de 100644 --- a/apps/studio/src/locales/en/translation.json +++ b/apps/studio/src/locales/en/translation.json @@ -73,16 +73,6 @@ "fileReference": "File Reference", "submit": "Start building your site" }, - "crawl": { - "title": "Duplicate a website", - "description": "Paste a link to a website that you want to duplicate", - "input": { - "placeholder": "Paste a link to a website", - "ariaLabel": "URL input for web page crawling", - "crawling": "Getting Data", - "submit": "Get Data" - } - }, "blankStart": "Start from a blank page" } }, diff --git a/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx b/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx index f2c4fcc3fb..402dc6533b 100644 --- a/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx +++ b/apps/studio/src/routes/projects/PromptCreation/PromptingCard.tsx @@ -14,9 +14,6 @@ import { useEffect, useRef, useState } from 'react'; import useResizeObserver from 'use-resize-observer'; import { DraftImagePill } from '../../editor/EditPanel/ChatTab/ContextPills/DraftingImagePill'; import { useTranslation } from 'react-i18next'; -import { CrawlerService, validateCrawlerResponse } from '@/lib/services/crawler'; -import type { CrawledContent } from '@/lib/services/crawler'; -import { toast } from '@onlook/ui/use-toast'; export const PromptingCard = () => { const projectsManager = useProjectsManager(); @@ -31,9 +28,6 @@ export const PromptingCard = () => { const [isComposing, setIsComposing] = useState(false); const imageRef = useRef(null); const { t } = useTranslation(); - const [urlInput, setUrlInput] = useState(''); - const [isCrawling, setIsCrawling] = useState(false); - const [crawledValue, setCrawledValue] = useState({ markdown: '', html: '' }); useEffect(() => { const handleEscapeKey = (e: KeyboardEvent) => { @@ -51,12 +45,11 @@ export const PromptingCard = () => { console.warn('Input is too short'); return; } - projectsManager.create.sendPrompt(inputValue, selectedImages, crawledValue, false); - setCrawledValue({ markdown: '', html: '' }); + projectsManager.create.sendPrompt(inputValue, selectedImages, false); }; const handleBlankSubmit = async () => { - projectsManager.create.sendPrompt('', [], { html: '', markdown: '' }, true); + projectsManager.create.sendPrompt('', [], true); }; const handleDragOver = (e: React.DragEvent) => { @@ -186,72 +179,6 @@ export const PromptingCard = () => { } }; - const handleCrawlSubmit = async () => { - const trimmedUrlInput = urlInput.trim(); - if (!trimmedUrlInput) { - toast({ - title: 'URL Required', - description: 'Please enter a URL before submitting.', - variant: 'destructive', - }); - return; - } - - try { - const url = new URL(trimmedUrlInput); - if (!['http:', 'https:'].includes(url.protocol)) { - console.warn('URL must start with http or https'); - toast({ - title: 'Invalid URL', - description: 'Please enter a URL that starts with http or https.', - variant: 'destructive', - }); - return; - } - } catch (error) { - console.warn('Invalid URL:', trimmedUrlInput); - toast({ - title: 'Invalid URL', - description: 'Please enter a valid URL format.', - variant: 'destructive', - }); - return; - } - - setIsCrawling(true); - - try { - const crawler = CrawlerService.getInstance(); - const response = await crawler.crawlUrl(trimmedUrlInput); - - if (!validateCrawlerResponse(response)) { - throw new Error('Invalid response format from crawler'); - } - - const responseData = response.data; - setCrawledValue({ - html: responseData[0]?.html || '', - markdown: responseData[0]?.markdown || '', - }); - - toast({ - title: 'URL Crawled', - description: `Data for ${trimmedUrlInput} has been crawled successfully.`, - }); - - setUrlInput(''); - } catch (error) { - console.error('Failed to crawl URL:', error); - toast({ - title: 'Failed to Crawl URL', - description: error instanceof Error ? error.message : 'An unknown error occurred', - variant: 'destructive', - }); - } finally { - setIsCrawling(false); - } - }; - return (
@@ -455,69 +382,6 @@ export const PromptingCard = () => { - - - - {t('projects.prompt.crawl.title')} - - - {t('projects.prompt.crawl.description')} - - - -
-
- setUrlInput(e.target.value)} - aria-label={t('projects.prompt.crawl.input.ariaLabel')} - placeholder={t('projects.prompt.crawl.input.placeholder')} - className={cn( - 'flex-1 h-9 px-3 rounded-md', - 'bg-background-secondary/80 backdrop-blur-sm', - 'border border-border', - 'text-sm text-foreground-primary', - 'placeholder:text-foreground-secondary', - 'focus:outline-none focus:ring-2 focus:ring-ring', - )} - /> - -
-
- -