From 8f910dee22fefad70068e26632764cf7e17f4c6f Mon Sep 17 00:00:00 2001 From: Brian Murray Date: Tue, 4 Mar 2025 16:51:22 -0800 Subject: [PATCH 1/6] add search --- firecrawl.go | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/firecrawl.go b/firecrawl.go index 67a50fd..9d5da92 100644 --- a/firecrawl.go +++ b/firecrawl.go @@ -146,6 +146,36 @@ type MapResponse struct { Error string `json:"error,omitempty"` } +// SearchMetadata represents metadata for a search result +type SearchMetadata struct { + Title string `json:"title"` + Description string `json:"description"` + SourceURL string `json:"sourceURL"` + StatusCode int `json:"statusCode"` + Error string `json:"error"` +} + +// SearchDocument represents a document in search results +type SearchDocument struct { + Title string `json:"title"` + Description string `json:"description"` + URL string `json:"url"` + Markdown string `json:"markdown"` + HTML string `json:"html"` + RawHTML string `json:"rawHtml"` + Links []string `json:"links"` + Screenshot string `json:"screenshot"` + Metadata SearchMetadata `json:"metadata"` +} + +// SearchResponse represents the response for search operations +type SearchResponse struct { + Success bool `json:"success"` + Error string `json:"error,omitempty"` + Warning string `json:"warning,omitempty"` + Data []SearchDocument `json:"data,omitempty"` +} + // requestOptions represents options for making requests. type requestOptions struct { retries int @@ -602,8 +632,34 @@ func (app *FirecrawlApp) MapURL(url string, params *MapParams) (*MapResponse, er // - error: An error if the search request fails. // // Search is not implemented in API version 1.0.0. -func (app *FirecrawlApp) Search(query string, params *any) (any, error) { - return nil, fmt.Errorf("Search is not implemented in API version 1.0.0") +func (app *FirecrawlApp) Search(query string, params *any) (*SearchResponse, error) { + headers := app.prepareHeaders(nil) + jsonData := map[string]any{"query": query} + + resp, err := app.makeRequest( + http.MethodPost, + fmt.Sprintf("%s/v1/search", app.APIURL), + jsonData, + headers, + "search", + ) + if err != nil { + return nil, err + } + + var searchResponse SearchResponse + err = json.Unmarshal(resp, &searchResponse) + if err != nil { + return nil, err + } + + if searchResponse.Success { + return &searchResponse, nil + } else { + return nil, fmt.Errorf("search operation failed: %s", searchResponse.Error) + } + + // return nil, fmt.Errorf("Search is not implemented in API version 1.0.0") } // prepareHeaders prepares the headers for an HTTP request. From cc251dab80ba9adb8173aa0fbf806aa5628af6f5 Mon Sep 17 00:00:00 2001 From: Brian Murray Date: Tue, 4 Mar 2025 16:57:35 -0800 Subject: [PATCH 2/6] Add support for search query params --- firecrawl.go | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/firecrawl.go b/firecrawl.go index 9d5da92..c5a32ac 100644 --- a/firecrawl.go +++ b/firecrawl.go @@ -146,6 +146,17 @@ type MapResponse struct { Error string `json:"error,omitempty"` } +// SearchParams represents the parameters for a search request. +type SearchParams struct { + Limit int `json:"limit"` + TimeBased string `json:"tbs"` + Lang string `json:"lang"` + Country string `json:"country"` + Location string `json:"location"` + Timeout int `json:"timeout"` + ScrapeOptions ScrapeParams `json:"scrapeOptions"` +} + // SearchMetadata represents metadata for a search result type SearchMetadata struct { Title string `json:"title"` @@ -627,14 +638,38 @@ func (app *FirecrawlApp) MapURL(url string, params *MapParams) (*MapResponse, er // SearchURL searches for a URL using the Firecrawl API. // // Parameters: -// - url: The URL to search for. +// - query: The search query. // - params: Optional parameters for the search request. // - error: An error if the search request fails. -// -// Search is not implemented in API version 1.0.0. -func (app *FirecrawlApp) Search(query string, params *any) (*SearchResponse, error) { +func (app *FirecrawlApp) Search(query string, params *SearchParams) (*SearchResponse, error) { headers := app.prepareHeaders(nil) jsonData := map[string]any{"query": query} + if params != nil { + if params.Limit != 0 { + if params.Limit < 1 || params.Limit > 10 { + return nil, fmt.Errorf("limit must be between 1 and 10") + } + jsonData["limit"] = params.Limit + } + if params.TimeBased != "" { + jsonData["tbs"] = params.TimeBased + } + if params.Lang != "" { + jsonData["lang"] = params.Lang + } + if params.Country != "" { + jsonData["country"] = params.Country + } + if params.Location != "" { + jsonData["location"] = params.Location + } + if params.Timeout != 0 { + jsonData["timeout"] = params.Timeout + } + if params.ScrapeOptions.Formats != nil { + jsonData["scrapeOptions"] = params.ScrapeOptions + } + } resp, err := app.makeRequest( http.MethodPost, From 6216c36977a5cf014289664307f74e57ffdc914b Mon Sep 17 00:00:00 2001 From: Brian Murray Date: Tue, 4 Mar 2025 17:26:04 -0800 Subject: [PATCH 3/6] Adds context support and app configurations --- firecrawl.go | 79 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 7 deletions(-) diff --git a/firecrawl.go b/firecrawl.go index c5a32ac..8426acf 100644 --- a/firecrawl.go +++ b/firecrawl.go @@ -3,6 +3,7 @@ package firecrawl import ( "bytes" + "context" "encoding/json" "fmt" "io" @@ -245,6 +246,23 @@ type FirecrawlApp struct { Version string } +// FirecrawlOption is a functional option type for FirecrawlApp. +type FirecrawlOption func(*FirecrawlApp) + +// WithVersion sets the API version for the Firecrawl client. +func WithVersion(version string) FirecrawlOption { + return func(app *FirecrawlApp) { + app.Version = version + } +} + +// WithClient sets the HTTP client for the Firecrawl client. +func WithClient(client *http.Client) FirecrawlOption { + return func(app *FirecrawlApp) { + app.Client = client + } +} + // NewFirecrawlApp creates a new instance of FirecrawlApp with the provided API key and API URL. // If the API key or API URL is not provided, it attempts to retrieve them from environment variables. // If the API key is still not found, it returns an error. @@ -256,7 +274,7 @@ type FirecrawlApp struct { // Returns: // - *FirecrawlApp: A new instance of FirecrawlApp configured with the provided or retrieved API key and API URL. // - error: An error if the API key is not provided or retrieved. -func NewFirecrawlApp(apiKey, apiURL string) (*FirecrawlApp, error) { +func NewFirecrawlApp(apiKey, apiURL string, opts ...FirecrawlOption) (*FirecrawlApp, error) { if apiKey == "" { apiKey = os.Getenv("FIRECRAWL_API_KEY") if apiKey == "" { @@ -275,11 +293,16 @@ func NewFirecrawlApp(apiKey, apiURL string) (*FirecrawlApp, error) { Timeout: 60 * time.Second, } - return &FirecrawlApp{ + fca := &FirecrawlApp{ APIKey: apiKey, APIURL: apiURL, Client: client, - }, nil + } + for _, opt := range opts { + opt(fca) + } + + return fca, nil } // ScrapeURL scrapes the content of the specified URL using the Firecrawl API. @@ -292,6 +315,11 @@ func NewFirecrawlApp(apiKey, apiURL string) (*FirecrawlApp, error) { // - *FirecrawlDocument or *FirecrawlDocumentV0: The scraped document data depending on the API version. // - error: An error if the scrape request fails. func (app *FirecrawlApp) ScrapeURL(url string, params *ScrapeParams) (*FirecrawlDocument, error) { + return app.ScrapeURLWithContext(context.Background(), url, params) +} + +// ScrapeURLWithContext scrapes the content of the specified URL using the Firecrawl API. See ScrapeURL for more information. +func (app *FirecrawlApp) ScrapeURLWithContext(ctx context.Context, url string, params *ScrapeParams) (*FirecrawlDocument, error) { headers := app.prepareHeaders(nil) scrapeBody := map[string]any{"url": url} @@ -341,6 +369,7 @@ func (app *FirecrawlApp) ScrapeURL(url string, params *ScrapeParams) (*Firecrawl } resp, err := app.makeRequest( + ctx, http.MethodPost, fmt.Sprintf("%s/v1/scrape", app.APIURL), scrapeBody, @@ -377,6 +406,11 @@ func (app *FirecrawlApp) ScrapeURL(url string, params *ScrapeParams) (*Firecrawl // - CrawlStatusResponse: The crawl result if the job is completed. // - error: An error if the crawl request fails. func (app *FirecrawlApp) CrawlURL(url string, params *CrawlParams, idempotencyKey *string, pollInterval ...int) (*CrawlStatusResponse, error) { + return app.CrawlURLWithContext(context.Background(), url, params, idempotencyKey, pollInterval...) +} + +// CrawlURLWithContext starts a crawl job for the specified URL using the Firecrawl API. See CrawlURL for more information. +func (app *FirecrawlApp) CrawlURLWithContext(ctx context.Context, url string, params *CrawlParams, idempotencyKey *string, pollInterval ...int) (*CrawlStatusResponse, error) { var key string if idempotencyKey != nil { key = *idempotencyKey @@ -421,6 +455,7 @@ func (app *FirecrawlApp) CrawlURL(url string, params *CrawlParams, idempotencyKe } resp, err := app.makeRequest( + ctx, http.MethodPost, fmt.Sprintf("%s/v1/crawl", app.APIURL), crawlBody, @@ -439,7 +474,7 @@ func (app *FirecrawlApp) CrawlURL(url string, params *CrawlParams, idempotencyKe return nil, err } - return app.monitorJobStatus(crawlResponse.ID, headers, actualPollInterval) + return app.monitorJobStatus(ctx, crawlResponse.ID, headers, actualPollInterval) } // CrawlURL starts a crawl job for the specified URL using the Firecrawl API. @@ -453,6 +488,11 @@ func (app *FirecrawlApp) CrawlURL(url string, params *CrawlParams, idempotencyKe // - *CrawlResponse: The crawl response with id. // - error: An error if the crawl request fails. func (app *FirecrawlApp) AsyncCrawlURL(url string, params *CrawlParams, idempotencyKey *string) (*CrawlResponse, error) { + return app.AsyncCrawlURLWithContext(context.Background(), url, params, idempotencyKey) +} + +// AsyncCrawlURLWithContext starts a crawl job for the specified URL using the Firecrawl API. See AsyncCrawlURL for more information. +func (app *FirecrawlApp) AsyncCrawlURLWithContext(ctx context.Context, url string, params *CrawlParams, idempotencyKey *string) (*CrawlResponse, error) { var key string if idempotencyKey != nil { key = *idempotencyKey @@ -492,6 +532,7 @@ func (app *FirecrawlApp) AsyncCrawlURL(url string, params *CrawlParams, idempote } resp, err := app.makeRequest( + ctx, http.MethodPost, fmt.Sprintf("%s/v1/crawl", app.APIURL), crawlBody, @@ -527,10 +568,16 @@ func (app *FirecrawlApp) AsyncCrawlURL(url string, params *CrawlParams, idempote // - *CrawlStatusResponse: The status of the crawl job. // - error: An error if the crawl status check request fails. func (app *FirecrawlApp) CheckCrawlStatus(ID string) (*CrawlStatusResponse, error) { + return app.CheckCrawlStatusWithContext(context.Background(), ID) +} + +// CheckCrawlStatusWithContext checks the status of a crawl job using the Firecrawl API. See CheckCrawlStatus for more information. +func (app *FirecrawlApp) CheckCrawlStatusWithContext(ctx context.Context, ID string) (*CrawlStatusResponse, error) { headers := app.prepareHeaders(nil) apiURL := fmt.Sprintf("%s/v1/crawl/%s", app.APIURL, ID) resp, err := app.makeRequest( + ctx, http.MethodGet, apiURL, nil, @@ -561,9 +608,15 @@ func (app *FirecrawlApp) CheckCrawlStatus(ID string) (*CrawlStatusResponse, erro // - string: The status of the crawl job after cancellation. // - error: An error if the crawl job cancellation request fails. func (app *FirecrawlApp) CancelCrawlJob(ID string) (string, error) { + return app.CancelCrawlJobWithContext(context.Background(), ID) +} + +// CancelCrawlJobWithContext cancels a crawl job using the Firecrawl API. See CancelCrawlJob for more information. +func (app *FirecrawlApp) CancelCrawlJobWithContext(ctx context.Context, ID string) (string, error) { headers := app.prepareHeaders(nil) apiURL := fmt.Sprintf("%s/v1/crawl/%s", app.APIURL, ID) resp, err := app.makeRequest( + ctx, http.MethodDelete, apiURL, nil, @@ -593,6 +646,9 @@ func (app *FirecrawlApp) CancelCrawlJob(ID string) (string, error) { // - *MapResponse: The response from the mapping operation. // - error: An error if the mapping request fails. func (app *FirecrawlApp) MapURL(url string, params *MapParams) (*MapResponse, error) { + return app.MapURLWithContext(context.Background(), url, params) +} +func (app *FirecrawlApp) MapURLWithContext(ctx context.Context, url string, params *MapParams) (*MapResponse, error) { headers := app.prepareHeaders(nil) jsonData := map[string]any{"url": url} @@ -612,6 +668,7 @@ func (app *FirecrawlApp) MapURL(url string, params *MapParams) (*MapResponse, er } resp, err := app.makeRequest( + ctx, http.MethodPost, fmt.Sprintf("%s/v1/map", app.APIURL), jsonData, @@ -642,6 +699,11 @@ func (app *FirecrawlApp) MapURL(url string, params *MapParams) (*MapResponse, er // - params: Optional parameters for the search request. // - error: An error if the search request fails. func (app *FirecrawlApp) Search(query string, params *SearchParams) (*SearchResponse, error) { + return app.SearchWithContext(context.Background(), query, params) +} + +// SearchURLWithContext searches for a URL using the Firecrawl API. See SearchURL for more information. +func (app *FirecrawlApp) SearchWithContext(ctx context.Context, query string, params *SearchParams) (*SearchResponse, error) { headers := app.prepareHeaders(nil) jsonData := map[string]any{"query": query} if params != nil { @@ -672,6 +734,7 @@ func (app *FirecrawlApp) Search(query string, params *SearchParams) (*SearchResp } resp, err := app.makeRequest( + ctx, http.MethodPost, fmt.Sprintf("%s/v1/search", app.APIURL), jsonData, @@ -729,7 +792,7 @@ func (app *FirecrawlApp) prepareHeaders(idempotencyKey *string) map[string]strin // Returns: // - []byte: The response body from the request. // - error: An error if the request fails. -func (app *FirecrawlApp) makeRequest(method, url string, data map[string]any, headers map[string]string, action string, opts ...requestOption) ([]byte, error) { +func (app *FirecrawlApp) makeRequest(ctx context.Context, method, url string, data map[string]any, headers map[string]string, action string, opts ...requestOption) ([]byte, error) { var body []byte var err error if data != nil { @@ -739,7 +802,7 @@ func (app *FirecrawlApp) makeRequest(method, url string, data map[string]any, he } } - req, err := http.NewRequest(method, url, bytes.NewBuffer(body)) + req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewBuffer(body)) if err != nil { return nil, err } @@ -787,11 +850,12 @@ func (app *FirecrawlApp) makeRequest(method, url string, data map[string]any, he // Returns: // - *CrawlStatusResponse: The crawl result if the job is completed. // - error: An error if the crawl status check request fails. -func (app *FirecrawlApp) monitorJobStatus(ID string, headers map[string]string, pollInterval int) (*CrawlStatusResponse, error) { +func (app *FirecrawlApp) monitorJobStatus(ctx context.Context, ID string, headers map[string]string, pollInterval int) (*CrawlStatusResponse, error) { attempts := 3 for { resp, err := app.makeRequest( + ctx, http.MethodGet, fmt.Sprintf("%s/v1/crawl/%s", app.APIURL, ID), nil, @@ -819,6 +883,7 @@ func (app *FirecrawlApp) monitorJobStatus(ID string, headers map[string]string, allData := statusData.Data for statusData.Next != nil { resp, err := app.makeRequest( + ctx, http.MethodGet, *statusData.Next, nil, From 0f55c4612a4ca37a8ce8ddd8b87fd75d67d8d989 Mon Sep 17 00:00:00 2001 From: Brian Murray Date: Fri, 28 Mar 2025 12:11:20 -0600 Subject: [PATCH 4/6] Allow continue if theres no key --- firecrawl.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/firecrawl.go b/firecrawl.go index 8426acf..18a76e5 100644 --- a/firecrawl.go +++ b/firecrawl.go @@ -278,7 +278,8 @@ func NewFirecrawlApp(apiKey, apiURL string, opts ...FirecrawlOption) (*Firecrawl if apiKey == "" { apiKey = os.Getenv("FIRECRAWL_API_KEY") if apiKey == "" { - return nil, fmt.Errorf("no API key provided") + fmt.Println("no API key provided") + // return nil, fmt.Errorf("no API key provided") } } From af84caf59ca50ed6e9aa8b19842e1ec7ee6eda83 Mon Sep 17 00:00:00 2001 From: rafaelmmiller <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 22 Jul 2025 17:33:09 -0300 Subject: [PATCH 5/6] added tests --- firecrawl_test.go | 276 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 276 insertions(+) diff --git a/firecrawl_test.go b/firecrawl_test.go index cf2b062..ab8b589 100644 --- a/firecrawl_test.go +++ b/firecrawl_test.go @@ -1,6 +1,8 @@ package firecrawl import ( + "context" + "fmt" "log" "os" "testing" @@ -424,3 +426,277 @@ func TestMapURLWithSearchParameter(t *testing.T) { assert.Error(t, err) assert.Contains(t, err.Error(), "Search is not implemented in API version 1.0.0") } + +func TestSearchE2E(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + params := &SearchParams{ + Limit: 3, + Lang: "en", + } + + response, err := app.Search("firecrawl", params) + require.NoError(t, err) + assert.NotNil(t, response) + assert.True(t, response.Success) + assert.Greater(t, len(response.Data), 0) + + // Verify the search results contain expected data + for _, result := range response.Data { + assert.NotEmpty(t, result.URL) + assert.NotEmpty(t, result.Title) + assert.NotEmpty(t, result.Description) + } +} + +// Context-related tests +func TestScrapeURLWithContextTimeout(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // Test with a very short timeout that should cause the request to fail + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + defer cancel() + + // Wait a bit to ensure the context is already expired + time.Sleep(1 * time.Millisecond) + + _, err = app.ScrapeURLWithContext(ctx, "https://roastmywebsite.ai", nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "context deadline exceeded") +} + +func TestScrapeURLWithContextCancellation(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + + // Start the request in a goroutine + var response *FirecrawlDocument + var requestErr error + done := make(chan bool) + + go func() { + response, requestErr = app.ScrapeURLWithContext(ctx, "https://roastmywebsite.ai", nil) + done <- true + }() + + // Cancel the context immediately + cancel() + + // Wait for the goroutine to finish + <-done + + assert.Error(t, requestErr) + assert.Contains(t, requestErr.Error(), "context canceled") + assert.Nil(t, response) +} + +func TestScrapeURLWithContextSuccess(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + response, err := app.ScrapeURLWithContext(ctx, "https://roastmywebsite.ai", nil) + require.NoError(t, err) + assert.NotNil(t, response) + assert.Contains(t, response.Markdown, "_Roast_") +} + +func TestCrawlURLWithContextTimeout(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // Test with a very short timeout + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + defer cancel() + + // Wait a bit to ensure the context is already expired + time.Sleep(1 * time.Millisecond) + + _, err = app.CrawlURLWithContext(ctx, "https://roastmywebsite.ai", nil, nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "context deadline exceeded") +} + +func TestCrawlURLWithContextSuccess(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + // Use a small limit to make the test faster + params := &CrawlParams{ + Limit: ptr(2), + } + + response, err := app.CrawlURLWithContext(ctx, "https://roastmywebsite.ai", params, nil) + require.NoError(t, err) + assert.NotNil(t, response) + assert.Equal(t, "completed", response.Status) + assert.Greater(t, len(response.Data), 0) +} + +func TestAsyncCrawlURLWithContext(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + response, err := app.AsyncCrawlURLWithContext(ctx, "https://roastmywebsite.ai", nil, nil) + require.NoError(t, err) + assert.NotNil(t, response) + assert.NotEmpty(t, response.ID) + assert.Contains(t, response.URL, "https://api.firecrawl.dev/v1/crawl/") +} + +func TestCheckCrawlStatusWithContext(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // First start an async crawl + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + crawlResponse, err := app.AsyncCrawlURLWithContext(ctx, "https://roastmywebsite.ai", nil, nil) + require.NoError(t, err) + + // Check the status with context + statusResponse, err := app.CheckCrawlStatusWithContext(ctx, crawlResponse.ID) + require.NoError(t, err) + assert.NotNil(t, statusResponse) + assert.Contains(t, []string{"scraping", "completed"}, statusResponse.Status) +} + +func TestCancelCrawlJobWithContext(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Start an async crawl + crawlResponse, err := app.AsyncCrawlURLWithContext(ctx, "https://roastmywebsite.ai", nil, nil) + require.NoError(t, err) + + // Cancel the job + status, err := app.CancelCrawlJobWithContext(ctx, crawlResponse.ID) + require.NoError(t, err) + assert.Equal(t, "cancelled", status) +} + +func TestMapURLWithContext(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + response, err := app.MapURLWithContext(ctx, "https://roastmywebsite.ai", nil) + require.NoError(t, err) + assert.NotNil(t, response) + assert.Greater(t, len(response.Links), 0) +} + +func TestMapURLWithContextTimeout(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // Test with a very short timeout + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + defer cancel() + + // Wait a bit to ensure the context is already expired + time.Sleep(1 * time.Millisecond) + + _, err = app.MapURLWithContext(ctx, "https://roastmywebsite.ai", nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "context deadline exceeded") +} + +func TestSearchWithContext(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + params := &SearchParams{ + Limit: 3, + Lang: "en", + } + + response, err := app.SearchWithContext(ctx, "firecrawl", params) + require.NoError(t, err) + assert.NotNil(t, response) + assert.True(t, response.Success) + assert.Greater(t, len(response.Data), 0) +} + +func TestSearchWithContextTimeout(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // Test with a very short timeout + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + defer cancel() + + // Wait a bit to ensure the context is already expired + time.Sleep(1 * time.Millisecond) + + _, err = app.SearchWithContext(ctx, "firecrawl", nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "context deadline exceeded") +} + +func TestContextWithValues(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // Test that context values are preserved (though not used by the API) + ctx := context.WithValue(context.Background(), "test_key", "test_value") + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + response, err := app.ScrapeURLWithContext(ctx, "https://roastmywebsite.ai", nil) + require.NoError(t, err) + assert.NotNil(t, response) + assert.Contains(t, response.Markdown, "_Roast_") + + // Verify the context value is still there + assert.Equal(t, "test_value", ctx.Value("test_key")) +} + +func TestConcurrentContextRequests(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // Test multiple concurrent requests with different contexts + const numRequests = 3 + results := make(chan error, numRequests) + + for i := 0; i < numRequests; i++ { + go func(id int) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Add a unique value to each context + ctx = context.WithValue(ctx, "request_id", fmt.Sprintf("req_%d", id)) + + _, err := app.ScrapeURLWithContext(ctx, "https://roastmywebsite.ai", nil) + results <- err + }(i) + } + + // Wait for all requests to complete + for i := 0; i < numRequests; i++ { + err := <-results + assert.NoError(t, err) + } +} From ef8a7f11a031f9413e4852c079af192bf57919fb Mon Sep 17 00:00:00 2001 From: rafaelmmiller <150964962+rafaelsideguide@users.noreply.github.com> Date: Tue, 22 Jul 2025 17:45:01 -0300 Subject: [PATCH 6/6] added tests to search and contexts --- firecrawl_test.go | 138 ++++++++++++++++++++++++++++++---------------- 1 file changed, 90 insertions(+), 48 deletions(-) diff --git a/firecrawl_test.go b/firecrawl_test.go index ab8b589..fdabcc8 100644 --- a/firecrawl_test.go +++ b/firecrawl_test.go @@ -51,29 +51,18 @@ func TestBlocklistedURL(t *testing.T) { _, err = app.ScrapeURL("https://facebook.com/fake-test", nil) assert.Error(t, err) - assert.Contains(t, err.Error(), "URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.") -} - -func TestSuccessfulResponseWithValidPreviewToken(t *testing.T) { - app, err := NewFirecrawlApp("this_is_just_a_preview_token", API_URL) - require.NoError(t, err) - - response, err := app.ScrapeURL("https://roastmywebsite.ai", nil) - require.NoError(t, err) - assert.NotNil(t, response) - - assert.Contains(t, response.Markdown, "_Roast_") + assert.Contains(t, err.Error(), "Status code 403") } func TestScrapeURLE2E(t *testing.T) { app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) require.NoError(t, err) - response, err := app.ScrapeURL("https://roastmywebsite.ai", nil) + response, err := app.ScrapeURL("https://www.scrapethissite.com", nil) require.NoError(t, err) assert.NotNil(t, response) - assert.Contains(t, response.Markdown, "_Roast_") + assert.Contains(t, response.Markdown, "# Scrape This Site") assert.NotEqual(t, response.Markdown, "") assert.NotNil(t, response.Metadata) assert.Equal(t, response.HTML, "") @@ -93,19 +82,17 @@ func TestSuccessfulResponseWithValidAPIKeyAndIncludeHTML(t *testing.T) { WaitFor: ptr(1000), } - response, err := app.ScrapeURL("https://roastmywebsite.ai", ¶ms) + response, err := app.ScrapeURL("https://www.scrapethissite.com", ¶ms) require.NoError(t, err) assert.NotNil(t, response) - assert.Contains(t, response.Markdown, "_Roast_") + assert.Contains(t, response.Markdown, "# Scrape This Site") assert.Contains(t, response.HTML, "