diff --git a/README.md b/README.md index 086db7f..4655427 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ The Firecrawl Go SDK is a library that allows you to easily scrape and crawl web To install the Firecrawl Go SDK, you can ```bash -go get github.com/mendableai/firecrawl-go/v2 +go get github.com/mendableai/firecrawl-go ``` ## Usage @@ -19,19 +19,16 @@ go get github.com/mendableai/firecrawl-go/v2 Here's an example of how to use the SDK with error handling: ```go -package main - import ( - "encoding/json" "fmt" "log" - "github.com/mendableai/firecrawl-go/v2" + "github.com/mendableai/firecrawl-go" ) func main() { - // Initialize the FirecrawlApp with your API key and optional URL - app, err := firecrawl.NewFirecrawlApp("YOUR_API_KEY", "YOUR_API_URL") + // Initialize the FirecrawlApp with your API key + app, err := firecrawl.NewFirecrawlApp("YOUR_API_KEY") if err != nil { log.Fatalf("Failed to initialize FirecrawlApp: %v", err) } @@ -44,7 +41,7 @@ func main() { fmt.Println(scrapeResult.Markdown) // Crawl a website - idempotencyKey := "idempotency-key" // optional idempotency key + idempotencyKey := uuid.New().String() // optional idempotency key crawlParams := &firecrawl.CrawlParams{ ExcludePaths: []string{"blog/*"}, MaxDepth: prt(2), diff --git a/firecrawl.go b/firecrawl.go index 8964d44..753efe6 100644 --- a/firecrawl.go +++ b/firecrawl.go @@ -3,6 +3,7 @@ package firecrawl import ( "bytes" + "context" "encoding/json" "fmt" "io" @@ -158,6 +159,47 @@ type MapResponse struct { Error string `json:"error,omitempty"` } +// SearchParams represents the parameters for a search request. +type SearchParams struct { + Limit int `json:"limit"` + TimeBased string `json:"tbs"` + Lang string `json:"lang"` + Country string `json:"country"` + Location string `json:"location"` + Timeout int `json:"timeout"` + ScrapeOptions ScrapeParams `json:"scrapeOptions"` +} + +// SearchMetadata represents metadata for a search result +type SearchMetadata struct { + Title string `json:"title"` + Description string `json:"description"` + SourceURL string `json:"sourceURL"` + StatusCode int `json:"statusCode"` + Error string `json:"error"` +} + +// SearchDocument represents a document in search results +type SearchDocument struct { + Title string `json:"title"` + Description string `json:"description"` + URL string `json:"url"` + Markdown string `json:"markdown"` + HTML string `json:"html"` + RawHTML string `json:"rawHtml"` + Links []string `json:"links"` + Screenshot string `json:"screenshot"` + Metadata SearchMetadata `json:"metadata"` +} + +// SearchResponse represents the response for search operations +type SearchResponse struct { + Success bool `json:"success"` + Error string `json:"error,omitempty"` + Warning string `json:"warning,omitempty"` + Data []SearchDocument `json:"data,omitempty"` +} + // requestOptions represents options for making requests. type requestOptions struct { retries int @@ -216,6 +258,30 @@ type FirecrawlApp struct { Version string } +// FirecrawlOption is a functional option type for FirecrawlApp. +type FirecrawlOption func(*FirecrawlApp) + +// WithVersion sets the API version for the Firecrawl client. +func WithVersion(version string) FirecrawlOption { + return func(app *FirecrawlApp) { + app.Version = version + } +} + +// WithClient sets the HTTP client for the Firecrawl client. +func WithClient(client *http.Client) FirecrawlOption { + return func(app *FirecrawlApp) { + app.Client = client + } +} + +// WithTimeout sets the timeout for the HTTP client. +func WithTimeout(timeout time.Duration) FirecrawlOption { + return func(app *FirecrawlApp) { + app.Client.Timeout = timeout + } +} + // NewFirecrawlApp creates a new instance of FirecrawlApp with the provided API key and API URL. // If the API key or API URL is not provided, it attempts to retrieve them from environment variables. // If the API key is still not found, it returns an error. @@ -228,11 +294,12 @@ type FirecrawlApp struct { // Returns: // - *FirecrawlApp: A new instance of FirecrawlApp configured with the provided or retrieved API key and API URL. // - error: An error if the API key is not provided or retrieved. -func NewFirecrawlApp(apiKey, apiURL string, timeout ...time.Duration) (*FirecrawlApp, error) { +func NewFirecrawlApp(apiKey, apiURL string, opts ...FirecrawlOption) (*FirecrawlApp, error) { if apiKey == "" { apiKey = os.Getenv("FIRECRAWL_API_KEY") if apiKey == "" { - return nil, fmt.Errorf("no API key provided") + fmt.Println("no API key provided") + // return nil, fmt.Errorf("no API key provided") } } @@ -243,21 +310,21 @@ func NewFirecrawlApp(apiKey, apiURL string, timeout ...time.Duration) (*Firecraw } } - t := 120 * time.Second // default - if len(timeout) > 0 { - t = timeout[0] + fca := &FirecrawlApp{ + APIKey: apiKey, + APIURL: apiURL, + Client: &http.Client{ + Timeout: 60 * time.Second, // default timeout + Transport: http.DefaultTransport, + }, } - client := &http.Client{ - Timeout: t, - Transport: http.DefaultTransport, + // Apply any custom options + for _, opt := range opts { + opt(fca) } - return &FirecrawlApp{ - APIKey: apiKey, - APIURL: apiURL, - Client: client, - }, nil + return fca, nil } // ScrapeURL scrapes the content of the specified URL using the Firecrawl API. @@ -270,6 +337,11 @@ func NewFirecrawlApp(apiKey, apiURL string, timeout ...time.Duration) (*Firecraw // - *FirecrawlDocument or *FirecrawlDocumentV0: The scraped document data depending on the API version. // - error: An error if the scrape request fails. func (app *FirecrawlApp) ScrapeURL(url string, params *ScrapeParams) (*FirecrawlDocument, error) { + return app.ScrapeURLWithContext(context.Background(), url, params) +} + +// ScrapeURLWithContext scrapes the content of the specified URL using the Firecrawl API. See ScrapeURL for more information. +func (app *FirecrawlApp) ScrapeURLWithContext(ctx context.Context, url string, params *ScrapeParams) (*FirecrawlDocument, error) { headers := app.prepareHeaders(nil) scrapeBody := map[string]any{"url": url} @@ -325,6 +397,7 @@ func (app *FirecrawlApp) ScrapeURL(url string, params *ScrapeParams) (*Firecrawl } resp, err := app.makeRequest( + ctx, http.MethodPost, fmt.Sprintf("%s/v1/scrape", app.APIURL), scrapeBody, @@ -361,6 +434,11 @@ func (app *FirecrawlApp) ScrapeURL(url string, params *ScrapeParams) (*Firecrawl // - CrawlStatusResponse: The crawl result if the job is completed. // - error: An error if the crawl request fails. func (app *FirecrawlApp) CrawlURL(url string, params *CrawlParams, idempotencyKey *string, pollInterval ...int) (*CrawlStatusResponse, error) { + return app.CrawlURLWithContext(context.Background(), url, params, idempotencyKey, pollInterval...) +} + +// CrawlURLWithContext starts a crawl job for the specified URL using the Firecrawl API. See CrawlURL for more information. +func (app *FirecrawlApp) CrawlURLWithContext(ctx context.Context, url string, params *CrawlParams, idempotencyKey *string, pollInterval ...int) (*CrawlStatusResponse, error) { var key string if idempotencyKey != nil { key = *idempotencyKey @@ -405,6 +483,7 @@ func (app *FirecrawlApp) CrawlURL(url string, params *CrawlParams, idempotencyKe } resp, err := app.makeRequest( + ctx, http.MethodPost, fmt.Sprintf("%s/v1/crawl", app.APIURL), crawlBody, @@ -423,7 +502,7 @@ func (app *FirecrawlApp) CrawlURL(url string, params *CrawlParams, idempotencyKe return nil, err } - return app.monitorJobStatus(crawlResponse.ID, headers, actualPollInterval) + return app.monitorJobStatus(ctx, crawlResponse.ID, headers, actualPollInterval) } // CrawlURL starts a crawl job for the specified URL using the Firecrawl API. @@ -437,6 +516,11 @@ func (app *FirecrawlApp) CrawlURL(url string, params *CrawlParams, idempotencyKe // - *CrawlResponse: The crawl response with id. // - error: An error if the crawl request fails. func (app *FirecrawlApp) AsyncCrawlURL(url string, params *CrawlParams, idempotencyKey *string) (*CrawlResponse, error) { + return app.AsyncCrawlURLWithContext(context.Background(), url, params, idempotencyKey) +} + +// AsyncCrawlURLWithContext starts a crawl job for the specified URL using the Firecrawl API. See AsyncCrawlURL for more information. +func (app *FirecrawlApp) AsyncCrawlURLWithContext(ctx context.Context, url string, params *CrawlParams, idempotencyKey *string) (*CrawlResponse, error) { var key string if idempotencyKey != nil { key = *idempotencyKey @@ -476,6 +560,7 @@ func (app *FirecrawlApp) AsyncCrawlURL(url string, params *CrawlParams, idempote } resp, err := app.makeRequest( + ctx, http.MethodPost, fmt.Sprintf("%s/v1/crawl", app.APIURL), crawlBody, @@ -511,10 +596,16 @@ func (app *FirecrawlApp) AsyncCrawlURL(url string, params *CrawlParams, idempote // - *CrawlStatusResponse: The status of the crawl job. // - error: An error if the crawl status check request fails. func (app *FirecrawlApp) CheckCrawlStatus(ID string) (*CrawlStatusResponse, error) { + return app.CheckCrawlStatusWithContext(context.Background(), ID) +} + +// CheckCrawlStatusWithContext checks the status of a crawl job using the Firecrawl API. See CheckCrawlStatus for more information. +func (app *FirecrawlApp) CheckCrawlStatusWithContext(ctx context.Context, ID string) (*CrawlStatusResponse, error) { headers := app.prepareHeaders(nil) apiURL := fmt.Sprintf("%s/v1/crawl/%s", app.APIURL, ID) resp, err := app.makeRequest( + ctx, http.MethodGet, apiURL, nil, @@ -545,9 +636,15 @@ func (app *FirecrawlApp) CheckCrawlStatus(ID string) (*CrawlStatusResponse, erro // - string: The status of the crawl job after cancellation. // - error: An error if the crawl job cancellation request fails. func (app *FirecrawlApp) CancelCrawlJob(ID string) (string, error) { + return app.CancelCrawlJobWithContext(context.Background(), ID) +} + +// CancelCrawlJobWithContext cancels a crawl job using the Firecrawl API. See CancelCrawlJob for more information. +func (app *FirecrawlApp) CancelCrawlJobWithContext(ctx context.Context, ID string) (string, error) { headers := app.prepareHeaders(nil) apiURL := fmt.Sprintf("%s/v1/crawl/%s", app.APIURL, ID) resp, err := app.makeRequest( + ctx, http.MethodDelete, apiURL, nil, @@ -577,6 +674,9 @@ func (app *FirecrawlApp) CancelCrawlJob(ID string) (string, error) { // - *MapResponse: The response from the mapping operation. // - error: An error if the mapping request fails. func (app *FirecrawlApp) MapURL(url string, params *MapParams) (*MapResponse, error) { + return app.MapURLWithContext(context.Background(), url, params) +} +func (app *FirecrawlApp) MapURLWithContext(ctx context.Context, url string, params *MapParams) (*MapResponse, error) { headers := app.prepareHeaders(nil) jsonData := map[string]any{"url": url} @@ -596,6 +696,7 @@ func (app *FirecrawlApp) MapURL(url string, params *MapParams) (*MapResponse, er } resp, err := app.makeRequest( + ctx, http.MethodPost, fmt.Sprintf("%s/v1/map", app.APIURL), jsonData, @@ -622,13 +723,69 @@ func (app *FirecrawlApp) MapURL(url string, params *MapParams) (*MapResponse, er // SearchURL searches for a URL using the Firecrawl API. // // Parameters: -// - url: The URL to search for. +// - query: The search query. // - params: Optional parameters for the search request. // - error: An error if the search request fails. -// -// Search is not implemented in API version 1.0.0. -func (app *FirecrawlApp) Search(query string, params *any) (any, error) { - return nil, fmt.Errorf("Search is not implemented in API version 1.0.0") +func (app *FirecrawlApp) Search(query string, params *SearchParams) (*SearchResponse, error) { + return app.SearchWithContext(context.Background(), query, params) +} + +// SearchURLWithContext searches for a URL using the Firecrawl API. See SearchURL for more information. +func (app *FirecrawlApp) SearchWithContext(ctx context.Context, query string, params *SearchParams) (*SearchResponse, error) { + headers := app.prepareHeaders(nil) + jsonData := map[string]any{"query": query} + if params != nil { + if params.Limit != 0 { + if params.Limit < 1 || params.Limit > 10 { + return nil, fmt.Errorf("limit must be between 1 and 10") + } + jsonData["limit"] = params.Limit + } + if params.TimeBased != "" { + jsonData["tbs"] = params.TimeBased + } + if params.Lang != "" { + jsonData["lang"] = params.Lang + } + if params.Country != "" { + jsonData["country"] = params.Country + } + if params.Location != "" { + jsonData["location"] = params.Location + } + if params.Timeout != 0 { + jsonData["timeout"] = params.Timeout + } + if params.ScrapeOptions.Formats != nil { + jsonData["scrapeOptions"] = params.ScrapeOptions + } + } + + resp, err := app.makeRequest( + ctx, + http.MethodPost, + fmt.Sprintf("%s/v1/search", app.APIURL), + jsonData, + headers, + "search", + ) + if err != nil { + return nil, err + } + + var searchResponse SearchResponse + err = json.Unmarshal(resp, &searchResponse) + if err != nil { + return nil, err + } + + if searchResponse.Success { + return &searchResponse, nil + } else { + return nil, fmt.Errorf("search operation failed: %s", searchResponse.Error) + } + + // return nil, fmt.Errorf("Search is not implemented in API version 1.0.0") } // prepareHeaders prepares the headers for an HTTP request. @@ -663,7 +820,7 @@ func (app *FirecrawlApp) prepareHeaders(idempotencyKey *string) map[string]strin // Returns: // - []byte: The response body from the request. // - error: An error if the request fails. -func (app *FirecrawlApp) makeRequest(method, url string, data map[string]any, headers map[string]string, action string, opts ...requestOption) ([]byte, error) { +func (app *FirecrawlApp) makeRequest(ctx context.Context, method, url string, data map[string]any, headers map[string]string, action string, opts ...requestOption) ([]byte, error) { var body []byte var err error if data != nil { @@ -673,7 +830,7 @@ func (app *FirecrawlApp) makeRequest(method, url string, data map[string]any, he } } - req, err := http.NewRequest(method, url, bytes.NewBuffer(body)) + req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewBuffer(body)) if err != nil { return nil, err } @@ -721,11 +878,12 @@ func (app *FirecrawlApp) makeRequest(method, url string, data map[string]any, he // Returns: // - *CrawlStatusResponse: The crawl result if the job is completed. // - error: An error if the crawl status check request fails. -func (app *FirecrawlApp) monitorJobStatus(ID string, headers map[string]string, pollInterval int) (*CrawlStatusResponse, error) { +func (app *FirecrawlApp) monitorJobStatus(ctx context.Context, ID string, headers map[string]string, pollInterval int) (*CrawlStatusResponse, error) { attempts := 3 for { resp, err := app.makeRequest( + ctx, http.MethodGet, fmt.Sprintf("%s/v1/crawl/%s", app.APIURL, ID), nil, @@ -753,6 +911,7 @@ func (app *FirecrawlApp) monitorJobStatus(ID string, headers map[string]string, allData := statusData.Data for statusData.Next != nil { resp, err := app.makeRequest( + ctx, http.MethodGet, *statusData.Next, nil, diff --git a/firecrawl_test.go b/firecrawl_test.go index d012bf8..00842fd 100644 --- a/firecrawl_test.go +++ b/firecrawl_test.go @@ -1,6 +1,8 @@ package firecrawl import ( + "context" + "fmt" "log" "os" "testing" @@ -399,6 +401,280 @@ func TestMapURLWithSearchParameter(t *testing.T) { assert.Contains(t, err.Error(), "Search is not implemented in API version 1.0.0") } +func TestSearchE2E(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + params := &SearchParams{ + Limit: 3, + Lang: "en", + } + + response, err := app.Search("firecrawl", params) + require.NoError(t, err) + assert.NotNil(t, response) + assert.True(t, response.Success) + assert.Greater(t, len(response.Data), 0) + + // Verify the search results contain expected data + for _, result := range response.Data { + assert.NotEmpty(t, result.URL) + assert.NotEmpty(t, result.Title) + assert.NotEmpty(t, result.Description) + } +} + +// Context-related tests +func TestScrapeURLWithContextTimeout(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // Test with a very short timeout that should cause the request to fail + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + defer cancel() + + // Wait a bit to ensure the context is already expired + time.Sleep(1 * time.Millisecond) + + _, err = app.ScrapeURLWithContext(ctx, "https://roastmywebsite.ai", nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "context deadline exceeded") +} + +func TestScrapeURLWithContextCancellation(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + + // Start the request in a goroutine + var response *FirecrawlDocument + var requestErr error + done := make(chan bool) + + go func() { + response, requestErr = app.ScrapeURLWithContext(ctx, "https://roastmywebsite.ai", nil) + done <- true + }() + + // Cancel the context immediately + cancel() + + // Wait for the goroutine to finish + <-done + + assert.Error(t, requestErr) + assert.Contains(t, requestErr.Error(), "context canceled") + assert.Nil(t, response) +} + +func TestScrapeURLWithContextSuccess(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + response, err := app.ScrapeURLWithContext(ctx, "https://roastmywebsite.ai", nil) + require.NoError(t, err) + assert.NotNil(t, response) + assert.Contains(t, response.Markdown, "_Roast_") +} + +func TestCrawlURLWithContextTimeout(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // Test with a very short timeout + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + defer cancel() + + // Wait a bit to ensure the context is already expired + time.Sleep(1 * time.Millisecond) + + _, err = app.CrawlURLWithContext(ctx, "https://roastmywebsite.ai", nil, nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "context deadline exceeded") +} + +func TestCrawlURLWithContextSuccess(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + // Use a small limit to make the test faster + params := &CrawlParams{ + Limit: ptr(2), + } + + response, err := app.CrawlURLWithContext(ctx, "https://roastmywebsite.ai", params, nil) + require.NoError(t, err) + assert.NotNil(t, response) + assert.Equal(t, "completed", response.Status) + assert.Greater(t, len(response.Data), 0) +} + +func TestAsyncCrawlURLWithContext(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + response, err := app.AsyncCrawlURLWithContext(ctx, "https://roastmywebsite.ai", nil, nil) + require.NoError(t, err) + assert.NotNil(t, response) + assert.NotEmpty(t, response.ID) + assert.Contains(t, response.URL, "https://api.firecrawl.dev/v1/crawl/") +} + +func TestCheckCrawlStatusWithContext(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // First start an async crawl + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + crawlResponse, err := app.AsyncCrawlURLWithContext(ctx, "https://roastmywebsite.ai", nil, nil) + require.NoError(t, err) + + // Check the status with context + statusResponse, err := app.CheckCrawlStatusWithContext(ctx, crawlResponse.ID) + require.NoError(t, err) + assert.NotNil(t, statusResponse) + assert.Contains(t, []string{"scraping", "completed"}, statusResponse.Status) +} + +func TestCancelCrawlJobWithContext(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Start an async crawl + crawlResponse, err := app.AsyncCrawlURLWithContext(ctx, "https://roastmywebsite.ai", nil, nil) + require.NoError(t, err) + + // Cancel the job + status, err := app.CancelCrawlJobWithContext(ctx, crawlResponse.ID) + require.NoError(t, err) + assert.Equal(t, "cancelled", status) +} + +func TestMapURLWithContext(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + response, err := app.MapURLWithContext(ctx, "https://roastmywebsite.ai", nil) + require.NoError(t, err) + assert.NotNil(t, response) + assert.Greater(t, len(response.Links), 0) +} + +func TestMapURLWithContextTimeout(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // Test with a very short timeout + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + defer cancel() + + // Wait a bit to ensure the context is already expired + time.Sleep(1 * time.Millisecond) + + _, err = app.MapURLWithContext(ctx, "https://roastmywebsite.ai", nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "context deadline exceeded") +} + +func TestSearchWithContext(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + params := &SearchParams{ + Limit: 3, + Lang: "en", + } + + response, err := app.SearchWithContext(ctx, "firecrawl", params) + require.NoError(t, err) + assert.NotNil(t, response) + assert.True(t, response.Success) + assert.Greater(t, len(response.Data), 0) +} + +func TestSearchWithContextTimeout(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // Test with a very short timeout + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) + defer cancel() + + // Wait a bit to ensure the context is already expired + time.Sleep(1 * time.Millisecond) + + _, err = app.SearchWithContext(ctx, "firecrawl", nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "context deadline exceeded") +} + +func TestContextWithValues(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // Test that context values are preserved (though not used by the API) + ctx := context.WithValue(context.Background(), "test_key", "test_value") + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + response, err := app.ScrapeURLWithContext(ctx, "https://roastmywebsite.ai", nil) + require.NoError(t, err) + assert.NotNil(t, response) + assert.Contains(t, response.Markdown, "_Roast_") + + // Verify the context value is still there + assert.Equal(t, "test_value", ctx.Value("test_key")) +} + +func TestConcurrentContextRequests(t *testing.T) { + app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) + require.NoError(t, err) + + // Test multiple concurrent requests with different contexts + const numRequests = 3 + results := make(chan error, numRequests) + + for i := 0; i < numRequests; i++ { + go func(id int) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Add a unique value to each context + ctx = context.WithValue(ctx, "request_id", fmt.Sprintf("req_%d", id)) + + _, err := app.ScrapeURLWithContext(ctx, "https://roastmywebsite.ai", nil) + results <- err + }(i) + } + + // Wait for all requests to complete + for i := 0; i < numRequests; i++ { + err := <-results + assert.NoError(t, err) + } +} + func TestScrapeURLWithMaxAge(t *testing.T) { app, err := NewFirecrawlApp(TEST_API_KEY, API_URL) require.NoError(t, err) diff --git a/go.mod b/go.mod index 02ca7ce..0a9125f 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/mendableai/firecrawl-go/v2.1.0 +module github.com/mendableai/firecrawl-go/v2 go 1.22.5