diff --git a/.changeset/empty-dolls-judge.md b/.changeset/empty-dolls-judge.md new file mode 100644 index 0000000000..477cc9ef47 --- /dev/null +++ b/.changeset/empty-dolls-judge.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Switch to profile after successful login diff --git a/.changeset/rare-mails-fail.md b/.changeset/rare-mails-fail.md new file mode 100644 index 0000000000..ef4e9861b0 --- /dev/null +++ b/.changeset/rare-mails-fail.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/build": patch +--- + +Add Lightpanda extension diff --git a/docs/config/extensions/lightpanda.mdx b/docs/config/extensions/lightpanda.mdx new file mode 100644 index 0000000000..0408d45ad5 --- /dev/null +++ b/docs/config/extensions/lightpanda.mdx @@ -0,0 +1,60 @@ +--- +title: "Lightpanda" +sidebarTitle: "lightpanda" +description: "Use the lightpanda build extension to add Lightpanda browser to your project" +tag: "v4" +--- + +import UpgradeToV4Note from "/snippets/upgrade-to-v4-note.mdx"; + + + +To use the Lightpanda browser in your project, add the extension to your `trigger.config.ts` file: + +```ts trigger.config.ts +import { defineConfig } from "@trigger.dev/sdk"; +import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; + +export default defineConfig({ + project: "", + build: { + extensions: [lightpanda()], + }, +}); +``` + +## Options + +- `version`: The version of the browser to install. Default: `"latest"`. +- `disableTelemetry`: Whether to disable telemetry. Default: `false`. + +For example: + +```ts trigger.config.ts +import { defineConfig } from "@trigger.dev/sdk"; +import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; + +export default defineConfig({ + project: "", + build: { + extensions: [ + lightpanda({ + version: "nightly", + disableTelemetry: true, + }), + ], + }, +}); +``` + +## Development + +When running in dev, you will first have to download the Lightpanda browser binary and make sure it's in your `PATH`. See [Lightpanda's installation guide](https://lightpanda.io/docs/getting-started/installation). + +## Next steps + + + + Learn how to use Lightpanda in your project. + + diff --git a/docs/config/extensions/overview.mdx b/docs/config/extensions/overview.mdx index abba56694e..412a11062b 100644 --- a/docs/config/extensions/overview.mdx +++ b/docs/config/extensions/overview.mdx @@ -50,7 +50,6 @@ Trigger.dev provides a set of built-in extensions that you can use to customize | :-------------------------------------------------------------------- | :----------------------------------------------------------------------------- | | [prismaExtension](/config/extensions/prismaExtension) | Using prisma in your Trigger.dev tasks | | [pythonExtension](/config/extensions/pythonExtension) | Execute Python scripts in your project | -| [playwright](/config/extensions/playwright) | Use Playwright in your Trigger.dev tasks | | [puppeteer](/config/extensions/puppeteer) | Use Puppeteer in your Trigger.dev tasks | | [ffmpeg](/config/extensions/ffmpeg) | Use FFmpeg in your Trigger.dev tasks | | [aptGet](/config/extensions/aptGet) | Install system packages in your build image | diff --git a/docs/docs.json b/docs/docs.json index b88d903a93..6f09113887 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -78,6 +78,7 @@ "config/extensions/pythonExtension", "config/extensions/playwright", "config/extensions/puppeteer", + "config/extensions/lightpanda", "config/extensions/ffmpeg", "config/extensions/aptGet", "config/extensions/additionalFiles", @@ -362,6 +363,7 @@ "guides/examples/fal-ai-realtime", "guides/examples/ffmpeg-video-processing", "guides/examples/firecrawl-url-crawl", + "guides/examples/lightpanda", "guides/examples/libreoffice-pdf-conversion", "guides/examples/open-ai-with-retrying", "guides/examples/pdf-to-image", diff --git a/docs/guides/examples/lightpanda.mdx b/docs/guides/examples/lightpanda.mdx new file mode 100644 index 0000000000..9eab531176 --- /dev/null +++ b/docs/guides/examples/lightpanda.mdx @@ -0,0 +1,227 @@ +--- +title: "Lightpanda" +sidebarTitle: "Lightpanda" +description: "These examples demonstrate how to use Lightpanda with Trigger.dev." +tag: "v4" +--- + +import ScrapingWarning from "/snippets/web-scraping-warning.mdx"; +import UpgradeToV4Note from "/snippets/upgrade-to-v4-note.mdx"; + + + +## Overview + +Lightpanda is a purpose-built browser for AI and automation workflows. It is 10x faster, uses 10x less RAM than Chrome headless. + +Here are a few examples of how to use Lightpanda with Trigger.dev. + + + +## Limitations + +- Lightpanda does not support the `puppeteer` screenshot feature. + +## Using Lightpanda Cloud + +### Prerequisites + +- A [Lightpanda](https://lightpanda.io/) cloud token + +### Get links from a website +In this task we use Lightpanda browser to get links from a provided URL. You will have to pass the URL as a payload when triggering the task. + +Make sure to add `LIGHTPANDA_TOKEN` to your Trigger.dev dashboard on the Environment Variables page: +```bash +LIGHTPANDA_TOKEN="" +``` + +```ts trigger/lightpanda-cloud-puppeteer.ts +import { logger, task } from "@trigger.dev/sdk"; +import puppeteer from "puppeteer-core"; + +export const lightpandaCloudPuppeteer = task({ + id: "lightpanda-cloud-puppeteer", + machine: { + preset: "micro", + }, + run: async (payload: { url: string }, { ctx }) => { + logger.log("Lets get a page's links with Lightpanda!", { payload, ctx }); + + if (!payload.url) { + logger.warn("Please define the payload url"); + throw new Error("payload.url is undefined"); + } + + const token = process.env.LIGHTPANDA_TOKEN; + if (!token) { + logger.warn("Please define the env variable LIGHTPANDA_TOKEN"); + throw new Error("LIGHTPANDA_TOKEN is undefined"); + } + + // Connect to Lightpanda's cloud + const browser = await puppeteer.connect({ + browserWSEndpoint: `wss://cloud.lightpanda.io/ws?browser=lightpanda&token=${token}`, + }); + const context = await browser.createBrowserContext(); + const page = await context.newPage(); + + // Dump all the links from the page. + await page.goto(payload.url); + + const links = await page.evaluate(() => { + return Array.from(document.querySelectorAll("a")).map((row) => { + return row.getAttribute("href"); + }); + }); + + logger.info("Processing done, shutting down…"); + + await page.close(); + await context.close(); + await browser.disconnect(); + + logger.info("✅ Completed"); + + return { + links, + }; + }, +}); +``` + +### Proxies + +Proxies can be used with your browser via the proxy query string parameter. By default, the proxy used is "datacenter" which is a pool of shared datacenter IPs. +`datacenter` accepts an optional `country` query string parameter which is an [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) country code. + +```bash +# This example will use a German IP +wss://cloud.lightpanda.io/ws?proxy=datacenter&country=de&token=${token} +``` + +### Session + +A session is alive until you close it or the connection is closed. The max duration of a session is 15 minutes. + +## Using Lightpanda browser directly + +### Prerequisites + +- Setup the [Lightpanda build extension](/config/extensions/lightpanda) + +### Get the HTML of a webpage + +This task will dump the HTML of a provided URL using the Lightpanda browser binary. You will have to pass the URL as a payload when triggering the task. + +```ts trigger/lightpanda-fetch.ts +import { logger, task } from "@trigger.dev/sdk"; +import { execSync } from "node:child_process"; + +export const lightpandaFetch = task({ + id: "lightpanda-fetch", + machine: { + preset: "micro", + }, + run: async (payload: { url: string }, { ctx }) => { + logger.log("Lets get a page's content with Lightpanda!", { payload, ctx }); + + if (!payload.url) { + logger.warn("Please define the payload url"); + throw new Error("payload.url is undefined"); + } + + const buffer = execSync(`lightpanda fetch --dump ${payload.url}`); + + logger.info("✅ Completed"); + + return { + message: buffer.toString(), + }; + }, +}); +``` + +### Lightpanda CDP with Puppeteer + +This task initializes a Lightpanda CDP server and uses it with `puppeteer-core` to scrape a provided URL. + +```ts trigger/lightpanda-cdp.ts +import { logger, task } from "@trigger.dev/sdk"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import puppeteer from "puppeteer-core"; + +const spawnLightpanda = async (host: string, port: string) => + new Promise((resolve, reject) => { + const child = spawn("lightpanda", [ + "serve", + "--host", + host, + "--port", + port, + "--log_level", + "info", + ]); + + child.on("spawn", async () => { + logger.info("Running Lightpanda's CDP server…", { + pid: child.pid, + }); + + await new Promise((resolve) => setTimeout(resolve, 250)); + resolve(child); + }); + child.on("error", (e) => reject(e)); + }); + +export const lightpandaCDP = task({ + id: "lightpanda-cdp", + machine: { + preset: "micro", + }, + run: async (payload: { url: string }, { ctx }) => { + logger.log("Lets get a page's links with Lightpanda!", { payload, ctx }); + + if (!payload.url) { + logger.warn("Please define the payload url"); + throw new Error("payload.url is undefined"); + } + + const host = process.env.LIGHTPANDA_CDP_HOST ?? "127.0.0.1"; + const port = process.env.LIGHTPANDA_CDP_PORT ?? "9222"; + + // Launch Lightpanda's CDP server + const lpProcess = await spawnLightpanda(host, port); + + const browser = await puppeteer.connect({ + browserWSEndpoint: `ws://${host}:${port}`, + }); + const context = await browser.createBrowserContext(); + const page = await context.newPage(); + + // Dump all the links from the page. + await page.goto(payload.url); + + const links = await page.evaluate(() => { + return Array.from(document.querySelectorAll("a")).map((row) => { + return row.getAttribute("href"); + }); + }); + + logger.info("Processing done"); + logger.info("Shutting down…"); + + // Close Puppeteer instance + await browser.close(); + + // Stop Lightpanda's CDP Server + lpProcess.kill(); + + logger.info("✅ Completed"); + + return { + links, + }; + }, +}); +``` diff --git a/docs/guides/introduction.mdx b/docs/guides/introduction.mdx index 12fc3e0faa..79afeee86a 100644 --- a/docs/guides/introduction.mdx +++ b/docs/guides/introduction.mdx @@ -71,6 +71,7 @@ Task code you can copy and paste to use in your project. They can all be extende | [FFmpeg video processing](/guides/examples/ffmpeg-video-processing) | Use FFmpeg to process a video in various ways and save it to Cloudflare R2. | | [Firecrawl URL crawl](/guides/examples/firecrawl-url-crawl) | Learn how to use Firecrawl to crawl a URL and return LLM-ready markdown. | | [LibreOffice PDF conversion](/guides/examples/libreoffice-pdf-conversion) | Convert a document to PDF using LibreOffice. | +| [Lightpanda](/guides/examples/lightpanda) | Use Lightpanda browser (or cloud version) to get a webpage's content. | | [OpenAI with retrying](/guides/examples/open-ai-with-retrying) | Create a reusable OpenAI task with custom retry options. | | [PDF to image](/guides/examples/pdf-to-image) | Use `MuPDF` to turn a PDF into images and save them to Cloudflare R2. | | [Puppeteer](/guides/examples/puppeteer) | Use Puppeteer to generate a PDF or scrape a webpage. | diff --git a/docs/images/intro-lightpanda.jpg b/docs/images/intro-lightpanda.jpg new file mode 100644 index 0000000000..8fc2102bb0 Binary files /dev/null and b/docs/images/intro-lightpanda.jpg differ diff --git a/docs/introduction.mdx b/docs/introduction.mdx index c8ed0f3480..1ab5baaf7a 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -83,23 +83,24 @@ We provide everything you need to build and manage background tasks: a CLI and S + ## Explore by build extension -| Extension | What it does | Docs | -|:----------|:------------|:--------------| -| prismaExtension | Use Prisma with Trigger.dev | [Learn more](/config/extensions/prismaExtension) | -| pythonExtension | Execute Python scripts in Trigger.dev | [Learn more](/config/extensions/pythonExtension) | -| puppeteer | Use Puppeteer with Trigger.dev | [Learn more](/config/extensions/puppeteer) | -| ffmpeg | Use FFmpeg with Trigger.dev | [Learn more](/config/extensions/ffmpeg) | -| aptGet | Install system packages with aptGet | [Learn more](/config/extensions/aptGet) | -| additionalFiles | Copy additional files to the build directory | [Learn more](/config/extensions/additionalFiles) | -| additionalPackages | Include additional packages in the build | [Learn more](/config/extensions/additionalPackages) | -| syncEnvVars | Automatically sync environment variables to Trigger.dev | [Learn more](/config/extensions/syncEnvVars) | -| esbuildPlugin | Add existing or custom esbuild plugins to your build process | [Learn more](/config/extensions/esbuildPlugin) | -| emitDecoratorMetadata | Support for the emitDecoratorMetadata TypeScript compiler | [Learn more](/config/extensions/emitDecoratorMetadata) | -| audioWaveform | Support for Audio Waveform in your project | [Learn more](/config/extensions/audioWaveform) | +| Extension | What it does | Docs | +| :-------------------- | :----------------------------------------------------------- | :----------------------------------------------------- | +| prismaExtension | Use Prisma with Trigger.dev | [Learn more](/config/extensions/prismaExtension) | +| pythonExtension | Execute Python scripts in Trigger.dev | [Learn more](/config/extensions/pythonExtension) | +| puppeteer | Use Puppeteer with Trigger.dev | [Learn more](/config/extensions/puppeteer) | +| ffmpeg | Use FFmpeg with Trigger.dev | [Learn more](/config/extensions/ffmpeg) | +| aptGet | Install system packages with aptGet | [Learn more](/config/extensions/aptGet) | +| additionalFiles | Copy additional files to the build directory | [Learn more](/config/extensions/additionalFiles) | +| additionalPackages | Include additional packages in the build | [Learn more](/config/extensions/additionalPackages) | +| syncEnvVars | Automatically sync environment variables to Trigger.dev | [Learn more](/config/extensions/syncEnvVars) | +| esbuildPlugin | Add existing or custom esbuild plugins to your build process | [Learn more](/config/extensions/esbuildPlugin) | +| emitDecoratorMetadata | Support for the emitDecoratorMetadata TypeScript compiler | [Learn more](/config/extensions/emitDecoratorMetadata) | +| audioWaveform | Support for Audio Waveform in your project | [Learn more](/config/extensions/audioWaveform) | ## Getting help diff --git a/packages/build/package.json b/packages/build/package.json index 6751f8bae2..78f3558e11 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -30,7 +30,8 @@ "./extensions/audioWaveform": "./src/extensions/audioWaveform.ts", "./extensions/typescript": "./src/extensions/typescript.ts", "./extensions/puppeteer": "./src/extensions/puppeteer.ts", - "./extensions/playwright": "./src/extensions/playwright.ts" + "./extensions/playwright": "./src/extensions/playwright.ts", + "./extensions/lightpanda": "./src/extensions/lightpanda.ts" }, "sourceDialects": [ "@triggerdotdev/source" @@ -61,6 +62,9 @@ ], "extensions/playwright": [ "dist/commonjs/extensions/playwright.d.ts" + ], + "extensions/lightpanda": [ + "dist/commonjs/extensions/lightpanda.d.ts" ] } }, @@ -188,6 +192,17 @@ "types": "./dist/commonjs/extensions/playwright.d.ts", "default": "./dist/commonjs/extensions/playwright.js" } + }, + "./extensions/lightpanda": { + "import": { + "@triggerdotdev/source": "./src/extensions/lightpanda.ts", + "types": "./dist/esm/extensions/lightpanda.d.ts", + "default": "./dist/esm/extensions/lightpanda.js" + }, + "require": { + "types": "./dist/commonjs/extensions/lightpanda.d.ts", + "default": "./dist/commonjs/extensions/lightpanda.js" + } } }, "main": "./dist/commonjs/index.js", diff --git a/packages/build/src/extensions/lightpanda.ts b/packages/build/src/extensions/lightpanda.ts new file mode 100644 index 0000000000..16c62a08b4 --- /dev/null +++ b/packages/build/src/extensions/lightpanda.ts @@ -0,0 +1,38 @@ +import type { BuildExtension } from "@trigger.dev/core/v3/build"; + +type LightpandaOpts = { + version?: "nightly" | "latest"; + disableTelemetry?: boolean; +}; + +export const lightpanda = ({ + version = "latest", + disableTelemetry = false, +}: LightpandaOpts = {}): BuildExtension => ({ + name: "lightpanda", + onBuildComplete: async (context) => { + if (context.target === "dev") { + return; + } + + context.logger.debug(`Adding lightpanda`, { version, disableTelemetry }); + + const instructions = [ + `COPY --from=lightpanda/browser:${version} /usr/bin/lightpanda /usr/local/bin/lightpanda`, + `RUN /usr/local/bin/lightpanda version || (echo "lightpanda binary is not functional" && exit 1)`, + ] satisfies string[]; + + context.addLayer({ + id: "lightpanda", + image: { + instructions, + }, + deploy: { + env: { + ...(disableTelemetry ? { LIGHTPANDA_DISABLE_TELEMETRY: "true" } : {}), + }, + override: true, + }, + }); + }, +}); diff --git a/packages/cli-v3/src/commands/login.ts b/packages/cli-v3/src/commands/login.ts index 687b6e8f6d..953a0c796f 100644 --- a/packages/cli-v3/src/commands/login.ts +++ b/packages/cli-v3/src/commands/login.ts @@ -14,7 +14,11 @@ import { wrapCommandAction, } from "../cli/common.js"; import { chalkLink, prettyError } from "../utilities/cliOutput.js"; -import { readAuthConfigProfile, writeAuthConfigProfile } from "../utilities/configFiles.js"; +import { + readAuthConfigProfile, + writeAuthConfigProfile, + writeAuthConfigCurrentProfileName, +} from "../utilities/configFiles.js"; import { printInitialBanner } from "../utilities/initialBanner.js"; import { LoginResult } from "../utilities/session.js"; import { whoAmI } from "./whoami.js"; @@ -283,6 +287,11 @@ export async function login(options?: LoginOptions): Promise { throw new Error(whoAmIResult.error); } + const profileName = options?.profile ?? "default"; + + // Set this profile as the current default + writeAuthConfigCurrentProfileName(profileName); + if (opts.embedded) { log.step("Logged in successfully"); } else { @@ -293,7 +302,7 @@ export async function login(options?: LoginOptions): Promise { return { ok: true as const, - profile: options?.profile ?? "default", + profile: profileName, userId: whoAmIResult.data.userId, email: whoAmIResult.data.email, dashboardUrl: whoAmIResult.data.dashboardUrl, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cecbf40d53..37ec88c236 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2105,6 +2105,9 @@ importers: openai: specifier: ^4.97.0 version: 4.97.0(ws@8.12.0)(zod@3.23.8) + puppeteer-core: + specifier: ^24.15.0 + version: 24.15.0 replicate: specifier: ^1.0.1 version: 1.0.1 @@ -11401,6 +11404,23 @@ packages: /@protobufjs/utf8@1.1.0: resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + /@puppeteer/browsers@2.10.6: + resolution: {integrity: sha512-pHUn6ZRt39bP3698HFQlu2ZHCkS/lPcpv7fVQcGBSzNNygw171UXAKrCUhy+TEMw4lEttOKDgNpb04hwUAJeiQ==} + engines: {node: '>=18'} + hasBin: true + dependencies: + debug: 4.4.1 + extract-zip: 2.0.1 + progress: 2.0.3 + proxy-agent: 6.5.0 + semver: 7.7.2 + tar-fs: 3.1.0 + yargs: 17.7.2 + transitivePeerDependencies: + - bare-buffer + - supports-color + dev: false + /@puppeteer/browsers@2.4.0: resolution: {integrity: sha512-x8J1csfIygOwf6D6qUAZ0ASk3z63zPb7wkNeHRerCMh82qWKUrOgkuP005AJC8lDL6/evtXETGEJVcwykKT4/g==} engines: {node: '>=18'} @@ -21250,6 +21270,11 @@ packages: transitivePeerDependencies: - supports-color + /agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + dev: false + /agentkeepalive@4.5.0: resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==} engines: {node: '>= 8.0.0'} @@ -22718,6 +22743,16 @@ packages: zod: 3.23.8 dev: false + /chromium-bidi@7.2.0(devtools-protocol@0.0.1464554): + resolution: {integrity: sha512-gREyhyBstermK+0RbcJLbFhcQctg92AGgDe/h/taMJEOLRdtSswBAO9KmvltFSQWgM2LrwWu5SIuEUbdm3JsyQ==} + peerDependencies: + devtools-protocol: '*' + dependencies: + devtools-protocol: 0.0.1464554 + mitt: 3.0.1 + zod: 3.25.76 + dev: false + /ci-info@3.8.0: resolution: {integrity: sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==} engines: {node: '>=8'} @@ -23650,6 +23685,18 @@ packages: ms: 2.1.3 supports-color: 10.0.0 + /debug@4.4.1: + resolution: {integrity: sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.3 + dev: false + /decamelize-keys@1.1.1: resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} engines: {node: '>=0.10.0'} @@ -23866,6 +23913,10 @@ packages: resolution: {integrity: sha512-75fMas7PkYNDTmDyb6PRJCH7ILmHLp+BhrZGeMsa4bCh40DTxgCz2NRy5UDzII4C5KuD0oBMZ9vXKhEl6UD/3w==} dev: false + /devtools-protocol@0.0.1464554: + resolution: {integrity: sha512-CAoP3lYfwAGQTaAXYvA6JZR0fjGUb7qec1qf4mToyoH2TZgUFeIqYcjh6f9jNuhHfuZiEdH+PONHYrLhRQX6aw==} + dev: false + /dezalgo@1.0.4: resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==} dependencies: @@ -25658,7 +25709,7 @@ packages: engines: {node: '>= 10.17.0'} hasBin: true dependencies: - debug: 4.4.0(supports-color@10.0.0) + debug: 4.4.1 get-stream: 5.2.0 yauzl: 2.10.0 optionalDependencies: @@ -26958,8 +27009,8 @@ packages: resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} engines: {node: '>= 14'} dependencies: - agent-base: 7.1.1 - debug: 4.4.0(supports-color@10.0.0) + agent-base: 7.1.4 + debug: 4.4.1 transitivePeerDependencies: - supports-color dev: false @@ -27002,6 +27053,16 @@ packages: - supports-color dev: false + /https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + dependencies: + agent-base: 7.1.4 + debug: 4.4.1 + transitivePeerDependencies: + - supports-color + dev: false + /human-id@1.0.2: resolution: {integrity: sha512-UNopramDEhHJD+VR+ehk8rOslwSfByxPIZyJRfV739NDhN5LF1fa1MqnzKm2lGTQRjNrjK19Q5fhkgIfjlVUKw==} dev: false @@ -31065,6 +31126,22 @@ packages: - supports-color dev: false + /pac-proxy-agent@7.2.0: + resolution: {integrity: sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==} + engines: {node: '>= 14'} + dependencies: + '@tootallnate/quickjs-emscripten': 0.23.0 + agent-base: 7.1.4 + debug: 4.4.1 + get-uri: 6.0.1 + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + pac-resolver: 7.0.1 + socks-proxy-agent: 8.0.5 + transitivePeerDependencies: + - supports-color + dev: false + /pac-resolver@7.0.1: resolution: {integrity: sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==} engines: {node: '>= 14'} @@ -32273,6 +32350,22 @@ packages: - supports-color dev: false + /proxy-agent@6.5.0: + resolution: {integrity: sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==} + engines: {node: '>= 14'} + dependencies: + agent-base: 7.1.4 + debug: 4.4.1 + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + lru-cache: 7.18.3 + pac-proxy-agent: 7.2.0 + proxy-from-env: 1.1.0 + socks-proxy-agent: 8.0.5 + transitivePeerDependencies: + - supports-color + dev: false + /proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} @@ -32335,6 +32428,23 @@ packages: - utf-8-validate dev: false + /puppeteer-core@24.15.0: + resolution: {integrity: sha512-2iy0iBeWbNyhgiCGd/wvGrDSo73emNFjSxYOcyAqYiagkYt5q4cPfVXaVDKBsukgc2fIIfLAalBZlaxldxdDYg==} + engines: {node: '>=18'} + dependencies: + '@puppeteer/browsers': 2.10.6 + chromium-bidi: 7.2.0(devtools-protocol@0.0.1464554) + debug: 4.4.1 + devtools-protocol: 0.0.1464554 + typed-query-selector: 2.12.0 + ws: 8.18.3 + transitivePeerDependencies: + - bare-buffer + - bufferutil + - supports-color + - utf-8-validate + dev: false + /puppeteer@23.4.0(typescript@5.5.4): resolution: {integrity: sha512-FxgFFJI7NAsX8uebiEDSjS86vufz9TaqERQHShQT0lCbSRI3jUPEcz/0HdwLiYvfYNsc1zGjqY3NsGZya4PvUA==} engines: {node: '>=18'} @@ -34474,6 +34584,17 @@ packages: - supports-color dev: false + /socks-proxy-agent@8.0.5: + resolution: {integrity: sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==} + engines: {node: '>= 14'} + dependencies: + agent-base: 7.1.4 + debug: 4.4.1 + socks: 2.8.3 + transitivePeerDependencies: + - supports-color + dev: false + /socks@2.8.3: resolution: {integrity: sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==} engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} @@ -35419,6 +35540,18 @@ packages: transitivePeerDependencies: - bare-buffer + /tar-fs@3.1.0: + resolution: {integrity: sha512-5Mty5y/sOF1YWj1J6GiBodjlDc05CUR8PKXrsnFAiSG0xA+GHeWLovaZPYUDXkH/1iKRf2+M5+OrRgzC7O9b7w==} + dependencies: + pump: 3.0.2 + tar-stream: 3.1.7 + optionalDependencies: + bare-fs: 4.1.5 + bare-path: 3.0.0 + transitivePeerDependencies: + - bare-buffer + dev: false + /tar-stream@2.2.0: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} engines: {node: '>=6'} @@ -37796,6 +37929,19 @@ packages: dependencies: bufferutil: 4.0.9 + /ws@8.18.3: + resolution: {integrity: sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + dev: false + /xdg-app-paths@8.3.0: resolution: {integrity: sha512-mgxlWVZw0TNWHoGmXq+NC3uhCIc55dDpAlDkMQUaIAcQzysb0kxctwv//fvuW61/nAAeUBJMQ8mnZjMmuYwOcQ==} engines: {node: '>= 4.0'} @@ -38030,6 +38176,10 @@ packages: /zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} + /zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + dev: false + /zustand@4.5.5(@types/react@18.2.69)(react@18.2.0): resolution: {integrity: sha512-+0PALYNJNgK6hldkgDq2vLrw5f6g/jCInz52n9RTpropGgeAf/ioFUCdtsjCqu4gNhW9D01rUQBROoRjdzyn2Q==} engines: {node: '>=12.7.0'} diff --git a/references/hello-world/package.json b/references/hello-world/package.json index b6a8d799f4..89dbeea911 100644 --- a/references/hello-world/package.json +++ b/references/hello-world/package.json @@ -9,6 +9,7 @@ "@trigger.dev/build": "workspace:*", "@trigger.dev/sdk": "workspace:*", "openai": "^4.97.0", + "puppeteer-core": "^24.15.0", "replicate": "^1.0.1", "zod": "3.23.8" }, diff --git a/references/hello-world/src/trigger/lightpanda.ts b/references/hello-world/src/trigger/lightpanda.ts new file mode 100644 index 0000000000..830f6391a2 --- /dev/null +++ b/references/hello-world/src/trigger/lightpanda.ts @@ -0,0 +1,149 @@ +import { logger, task } from "@trigger.dev/sdk"; +import { execSync, spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import puppeteer from "puppeteer-core"; + +const spawnLightpanda = async (host: string, port: string) => + new Promise((resolve, reject) => { + const child = spawn("lightpanda", [ + "serve", + "--host", + host, + "--port", + port, + "--log_level", + "info", + ]); + + child.on("spawn", async () => { + logger.info("Running Lightpanda's CDP server…", { + pid: child.pid, + }); + + await new Promise((resolve) => setTimeout(resolve, 250)); + resolve(child); + }); + child.on("error", (e) => reject(e)); + }); + +export const lightpandaCdp = task({ + id: "lightpanda-cdp", + machine: { + preset: "micro", + }, + run: async (payload: { url: string }, { ctx }) => { + logger.log("Lets get a page's links with Lightpanda!", { payload, ctx }); + + if (!payload.url) { + logger.warn("Please define the payload url"); + throw new Error("payload.url is undefined"); + } + + const host = process.env.LIGHTPANDA_CDP_HOST ?? "127.0.0.1"; + const port = process.env.LIGHTPANDA_CDP_PORT ?? "9222"; + + // Launch Lightpanda's CDP server + const lpProcess = await spawnLightpanda(host, port); + + const browser = await puppeteer.connect({ + browserWSEndpoint: `ws://${host}:${port}`, + }); + const context = await browser.createBrowserContext(); + const page = await context.newPage(); + + // Dump all the links from the page. + await page.goto(payload.url); + + const links = await page.evaluate(() => { + return Array.from(document.querySelectorAll("a")).map((row) => { + return row.getAttribute("href"); + }); + }); + + logger.info("Processing done"); + logger.info("Shutting down…"); + + // Close Puppeteer instance + await browser.close(); + + // Stop Lightpanda's CDP Server + lpProcess.kill(); + + logger.info("✅ Completed"); + + return { + links, + }; + }, +}); + +export const lightpandaFetch = task({ + id: "lightpanda-fetch", + machine: { + preset: "micro", + }, + run: async (payload: { url: string }, { ctx }) => { + logger.log("Lets get a page's content with Lightpanda!", { payload, ctx }); + + if (!payload.url) { + logger.warn("Please define the payload url"); + throw new Error("payload.url is undefined"); + } + + const buffer = execSync(`lightpanda fetch --dump ${payload.url}`); + + logger.info("✅ Completed"); + + return { + message: buffer.toString(), + }; + }, +}); + +export const lightpandaCloudPuppeteer = task({ + id: "lightpanda-cloud-puppeteer", + machine: { + preset: "micro", + }, + run: async (payload: { url: string }, { ctx }) => { + logger.log("Lets get a page's links with Lightpanda!", { payload, ctx }); + + if (!payload.url) { + logger.warn("Please define the payload url"); + throw new Error("payload.url is undefined"); + } + + const token = process.env.LIGHTPANDA_TOKEN; + if (!token) { + logger.warn("Please define the env variable LIGHTPANDA_TOKEN"); + throw new Error("LIGHTPANDA_TOKEN is undefined"); + } + + // Connect to Lightpanda's cloud + const browser = await puppeteer.connect({ + browserWSEndpoint: `wss://cloud.lightpanda.io/ws?browser=lightpanda&token=${token}`, + }); + const context = await browser.createBrowserContext(); + const page = await context.newPage(); + + // Dump all the links from the page. + await page.goto(payload.url); + + const links = await page.evaluate(() => { + return Array.from(document.querySelectorAll("a")).map((row) => { + return row.getAttribute("href"); + }); + }); + + logger.info("Processing done, shutting down…"); + + await page.close(); + await context.close(); + await browser.disconnect(); + + logger.info("✅ Completed"); + + return { + links, + }; + }, +}); diff --git a/references/hello-world/trigger.config.ts b/references/hello-world/trigger.config.ts index 33935a4a13..2b4a68912f 100644 --- a/references/hello-world/trigger.config.ts +++ b/references/hello-world/trigger.config.ts @@ -1,5 +1,6 @@ import { defineConfig } from "@trigger.dev/sdk/v3"; import { syncEnvVars } from "@trigger.dev/build/extensions/core"; +import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; export default defineConfig({ compatibilityFlags: ["run_engine_v2"], @@ -23,6 +24,7 @@ export default defineConfig({ machine: "small-2x", build: { extensions: [ + lightpanda(), syncEnvVars(async (ctx) => { console.log("syncEnvVars", { environment: ctx.environment, branch: ctx.branch }); return [