From b3c5b9ba1dccc5abae8e7f27f06667acd653215b Mon Sep 17 00:00:00 2001 From: Nicolas Rigaudiere Date: Tue, 20 May 2025 17:35:26 +0200 Subject: [PATCH 01/23] feat: add lightpanda structure --- packages/build/package.json | 17 ++++++++++++- packages/build/src/extensions/lightpanda.ts | 27 +++++++++++++++++++++ references/v3-catalog/trigger.config.ts | 2 ++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 packages/build/src/extensions/lightpanda.ts diff --git a/packages/build/package.json b/packages/build/package.json index 6df2b1a0f9..71a055ea32 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -30,7 +30,8 @@ "./extensions/audioWaveform": "./src/extensions/audioWaveform.ts", "./extensions/typescript": "./src/extensions/typescript.ts", "./extensions/puppeteer": "./src/extensions/puppeteer.ts", - "./extensions/playwright": "./src/extensions/playwright.ts" + "./extensions/playwright": "./src/extensions/playwright.ts", + "./extensions/lightpanda": "./src/extensions/lightpanda.ts" }, "sourceDialects": [ "@triggerdotdev/source" @@ -61,6 +62,9 @@ ], "extensions/playwright": [ "dist/commonjs/extensions/playwright.d.ts" + ], + "extensions/lightpanda": [ + "dist/commonjs/extensions/lightpanda.d.ts" ] } }, @@ -188,6 +192,17 @@ "types": "./dist/commonjs/extensions/playwright.d.ts", "default": "./dist/commonjs/extensions/playwright.js" } + }, + "./extensions/lightpanda": { + "import": { + "@triggerdotdev/source": "./src/extensions/lightpanda.ts", + "types": "./dist/esm/extensions/lightpanda.d.ts", + "default": "./dist/esm/extensions/lightpanda.js" + }, + "require": { + "types": "./dist/commonjs/extensions/lightpanda.d.ts", + "default": "./dist/commonjs/extensions/lightpanda.js" + } } }, "main": "./dist/commonjs/index.js", diff --git a/packages/build/src/extensions/lightpanda.ts b/packages/build/src/extensions/lightpanda.ts new file mode 100644 index 0000000000..c9ee906d42 --- /dev/null +++ b/packages/build/src/extensions/lightpanda.ts @@ -0,0 +1,27 @@ +import type { BuildExtension } from "@trigger.dev/core/v3/build"; + +type LightpandaOpts = { + arch?: 'aarch64' | 'x86_64' + version?: 'nightly' +} + +export const lightpanda = ({ arch = 'x86_64', version = 'nightly' }: LightpandaOpts = {}): BuildExtension => ({ + name: "LightpandaExtension", + onBuildComplete: async (context) => { + if (context.target === "dev") { + return + } + + context.logger.debug(lightpanda.name); + context.addLayer({ + id: "lightpanda", + image: { + instructions: [ + `RUN apt-get update && apt-get install curl -y \ && + curl -L -o lightpanda https://github.com/lightpanda-io/browser/releases/download/${version}/lightpanda-${arch}-linux \ && + chmod a+x ./lightpanda`, + ], + }, + }) + }, +}) diff --git a/references/v3-catalog/trigger.config.ts b/references/v3-catalog/trigger.config.ts index 62698d14fb..055c75d6a4 100644 --- a/references/v3-catalog/trigger.config.ts +++ b/references/v3-catalog/trigger.config.ts @@ -6,6 +6,7 @@ import { audioWaveform } from "@trigger.dev/build/extensions/audioWaveform"; import { additionalFiles, ffmpeg, syncEnvVars } from "@trigger.dev/build/extensions/core"; import { puppeteer } from "@trigger.dev/build/extensions/puppeteer"; import { playwright } from "@trigger.dev/build/extensions/playwright"; +import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript"; import { defineConfig } from "@trigger.dev/sdk/v3"; @@ -83,6 +84,7 @@ export default defineConfig({ }), puppeteer(), playwright(), + lightpanda(), ], external: ["re2"], }, From 85198b8fbe1c9dbd2c58baffcee7dfb8d53ff5dd Mon Sep 17 00:00:00 2001 From: Nicolas Rigaudiere Date: Wed, 4 Jun 2025 16:42:17 +0200 Subject: [PATCH 02/23] chore: add lightpanda doc links --- docs/config/config-file.mdx | 4 ++++ docs/config/extensions/overview.mdx | 1 + docs/docs.json | 1 + docs/introduction.mdx | 1 + 4 files changed, 7 insertions(+) diff --git a/docs/config/config-file.mdx b/docs/config/config-file.mdx index 62da85593e..27ed9ff3c4 100644 --- a/docs/config/config-file.mdx +++ b/docs/config/config-file.mdx @@ -428,6 +428,10 @@ See the [syncEnvVars documentation](/config/extensions/syncEnvVars) for more inf See the [puppeteer documentation](/config/extensions/puppeteer) for more information. +#### lightpanda + +See the [Lightpanda documentation](/config/extensions/lightpanda) for more information. + #### ffmpeg See the [ffmpeg documentation](/config/extensions/ffmpeg) for more information. diff --git a/docs/config/extensions/overview.mdx b/docs/config/extensions/overview.mdx index abba56694e..a3c2542a2f 100644 --- a/docs/config/extensions/overview.mdx +++ b/docs/config/extensions/overview.mdx @@ -52,6 +52,7 @@ Trigger.dev provides a set of built-in extensions that you can use to customize | [pythonExtension](/config/extensions/pythonExtension) | Execute Python scripts in your project | | [playwright](/config/extensions/playwright) | Use Playwright in your Trigger.dev tasks | | [puppeteer](/config/extensions/puppeteer) | Use Puppeteer in your Trigger.dev tasks | +| [lightpanda](/config/extensions/lightpanda) | Use Lightpanda in your Trigger.dev tasks | | [ffmpeg](/config/extensions/ffmpeg) | Use FFmpeg in your Trigger.dev tasks | | [aptGet](/config/extensions/aptGet) | Install system packages in your build image | | [additionalFiles](/config/extensions/additionalFiles) | Copy additional files to your build image | diff --git a/docs/docs.json b/docs/docs.json index 88b0d12a1d..871cbae6c7 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -78,6 +78,7 @@ "config/extensions/pythonExtension", "config/extensions/playwright", "config/extensions/puppeteer", + "config/extensions/lightpanda", "config/extensions/ffmpeg", "config/extensions/aptGet", "config/extensions/additionalFiles", diff --git a/docs/introduction.mdx b/docs/introduction.mdx index c8ed0f3480..443eec82c0 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -92,6 +92,7 @@ We provide everything you need to build and manage background tasks: a CLI and S | prismaExtension | Use Prisma with Trigger.dev | [Learn more](/config/extensions/prismaExtension) | | pythonExtension | Execute Python scripts in Trigger.dev | [Learn more](/config/extensions/pythonExtension) | | puppeteer | Use Puppeteer with Trigger.dev | [Learn more](/config/extensions/puppeteer) | +| lightpanda | Use Lightpanda Browser with Trigger.dev | [Learn more](/config/extensions/lightpanda) | | ffmpeg | Use FFmpeg with Trigger.dev | [Learn more](/config/extensions/ffmpeg) | | aptGet | Install system packages with aptGet | [Learn more](/config/extensions/aptGet) | | additionalFiles | Copy additional files to the build directory | [Learn more](/config/extensions/additionalFiles) | From 29dbf6674568d32cab8d9407c22c7ac805fa37e5 Mon Sep 17 00:00:00 2001 From: Nicolas Rigaudiere Date: Fri, 20 Jun 2025 11:36:26 +0200 Subject: [PATCH 03/23] fix: lightpanda extension instructions --- docs/config/extensions/lightpanda.mdx | 30 ++++++++++++++ packages/build/src/extensions/lightpanda.ts | 45 +++++++++++++++++---- 2 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 docs/config/extensions/lightpanda.mdx diff --git a/docs/config/extensions/lightpanda.mdx b/docs/config/extensions/lightpanda.mdx new file mode 100644 index 0000000000..1ed7a55d5f --- /dev/null +++ b/docs/config/extensions/lightpanda.mdx @@ -0,0 +1,30 @@ +--- +title: "Lightpanda" +sidebarTitle: "lightpanda" +description: "Use the lightpanda build extension to be able to use Lightpanda Browser in your project" +--- + + + +To use Lightpanda in your project, add these build settings to your `trigger.config.ts` file: + +```ts trigger.config.ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [lightpanda()], + }, +}); +``` + +And add the following environment variable in your Trigger.dev dashboard on the Environment Variables page: + +```bash +LIGHTPANDA_BROWSER_PATH: "/usr/bin/lightpanda", +``` + +Follow [this example](/guides/examples/lightpanda) to get setup with Trigger.dev and Lightpanda in your project. diff --git a/packages/build/src/extensions/lightpanda.ts b/packages/build/src/extensions/lightpanda.ts index c9ee906d42..cf8d905d4c 100644 --- a/packages/build/src/extensions/lightpanda.ts +++ b/packages/build/src/extensions/lightpanda.ts @@ -1,26 +1,55 @@ import type { BuildExtension } from "@trigger.dev/core/v3/build"; +const NAME = 'LightpandaExtension' + type LightpandaOpts = { arch?: 'aarch64' | 'x86_64' version?: 'nightly' + disableTelemetry?: boolean } -export const lightpanda = ({ arch = 'x86_64', version = 'nightly' }: LightpandaOpts = {}): BuildExtension => ({ - name: "LightpandaExtension", +export const lightpanda = ({ arch = 'x86_64', version = 'nightly', disableTelemetry = false }: LightpandaOpts = {}): BuildExtension => ({ + name: NAME, onBuildComplete: async (context) => { + context.logger.progress(`Running ${NAME} on ${context.target} env for arch ${arch}`); + context.logger.progress(`version: ${version}`); + if (context.target === "dev") { return } - context.logger.debug(lightpanda.name); + const instructions: string[] = [] + + if (disableTelemetry) { + instructions.push('RUN export LIGHTPANDA_DISABLE_TELEMETRY=true') + } + + /* Update / install required packages */ + instructions.push( + `RUN apt-get update && apt-get install --no-install-recommends -y \ + curl \ + ca-certificates \ + && update-ca-certificates \ + && apt-get clean && rm -rf /var/lib/apt/lists/*`, + ) + + /* Install Lightpanda */ + instructions.push( + `RUN curl -L -o lightpanda https://github.com/lightpanda-io/browser/releases/download/${version}/lightpanda-${arch}-linux`, + 'RUN chmod a+x ./lightpanda', + 'RUN mv ./lightpanda /usr/bin/lightpanda', + ) + context.addLayer({ id: "lightpanda", image: { - instructions: [ - `RUN apt-get update && apt-get install curl -y \ && - curl -L -o lightpanda https://github.com/lightpanda-io/browser/releases/download/${version}/lightpanda-${arch}-linux \ && - chmod a+x ./lightpanda`, - ], + instructions, + }, + deploy: { + env: { + LIGHTPANDA_BROWSER_PATH: "/usr/bin/lightpanda", + }, + override: true, }, }) }, From 33ffbcf54d9af638229335338cb13f8b0dadea1f Mon Sep 17 00:00:00 2001 From: Nicolas Rigaudiere Date: Fri, 20 Jun 2025 11:37:54 +0200 Subject: [PATCH 04/23] feat: add Lightpanda guide and examples --- .../route.tsx | 1 + docs/docs.json | 1 + docs/guides/examples/lightpanda.mdx | 122 ++++++++++++++++++ docs/guides/introduction.mdx | 1 + docs/images/intro-lightpanda.jpg | Bin 0 -> 11853 bytes docs/introduction.mdx | 1 + 6 files changed, 126 insertions(+) create mode 100644 docs/guides/examples/lightpanda.mdx create mode 100644 docs/images/intro-lightpanda.jpg diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx index 449ef16dcb..78ff19fd9b 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx @@ -715,6 +715,7 @@ function HelpfulInfoHasTasks({ onClose }: { onClose: () => void }) { isExternal /> + + When using Lightpanda, we recommend that you respect robots.txt files and avoid high frequency requesting websites. + DDOS could happen fast for small infrastructures. + + +## Prerequisites + +- A project with [Trigger.dev initialized](/quick-start) +- A [Lightpanda](https://lightpanda.io/) cloud token (for the 1st example) + +## Example \#1 - Get links from a website using Lightpanda cloud & Puppeteer + +In this task, we use Lightpanda browser to get links from a provided URL. +You will have to pass the URL as a payload when triggering the task. + +```ts trigger/lightpanda-cloud-puppeteer.ts +import puppeteer from "puppeteer" + +export const lightpandaCloudPuppeteer = task({ + id: "lightpanda-cloud-puppeteer", + run: async (payload: { url: string }) => { + const { url } = payload + + const browser = await puppeteer.connect({ + browserWSEndpoint: "wss://cloud.lightpanda.io/ws?browser=lightpanda&token=TOKEN", + }) + const context = await browser.createBrowserContext() + const page = await context.newPage() + + // Dump all the links from the page. + await page.goto(url) + + const links = await page.evaluate(() => { + return Array.from(document.querySelectorAll('a')).map(row => { + return row.getAttribute('href') + }) + }) + + await page.close() + await context.close() + await browser.disconnect() + + return { + links, + } + }, +}) +``` +### Proxies + +Proxies can be used with your browser via the proxy query string parameter. By default, the proxy used is "datacenter" which is a pool of shared datacenter IPs. +`datacenter` accepts an optional `country` query string parameter, an [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) country code. + +_Example using a German IP :_ + +```wss://cloud.lightpanda.io/ws?proxy=datacenter&country=de&token=TOKEN``` + + + +### Session +A session is alive until you close it or the connection is closed. The max time duration of a session is 15 min. + + +## Example \#2 - Launch and use a Lightpanda CDP server + +This task initialises a Lightpanda CDP server to allow you to scrape directly via Trigger.dev. + +### Configuration + +To use this example, you will need to add these build settings to your `trigger.config.ts` file: + +```ts trigger.config.ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + // This is required to use the Puppeteer library + extensions: [lightpanda()], + }, +}); +``` +That will set a `LIGHTPANDA_BROWSER_PATH` env variable that will be needed to get access to the binary. + +### Task + +Your task will have to launch a child process in order to have the websocket available to scrape using Puppeteer. + +```ts trigger/lightpandaLaunch.ts +import puppeteer from "puppeteer"; + +export const lightpandaLaunch = task({ + id: "lightpanda-launch", + run: async (payload: { url: string }) => { + + // use browserWSEndpoint to pass the Lightpanda's CDP server address. + const browser = await puppeteer.connect({ + browserWSEndpoint: "ws://127.0.0.1:9222", + }) + + const page = await browser.newPage(); + + return { + data: scrapeResult, + }; + }, +}); +``` diff --git a/docs/guides/introduction.mdx b/docs/guides/introduction.mdx index 3fba744586..a9b1061ce1 100644 --- a/docs/guides/introduction.mdx +++ b/docs/guides/introduction.mdx @@ -69,6 +69,7 @@ Task code you can copy and paste to use in your project. They can all be extende | [FFmpeg video processing](/guides/examples/ffmpeg-video-processing) | Use FFmpeg to process a video in various ways and save it to Cloudflare R2. | | [Firecrawl URL crawl](/guides/examples/firecrawl-url-crawl) | Learn how to use Firecrawl to crawl a URL and return LLM-ready markdown. | | [LibreOffice PDF conversion](/guides/examples/libreoffice-pdf-conversion) | Convert a document to PDF using LibreOffice. | +| [Lightpanda](/guides/examples/lightpanda) | Use Lightpanda browser (or cloud version) to get a webpage's content. | | [OpenAI with retrying](/guides/examples/open-ai-with-retrying) | Create a reusable OpenAI task with custom retry options. | | [PDF to image](/guides/examples/pdf-to-image) | Use `MuPDF` to turn a PDF into images and save them to Cloudflare R2. | | [Puppeteer](/guides/examples/puppeteer) | Use Puppeteer to generate a PDF or scrape a webpage. | diff --git a/docs/images/intro-lightpanda.jpg b/docs/images/intro-lightpanda.jpg new file mode 100644 index 0000000000000000000000000000000000000000..8fc2102bb088f84c172e50cae7f71e2933c53806 GIT binary patch literal 11853 zcmc(FV{|56v-TZ36Wg|}JGL`1@7Ojc#>B?NwvCzCnOGBRVw+#yXPxIg|IeRqcduT1 z^{(38U8}3EYwxO$<&P}@imarJBmfK?3?Th^06zWzLI7a@zuQM202LOH2gU#ah6(^j z1%p5Z`xpT50Kfp?5dV2Mz`qR^4h9|p8WQS1_rU(S{yPGI00W1Ff`);G`&b1aLVRkX zK%jgM&Y76}SLFZH0V5&rbT|`u!8Dp)Hti+Yda%-<(vNBDH%7>6jKGJ z^zx&&02o&p#O&u;Xh0du{|cb9;dk_2d#xPv&HLUTlV=nCa$Ki!CXVddp zFI(-#q-M0dp?AUdFS6aLXkRf>wbyCJFrkS+>ebX~-M``hMj+OEtzM2}C2sXWc4u(4 zcybB_89hz`r$exm;FgI#K~X7z(73_B@?a^awPwiNpu@6IG__3g4IH}^^X-r>&Oy%h z;zfE)@2x|NveS_#dp928e^ns-i5dqDC49AI-#p;Zc$uZTa-p9~r3_i0xmR=<0;+%d zp~<*EcVwLem?4+^`Csc&{PfvPvpWM)?PV*z^zUy;D*pWK7vADqALeB!T&YY0?7Tt8t3I4>o(*D0sh1OcO{QCjW-@l5JT7I0c3$tFinH#RW0fO51>ohAK zvD6=klj_ft%6d2SFIHZ1P6TVr3ALWWr)QFe-!q>;qyJ6>0Dxt<>mWLqUgIzHxR0z` z{CIr(0>XdII{W}I79anw*#D&$@aYruC;%`>2mm-Z7z6~=Ka;?G{`Iin5O84d0AW-# z3``U(M{FEi60%P>fc=k`AOb!B=!?hoAjTtHoSas8Gl|GanmcyT^u`=ZPgv+V^47bw zi$Q0p)VHxrh`U;&sKK2`77?tNI?3sY0WxEum?LT9tF_yK~+XNS;O_S5P&>U^B?0O~{UG7$RNG@q*cAyN^6Mt%>IQR{2~VgArMh|f9;LjrYkzUz&98EXI{4u}VCLKTE7j`biDlG%>oSfzN;y{z)SKyr<*09NyF7cdnSOY5&B}(Kk6Zz>83Il%l@YlNL zX>uF!#7p&Xd`a298oRzFXUS1Mrw!lM{yO$&AMqJixBK;>YI(v&43iH)137*VF8nSk zqRH(C!1)$pW!4jllwSqeSZr{bwY}uY6@KMhv+Ze`9!$XY8jp}_sa^KqPS)4^C@^mk zyT191e!xeq2p%J8C`^z09r$o&v!csbup{7d+>-VeED%`fAk8?TTbAE3@@`0n2`)6| zwsWa{ZrP1MQ(#4Y`p4D&ZH_5`fG+#jTkXpPd~!OnNIh)>ee49;2S6~&X2)fdKGw#y zxK}*x^h$>)(`iA6Z>&-h-p$QsexS#=&YYfEYz%^Ur|K_(w~JC&8zc9e3%R@tu4y3# zzpk&e!B5b!^mSlu6~79@dTcQ&;qp01YIk_|&Ij`O&GlRHj97ME6{*A`5B~ssrZLns zUMXBVrXvQ1kh{xrUd*UTzg7T|TSC-jCm|*qL4~H2Pnl{cdXNc`NnZ{O!A`!i3|3Xr zh>$){h0FqP3rq3ib|SEGAd=2ASK|nxAp5a&>5Zus>#5Ukw!MmPal{)9(Z`88$S+^! zM?(GnuDUxA39;W&Bt(J5DmUWJe&MG9zpL$1r+NjPPB~%;fkVmgfy$`)%=~-vZ3p~{ z;i;`hE+zrx8Rp#NAgUS+bw~)<6Hh{)LA6we!&kLk#&mEH`JRCQvriQ=r^@EM4 zfn)YY{ofoH$9L@&ui*6jz0ki;8Qbd+8UJJs7digR-S=mTE72?20JxS@3I0N%Q+t7V zT!b<_80&nGcX|W%W{}4*gMQf8=99T#VP53TVCEq&nBt3jpGw{CZ~8POE&G=B4K z^81?3Lj*1wC%J9ppIy;!nTM3#6mlxugF|dCu4CjA2-otmabB=ZEH$SAmu}qaVf-3~ z;7osUVaU#l@DY=v?0ltTcAN@+F9|db&1$YtSo7j151S)-8x($lC0dvZkm ztR{%E()p6|;@m7Bsg4VEHQHnVsuWbHbUP?69H`^EeWJ%ExABQZKBj%m$p>i!g~_{1 zn@x0MV+YApn+-H?XXo#$X&05nuq4M*Ggt220Ecsl<9^2p3H{2?Q_TNkGm|4jVi)Fl zd?a9j{sB<1Cdt#(gh|YLa5Lw3xAST|h#^MT9XF1%=bv3ULCke!FXiq*`l)fyNzCQ; z%Wxw&ODY;)0UWhgFlAML65sQuZ8|5=P=vt=+CaUE%@xH*3VAVH+*utzjeF|^$H&&s zAr}B)dX|MT2{ZFoidt3yabN>zQ{`7IPJUXy?%LnjzS~rdSU*Z0 zQjeO}Fp;mC%Ycv~APZoGjPfq#LtM`Mpv2kjtJBV zT8Fq8SmwK37z!0Tf74!7UUk3`ue3k^O(o>~TU^Xw%pG0gz?8-CcH1C+{9O2|@Cj+W zpO6Lx0D}OBfP#jA`wyo550yhjLx;rxkdk4NvkHsApnQ$paOcg%rB9h@_!g1C=@;8E0ESqU1$IbogxAsyII5{L1S zNpGtQ+x+^AjFpyrL9f6!D`%j=)d#J~sgB3GlaTWK$8y*-^_MXhSDaJQ7y6T+4?yjR z2$5CYEk8%fTHdBaTV zff6276Wc58YGDebkz2&_g0*JI zw4!aZs{iOA)mm}*bjyULc6^_sm5oj^vXH8dx>uhas49&?>jA#ZCl5ozWS7~P!22Gd z8sC34=T#N(+}L#r2SJmBSddw=LwK2*of?NMs%t!sDwwAK_UI_;CAS8IaCmS}VB{b259~U1}!2DiSU77#&FXNBK zO@kLM!SvUj@Fe!GIJ^vV)WPU?>E@n!a|`X3?p_3{QcTT3-(EDi${~mr3M(h%$uLdy z;U_SsaKg?0D=rD4!t6ck$g~JJp&f>fEL@CwIJyp7GHp#Y9cfwjk>6Tzuy!7`%+k*~ zV_8Rlvx<%dQ_JS2R;)Yqu@8VBJGX)#+V#XQLVDBrqQ*Z<)(kQ}Y6>L__N)X_d4#5= z4+xk+74gWh)B7i;7=e`L=27E()ITOu2nSN#xLjnacZj}rR0ShfbZS=i;lY{M?9MMm zpwR7GIk$Rs>b7BCh|Syqct!nPKX=Vg?k0k4p*(b^ z$M-*wCJ6XKdrteCD=XF~g(7@WzBl4g%OuJ3%t2#_NK_uPljX2$Fa+?T9TJvLUh$Nr zSMOuO8Um4zg~^^($H-<}L0ElO>-S8~u$Am9@+p+|E}UXI$US(^)^wiJ$H!aY?He@l z#z8#KwMKIl?`M;?QIp7WDhO*s-Z;>)^nB&{o{a3PbieucRE`d*77r~gNKR?1wkA_dGEV_X*GSM1$UbH1`#EN(yehTcGFAgTq7)thr(*VX|#` zCYGU81#^R*hWOjqvdY%wMO!s(EM7*j%-@5q#zJ%L7ACCtOIJVfGYD5Nu&*ED5M(0g zaTeX1#H!U&Kkh#O zr|NJ1JlMcQAnWp#f58K8$v%_Xnz@bpRO!$8zZOf{vVF$ad<+w??Kra#`qajL9B*`b zBh?cSoX0vp6P)=r`2N`4;nT_AO}WIn>BY&6v|Ep_+#BUjt1Aq)^xXF~*Zna`Y-l5_ z?eN8G=*^zMpaSIc%qd#lNt56(q|=r?r1-lZuNiGX%HD~k)2zYtCx5L8!zUB;)2q8FBY$m{{-*u z`TfP9c1NzU1G%K=_txk;{BTyu7^&<$&<4*rE$&mg-nK%W<>>C2ZH{1N>!ui=mr%}Q z!;Nq0Qh0g&XFpQ*A}bv<0<_j{fg769u_a zWG*Ue-gQO#499}LvvSo`jTcl&CXL;Z8PQ_TzZ2_Dw$=Xj>CmwH6k^@=6`8EBMFX+! zDDo_8X#Kjh_OaW%{07YUQPm=XKjntzB?T+0o$2UGhrZ8!VVzTF)#Iwl*lOQB0|OuM zS^eAI?5d%PSzrYl1CdiD!V-d`=h%&Y>mvV9YIvp69cy^8RC{uDOmrex8}W*H4$5L6 zd}PDbn>?`5P3&ncY#}jpQ|B8>yb&b2&Yy>lZdJ2k2&Q6t_AdUJx4R9J6DwYs|9i2O z*)a@iXsHOMs%0P3eSW&p9&v1V?v~dr{duNq!^4%@*s$^vg-37AteiO|pD@^^)Y8yc z!||dw1m4&AofnJou0?xfLbYYa_Oh))=BV!|`>lr%+|C9iXu-6d{{eutAc1~JxRZm2rLm=9tY*uJedAW$81#(5 z7Htb3z{S$;b)1M{<3rpO;8cyK@D4S&1e;QB+|@s-%w|_r&h|m!{sAhhCjU{k)ZQJF zSS!^W>w{T##rRIWEl%rVO=8gYa#zY$D`La21>A)DfjGb#CY-ByY#THcCjMOU0{@52nynGv8CCENn z^3vogzF&wVIXRxRYTqVIT%AV=M8u0RO&=v;-TR6myf1KP-bh@V$6vIxyrA{H4Q-j5 zGtpQs_Lk9aU3^QjwWKkeu40yeU-n2j?Dn`1?#zcEcfH2Cg82i0y^AUb%IMneu@5d= ze%AaO?R#WnPNzQ=z-a@cF!*w;Yj}L!wL45&KeH$z80(;!uz672Ur}DVVdSOpM{QQH%D zbOl2CPMHpahyswc4|kQeQ%JtCpiO@O?=7d1pBa?b(4WhiKDBIlppCDtul-BD{MU2D z?&@+-S#-lP`rx>;xpAHu$?l#@6#Y_sVE86q(eMEhi8=l&H;ha;n!@fU%b(*7F%~IeCY-Fkq?%;c-28K z11hxvqGa`Y-y5f9t#7zdJG+}}0wi?tb~49+J~|%&$m64gVubE$kLc=iYv!+Sv*tZ% z)^hGM z50PerRl%8}FeIjUIfP>vkknd|!Q4;49o!yF81F9tX1e051|*))(q;>Re#wZ_eeq|T zSp?T!eHf@Q_mP#hCD|ofR7JXxUhclKDr<{MO%y3^Gglc-16A$G@4KpS6rxj(#wKAf z#Lo`K@k%PUoC`#a+qpp}N*2yjg|)?L2$DI&f>brGio9lKqAVYyJgoCjWpTQ~Jc~$1 z?HF2FsWQf%&`UgZR%Z4HVpVY}SX#9BJKD#Ravtb6=9Lw&(C49%8-xp~$*#D9#5F9B zM#60h`KCo9kuP}Zq>PMC*)t*~T?&1A(-Iqa*Eh&jG$bh9gJ~**4akR|vl5R1F*z85E}_eXu=KDJX&&RC6t;B8gRcgBiPwCS(C*=itd zYgle|EVjf$m1-MMsyME&RH)mUcUS4)d+gIV;*+mO%HK@uH>R|dDmGQn@9C?aF(A#x z(1di~LDT$<`OKl+-ONpfmJ{5MKnREOx}GdOp54*;mf2a?!pW-gifRn_wL1mo+otxe z7+sr6OWIwD;<6efU)cJB3O7^;`i!246H`OC0{`zgO#@aprvoP8R4rm zaDX#myl^B}CXvBrg?$SP@o`;66>21G7Va?i!bN;k0dl0)!zG!HQ~jFra8sHf6U8i$ zQm8NLP9Z*~b{||KJ|vhs-J@Cg zyRNJxk^y=yR1k&XahM^6jlxE3qA~z08)XK)1*+)>i6R#2E9`WC&xDUs9?Kf@(Qf?txX+d*l*^=qWMi0)B6J zm6UA=4(QU^Q|d;e6As)q($yLP$_wH)KcT$o$qRp^bS0?xMpwk*kzgN|ORpScKo1cdt0Qalm!7RL9#xzwTBVh$B`iYXgsQ4p+B%g}z*y zwn1CzEESNoQ4`NvHsCG@BosxfLRL-(Tfg=Y3H`Ed9HYzTQLjG5%P|3$zTZXnPmt`2 zG$UZtq=Ck+Kn34H5ibV8l^2FBNZ1t3Fz5K7keMWps-EbCMaCvus0-BEsK=RBcuh#( zlPN~cAx=kC9+;@8hk9a(yB=3Lse6|WRpdQwsBUL7c)9BZjAnYcFJ@iPhhoU|>oWF(c(z=H0KY5M?BdyKud_oX?%4#)#p_AfasIuL~k z(6a*%q%$hkG|7|^Hkvc^ll3ofjJrh~CDO2;&9~O??m=9>8B;FYyxl@Rx(>q&nNoHY zp2OiZq4`N8Sei~Q&J>+WGU?^qe>?&i2wQ?S)hlOoFp*5W@_TZ8_x^&~9AdJDCpvejPIDoVe;A1z=4KTo9zwfp#tXa=4eF9;i2of<+xf-J5Q7gi|3 zofln|d{zxa6Au-OJN8g2M9br0aXUtyCzyDu5nK0R%92Wp`dFPa^_7{j+Sd9}pZmzX z@8<@v{YpVk0)bQZOsw@pvmbzYlf2#E5=i2E&&md0@U?18bfR4{t6yQ6Ece{`OOF>c zj?mj{j_z33b@rZkF@+2v$O#_*uuNY7+&MM7DD`6mS611Y^@h7(z?P%p;6gMIqe_kiR%BDm1O_yk`>A%#N7I!1 zJ^icBiLoKzax?o@Q(IfJ1^l+sEKjM=B$YE`2oJDEgMHc zrs+X~rAz>xPWu6%t1H_hM{g=qtkGmeElXQqu$qa;MipFOwU(DE-Q7=Yfc@*cQTtSI zs~AF<9TB>8XNVAHacejua9IAfv^{yaf?}g2dp>7-FVlg8jskVvpFun_+wp1{B% zAW;EOXr!#r=prgGC}eEHq99|Zz=Zl43=(!#=ltGFOmY*)pyX`|7O}5M1%(Zbv-AIK z+JS?C0l{8MRTBd_ap9Sm#Z0OVFw|=3F)J&!&sQO%;RvB4Bisb6UNql(p1CoSoClL? zzwZb+lyJuqob={bOMzVpB|AOcQYDyuhQOi@2 zrKgk~I3tC!P{zsh;<<}{_jf8ys?;h?y6K*uEKqIW7vh;2Sos=;6H}r;S()k4zq`;+ zmS*KH33oBK5;htBI;_y!oy~nb)?qt_=Mc(~vlL5{+*Y2)Ll?a8b(dl-K!4-~@M>8TxZ+oSy5dPg z4aail^R};*qFF)J$HTF`71%Rb zJb=(@9!iOy2us@*>;$PN0{!$GcT=$aaMh1ZneSzIq=<|nv2mzB&ok#7Q>_s`P4(t7 zRwFKwj;PA|AlX%r=r&5%)Hv0&m``fSe1&$Z?OuHtfKB-2YdtL$5fu2G6*d4Xqh_15 zJ+jg5m-75wK}@k3rK#qLZo~&*1?Y$GKxcFjTC0u*gT_HSE~dEzoAGH|jOX*Qi z$DF2OBA_BuMYWYFOQ^5`6i;(d?zJW4LHN;Eg;!VfwSwYpa9G(0KokR{K)WyA=Q!m* zb|@HW3gJQWR+dOt5NltgLSet4BxeO?#^&$1D#O7c!oY>g3s$PoYuO1%hO9e2ms2|i zo-leM9b?&-&Rx`PnAe0GgJol><#=AV?FK2*|Ip2eoN;S5KX~S&zT_(zHgT7LGSNaL zF9fnZydAE4=`?r8xaq}ed%TNRM4Oqa+DC?)-EP?`6lc5KMhIe|OOhWi>0d`Ru&zec z2{=e0pSIPKpd&V+pU7Ic7BRz%?q}7>iE}B7X;IHCqF6DBb!D}h)%(q5-lzeMGXkZh zm!%Jar=1W7nmi{NnAE28m_?h5jcBYe;>xVOr_8x(!`&>fMO~CW^X3KNN;>;e_q^o< z>}HRT{+4c~D17YEY5aN|216`zXwRxtIzS7{ge6-KWm`Yxh0`<>dYqz$^%E!iezDfa zeYjSfmJquJyA4=?g*{OmS#k;?^dudkM?kQjDsD`JUdZ~ zg=~e9PtzzbJ*KBRbIH8g8zF*NT*1!KbxI0l*=RlV6nzaXskE61CKI>03<6mebhd)U_F@{f z3~|>K_Je!X43l?Z4To$Jt~Y51)l$*3Dhgwlj(~!U?)Ac` zb5wl9)(f*CEj?UTL(qQsg4y}(+(`NF`ghHL1s zYeG22*T6P0d)?I(d%gSqMo$lQJZoNF7WkgUWaUwjr|#sr3zLo2-nC?VNvg}$q^~j_ zRGO>FcTqpd7n0NG)a^zH4+arEW1rISoz(8s0n3uN%E?XxS&dd`I%X_!MAwGm}@(Y1rNo%%>p zKeh_yQ%qCM8DQ<(-!iQleR3<-a`O8M^TGaDlLWV|5$OXp{TFy$f*@S)GxfrSR%N4% zhlz_E$rrn7ym2j&#Jz{LW#EA$5iZ2nW2Z;W z{3=4Y6)nG=!s(wang*yQ>Ps z5V_p&CgduPTb)>5RslJuo!OhQX%4*<_>hk@VrzFnXD36j&_a_ef2TD^&%I<=DnUWZ zAOEB+y6{_O^qTqlWstrVF@&-o0A(8D z-UtGmF{iKCjAPl-!JQ7HVu-}o16FB09{|fuj1EewsNX8Gr4m8wRWZ=jUD3 z;YG)b)$Q-2|8PB8)X(y{&sYZiSw{CiTo09$6(FJlazYUfOvtY%>AiGhnYsEenM3(p zl|}no_RX?@X32u1I^R?2Xd(gZvWC*-H0sO}3shzea&P%NMtZflv z3IcHY zUxNH{iO`Z0DYwN1VMHT92yTjv{*BvPW>cPXfh1@C$XZf?n|4)*ddUxq%5+&IHu|T| zw$SHiHC7#Nl0XFhnITq!;Z^Ss4syAs*oG9rMHBDiQR23;>tcR%%Z1n_^N)e2#t@|@ z&s7@a<|5+KZ}A>xjxIEcZItRz49`S=G^x`(k+4&7HA`Zff{k?xCR8+uIA4C~yp9Aes?ZC`1WEAj|cw-#ML(>|+w7$ZB?f=r*dk1w&%t7(; zmhk4er*$g$!&@>UJ8h=V8<;f1INEbkel3=939BuB9y}@Q=aLWfmY@ky;gLk&>=PJI z`9)BBeO4Bv-Yd#BK8h|0#1@TpZo>&sl9;6xCyVS;E7_iOh4Kxr&0=-hr-i$Z|CZC0 zFzVFao<#>|Y^9b)Q!}+vCzT{lLXrj@oQe^oIV(CM`4$uUn2D@WJ|` zSir!}YBz*>Ui&Z)6dD~X)HrplDSjoOv^bc-LnxLVm@4p6e=m}Zj~5896a_42I!=b8 zk+(%qs2!297Bj%*@x(96fqhFMUpt_dfRGJT784Pn!Qd>PrUdUAh!qb}g{27%5vZU= z>zfu1{3HLDiCrI=?g&faXu!-_1UHbIpEk^;vhkqV!7G}2p+4_RzZRLSG6GmS1GwxM z5McBP|6hc}vRd%$3+?QJ3*hM*_OH9`@vPcmjS>ivKv`ryoK=ewM-h!zcis^|7R^A}T1tAg4f*&y`kw zujBS5%YSJD$~!gMx3sr2-CCH78CSu%%%qEpvghtLh9)zK;5L-`9!Q9eQ5&LtWy0YtO3ahx6G;pcL&PV?j2=6jv3@H{PU(w+ zRH4I2(6^zaC3lF*fX+C7b5S9*LBpcea%8LHuwk0ecA@z#U==z{GKK({7yq$p&a|tS zL(0M-I*frK!f6VwM#3UZpzDfW*9=~#Wi8FDA+it{>>H0lPQ`dtpvfrgWE$6kf}#5<`$CTu?wTzdRlBPc~5Lz_|M&9^~82`nqn*{ceqzNWW!COXNwU z70V6*_ HV&(q + ## Explore by build extension From d5505b515f9b6d8bec0a78cb66212e5ebc6da38c Mon Sep 17 00:00:00 2001 From: Nicolas Rigaudiere Date: Mon, 23 Jun 2025 11:44:45 +0200 Subject: [PATCH 05/23] feat: lightpanda - add 3rd example --- docs/guides/examples/lightpanda.mdx | 60 +++++++++++++++++++---------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/docs/guides/examples/lightpanda.mdx b/docs/guides/examples/lightpanda.mdx index f278740c53..b0c6769839 100644 --- a/docs/guides/examples/lightpanda.mdx +++ b/docs/guides/examples/lightpanda.mdx @@ -68,40 +68,58 @@ _Example using a German IP :_ ```wss://cloud.lightpanda.io/ws?proxy=datacenter&country=de&token=TOKEN``` - ### Session A session is alive until you close it or the connection is closed. The max time duration of a session is 15 min. -## Example \#2 - Launch and use a Lightpanda CDP server +## Example \#2 - Get a webpage using Lightpanda -This task initialises a Lightpanda CDP server to allow you to scrape directly via Trigger.dev. +Using the Lightpanda binary we will dump the HTML for a provided URL. +You will have to pass the URL as a payload when triggering the task. + + +### Prerequisites +- Setup the [Lightpanda build extension](/config/extensions/lightpanda) -### Configuration +### Task +```ts trigger/lightpanda-lightpanda-fetch.ts +import { logger, task } from '@trigger.dev/sdk/v3' +import { execSync } from 'node:child_process' + +export const lightpandaFetch = task({ + id: 'lightpanda-fetch', + machine: { + preset: "micro", + }, + run: async (payload: { url: string }, { ctx }) => { + logger.log("Lets get a page's content with Lightpanda!", { payload, ctx }) -To use this example, you will need to add these build settings to your `trigger.config.ts` file: + if (!payload.url) { + logger.warn('Please define the payload url') + throw new Error('payload.url is undefined') + } -```ts trigger.config.ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; + const e = execSync(`${process.env.LIGHTPANDA_BROWSER_PATH} fetch --dump ${payload.url}`) -export default defineConfig({ - project: "", - // Your other config settings... - build: { - // This is required to use the Puppeteer library - extensions: [lightpanda()], + return { + message: e.toString(), + } }, -}); +}) ``` -That will set a `LIGHTPANDA_BROWSER_PATH` env variable that will be needed to get access to the binary. -### Task +## Example \#3 - Launch and use a Lightpanda CDP server + +This task initialises a Lightpanda CDP server to allow you to scrape directly via Trigger.dev. + +### Prerequisites +- Setup the [Lightpanda build extension](/config/extensions/lightpanda) +### Task Your task will have to launch a child process in order to have the websocket available to scrape using Puppeteer. ```ts trigger/lightpandaLaunch.ts -import puppeteer from "puppeteer"; +import puppeteer from "puppeteer" export const lightpandaLaunch = task({ id: "lightpanda-launch", @@ -112,11 +130,11 @@ export const lightpandaLaunch = task({ browserWSEndpoint: "ws://127.0.0.1:9222", }) - const page = await browser.newPage(); + const page = await browser.newPage() return { data: scrapeResult, - }; + } }, -}); +}) ``` From f43f8af751e66cc42ee3904e8484a394ba7b85e2 Mon Sep 17 00:00:00 2001 From: Nicolas Rigaudiere Date: Mon, 23 Jun 2025 11:48:41 +0200 Subject: [PATCH 06/23] feat: add lightpandaTask --- .../v3-catalog/src/trigger/lightpandaTask.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 references/v3-catalog/src/trigger/lightpandaTask.ts diff --git a/references/v3-catalog/src/trigger/lightpandaTask.ts b/references/v3-catalog/src/trigger/lightpandaTask.ts new file mode 100644 index 0000000000..b0cb3699e5 --- /dev/null +++ b/references/v3-catalog/src/trigger/lightpandaTask.ts @@ -0,0 +1,25 @@ +import { logger, task } from "@trigger.dev/sdk/v3"; +import { execSync } from "node:child_process"; + +export const lightpandaFetch = task({ + id: "lightpanda-fetch", + machine: { + preset: "micro", + }, + run: async (payload: { url: string }, { ctx }) => { + logger.log("Lets get a page's content with Lightpanda!", { payload, ctx }); + + if (!payload.url) { + logger.warn("Please define the payload url"); + throw new Error("payload.url is undefined"); + } + + const e = execSync( + `${process.env.LIGHTPANDA_BROWSER_PATH} fetch --dump ${payload.url}`, + ); + + return { + message: e.toString(), + }; + }, +}); \ No newline at end of file From 11efe5c9cea3f4d1ad04a22354192c5302cdf4d2 Mon Sep 17 00:00:00 2001 From: Nicolas Rigaudiere Date: Mon, 23 Jun 2025 16:52:34 +0200 Subject: [PATCH 07/23] fix: lightpanda 3rd example --- docs/guides/examples/lightpanda.mdx | 94 +++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 12 deletions(-) diff --git a/docs/guides/examples/lightpanda.mdx b/docs/guides/examples/lightpanda.mdx index b0c6769839..59bcd81526 100644 --- a/docs/guides/examples/lightpanda.mdx +++ b/docs/guides/examples/lightpanda.mdx @@ -118,22 +118,92 @@ This task initialises a Lightpanda CDP server to allow you to scrape directly vi ### Task Your task will have to launch a child process in order to have the websocket available to scrape using Puppeteer. -```ts trigger/lightpandaLaunch.ts -import puppeteer from "puppeteer" - -export const lightpandaLaunch = task({ - id: "lightpanda-launch", - run: async (payload: { url: string }) => { +```ts trigger/lightpandaCDP.ts +import { logger, task } from '@trigger.dev/sdk/v3' +import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process' +import puppeteer from 'puppeteer' + +const spawnLightpanda = async (log: typeof logger) => + new Promise((resolve, reject) => { + const child = spawn(process.env.LIGHTPANDA_BROWSER_PATH as string, [ + 'serve', + '--host', + '127.0.0.1', + '--port', + '9222', + '--log_level', + 'info', + ]) + + child.on('spawn', async () => { + log.info("Running Lightpanda's CDP server…", { + pid: child.pid, + }) - // use browserWSEndpoint to pass the Lightpanda's CDP server address. - const browser = await puppeteer.connect({ - browserWSEndpoint: "ws://127.0.0.1:9222", + await new Promise(resolve => setTimeout(resolve, 250)) + resolve(child) }) + child.on('error', e => reject(e)) + }) - const page = await browser.newPage() +export const lightpandaCDP = task({ + id: 'lightpanda-cdp', + machine: { + preset: 'micro', + }, + run: async (payload: { url: string }, { ctx }) => { + logger.log("Lets get a page's links with Lightpanda!", { payload, ctx }) - return { - data: scrapeResult, + if (!payload.url) { + logger.warn('Please define the payload url') + throw new Error('payload.url is undefined') + } + + if (typeof process.env.LIGHTPANDA_BROWSER_PATH === 'undefined') { + logger.warn('Please define the env variable $LIGHTPANDA_BROWSER_PATH', { + env: process.env, + }) + + throw new Error('$LIGHTPANDA_BROWSER_PATH is undefined') + } + + try { + // Launch Lightpanda's CDP server + const lpProcess = await spawnLightpanda(logger) + + const browser = await puppeteer.connect({ + browserWSEndpoint: 'ws://127.0.0.1:9222', + }) + const context = await browser.createBrowserContext() + const page = await context.newPage() + + // Dump all the links from the page. + await page.goto(payload.url) + + const links = await page.evaluate(() => { + return Array.from(document.querySelectorAll('a')).map(row => { + return row.getAttribute('href') + }) + }) + + logger.info('Processing done') + logger.info('Shutting down…') + + // Close Puppeteer instance + await browser.close() + + // Stop Lightpanda's CDP Server + lpProcess.stdout.destroy() + lpProcess.stderr.destroy() + lpProcess.kill() + + logger.info('✅ Completed') + + return { + links, + } + } catch (e: any) { + throw new Error(e) } }, }) From 806aeff3b19e70052b74241e99c617716acd31b2 Mon Sep 17 00:00:00 2001 From: Nicolas Rigaudiere Date: Mon, 23 Jun 2025 17:05:55 +0200 Subject: [PATCH 08/23] fix: lightpanda 1st example --- docs/guides/examples/lightpanda.mdx | 48 ++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/docs/guides/examples/lightpanda.mdx b/docs/guides/examples/lightpanda.mdx index 59bcd81526..9e50aedb4e 100644 --- a/docs/guides/examples/lightpanda.mdx +++ b/docs/guides/examples/lightpanda.mdx @@ -25,22 +25,43 @@ You will find here are a couple of examples of how to use Lightpanda with Trigge In this task, we use Lightpanda browser to get links from a provided URL. You will have to pass the URL as a payload when triggering the task. +Make sure to add `$LIGHTPANDA_TOKEN` to your Trigger.dev dashboard on the Environment Variables page: +```bash +LIGHTPANDA_TOKEN: "", +``` + ```ts trigger/lightpanda-cloud-puppeteer.ts -import puppeteer from "puppeteer" +import { logger, task } from '@trigger.dev/sdk/v3' +import puppeteer from 'puppeteer' export const lightpandaCloudPuppeteer = task({ - id: "lightpanda-cloud-puppeteer", - run: async (payload: { url: string }) => { - const { url } = payload + id: 'lightpanda-cloud-puppeteer', + machine: { + preset: 'micro', + }, + run: async (payload: { url: string }, { ctx }) => { + logger.log("Lets get a page's links with Lightpanda!", { payload, ctx }) + if (!payload.url) { + logger.warn('Please define the payload url') + throw new Error('payload.url is undefined') + } + + if (typeof process.env.LIGHTPANDA_TOKEN === 'undefined') { + logger.warn('Please define the env variable $LIGHTPANDA_TOKEN', { + env: process.env, + }) + throw new Error('$LIGHTPANDA_TOKEN is undefined') + } + // Connect to Lightpanda's cloud const browser = await puppeteer.connect({ - browserWSEndpoint: "wss://cloud.lightpanda.io/ws?browser=lightpanda&token=TOKEN", + browserWSEndpoint: `wss://cloud.lightpanda.io/ws?browser=lightpanda&token=${process.env.LIGHTPANDA_TOKEN}`, }) const context = await browser.createBrowserContext() const page = await context.newPage() // Dump all the links from the page. - await page.goto(url) + await page.goto(payload.url) const links = await page.evaluate(() => { return Array.from(document.querySelectorAll('a')).map(row => { @@ -48,10 +69,15 @@ export const lightpandaCloudPuppeteer = task({ }) }) + logger.info('Processing done') + logger.info('Shutting down…') + await page.close() await context.close() await browser.disconnect() + logger.info('✅ Completed') + return { links, } @@ -99,8 +125,17 @@ export const lightpandaFetch = task({ throw new Error('payload.url is undefined') } + if (typeof process.env.LIGHTPANDA_BROWSER_PATH === 'undefined') { + logger.warn('Please define the env variable $LIGHTPANDA_BROWSER_PATH', { + env: process.env, + }) + throw new Error('$LIGHTPANDA_BROWSER_PATH is undefined') + } + const e = execSync(`${process.env.LIGHTPANDA_BROWSER_PATH} fetch --dump ${payload.url}`) + logger.info('✅ Completed') + return { message: e.toString(), } @@ -163,7 +198,6 @@ export const lightpandaCDP = task({ logger.warn('Please define the env variable $LIGHTPANDA_BROWSER_PATH', { env: process.env, }) - throw new Error('$LIGHTPANDA_BROWSER_PATH is undefined') } From 76d936c17206a9098acf3192939cdf5a0952071a Mon Sep 17 00:00:00 2001 From: Nicolas Rigaudiere Date: Tue, 24 Jun 2025 10:14:20 +0200 Subject: [PATCH 09/23] chore: add changeset --- .changeset/rare-mails-fail.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/rare-mails-fail.md diff --git a/.changeset/rare-mails-fail.md b/.changeset/rare-mails-fail.md new file mode 100644 index 0000000000..fadbf8b95b --- /dev/null +++ b/.changeset/rare-mails-fail.md @@ -0,0 +1,8 @@ +--- +"@trigger.dev/build": patch +"trigger.dev": patch +"@trigger.dev/core": patch +"@trigger.dev/sdk": patch +--- + +Adding Lightpanda extension From 72225e789a50ff07d5546e0611db7e8bb13dc3a0 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Tue, 24 Jun 2025 09:37:15 +0100 Subject: [PATCH 10/23] add v4 tag to guide --- docs/guides/examples/lightpanda.mdx | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/guides/examples/lightpanda.mdx b/docs/guides/examples/lightpanda.mdx index 9e50aedb4e..9f70d81df2 100644 --- a/docs/guides/examples/lightpanda.mdx +++ b/docs/guides/examples/lightpanda.mdx @@ -2,6 +2,7 @@ title: "Get a webpage's content using Lightpanda browser" sidebarTitle: "Lightpanda" description: "In these examples, we will show you how to crawl using Lightpanda browser and Trigger.dev." +tag: "v4" --- ## Overview From fcc50a6a663edb8950e22b0b725c0c4dce0ebe6c Mon Sep 17 00:00:00 2001 From: Nicolas Rigaudiere Date: Wed, 25 Jun 2025 09:58:27 +0200 Subject: [PATCH 11/23] fix: merge lightpanda docker instructions --- packages/build/src/extensions/lightpanda.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/build/src/extensions/lightpanda.ts b/packages/build/src/extensions/lightpanda.ts index cf8d905d4c..41a11b65b1 100644 --- a/packages/build/src/extensions/lightpanda.ts +++ b/packages/build/src/extensions/lightpanda.ts @@ -35,9 +35,9 @@ export const lightpanda = ({ arch = 'x86_64', version = 'nightly', disableTeleme /* Install Lightpanda */ instructions.push( - `RUN curl -L -o lightpanda https://github.com/lightpanda-io/browser/releases/download/${version}/lightpanda-${arch}-linux`, - 'RUN chmod a+x ./lightpanda', - 'RUN mv ./lightpanda /usr/bin/lightpanda', + `RUN curl -L -o lightpanda https://github.com/lightpanda-io/browser/releases/download/${version}/lightpanda-${arch}-linux \ + && chmod a+x ./lightpanda \ + && mv ./lightpanda /usr/bin/lightpanda \ ) context.addLayer({ From 60f33573729d8b53a4a570c2c916ac6f5da2f2d9 Mon Sep 17 00:00:00 2001 From: Nicolas Rigaudiere Date: Wed, 25 Jun 2025 10:19:18 +0200 Subject: [PATCH 12/23] fix: add failsafes --- packages/build/src/extensions/lightpanda.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/build/src/extensions/lightpanda.ts b/packages/build/src/extensions/lightpanda.ts index 41a11b65b1..248aa792d1 100644 --- a/packages/build/src/extensions/lightpanda.ts +++ b/packages/build/src/extensions/lightpanda.ts @@ -1,4 +1,4 @@ -import type { BuildExtension } from "@trigger.dev/core/v3/build"; +import type { BuildExtension } from "@trigger.dev/core/v3/build" const NAME = 'LightpandaExtension' @@ -11,8 +11,8 @@ type LightpandaOpts = { export const lightpanda = ({ arch = 'x86_64', version = 'nightly', disableTelemetry = false }: LightpandaOpts = {}): BuildExtension => ({ name: NAME, onBuildComplete: async (context) => { - context.logger.progress(`Running ${NAME} on ${context.target} env for arch ${arch}`); - context.logger.progress(`version: ${version}`); + context.logger.progress(`Running ${NAME} on ${context.target} env for arch ${arch}`) + context.logger.progress(`version: ${version}`) if (context.target === "dev") { return @@ -35,9 +35,10 @@ export const lightpanda = ({ arch = 'x86_64', version = 'nightly', disableTeleme /* Install Lightpanda */ instructions.push( - `RUN curl -L -o lightpanda https://github.com/lightpanda-io/browser/releases/download/${version}/lightpanda-${arch}-linux \ + `RUN curl -L -f --retry 3 -o lightpanda https://github.com/lightpanda-io/browser/releases/download/${version}/lightpanda-${arch}-linux || (echo "Failed to download Lightpanda binary" && exit 1) \ && chmod a+x ./lightpanda \ && mv ./lightpanda /usr/bin/lightpanda \ + && /usr/bin/lightpanda version || (echo "Downloaded binary is not functional" && exit 1)`, ) context.addLayer({ From 281de09adb1a4229f8a0e3068f63f2c4974e8daf Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Thu, 26 Jun 2025 17:09:45 +0100 Subject: [PATCH 13/23] add scrape warning --- docs/guides/examples/lightpanda.mdx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/guides/examples/lightpanda.mdx b/docs/guides/examples/lightpanda.mdx index 9f70d81df2..631846c1c0 100644 --- a/docs/guides/examples/lightpanda.mdx +++ b/docs/guides/examples/lightpanda.mdx @@ -5,16 +5,15 @@ description: "In these examples, we will show you how to crawl using Lightpanda tag: "v4" --- +import ScrapingWarning from "/snippets/web-scraping-warning.mdx"; + ## Overview Lightpanda is a purpose-built browser for AI and automation workflows. It is 10x faster, uses 10x less RAM than Chrome headless. You will find here are a couple of examples of how to use Lightpanda with Trigger.dev. - - When using Lightpanda, we recommend that you respect robots.txt files and avoid high frequency requesting websites. - DDOS could happen fast for small infrastructures. - + ## Prerequisites From 5aec497aec61d10fae62ebfb9b94a5c8daabefa2 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Thu, 26 Jun 2025 20:18:46 +0100 Subject: [PATCH 14/23] lint --- .changeset/rare-mails-fail.md | 2 +- packages/build/src/extensions/lightpanda.ts | 42 ++++++++++--------- .../v3-catalog/src/trigger/lightpandaTask.ts | 6 +-- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/.changeset/rare-mails-fail.md b/.changeset/rare-mails-fail.md index fadbf8b95b..b35df51656 100644 --- a/.changeset/rare-mails-fail.md +++ b/.changeset/rare-mails-fail.md @@ -5,4 +5,4 @@ "@trigger.dev/sdk": patch --- -Adding Lightpanda extension +Add Lightpanda extension diff --git a/packages/build/src/extensions/lightpanda.ts b/packages/build/src/extensions/lightpanda.ts index 248aa792d1..c47ddfdd85 100644 --- a/packages/build/src/extensions/lightpanda.ts +++ b/packages/build/src/extensions/lightpanda.ts @@ -1,27 +1,31 @@ -import type { BuildExtension } from "@trigger.dev/core/v3/build" +import type { BuildExtension } from "@trigger.dev/core/v3/build"; -const NAME = 'LightpandaExtension' +const NAME = "LightpandaExtension"; type LightpandaOpts = { - arch?: 'aarch64' | 'x86_64' - version?: 'nightly' - disableTelemetry?: boolean -} - -export const lightpanda = ({ arch = 'x86_64', version = 'nightly', disableTelemetry = false }: LightpandaOpts = {}): BuildExtension => ({ + arch?: "aarch64" | "x86_64"; + version?: "nightly"; + disableTelemetry?: boolean; +}; + +export const lightpanda = ({ + arch = "x86_64", + version = "nightly", + disableTelemetry = false, +}: LightpandaOpts = {}): BuildExtension => ({ name: NAME, onBuildComplete: async (context) => { - context.logger.progress(`Running ${NAME} on ${context.target} env for arch ${arch}`) - context.logger.progress(`version: ${version}`) + context.logger.progress(`Running ${NAME} on ${context.target} env for arch ${arch}`); + context.logger.progress(`version: ${version}`); if (context.target === "dev") { - return + return; } - const instructions: string[] = [] + const instructions: string[] = []; if (disableTelemetry) { - instructions.push('RUN export LIGHTPANDA_DISABLE_TELEMETRY=true') + instructions.push("RUN export LIGHTPANDA_DISABLE_TELEMETRY=true"); } /* Update / install required packages */ @@ -30,16 +34,16 @@ export const lightpanda = ({ arch = 'x86_64', version = 'nightly', disableTeleme curl \ ca-certificates \ && update-ca-certificates \ - && apt-get clean && rm -rf /var/lib/apt/lists/*`, - ) + && apt-get clean && rm -rf /var/lib/apt/lists/*` + ); /* Install Lightpanda */ instructions.push( `RUN curl -L -f --retry 3 -o lightpanda https://github.com/lightpanda-io/browser/releases/download/${version}/lightpanda-${arch}-linux || (echo "Failed to download Lightpanda binary" && exit 1) \ && chmod a+x ./lightpanda \ && mv ./lightpanda /usr/bin/lightpanda \ - && /usr/bin/lightpanda version || (echo "Downloaded binary is not functional" && exit 1)`, - ) + && /usr/bin/lightpanda version || (echo "Downloaded binary is not functional" && exit 1)` + ); context.addLayer({ id: "lightpanda", @@ -52,6 +56,6 @@ export const lightpanda = ({ arch = 'x86_64', version = 'nightly', disableTeleme }, override: true, }, - }) + }); }, -}) +}); diff --git a/references/v3-catalog/src/trigger/lightpandaTask.ts b/references/v3-catalog/src/trigger/lightpandaTask.ts index b0cb3699e5..3175989ad6 100644 --- a/references/v3-catalog/src/trigger/lightpandaTask.ts +++ b/references/v3-catalog/src/trigger/lightpandaTask.ts @@ -14,12 +14,10 @@ export const lightpandaFetch = task({ throw new Error("payload.url is undefined"); } - const e = execSync( - `${process.env.LIGHTPANDA_BROWSER_PATH} fetch --dump ${payload.url}`, - ); + const e = execSync(`${process.env.LIGHTPANDA_BROWSER_PATH} fetch --dump ${payload.url}`); return { message: e.toString(), }; }, -}); \ No newline at end of file +}); From 32e4175adcf989bd6b2c27a460442150c3d3184b Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Fri, 27 Jun 2025 23:49:09 +0100 Subject: [PATCH 15/23] successful login also switches to that profile --- .changeset/empty-dolls-judge.md | 5 +++++ packages/cli-v3/src/commands/login.ts | 13 +++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 .changeset/empty-dolls-judge.md diff --git a/.changeset/empty-dolls-judge.md b/.changeset/empty-dolls-judge.md new file mode 100644 index 0000000000..477cc9ef47 --- /dev/null +++ b/.changeset/empty-dolls-judge.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Switch to profile after successful login diff --git a/packages/cli-v3/src/commands/login.ts b/packages/cli-v3/src/commands/login.ts index 484ada22a1..fa4fc510e0 100644 --- a/packages/cli-v3/src/commands/login.ts +++ b/packages/cli-v3/src/commands/login.ts @@ -14,7 +14,11 @@ import { wrapCommandAction, } from "../cli/common.js"; import { chalkLink, prettyError } from "../utilities/cliOutput.js"; -import { readAuthConfigProfile, writeAuthConfigProfile } from "../utilities/configFiles.js"; +import { + readAuthConfigProfile, + writeAuthConfigProfile, + writeAuthConfigCurrentProfileName, +} from "../utilities/configFiles.js"; import { printInitialBanner } from "../utilities/initialBanner.js"; import { LoginResult } from "../utilities/session.js"; import { whoAmI } from "./whoami.js"; @@ -264,6 +268,11 @@ export async function login(options?: LoginOptions): Promise { throw new Error(whoAmIResult.error); } + const profileName = options?.profile ?? "default"; + + // Set this profile as the current default + writeAuthConfigCurrentProfileName(profileName); + if (opts.embedded) { log.step("Logged in successfully"); } else { @@ -274,7 +283,7 @@ export async function login(options?: LoginOptions): Promise { return { ok: true as const, - profile: options?.profile ?? "default", + profile: profileName, userId: whoAmIResult.data.userId, email: whoAmIResult.data.email, dashboardUrl: whoAmIResult.data.dashboardUrl, From c9511b1c6044c8da0966e1f4abcdf3a2c3c200ad Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Thu, 3 Jul 2025 17:53:55 +0100 Subject: [PATCH 16/23] update docs and links as this is v4 only --- .../route.tsx | 1 - docs/config/config-file.mdx | 4 -- docs/config/extensions/lightpanda.mdx | 51 ++++++++++++++++--- docs/config/extensions/overview.mdx | 2 - docs/guides/examples/lightpanda.mdx | 11 ++-- 5 files changed, 50 insertions(+), 19 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx index 78ff19fd9b..449ef16dcb 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam._index/route.tsx @@ -715,7 +715,6 @@ function HelpfulInfoHasTasks({ onClose }: { onClose: () => void }) { isExternal /> - +import UpgradeToV4Note from "/snippets/upgrade-to-v4-note.mdx"; -To use Lightpanda in your project, add these build settings to your `trigger.config.ts` file: + + +To use the Lightpanda browser in your project, add the extension to your `trigger.config.ts` file: ```ts trigger.config.ts -import { defineConfig } from "@trigger.dev/sdk/v3"; +import { defineConfig } from "@trigger.dev/sdk"; import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; export default defineConfig({ @@ -21,10 +24,42 @@ export default defineConfig({ }); ``` -And add the following environment variable in your Trigger.dev dashboard on the Environment Variables page: +### Options -```bash -LIGHTPANDA_BROWSER_PATH: "/usr/bin/lightpanda", +- `version`: The version of the browser to install. Default: `"nightly"`. +- `disableTelemetry`: Whether to disable telemetry. Default: `false`. + +For example: + +```ts trigger.config.ts +import { defineConfig } from "@trigger.dev/sdk"; +import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; + +export default defineConfig({ + project: "", + build: { + extensions: [ + lightpanda({ + version: "nightly", + disableTelemetry: true, + }), + ], + }, +}); ``` -Follow [this example](/guides/examples/lightpanda) to get setup with Trigger.dev and Lightpanda in your project. +### Environment variables + +The extension sets the following environment variables during the build: + +- `LIGHTPANDA_BROWSER_PATH`: Set to `/usr/bin/lightpanda` so the browser can be found at runtime + +#### Development + +When running in dev, you will first have to download the Lightpanda browser binary. See [Lightpanda's installation guide](https://lightpanda.io/docs/getting-started/installation). + +Then, set the `LIGHTPANDA_BROWSER_PATH` environment variable to the path of the Lightpanda browser binary. + +```bash +export LIGHTPANDA_BROWSER_PATH="/usr/local/bin/lightpanda" +``` diff --git a/docs/config/extensions/overview.mdx b/docs/config/extensions/overview.mdx index a3c2542a2f..412a11062b 100644 --- a/docs/config/extensions/overview.mdx +++ b/docs/config/extensions/overview.mdx @@ -50,9 +50,7 @@ Trigger.dev provides a set of built-in extensions that you can use to customize | :-------------------------------------------------------------------- | :----------------------------------------------------------------------------- | | [prismaExtension](/config/extensions/prismaExtension) | Using prisma in your Trigger.dev tasks | | [pythonExtension](/config/extensions/pythonExtension) | Execute Python scripts in your project | -| [playwright](/config/extensions/playwright) | Use Playwright in your Trigger.dev tasks | | [puppeteer](/config/extensions/puppeteer) | Use Puppeteer in your Trigger.dev tasks | -| [lightpanda](/config/extensions/lightpanda) | Use Lightpanda in your Trigger.dev tasks | | [ffmpeg](/config/extensions/ffmpeg) | Use FFmpeg in your Trigger.dev tasks | | [aptGet](/config/extensions/aptGet) | Install system packages in your build image | | [additionalFiles](/config/extensions/additionalFiles) | Copy additional files to your build image | diff --git a/docs/guides/examples/lightpanda.mdx b/docs/guides/examples/lightpanda.mdx index 631846c1c0..12d47b5b1d 100644 --- a/docs/guides/examples/lightpanda.mdx +++ b/docs/guides/examples/lightpanda.mdx @@ -6,6 +6,9 @@ tag: "v4" --- import ScrapingWarning from "/snippets/web-scraping-warning.mdx"; +import UpgradeToV4Note from "/snippets/upgrade-to-v4-note.mdx"; + + ## Overview @@ -31,7 +34,7 @@ LIGHTPANDA_TOKEN: "", ``` ```ts trigger/lightpanda-cloud-puppeteer.ts -import { logger, task } from '@trigger.dev/sdk/v3' +import { logger, task } from '@trigger.dev/sdk' import puppeteer from 'puppeteer' export const lightpandaCloudPuppeteer = task({ @@ -109,7 +112,7 @@ You will have to pass the URL as a payload when triggering the task. ### Task ```ts trigger/lightpanda-lightpanda-fetch.ts -import { logger, task } from '@trigger.dev/sdk/v3' +import { logger, task } from '@trigger.dev/sdk' import { execSync } from 'node:child_process' export const lightpandaFetch = task({ @@ -154,7 +157,7 @@ This task initialises a Lightpanda CDP server to allow you to scrape directly vi Your task will have to launch a child process in order to have the websocket available to scrape using Puppeteer. ```ts trigger/lightpandaCDP.ts -import { logger, task } from '@trigger.dev/sdk/v3' +import { logger, task } from '@trigger.dev/sdk' import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process' import puppeteer from 'puppeteer' @@ -171,7 +174,7 @@ const spawnLightpanda = async (log: typeof logger) => ]) child.on('spawn', async () => { - log.info("Running Lightpanda's CDP server…", { + logger.info("Running Lightpanda's CDP server…", { pid: child.pid, }) From 7011482b5d513fd16b42a38a8a9a4e3dc33c55fe Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Thu, 3 Jul 2025 21:56:45 +0100 Subject: [PATCH 17/23] extension tweaks --- packages/build/src/extensions/lightpanda.ts | 24 +++++++++------------ 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/packages/build/src/extensions/lightpanda.ts b/packages/build/src/extensions/lightpanda.ts index c47ddfdd85..a70ee6270b 100644 --- a/packages/build/src/extensions/lightpanda.ts +++ b/packages/build/src/extensions/lightpanda.ts @@ -1,7 +1,5 @@ import type { BuildExtension } from "@trigger.dev/core/v3/build"; -const NAME = "LightpandaExtension"; - type LightpandaOpts = { arch?: "aarch64" | "x86_64"; version?: "nightly"; @@ -13,31 +11,28 @@ export const lightpanda = ({ version = "nightly", disableTelemetry = false, }: LightpandaOpts = {}): BuildExtension => ({ - name: NAME, + name: "lightpanda", onBuildComplete: async (context) => { - context.logger.progress(`Running ${NAME} on ${context.target} env for arch ${arch}`); - context.logger.progress(`version: ${version}`); - if (context.target === "dev") { return; } - const instructions: string[] = []; + const arch = context.targetPlatform === "linux/arm64" ? "aarch64" : "x86_64"; - if (disableTelemetry) { - instructions.push("RUN export LIGHTPANDA_DISABLE_TELEMETRY=true"); - } + context.logger.debug(`Adding lightpanda`, { arch, version, disableTelemetry }); + + const instructions: string[] = []; - /* Update / install required packages */ + // Install required packages instructions.push( `RUN apt-get update && apt-get install --no-install-recommends -y \ - curl \ - ca-certificates \ + curl \ + ca-certificates \ && update-ca-certificates \ && apt-get clean && rm -rf /var/lib/apt/lists/*` ); - /* Install Lightpanda */ + // Install Lightpanda instructions.push( `RUN curl -L -f --retry 3 -o lightpanda https://github.com/lightpanda-io/browser/releases/download/${version}/lightpanda-${arch}-linux || (echo "Failed to download Lightpanda binary" && exit 1) \ && chmod a+x ./lightpanda \ @@ -53,6 +48,7 @@ export const lightpanda = ({ deploy: { env: { LIGHTPANDA_BROWSER_PATH: "/usr/bin/lightpanda", + ...(disableTelemetry ? { LIGHTPANDA_DISABLE_TELEMETRY: "true" } : {}), }, override: true, }, From 54585cb059b1acf7adafa5105fec4803dae3ae08 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 28 Jul 2025 16:13:23 +0100 Subject: [PATCH 18/23] simplify extension --- packages/build/src/extensions/lightpanda.ts | 33 +++++---------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/packages/build/src/extensions/lightpanda.ts b/packages/build/src/extensions/lightpanda.ts index a70ee6270b..16c62a08b4 100644 --- a/packages/build/src/extensions/lightpanda.ts +++ b/packages/build/src/extensions/lightpanda.ts @@ -1,14 +1,12 @@ import type { BuildExtension } from "@trigger.dev/core/v3/build"; type LightpandaOpts = { - arch?: "aarch64" | "x86_64"; - version?: "nightly"; + version?: "nightly" | "latest"; disableTelemetry?: boolean; }; export const lightpanda = ({ - arch = "x86_64", - version = "nightly", + version = "latest", disableTelemetry = false, }: LightpandaOpts = {}): BuildExtension => ({ name: "lightpanda", @@ -17,28 +15,12 @@ export const lightpanda = ({ return; } - const arch = context.targetPlatform === "linux/arm64" ? "aarch64" : "x86_64"; + context.logger.debug(`Adding lightpanda`, { version, disableTelemetry }); - context.logger.debug(`Adding lightpanda`, { arch, version, disableTelemetry }); - - const instructions: string[] = []; - - // Install required packages - instructions.push( - `RUN apt-get update && apt-get install --no-install-recommends -y \ - curl \ - ca-certificates \ - && update-ca-certificates \ - && apt-get clean && rm -rf /var/lib/apt/lists/*` - ); - - // Install Lightpanda - instructions.push( - `RUN curl -L -f --retry 3 -o lightpanda https://github.com/lightpanda-io/browser/releases/download/${version}/lightpanda-${arch}-linux || (echo "Failed to download Lightpanda binary" && exit 1) \ - && chmod a+x ./lightpanda \ - && mv ./lightpanda /usr/bin/lightpanda \ - && /usr/bin/lightpanda version || (echo "Downloaded binary is not functional" && exit 1)` - ); + const instructions = [ + `COPY --from=lightpanda/browser:${version} /usr/bin/lightpanda /usr/local/bin/lightpanda`, + `RUN /usr/local/bin/lightpanda version || (echo "lightpanda binary is not functional" && exit 1)`, + ] satisfies string[]; context.addLayer({ id: "lightpanda", @@ -47,7 +29,6 @@ export const lightpanda = ({ }, deploy: { env: { - LIGHTPANDA_BROWSER_PATH: "/usr/bin/lightpanda", ...(disableTelemetry ? { LIGHTPANDA_DISABLE_TELEMETRY: "true" } : {}), }, override: true, From 067f38aa020e537eef082bad78895d2abe5e9dd2 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 28 Jul 2025 16:55:15 +0100 Subject: [PATCH 19/23] update examples --- docs/guides/examples/lightpanda.mdx | 206 ++++++++---------- pnpm-lock.yaml | 156 ++++++++++++- references/hello-world/package.json | 1 + .../hello-world/src/trigger/lightpanda.ts | 149 +++++++++++++ references/hello-world/trigger.config.ts | 2 + 5 files changed, 399 insertions(+), 115 deletions(-) create mode 100644 references/hello-world/src/trigger/lightpanda.ts diff --git a/docs/guides/examples/lightpanda.mdx b/docs/guides/examples/lightpanda.mdx index 12d47b5b1d..1d83c66818 100644 --- a/docs/guides/examples/lightpanda.mdx +++ b/docs/guides/examples/lightpanda.mdx @@ -34,58 +34,57 @@ LIGHTPANDA_TOKEN: "", ``` ```ts trigger/lightpanda-cloud-puppeteer.ts -import { logger, task } from '@trigger.dev/sdk' -import puppeteer from 'puppeteer' +import { logger, task } from "@trigger.dev/sdk"; +import puppeteer from "puppeteer-core"; export const lightpandaCloudPuppeteer = task({ - id: 'lightpanda-cloud-puppeteer', + id: "lightpanda-cloud-puppeteer", machine: { - preset: 'micro', + preset: "micro", }, run: async (payload: { url: string }, { ctx }) => { - logger.log("Lets get a page's links with Lightpanda!", { payload, ctx }) + logger.log("Lets get a page's links with Lightpanda!", { payload, ctx }); + if (!payload.url) { - logger.warn('Please define the payload url') - throw new Error('payload.url is undefined') + logger.warn("Please define the payload url"); + throw new Error("payload.url is undefined"); } - if (typeof process.env.LIGHTPANDA_TOKEN === 'undefined') { - logger.warn('Please define the env variable $LIGHTPANDA_TOKEN', { - env: process.env, - }) - throw new Error('$LIGHTPANDA_TOKEN is undefined') + const token = process.env.LIGHTPANDA_TOKEN; + if (!token) { + logger.warn("Please define the env variable LIGHTPANDA_TOKEN"); + throw new Error("LIGHTPANDA_TOKEN is undefined"); } // Connect to Lightpanda's cloud const browser = await puppeteer.connect({ - browserWSEndpoint: `wss://cloud.lightpanda.io/ws?browser=lightpanda&token=${process.env.LIGHTPANDA_TOKEN}`, - }) - const context = await browser.createBrowserContext() - const page = await context.newPage() + browserWSEndpoint: `wss://cloud.lightpanda.io/ws?browser=lightpanda&token=${token}`, + }); + const context = await browser.createBrowserContext(); + const page = await context.newPage(); // Dump all the links from the page. - await page.goto(payload.url) + await page.goto(payload.url); const links = await page.evaluate(() => { - return Array.from(document.querySelectorAll('a')).map(row => { - return row.getAttribute('href') - }) - }) + return Array.from(document.querySelectorAll("a")).map((row) => { + return row.getAttribute("href"); + }); + }); - logger.info('Processing done') - logger.info('Shutting down…') + logger.info("Processing done, shutting down…"); - await page.close() - await context.close() - await browser.disconnect() + await page.close(); + await context.close(); + await browser.disconnect(); - logger.info('✅ Completed') + logger.info("✅ Completed"); return { links, - } + }; }, -}) +}); ``` ### Proxies @@ -111,39 +110,32 @@ You will have to pass the URL as a payload when triggering the task. - Setup the [Lightpanda build extension](/config/extensions/lightpanda) ### Task -```ts trigger/lightpanda-lightpanda-fetch.ts -import { logger, task } from '@trigger.dev/sdk' -import { execSync } from 'node:child_process' +```ts trigger/lightpanda-fetch.ts +import { logger, task } from "@trigger.dev/sdk"; +import { execSync } from "node:child_process"; export const lightpandaFetch = task({ - id: 'lightpanda-fetch', + id: "lightpanda-fetch", machine: { preset: "micro", }, run: async (payload: { url: string }, { ctx }) => { - logger.log("Lets get a page's content with Lightpanda!", { payload, ctx }) + logger.log("Lets get a page's content with Lightpanda!", { payload, ctx }); if (!payload.url) { - logger.warn('Please define the payload url') - throw new Error('payload.url is undefined') + logger.warn("Please define the payload url"); + throw new Error("payload.url is undefined"); } - if (typeof process.env.LIGHTPANDA_BROWSER_PATH === 'undefined') { - logger.warn('Please define the env variable $LIGHTPANDA_BROWSER_PATH', { - env: process.env, - }) - throw new Error('$LIGHTPANDA_BROWSER_PATH is undefined') - } + const buffer = execSync(`lightpanda fetch --dump ${payload.url}`); - const e = execSync(`${process.env.LIGHTPANDA_BROWSER_PATH} fetch --dump ${payload.url}`) - - logger.info('✅ Completed') + logger.info("✅ Completed"); return { - message: e.toString(), - } + message: buffer.toString(), + }; }, -}) +}); ``` ## Example \#3 - Launch and use a Lightpanda CDP server @@ -156,92 +148,82 @@ This task initialises a Lightpanda CDP server to allow you to scrape directly vi ### Task Your task will have to launch a child process in order to have the websocket available to scrape using Puppeteer. -```ts trigger/lightpandaCDP.ts -import { logger, task } from '@trigger.dev/sdk' -import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process' -import puppeteer from 'puppeteer' +```ts trigger/lightpanda-cdp.ts +import { logger, task } from "@trigger.dev/sdk"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import puppeteer from "puppeteer-core"; -const spawnLightpanda = async (log: typeof logger) => +const spawnLightpanda = async (host: string, port: string) => new Promise((resolve, reject) => { - const child = spawn(process.env.LIGHTPANDA_BROWSER_PATH as string, [ - 'serve', - '--host', - '127.0.0.1', - '--port', - '9222', - '--log_level', - 'info', - ]) - - child.on('spawn', async () => { + const child = spawn("lightpanda", [ + "serve", + "--host", + host, + "--port", + port, + "--log_level", + "info", + ]); + + child.on("spawn", async () => { logger.info("Running Lightpanda's CDP server…", { pid: child.pid, - }) + }); - await new Promise(resolve => setTimeout(resolve, 250)) - resolve(child) - }) - child.on('error', e => reject(e)) - }) + await new Promise((resolve) => setTimeout(resolve, 250)); + resolve(child); + }); + child.on("error", (e) => reject(e)); + }); export const lightpandaCDP = task({ - id: 'lightpanda-cdp', + id: "lightpanda-cdp", machine: { - preset: 'micro', + preset: "micro", }, run: async (payload: { url: string }, { ctx }) => { - logger.log("Lets get a page's links with Lightpanda!", { payload, ctx }) + logger.log("Lets get a page's links with Lightpanda!", { payload, ctx }); if (!payload.url) { - logger.warn('Please define the payload url') - throw new Error('payload.url is undefined') + logger.warn("Please define the payload url"); + throw new Error("payload.url is undefined"); } - if (typeof process.env.LIGHTPANDA_BROWSER_PATH === 'undefined') { - logger.warn('Please define the env variable $LIGHTPANDA_BROWSER_PATH', { - env: process.env, - }) - throw new Error('$LIGHTPANDA_BROWSER_PATH is undefined') - } + const host = process.env.LIGHTPANDA_CDP_HOST ?? "127.0.0.1"; + const port = process.env.LIGHTPANDA_CDP_PORT ?? "9222"; - try { - // Launch Lightpanda's CDP server - const lpProcess = await spawnLightpanda(logger) + // Launch Lightpanda's CDP server + const lpProcess = await spawnLightpanda(host, port); - const browser = await puppeteer.connect({ - browserWSEndpoint: 'ws://127.0.0.1:9222', - }) - const context = await browser.createBrowserContext() - const page = await context.newPage() + const browser = await puppeteer.connect({ + browserWSEndpoint: `ws://${host}:${port}`, + }); + const context = await browser.createBrowserContext(); + const page = await context.newPage(); - // Dump all the links from the page. - await page.goto(payload.url) + // Dump all the links from the page. + await page.goto(payload.url); - const links = await page.evaluate(() => { - return Array.from(document.querySelectorAll('a')).map(row => { - return row.getAttribute('href') - }) - }) + const links = await page.evaluate(() => { + return Array.from(document.querySelectorAll("a")).map((row) => { + return row.getAttribute("href"); + }); + }); - logger.info('Processing done') - logger.info('Shutting down…') + logger.info("Processing done"); + logger.info("Shutting down…"); - // Close Puppeteer instance - await browser.close() + // Close Puppeteer instance + await browser.close(); - // Stop Lightpanda's CDP Server - lpProcess.stdout.destroy() - lpProcess.stderr.destroy() - lpProcess.kill() + // Stop Lightpanda's CDP Server + lpProcess.kill(); - logger.info('✅ Completed') + logger.info("✅ Completed"); - return { - links, - } - } catch (e: any) { - throw new Error(e) - } + return { + links, + }; }, -}) +}); ``` diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cecbf40d53..37ec88c236 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2105,6 +2105,9 @@ importers: openai: specifier: ^4.97.0 version: 4.97.0(ws@8.12.0)(zod@3.23.8) + puppeteer-core: + specifier: ^24.15.0 + version: 24.15.0 replicate: specifier: ^1.0.1 version: 1.0.1 @@ -11401,6 +11404,23 @@ packages: /@protobufjs/utf8@1.1.0: resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + /@puppeteer/browsers@2.10.6: + resolution: {integrity: sha512-pHUn6ZRt39bP3698HFQlu2ZHCkS/lPcpv7fVQcGBSzNNygw171UXAKrCUhy+TEMw4lEttOKDgNpb04hwUAJeiQ==} + engines: {node: '>=18'} + hasBin: true + dependencies: + debug: 4.4.1 + extract-zip: 2.0.1 + progress: 2.0.3 + proxy-agent: 6.5.0 + semver: 7.7.2 + tar-fs: 3.1.0 + yargs: 17.7.2 + transitivePeerDependencies: + - bare-buffer + - supports-color + dev: false + /@puppeteer/browsers@2.4.0: resolution: {integrity: sha512-x8J1csfIygOwf6D6qUAZ0ASk3z63zPb7wkNeHRerCMh82qWKUrOgkuP005AJC8lDL6/evtXETGEJVcwykKT4/g==} engines: {node: '>=18'} @@ -21250,6 +21270,11 @@ packages: transitivePeerDependencies: - supports-color + /agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + dev: false + /agentkeepalive@4.5.0: resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==} engines: {node: '>= 8.0.0'} @@ -22718,6 +22743,16 @@ packages: zod: 3.23.8 dev: false + /chromium-bidi@7.2.0(devtools-protocol@0.0.1464554): + resolution: {integrity: sha512-gREyhyBstermK+0RbcJLbFhcQctg92AGgDe/h/taMJEOLRdtSswBAO9KmvltFSQWgM2LrwWu5SIuEUbdm3JsyQ==} + peerDependencies: + devtools-protocol: '*' + dependencies: + devtools-protocol: 0.0.1464554 + mitt: 3.0.1 + zod: 3.25.76 + dev: false + /ci-info@3.8.0: resolution: {integrity: sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==} engines: {node: '>=8'} @@ -23650,6 +23685,18 @@ packages: ms: 2.1.3 supports-color: 10.0.0 + /debug@4.4.1: + resolution: {integrity: sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.3 + dev: false + /decamelize-keys@1.1.1: resolution: {integrity: sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==} engines: {node: '>=0.10.0'} @@ -23866,6 +23913,10 @@ packages: resolution: {integrity: sha512-75fMas7PkYNDTmDyb6PRJCH7ILmHLp+BhrZGeMsa4bCh40DTxgCz2NRy5UDzII4C5KuD0oBMZ9vXKhEl6UD/3w==} dev: false + /devtools-protocol@0.0.1464554: + resolution: {integrity: sha512-CAoP3lYfwAGQTaAXYvA6JZR0fjGUb7qec1qf4mToyoH2TZgUFeIqYcjh6f9jNuhHfuZiEdH+PONHYrLhRQX6aw==} + dev: false + /dezalgo@1.0.4: resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==} dependencies: @@ -25658,7 +25709,7 @@ packages: engines: {node: '>= 10.17.0'} hasBin: true dependencies: - debug: 4.4.0(supports-color@10.0.0) + debug: 4.4.1 get-stream: 5.2.0 yauzl: 2.10.0 optionalDependencies: @@ -26958,8 +27009,8 @@ packages: resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} engines: {node: '>= 14'} dependencies: - agent-base: 7.1.1 - debug: 4.4.0(supports-color@10.0.0) + agent-base: 7.1.4 + debug: 4.4.1 transitivePeerDependencies: - supports-color dev: false @@ -27002,6 +27053,16 @@ packages: - supports-color dev: false + /https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + dependencies: + agent-base: 7.1.4 + debug: 4.4.1 + transitivePeerDependencies: + - supports-color + dev: false + /human-id@1.0.2: resolution: {integrity: sha512-UNopramDEhHJD+VR+ehk8rOslwSfByxPIZyJRfV739NDhN5LF1fa1MqnzKm2lGTQRjNrjK19Q5fhkgIfjlVUKw==} dev: false @@ -31065,6 +31126,22 @@ packages: - supports-color dev: false + /pac-proxy-agent@7.2.0: + resolution: {integrity: sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==} + engines: {node: '>= 14'} + dependencies: + '@tootallnate/quickjs-emscripten': 0.23.0 + agent-base: 7.1.4 + debug: 4.4.1 + get-uri: 6.0.1 + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + pac-resolver: 7.0.1 + socks-proxy-agent: 8.0.5 + transitivePeerDependencies: + - supports-color + dev: false + /pac-resolver@7.0.1: resolution: {integrity: sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==} engines: {node: '>= 14'} @@ -32273,6 +32350,22 @@ packages: - supports-color dev: false + /proxy-agent@6.5.0: + resolution: {integrity: sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==} + engines: {node: '>= 14'} + dependencies: + agent-base: 7.1.4 + debug: 4.4.1 + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + lru-cache: 7.18.3 + pac-proxy-agent: 7.2.0 + proxy-from-env: 1.1.0 + socks-proxy-agent: 8.0.5 + transitivePeerDependencies: + - supports-color + dev: false + /proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} @@ -32335,6 +32428,23 @@ packages: - utf-8-validate dev: false + /puppeteer-core@24.15.0: + resolution: {integrity: sha512-2iy0iBeWbNyhgiCGd/wvGrDSo73emNFjSxYOcyAqYiagkYt5q4cPfVXaVDKBsukgc2fIIfLAalBZlaxldxdDYg==} + engines: {node: '>=18'} + dependencies: + '@puppeteer/browsers': 2.10.6 + chromium-bidi: 7.2.0(devtools-protocol@0.0.1464554) + debug: 4.4.1 + devtools-protocol: 0.0.1464554 + typed-query-selector: 2.12.0 + ws: 8.18.3 + transitivePeerDependencies: + - bare-buffer + - bufferutil + - supports-color + - utf-8-validate + dev: false + /puppeteer@23.4.0(typescript@5.5.4): resolution: {integrity: sha512-FxgFFJI7NAsX8uebiEDSjS86vufz9TaqERQHShQT0lCbSRI3jUPEcz/0HdwLiYvfYNsc1zGjqY3NsGZya4PvUA==} engines: {node: '>=18'} @@ -34474,6 +34584,17 @@ packages: - supports-color dev: false + /socks-proxy-agent@8.0.5: + resolution: {integrity: sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==} + engines: {node: '>= 14'} + dependencies: + agent-base: 7.1.4 + debug: 4.4.1 + socks: 2.8.3 + transitivePeerDependencies: + - supports-color + dev: false + /socks@2.8.3: resolution: {integrity: sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==} engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} @@ -35419,6 +35540,18 @@ packages: transitivePeerDependencies: - bare-buffer + /tar-fs@3.1.0: + resolution: {integrity: sha512-5Mty5y/sOF1YWj1J6GiBodjlDc05CUR8PKXrsnFAiSG0xA+GHeWLovaZPYUDXkH/1iKRf2+M5+OrRgzC7O9b7w==} + dependencies: + pump: 3.0.2 + tar-stream: 3.1.7 + optionalDependencies: + bare-fs: 4.1.5 + bare-path: 3.0.0 + transitivePeerDependencies: + - bare-buffer + dev: false + /tar-stream@2.2.0: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} engines: {node: '>=6'} @@ -37796,6 +37929,19 @@ packages: dependencies: bufferutil: 4.0.9 + /ws@8.18.3: + resolution: {integrity: sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + dev: false + /xdg-app-paths@8.3.0: resolution: {integrity: sha512-mgxlWVZw0TNWHoGmXq+NC3uhCIc55dDpAlDkMQUaIAcQzysb0kxctwv//fvuW61/nAAeUBJMQ8mnZjMmuYwOcQ==} engines: {node: '>= 4.0'} @@ -38030,6 +38176,10 @@ packages: /zod@3.23.8: resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==} + /zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + dev: false + /zustand@4.5.5(@types/react@18.2.69)(react@18.2.0): resolution: {integrity: sha512-+0PALYNJNgK6hldkgDq2vLrw5f6g/jCInz52n9RTpropGgeAf/ioFUCdtsjCqu4gNhW9D01rUQBROoRjdzyn2Q==} engines: {node: '>=12.7.0'} diff --git a/references/hello-world/package.json b/references/hello-world/package.json index b6a8d799f4..89dbeea911 100644 --- a/references/hello-world/package.json +++ b/references/hello-world/package.json @@ -9,6 +9,7 @@ "@trigger.dev/build": "workspace:*", "@trigger.dev/sdk": "workspace:*", "openai": "^4.97.0", + "puppeteer-core": "^24.15.0", "replicate": "^1.0.1", "zod": "3.23.8" }, diff --git a/references/hello-world/src/trigger/lightpanda.ts b/references/hello-world/src/trigger/lightpanda.ts new file mode 100644 index 0000000000..830f6391a2 --- /dev/null +++ b/references/hello-world/src/trigger/lightpanda.ts @@ -0,0 +1,149 @@ +import { logger, task } from "@trigger.dev/sdk"; +import { execSync, spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import puppeteer from "puppeteer-core"; + +const spawnLightpanda = async (host: string, port: string) => + new Promise((resolve, reject) => { + const child = spawn("lightpanda", [ + "serve", + "--host", + host, + "--port", + port, + "--log_level", + "info", + ]); + + child.on("spawn", async () => { + logger.info("Running Lightpanda's CDP server…", { + pid: child.pid, + }); + + await new Promise((resolve) => setTimeout(resolve, 250)); + resolve(child); + }); + child.on("error", (e) => reject(e)); + }); + +export const lightpandaCdp = task({ + id: "lightpanda-cdp", + machine: { + preset: "micro", + }, + run: async (payload: { url: string }, { ctx }) => { + logger.log("Lets get a page's links with Lightpanda!", { payload, ctx }); + + if (!payload.url) { + logger.warn("Please define the payload url"); + throw new Error("payload.url is undefined"); + } + + const host = process.env.LIGHTPANDA_CDP_HOST ?? "127.0.0.1"; + const port = process.env.LIGHTPANDA_CDP_PORT ?? "9222"; + + // Launch Lightpanda's CDP server + const lpProcess = await spawnLightpanda(host, port); + + const browser = await puppeteer.connect({ + browserWSEndpoint: `ws://${host}:${port}`, + }); + const context = await browser.createBrowserContext(); + const page = await context.newPage(); + + // Dump all the links from the page. + await page.goto(payload.url); + + const links = await page.evaluate(() => { + return Array.from(document.querySelectorAll("a")).map((row) => { + return row.getAttribute("href"); + }); + }); + + logger.info("Processing done"); + logger.info("Shutting down…"); + + // Close Puppeteer instance + await browser.close(); + + // Stop Lightpanda's CDP Server + lpProcess.kill(); + + logger.info("✅ Completed"); + + return { + links, + }; + }, +}); + +export const lightpandaFetch = task({ + id: "lightpanda-fetch", + machine: { + preset: "micro", + }, + run: async (payload: { url: string }, { ctx }) => { + logger.log("Lets get a page's content with Lightpanda!", { payload, ctx }); + + if (!payload.url) { + logger.warn("Please define the payload url"); + throw new Error("payload.url is undefined"); + } + + const buffer = execSync(`lightpanda fetch --dump ${payload.url}`); + + logger.info("✅ Completed"); + + return { + message: buffer.toString(), + }; + }, +}); + +export const lightpandaCloudPuppeteer = task({ + id: "lightpanda-cloud-puppeteer", + machine: { + preset: "micro", + }, + run: async (payload: { url: string }, { ctx }) => { + logger.log("Lets get a page's links with Lightpanda!", { payload, ctx }); + + if (!payload.url) { + logger.warn("Please define the payload url"); + throw new Error("payload.url is undefined"); + } + + const token = process.env.LIGHTPANDA_TOKEN; + if (!token) { + logger.warn("Please define the env variable LIGHTPANDA_TOKEN"); + throw new Error("LIGHTPANDA_TOKEN is undefined"); + } + + // Connect to Lightpanda's cloud + const browser = await puppeteer.connect({ + browserWSEndpoint: `wss://cloud.lightpanda.io/ws?browser=lightpanda&token=${token}`, + }); + const context = await browser.createBrowserContext(); + const page = await context.newPage(); + + // Dump all the links from the page. + await page.goto(payload.url); + + const links = await page.evaluate(() => { + return Array.from(document.querySelectorAll("a")).map((row) => { + return row.getAttribute("href"); + }); + }); + + logger.info("Processing done, shutting down…"); + + await page.close(); + await context.close(); + await browser.disconnect(); + + logger.info("✅ Completed"); + + return { + links, + }; + }, +}); diff --git a/references/hello-world/trigger.config.ts b/references/hello-world/trigger.config.ts index 33935a4a13..2b4a68912f 100644 --- a/references/hello-world/trigger.config.ts +++ b/references/hello-world/trigger.config.ts @@ -1,5 +1,6 @@ import { defineConfig } from "@trigger.dev/sdk/v3"; import { syncEnvVars } from "@trigger.dev/build/extensions/core"; +import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; export default defineConfig({ compatibilityFlags: ["run_engine_v2"], @@ -23,6 +24,7 @@ export default defineConfig({ machine: "small-2x", build: { extensions: [ + lightpanda(), syncEnvVars(async (ctx) => { console.log("syncEnvVars", { environment: ctx.environment, branch: ctx.branch }); return [ From 3464e5e4617f7d3b008f6d168d96922f602940d2 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 28 Jul 2025 17:24:10 +0100 Subject: [PATCH 20/23] update docs --- docs/config/extensions/lightpanda.mdx | 25 +++++------ docs/guides/examples/lightpanda.mdx | 60 +++++++++++++-------------- 2 files changed, 39 insertions(+), 46 deletions(-) diff --git a/docs/config/extensions/lightpanda.mdx b/docs/config/extensions/lightpanda.mdx index 722f610b25..0408d45ad5 100644 --- a/docs/config/extensions/lightpanda.mdx +++ b/docs/config/extensions/lightpanda.mdx @@ -17,16 +17,15 @@ import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; export default defineConfig({ project: "", - // Your other config settings... build: { extensions: [lightpanda()], }, }); ``` -### Options +## Options -- `version`: The version of the browser to install. Default: `"nightly"`. +- `version`: The version of the browser to install. Default: `"latest"`. - `disableTelemetry`: Whether to disable telemetry. Default: `false`. For example: @@ -48,18 +47,14 @@ export default defineConfig({ }); ``` -### Environment variables +## Development -The extension sets the following environment variables during the build: +When running in dev, you will first have to download the Lightpanda browser binary and make sure it's in your `PATH`. See [Lightpanda's installation guide](https://lightpanda.io/docs/getting-started/installation). -- `LIGHTPANDA_BROWSER_PATH`: Set to `/usr/bin/lightpanda` so the browser can be found at runtime +## Next steps -#### Development - -When running in dev, you will first have to download the Lightpanda browser binary. See [Lightpanda's installation guide](https://lightpanda.io/docs/getting-started/installation). - -Then, set the `LIGHTPANDA_BROWSER_PATH` environment variable to the path of the Lightpanda browser binary. - -```bash -export LIGHTPANDA_BROWSER_PATH="/usr/local/bin/lightpanda" -``` + + + Learn how to use Lightpanda in your project. + + diff --git a/docs/guides/examples/lightpanda.mdx b/docs/guides/examples/lightpanda.mdx index 1d83c66818..9eab531176 100644 --- a/docs/guides/examples/lightpanda.mdx +++ b/docs/guides/examples/lightpanda.mdx @@ -1,7 +1,7 @@ --- -title: "Get a webpage's content using Lightpanda browser" +title: "Lightpanda" sidebarTitle: "Lightpanda" -description: "In these examples, we will show you how to crawl using Lightpanda browser and Trigger.dev." +description: "These examples demonstrate how to use Lightpanda with Trigger.dev." tag: "v4" --- @@ -14,23 +14,26 @@ import UpgradeToV4Note from "/snippets/upgrade-to-v4-note.mdx"; Lightpanda is a purpose-built browser for AI and automation workflows. It is 10x faster, uses 10x less RAM than Chrome headless. -You will find here are a couple of examples of how to use Lightpanda with Trigger.dev. +Here are a few examples of how to use Lightpanda with Trigger.dev. -## Prerequisites +## Limitations -- A project with [Trigger.dev initialized](/quick-start) -- A [Lightpanda](https://lightpanda.io/) cloud token (for the 1st example) +- Lightpanda does not support the `puppeteer` screenshot feature. -## Example \#1 - Get links from a website using Lightpanda cloud & Puppeteer +## Using Lightpanda Cloud -In this task, we use Lightpanda browser to get links from a provided URL. -You will have to pass the URL as a payload when triggering the task. +### Prerequisites + +- A [Lightpanda](https://lightpanda.io/) cloud token + +### Get links from a website +In this task we use Lightpanda browser to get links from a provided URL. You will have to pass the URL as a payload when triggering the task. -Make sure to add `$LIGHTPANDA_TOKEN` to your Trigger.dev dashboard on the Environment Variables page: +Make sure to add `LIGHTPANDA_TOKEN` to your Trigger.dev dashboard on the Environment Variables page: ```bash -LIGHTPANDA_TOKEN: "", +LIGHTPANDA_TOKEN="" ``` ```ts trigger/lightpanda-cloud-puppeteer.ts @@ -86,30 +89,31 @@ export const lightpandaCloudPuppeteer = task({ }, }); ``` + ### Proxies Proxies can be used with your browser via the proxy query string parameter. By default, the proxy used is "datacenter" which is a pool of shared datacenter IPs. -`datacenter` accepts an optional `country` query string parameter, an [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) country code. - -_Example using a German IP :_ - -```wss://cloud.lightpanda.io/ws?proxy=datacenter&country=de&token=TOKEN``` +`datacenter` accepts an optional `country` query string parameter which is an [ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) country code. +```bash +# This example will use a German IP +wss://cloud.lightpanda.io/ws?proxy=datacenter&country=de&token=${token} +``` ### Session -A session is alive until you close it or the connection is closed. The max time duration of a session is 15 min. - -## Example \#2 - Get a webpage using Lightpanda - -Using the Lightpanda binary we will dump the HTML for a provided URL. -You will have to pass the URL as a payload when triggering the task. +A session is alive until you close it or the connection is closed. The max duration of a session is 15 minutes. +## Using Lightpanda browser directly ### Prerequisites + - Setup the [Lightpanda build extension](/config/extensions/lightpanda) -### Task +### Get the HTML of a webpage + +This task will dump the HTML of a provided URL using the Lightpanda browser binary. You will have to pass the URL as a payload when triggering the task. + ```ts trigger/lightpanda-fetch.ts import { logger, task } from "@trigger.dev/sdk"; import { execSync } from "node:child_process"; @@ -138,15 +142,9 @@ export const lightpandaFetch = task({ }); ``` -## Example \#3 - Launch and use a Lightpanda CDP server - -This task initialises a Lightpanda CDP server to allow you to scrape directly via Trigger.dev. - -### Prerequisites -- Setup the [Lightpanda build extension](/config/extensions/lightpanda) +### Lightpanda CDP with Puppeteer -### Task -Your task will have to launch a child process in order to have the websocket available to scrape using Puppeteer. +This task initializes a Lightpanda CDP server and uses it with `puppeteer-core` to scrape a provided URL. ```ts trigger/lightpanda-cdp.ts import { logger, task } from "@trigger.dev/sdk"; From b11d9b74875e6b0b0a8e4fda94f4acad46ef991e Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 28 Jul 2025 17:33:44 +0100 Subject: [PATCH 21/23] remove from extensions list as v4 only --- docs/introduction.mdx | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/docs/introduction.mdx b/docs/introduction.mdx index 77aa189a97..1ab5baaf7a 100644 --- a/docs/introduction.mdx +++ b/docs/introduction.mdx @@ -88,20 +88,19 @@ We provide everything you need to build and manage background tasks: a CLI and S ## Explore by build extension -| Extension | What it does | Docs | -|:----------|:------------|:--------------| -| prismaExtension | Use Prisma with Trigger.dev | [Learn more](/config/extensions/prismaExtension) | -| pythonExtension | Execute Python scripts in Trigger.dev | [Learn more](/config/extensions/pythonExtension) | -| puppeteer | Use Puppeteer with Trigger.dev | [Learn more](/config/extensions/puppeteer) | -| lightpanda | Use Lightpanda Browser with Trigger.dev | [Learn more](/config/extensions/lightpanda) | -| ffmpeg | Use FFmpeg with Trigger.dev | [Learn more](/config/extensions/ffmpeg) | -| aptGet | Install system packages with aptGet | [Learn more](/config/extensions/aptGet) | -| additionalFiles | Copy additional files to the build directory | [Learn more](/config/extensions/additionalFiles) | -| additionalPackages | Include additional packages in the build | [Learn more](/config/extensions/additionalPackages) | -| syncEnvVars | Automatically sync environment variables to Trigger.dev | [Learn more](/config/extensions/syncEnvVars) | -| esbuildPlugin | Add existing or custom esbuild plugins to your build process | [Learn more](/config/extensions/esbuildPlugin) | -| emitDecoratorMetadata | Support for the emitDecoratorMetadata TypeScript compiler | [Learn more](/config/extensions/emitDecoratorMetadata) | -| audioWaveform | Support for Audio Waveform in your project | [Learn more](/config/extensions/audioWaveform) | +| Extension | What it does | Docs | +| :-------------------- | :----------------------------------------------------------- | :----------------------------------------------------- | +| prismaExtension | Use Prisma with Trigger.dev | [Learn more](/config/extensions/prismaExtension) | +| pythonExtension | Execute Python scripts in Trigger.dev | [Learn more](/config/extensions/pythonExtension) | +| puppeteer | Use Puppeteer with Trigger.dev | [Learn more](/config/extensions/puppeteer) | +| ffmpeg | Use FFmpeg with Trigger.dev | [Learn more](/config/extensions/ffmpeg) | +| aptGet | Install system packages with aptGet | [Learn more](/config/extensions/aptGet) | +| additionalFiles | Copy additional files to the build directory | [Learn more](/config/extensions/additionalFiles) | +| additionalPackages | Include additional packages in the build | [Learn more](/config/extensions/additionalPackages) | +| syncEnvVars | Automatically sync environment variables to Trigger.dev | [Learn more](/config/extensions/syncEnvVars) | +| esbuildPlugin | Add existing or custom esbuild plugins to your build process | [Learn more](/config/extensions/esbuildPlugin) | +| emitDecoratorMetadata | Support for the emitDecoratorMetadata TypeScript compiler | [Learn more](/config/extensions/emitDecoratorMetadata) | +| audioWaveform | Support for Audio Waveform in your project | [Learn more](/config/extensions/audioWaveform) | ## Getting help From f18166414f36dc9a78e5558e0c24deccacdcfe88 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 28 Jul 2025 17:39:43 +0100 Subject: [PATCH 22/23] remove from catalog --- .../v3-catalog/src/trigger/lightpandaTask.ts | 23 ------------------- references/v3-catalog/trigger.config.ts | 2 -- 2 files changed, 25 deletions(-) delete mode 100644 references/v3-catalog/src/trigger/lightpandaTask.ts diff --git a/references/v3-catalog/src/trigger/lightpandaTask.ts b/references/v3-catalog/src/trigger/lightpandaTask.ts deleted file mode 100644 index 3175989ad6..0000000000 --- a/references/v3-catalog/src/trigger/lightpandaTask.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { logger, task } from "@trigger.dev/sdk/v3"; -import { execSync } from "node:child_process"; - -export const lightpandaFetch = task({ - id: "lightpanda-fetch", - machine: { - preset: "micro", - }, - run: async (payload: { url: string }, { ctx }) => { - logger.log("Lets get a page's content with Lightpanda!", { payload, ctx }); - - if (!payload.url) { - logger.warn("Please define the payload url"); - throw new Error("payload.url is undefined"); - } - - const e = execSync(`${process.env.LIGHTPANDA_BROWSER_PATH} fetch --dump ${payload.url}`); - - return { - message: e.toString(), - }; - }, -}); diff --git a/references/v3-catalog/trigger.config.ts b/references/v3-catalog/trigger.config.ts index 055c75d6a4..62698d14fb 100644 --- a/references/v3-catalog/trigger.config.ts +++ b/references/v3-catalog/trigger.config.ts @@ -6,7 +6,6 @@ import { audioWaveform } from "@trigger.dev/build/extensions/audioWaveform"; import { additionalFiles, ffmpeg, syncEnvVars } from "@trigger.dev/build/extensions/core"; import { puppeteer } from "@trigger.dev/build/extensions/puppeteer"; import { playwright } from "@trigger.dev/build/extensions/playwright"; -import { lightpanda } from "@trigger.dev/build/extensions/lightpanda"; import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript"; import { defineConfig } from "@trigger.dev/sdk/v3"; @@ -84,7 +83,6 @@ export default defineConfig({ }), puppeteer(), playwright(), - lightpanda(), ], external: ["re2"], }, From 0a0c06a4cb05f923337943e3e98b1f8e0cda1bdd Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 28 Jul 2025 17:40:33 +0100 Subject: [PATCH 23/23] update changeset --- .changeset/rare-mails-fail.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/.changeset/rare-mails-fail.md b/.changeset/rare-mails-fail.md index b35df51656..ef4e9861b0 100644 --- a/.changeset/rare-mails-fail.md +++ b/.changeset/rare-mails-fail.md @@ -1,8 +1,5 @@ --- "@trigger.dev/build": patch -"trigger.dev": patch -"@trigger.dev/core": patch -"@trigger.dev/sdk": patch --- Add Lightpanda extension