From fa38ed97f8248641ad68c4f5ec6160841a0c6620 Mon Sep 17 00:00:00 2001 From: joncrangle <94425204+joncrangle@users.noreply.github.com> Date: Wed, 20 Dec 2023 00:03:04 -0500 Subject: [PATCH 01/12] Begin migrating README --- docs/developer/_category_.json | 4 + docs/developer/developer.md | 27 + docs/install/_category_.json | 4 + docs/install/binary.md | 11 + docs/install/docker.md | 18 + docs/install/environment-variables.md | 22 + docs/install/helm.md | 38 + docs/intro.md | 47 - .../_category_.json | 4 +- docs/modifiers/request-modifiers.md | 51 + docs/modifiers/response-modifiers.md | 32 + docs/overview/_category_.json | 4 + docs/overview/features.md | 27 + docs/overview/how-it-works.md | 15 + docs/overview/introduction.md | 15 + docs/tutorial-basics/_category_.json | 8 - docs/tutorial-basics/congratulations.md | 23 - docs/tutorial-basics/create-a-blog-post.md | 34 - docs/tutorial-basics/create-a-document.md | 57 - docs/tutorial-basics/create-a-page.md | 43 - docs/tutorial-basics/deploy-your-site.md | 31 - docs/tutorial-basics/markdown-features.mdx | 150 - .../img/docsVersionDropdown.png | Bin 25427 -> 0 bytes docs/tutorial-extras/img/localeDropdown.png | Bin 27841 -> 0 bytes docs/tutorial-extras/manage-docs-versions.md | 55 - docs/tutorial-extras/translate-your-site.md | 88 - docs/usage/_category_.json | 4 + docs/usage/browser.md | 18 + docs/usage/headless.md | 33 + docs/usage/playground.md | 26 + docs/usage/rulesets.md | 5 + docusaurus.config.ts | 87 +- package.json | 1 + pnpm-lock.yaml | 9725 +++++++++++++++++ src/components/HomepageFeatures/index.tsx | 36 +- static/img/docusaurus-social-card.jpg | Bin 55746 -> 0 bytes static/img/docusaurus.png | Bin 5142 -> 0 bytes static/img/logo.svg | 1 - static/img/undraw_docusaurus_mountain.svg | 171 - static/img/undraw_docusaurus_react.svg | 170 - static/img/undraw_docusaurus_tree.svg | 40 - 41 files changed, 10136 insertions(+), 989 deletions(-) create mode 100644 docs/developer/_category_.json create mode 100644 docs/developer/developer.md create mode 100644 docs/install/_category_.json create mode 100644 docs/install/binary.md create mode 100644 docs/install/docker.md create mode 100644 docs/install/environment-variables.md create mode 100644 docs/install/helm.md delete mode 100644 docs/intro.md rename docs/{tutorial-extras => modifiers}/_category_.json (50%) create mode 100644 docs/modifiers/request-modifiers.md create mode 100644 docs/modifiers/response-modifiers.md create mode 100644 docs/overview/_category_.json create mode 100644 docs/overview/features.md create mode 100644 docs/overview/how-it-works.md create mode 100644 docs/overview/introduction.md delete mode 100644 docs/tutorial-basics/_category_.json delete mode 100644 docs/tutorial-basics/congratulations.md delete mode 100644 docs/tutorial-basics/create-a-blog-post.md delete mode 100644 docs/tutorial-basics/create-a-document.md delete mode 100644 docs/tutorial-basics/create-a-page.md delete mode 100644 docs/tutorial-basics/deploy-your-site.md delete mode 100644 docs/tutorial-basics/markdown-features.mdx delete mode 100644 docs/tutorial-extras/img/docsVersionDropdown.png delete mode 100644 docs/tutorial-extras/img/localeDropdown.png delete mode 100644 docs/tutorial-extras/manage-docs-versions.md delete mode 100644 docs/tutorial-extras/translate-your-site.md create mode 100644 docs/usage/_category_.json create mode 100644 docs/usage/browser.md create mode 100644 docs/usage/headless.md create mode 100644 docs/usage/playground.md create mode 100644 docs/usage/rulesets.md create mode 100644 pnpm-lock.yaml delete mode 100644 static/img/docusaurus-social-card.jpg delete mode 100644 static/img/docusaurus.png delete mode 100644 static/img/logo.svg delete mode 100644 static/img/undraw_docusaurus_mountain.svg delete mode 100644 static/img/undraw_docusaurus_react.svg delete mode 100644 static/img/undraw_docusaurus_tree.svg diff --git a/docs/developer/_category_.json b/docs/developer/_category_.json new file mode 100644 index 0000000..502009d --- /dev/null +++ b/docs/developer/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Developer", + "position": 5 +} diff --git a/docs/developer/developer.md b/docs/developer/developer.md new file mode 100644 index 0000000..e2e9ee0 --- /dev/null +++ b/docs/developer/developer.md @@ -0,0 +1,27 @@ +--- +sidebar_position: 1 +--- + +# Developer + +To run a development server at http://localhost:8080: + +```bash +git clone git@github.com-ladddder:everywall/ladder.git +git submodule update --init --recursive +echo "dev " > handlers/VERSION +echo "dev " > cmd/VERSION +RULESET="./ruleset.yaml" go run cmd/main.go +``` + +## Optional: Live reloading development server with [cosmtrek/air](https://github.com/cosmtrek/air) + +Install air according to the [installation instructions](https://github.com/cosmtrek/air#installation). + +Run a development server at http://localhost:8080: + +```bash +air +``` + +This project uses [pnpm](https://pnpm.io/) to build a stylesheet with the [Tailwind CSS](https://tailwindcss.com/) classes. For local development, if you modify styles in `form.html`, run `pnpm build` to generate a new stylesheet. diff --git a/docs/install/_category_.json b/docs/install/_category_.json new file mode 100644 index 0000000..6346441 --- /dev/null +++ b/docs/install/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Install", + "position": 2 +} diff --git a/docs/install/binary.md b/docs/install/binary.md new file mode 100644 index 0000000..f7bad1f --- /dev/null +++ b/docs/install/binary.md @@ -0,0 +1,11 @@ +--- +sidebar_position: 2 +--- + +# Binary + +> **Warning:** If your instance will be publicly accessible, make sure to enable Basic Auth. This will prevent unauthorized users from using your proxy. If you do not enable Basic Auth, anyone can use your proxy to browse nasty/illegal stuff. And you will be responsible for it. + +1. Download binary [here](https://github.com/everywall/ladder/releases/latest) +2. Unpack and run the binary `./ladder -r https://t.ly/14PSf` +3. Open Browser (Default: http://localhost:8080) diff --git a/docs/install/docker.md b/docs/install/docker.md new file mode 100644 index 0000000..78292e1 --- /dev/null +++ b/docs/install/docker.md @@ -0,0 +1,18 @@ +--- +sidebar_position: 1 +--- + +# Docker + +> **Warning:** If your instance will be publicly accessible, make sure to enable Basic Auth. This will prevent unauthorized users from using your proxy. If you do not enable Basic Auth, anyone can use your proxy to browse nasty/illegal stuff. And you will be responsible for it. + +```bash +docker run -p 8080:8080 -d --env RULESET=https://t.ly/14PSf --name ladder ghcr.io/everywall/ladder:latest +``` + +## Docker Compose + +```bash +curl https://raw.githubusercontent.com/everywall/ladder/main/docker-compose.yaml --output docker-compose.yaml +docker-compose up -d +``` diff --git a/docs/install/environment-variables.md b/docs/install/environment-variables.md new file mode 100644 index 0000000..41d60a2 --- /dev/null +++ b/docs/install/environment-variables.md @@ -0,0 +1,22 @@ +--- +sidebar_position: 4 +--- + +# Environment variables + +| Variable | Description | Value | +| ------------------------- | --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `PORT` | Port to listen on | `8080` | +| `PREFORK` | Spawn multiple server instances | `false` | +| `USER_AGENT` | User agent to emulate | `Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)` | +| `X_FORWARDED_FOR` | IP forwarder address | `66.249.66.1` | +| `USERPASS` | Enables Basic Auth, format `admin:123456` | `` | +| `LOG_URLS` | Log fetched URL's | `true` | +| `DISABLE_FORM` | Disables URL Form Frontpage | `false` | +| `FORM_PATH` | Path to custom Form HTML | `` | +| `RULESET` | Path or URL to a ruleset file, accepts local directories | `https://raw.githubusercontent.com/everywall/ladder-rules/main/ruleset.yaml` or `/path/to/my/rules.yaml` or `/path/to/my/rules/` | +| `EXPOSE_RULESET` | Make your Ruleset available to other ladders | `true` | +| `ALLOWED_DOMAINS` | Comma separated list of allowed domains. Empty = no limitations | `` | +| `ALLOWED_DOMAINS_RULESET` | Allow Domains from Ruleset. false = no limitations | `false` | + +`ALLOWED_DOMAINS` and `ALLOWED_DOMAINS_RULESET` are joined together. If both are empty, no limitations are applied. diff --git a/docs/install/helm.md b/docs/install/helm.md new file mode 100644 index 0000000..6384050 --- /dev/null +++ b/docs/install/helm.md @@ -0,0 +1,38 @@ +--- +sidebar_position: 3 +--- + +# Helm + +> **Warning:** If your instance will be publicly accessible, make sure to enable Basic Auth. This will prevent unauthorized users from using your proxy. If you do not enable Basic Auth, anyone can use your proxy to browse nasty/illegal stuff. And you will be responsible for it. + +## Deployment prerequisites + +### Values + +Edit the values to your own preferences, with the only minimum requirement being `ingress.HOST` (line 19) being updated to your intended domain name. + +Other variables in `values.yaml` can be updated as to your preferences, with details on each variable being listed in the main [README.md](/README.md) in the root of this repo. + +### Defaults in K8s + +No ingress default has been specified. +You can set this manually by adding an annotation to the ingress.yaml - if needed. +For example, to use Traefik - + +```yaml +metadata: + name: ladder-ingress + annotations: + kubernetes.io/ingress.class: traefik +``` + +## Helm Install + +`helm install -n --create-namespace` +`helm install ladder .\ladder\ -n ladder --create-namespace` + +## Helm Upgrade + +`helm upgrade -n ` +`helm upgrade ladder .\ladder\ -n ladder` diff --git a/docs/intro.md b/docs/intro.md deleted file mode 100644 index 45e8604..0000000 --- a/docs/intro.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Tutorial Intro - -Let's discover **Docusaurus in less than 5 minutes**. - -## Getting Started - -Get started by **creating a new site**. - -Or **try Docusaurus immediately** with **[docusaurus.new](https://docusaurus.new)**. - -### What you'll need - -- [Node.js](https://nodejs.org/en/download/) version 18.0 or above: - - When installing Node.js, you are recommended to check all checkboxes related to dependencies. - -## Generate a new site - -Generate a new Docusaurus site using the **classic template**. - -The classic template will automatically be added to your project after you run the command: - -```bash -npm init docusaurus@latest my-website classic -``` - -You can type this command into Command Prompt, Powershell, Terminal, or any other integrated terminal of your code editor. - -The command also installs all necessary dependencies you need to run Docusaurus. - -## Start your site - -Run the development server: - -```bash -cd my-website -npm run start -``` - -The `cd` command changes the directory you're working with. In order to work with your newly created Docusaurus site, you'll need to navigate the terminal there. - -The `npm run start` command builds your website locally and serves it through a development server, ready for you to view at http://localhost:3000/. - -Open `docs/intro.md` (this page) and edit some lines: the site **reloads automatically** and displays your changes. diff --git a/docs/tutorial-extras/_category_.json b/docs/modifiers/_category_.json similarity index 50% rename from docs/tutorial-extras/_category_.json rename to docs/modifiers/_category_.json index a8ffcc1..2f80356 100644 --- a/docs/tutorial-extras/_category_.json +++ b/docs/modifiers/_category_.json @@ -1,6 +1,6 @@ { - "label": "Tutorial - Extras", - "position": 3, + "label": "Modifiers", + "position": 4, "link": { "type": "generated-index" } diff --git a/docs/modifiers/request-modifiers.md b/docs/modifiers/request-modifiers.md new file mode 100644 index 0000000..654155f --- /dev/null +++ b/docs/modifiers/request-modifiers.md @@ -0,0 +1,51 @@ +--- +sidebar_position: 1 +description: "Request modifier functions" +--- + +# Request modifiers + +| Function | Description | +| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `AddCacheBusterQuery()` | AddCacheBusterQuery modifies query params to add a random parameter key in order to get the upstream network stack to serve a fresh copy of the page. | +| `DeleteOutgoingCookie(name string)` | DeleteOutgoingCookie modifies the http request's cookies header to delete a specific request cookie going to the upstream server. If the cookie does not exist, it does not do anything. | +| `DeleteOutgoingCookies()` | DeleteOutgoingCookies removes the cookie header entirely, preventing any cookies from reaching the upstream server. | +| `DeleteOutgoingCookiesExcept(whitelist ...string)` | DeleteOutGoingCookiesExcept prevents non-whitelisted cookies from being sent from the client to the upstream proxy server. Cookies whose names are in the whitelist are not removed. | +| `DeleteRequestHeader(name string)` | DeleteRequestHeader modifies a specific outgoing header This is the header that the upstream server will see. | +| `ForwardRequestHeaders()` | ForwardRequestHeaders forwards the requests headers sent from the client to the upstream server. | +| `HideOrigin()` | HideOrigin modifies the origin header so that it is the original origin, not the proxy. | +| `HideReferrer()` | HideReferrer modifies the referrer header so that it is the original referrer, not the proxy | +| `MasqueradeAsBaiduBot()` | MasqueradeAsBaiduBot modifies user agent and x-forwarded for to appear to be a Baidu Spider Bot | +| `MasqueradeAsBingBot()` | MasqueradeAsBingBot modifies user agent and x-forwarded for to appear to be a Bing Bot | +| `MasqueradeAsDuckDuckBot()` | MasqueradeAsDuckDuckBot modifies user agent and x-forwarded for to appear to be a DuckDuckGo Bot | +| `MasqueradeAsFacebookBot()` | MasqueradeAsFacebookBot modifies user agent and x-forwarded for to appear to be a Facebook Bot (link previews?) | +| `MasqueradeAsGoogleBot()` | MasqueradeAsGoogleBot modifies user agent and x-forwarded for to appear to be a Google Bot | +| `MasqueradeAsWaybackMachineBot()` | MasqueradeAsWaybackMachineBot modifies user agent and x-forwarded for to appear to be a archive.org (wayback machine) Bot | +| `MasqueradeAsYahooBot()` | MasqueradeAsYahooBot modifies user agent and x-forwarded for to appear to be a Yahoo Bot | +| `MasqueradeAsYandexBot()` | MasqueradeAsYandexBot modifies user agent and x-forwarded for to appear to be a Yandex Spider Bot | +| `ModifyDomainWithRegex(matchRegex string, replacement string)` | ModifyDomainWithRegex replaces a domain with a replacement string | +| `ModifyPathWithRegex(matchRegex string, replacement string)` | ModifyPathWithRegex replaces the path with a replacement string | +| `ModifyQueryParams(key string, value string)` | ModifyQueryParams replaces query parameter values in URL's query params in a ProxyChain's URL. If the query param key doesn't exist, it is created. | +| `RequestArchiveIs()` | RequestArchiveIs modifies a ProxyChain's URL to request an archived version from archive.is | +| `RequestGoogleCache()` | RequestGoogleCache modifies a ProxyChain's URL to request its Google Cache version. | +| `RequestWaybackMachine()` | RequestWaybackMachine modifies a ProxyChain's URL to request the wayback machine (archive.org) version. | +| `ResolveWithGoogleDoH()` | ResolveWithGoogleDoH modifies a ProxyChain's client to make the request by resolving the URL using Google's DNS over HTTPs service. | +| `SetOutgoingCookie(name string, val string)` | SetOutgoingCookie modifes a specific cookie name by modifying the request cookie headers going to the upstream server. If the cookie name does not already exist, it is created. | +| `SetOutgoingCookies(cookies string)` | SetOutgoingCookies modifies a client request's cookie header to a raw Cookie string, overwriting existing cookies. | +| `SetRequestHeader(name string, val string)` | SetRequestHeader modifies a specific outgoing header This is the header that the upstream server will see. | +| `SpoofOrigin(url string)` | SpoofOrigin modifies the origin header if the upstream server returns a Vary header it means you might get a different response if you change this | +| `SpoofReferrer(url string)` | SpoofReferrer modifies the referrer header. It is useful if the page can be accessed from a search engine or social media site, but not by browsing the website itself. if url is "", then the referrer header is removed. | +| `SpoofReferrerFromBaiduSearch()` | SpoofReferrerFromBaiduSearch modifies the referrer header pretending to be from a BaiduSearch | +| `SpoofReferrerFromBingSearch()` | SpoofReferrerFromBingSearch modifies the referrer header pretending to be from a bing search site | +| `SpoofReferrerFromGoogleSearch()` | SpoofReferrerFromGoogleSearch modifies the referrer header pretending to be from a google search site | +| `SpoofReferrerFromLinkedInPost()` | SpoofReferrerFromLinkedInPost modifies the referrer header pretending to be from a linkedin post | +| `SpoofReferrerFromNaverSearch()` | SpoofReferrerFromNaverSearch modifies the referrer header pretending to be from a Naver search (popular in South Korea) | +| `SpoofReferrerFromPinterestPost()` | SpoofReferrerFromPinterestPost modifies the referrer header pretending to be from a pinterest post | +| `SpoofReferrerFromQQPost()` | SpoofReferrerFromQQPost modifies the referrer header pretending to be from a QQ post (popular social media in China) | +| `SpoofReferrerFromRedditPost()` | SpoofReferrerFromRedditPost modifies the referrer header pretending to be from a reddit post | +| `SpoofReferrerFromTumblrPost()` | SpoofReferrerFromTumblrPost modifies the referrer header pretending to be from a tumblr post | +| `SpoofReferrerFromTwitterPost()` | SpoofReferrerFromTwitterPost modifies the referrer header pretending to be from a twitter post | +| `SpoofReferrerFromVkontaktePost()` | SpoofReferrerFromVkontaktePost modifies the referrer header pretending to be from a vkontakte post (popular in Russia) | +| `SpoofReferrerFromWeiboPost()` | SpoofReferrerFromWeiboPost modifies the referrer header pretending to be from a Weibo post (popular in China) | +| `SpoofUserAgent(ua string)` | SpoofUserAgent modifies the user agent. | +| `SpoofXForwardedFor(ip string)` | SpoofXForwardedFor modifies the X-Forwarded-For header in some cases, a forward proxy may interpret this as the source IP. | diff --git a/docs/modifiers/response-modifiers.md b/docs/modifiers/response-modifiers.md new file mode 100644 index 0000000..e697450 --- /dev/null +++ b/docs/modifiers/response-modifiers.md @@ -0,0 +1,32 @@ +--- +sidebar_position: 2 +description: "Response modifier functions" +--- + +# Response modifiers + +| Function | Description | +| ----------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `APIContent()` | APIContent creates an JSON representation of the article and returns it as an API response. | +| `BlockElementRemoval(cssSelector string)` | BlockElementRemoval prevents paywall javascript from removing a particular element by detecting the removal, then immediately reinserting it. This is useful when a page will return a "fake" 404, after flashing the content briefly. If the /outline/ API works, but the regular API doesn't, try this modifier. | +| `BlockThirdPartyScripts()` | BlockThirdPartyScripts rewrites HTML and injects JS to block all third party JS from loading. | +| `BypassCORS()` | BypassCORS modifies response headers to prevent the browser from enforcing any CORS restrictions. This should run at the end of the chain. | +| `BypassContentSecurityPolicy()` | BypassContentSecurityPolicy modifies response headers to prevent the browser from enforcing any CSP restrictions. This should run at the end of the chain. | +| `DeleteIncomingCookies(_ ...string)` | DeleteIncomingCookies prevents ALL cookies from being sent from the proxy server back down to the client. | +| `DeleteIncomingCookiesExcept(whitelist ...string)` | DeleteIncomingCookiesExcept prevents non-whitelisted cookies from being sent from the proxy server to the client. Cookies whose names are in the whitelist are not removed. | +| `DeleteLocalStorageData()` | DeleteLocalStorageData deletes localstorage cookies. If the page works once in a fresh incognito window, but fails for subsequent loads, try this response modifier alongside DeleteSessionStorageData and DeleteIncomingCookies | +| `DeleteResponseHeader(key string)` | DeleteResponseHeader removes response headers from the upstream server | +| `DeleteSessionStorageData()` | DeleteSessionStorageData deletes localstorage cookies. If the page works once in a fresh incognito window, but fails for subsequent loads, try this response modifier alongside DeleteLocalStorageData and DeleteIncomingCookies | +| `ForwardResponseHeaders()` | ForwardResponseHeaders forwards the response headers from the upstream server to the client | +| `GenerateReadableOutline()` | GenerateReadableOutline creates an reader-friendly distilled representation of the article. This is a reliable way of bypassing soft-paywalled articles, where the content is hidden, but still present in the DOM. | +| `InjectScriptAfterDOMContentLoaded(js string)` | InjectScriptAfterDOMContentLoaded modifies HTTP responses to inject a JS after DOM Content is loaded (script tag in head) | +| `InjectScriptAfterDOMIdle(js string)` | InjectScriptAfterDOMIdle modifies HTTP responses to inject a JS after the DOM is idle (ie: js framework loaded) | +| `InjectScriptBeforeDOMContentLoaded(js string)` | InjectScriptBeforeDOMContentLoaded modifies HTTP responses to inject a JS before DOM Content is loaded (script tag in head) | +| `ModifyIncomingScriptsWithRegex(matchRegex string, replacement string)` | ModifyIncomingScriptsWithRegex modifies all incoming javascript (application/javascript and inline `