Extract plugin URLs & build sgmodule #29206
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Extract plugin URLs & build sgmodule | |
| on: | |
| push: | |
| branches: [main] | |
| schedule: | |
| - cron: '5,30,55 * * * *' | |
| workflow_dispatch: | |
| inputs: | |
| list_url_primary: | |
| description: "Primary plugin list URL (e.g. https://hub.kelee.one/list.json)" | |
| required: false | |
| default: "" | |
| list_url_backup: | |
| description: "Backup plugin list URL" | |
| required: false | |
| default: "" | |
| proxy_base: | |
| description: "Proxy base URL (include trailing ?url= if needed)" | |
| required: false | |
| default: "" | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| services: | |
| docker: | |
| image: xream/script-hub:latest | |
| ports: ['9100:9100', '9101:9101'] | |
| env: | |
| MIRROR_RAW: https://github.com/bunizao/Mirrored/raw/main/Chores/js | |
| MIRROR_KEY: github.com/bunizao/Mirrored | |
| PROXY_BASE: ${{ inputs.proxy_base || vars.PROXY_BASE || '' }} | |
| LIST_URL_PRIMARY: ${{ inputs.list_url_primary || vars.LIST_URL_PRIMARY }} | |
| LIST_URL_BACKUP: ${{ inputs.list_url_backup || vars.LIST_URL_BACKUP }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup git identity | |
| run: | | |
| git config --global user.name "github-actions[bot]" | |
| git config --global user.email \ | |
| "github-actions[bot]@users.noreply.github.com" | |
| - name: Install workflow tooling | |
| run: | | |
| sudo apt-get update -qq | |
| sudo apt-get install -y \ | |
| curl \ | |
| jq \ | |
| netcat-openbsd \ | |
| python3-pip \ | |
| nodejs | |
| - name: Wait for Script-Hub container (:9101) | |
| run: | | |
| for i in {1..20}; do | |
| nc -z localhost 9101 && exit 0 | |
| sleep 5 | |
| done | |
| echo "::error ::container not ready"; exit 1 | |
| # ---------- 1. Fetch the plugin list and extract source URLs ---------- | |
| - name: Download plugin list | |
| run: | | |
| set -euo pipefail | |
| python3 -m pip install --quiet --upgrade pip | |
| python3 -m pip install --quiet cloudscraper | |
| candidate_urls=() | |
| if [ -n "${LIST_URL_PRIMARY:-}" ]; then | |
| candidate_urls+=("$LIST_URL_PRIMARY") | |
| fi | |
| if [ -n "${LIST_URL_BACKUP:-}" ]; then | |
| candidate_urls+=("$LIST_URL_BACKUP") | |
| fi | |
| if [ "${#candidate_urls[@]}" -eq 0 ]; then | |
| echo "::warning ::LIST_URL_* vars missing; using default catalog URLs" | |
| candidate_urls=( | |
| "https://hub.kelee.one/list.json" | |
| "https://pluginhub.kelee.one/list.json" | |
| ) | |
| fi | |
| success=0 | |
| for url in "${candidate_urls[@]}"; do | |
| echo "Trying: $url" | |
| if python3 scripts/download_plugin_list.py \ | |
| --url "$url" \ | |
| --output plugin_data.json; then | |
| success=1 | |
| break | |
| fi | |
| done | |
| if [ "$success" -ne 1 ] || [ ! -s plugin_data.json ]; then | |
| echo "::error ::Failed to download plugin list from all configured URLs" | |
| exit 1 | |
| fi | |
| - name: Extract plugin source URLs | |
| run: | | |
| set -euo pipefail | |
| python3 scripts/extract_lnplugin_urls.py \ | |
| --input plugin_data.json \ | |
| --output plugin_urls.txt \ | |
| --extensions .lpx .plugin | |
| plugin_count=$(wc -l < plugin_urls.txt) | |
| if [ "$plugin_count" -eq 0 ]; then | |
| echo "::error ::Plugin list JSON did not contain any .plugin/.lpx URLs" | |
| exit 1 | |
| fi | |
| echo "Found $plugin_count plugin links:" | |
| cat plugin_urls.txt | |
| # ---------- 2. Ask Script-Hub to render .sgmodule files ---------- | |
| - name: Generate sgmodule files | |
| run: | | |
| mkdir -p Chores/sgmodule | |
| find Chores/sgmodule -maxdepth 1 -type f \ | |
| -name "*.lpx.sgmodule" -delete | |
| category="🚫 AD Block" | |
| enc_cat=$(printf '%s' "$category" | jq -sRr @uri) | |
| trim_control_chars() { | |
| printf '%s' "$1" | tr -d '\r\n\t' | |
| } | |
| strip_surrounding_whitespace() { | |
| printf '%s' "$1" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | |
| } | |
| sanitize_field() { | |
| strip_surrounding_whitespace "$(trim_control_chars "$1")" | |
| } | |
| script_hub_host="localhost" | |
| script_hub_port=9101 | |
| proxy_base="${PROXY_BASE:-}" | |
| proxy_base="$(sanitize_field "$proxy_base")" | |
| header_value=$(printf 'User-Agent: Surge Mac/2985' | jq -sRr @uri) | |
| while read -r plugin_url; do | |
| plugin_url="$(sanitize_field "$plugin_url")" | |
| [ -z "$plugin_url" ] && continue | |
| source_url="$plugin_url" | |
| url_no_query="${source_url%%[\?#]*}" | |
| base_name=$(basename "$url_no_query") | |
| base_name="$(sanitize_field "$base_name")" | |
| name="$base_name" | |
| if [[ "$name" == *.lpx ]]; then | |
| name="${name%.lpx}" | |
| fi | |
| if [[ "$name" == *.plugin ]]; then | |
| name="${name%.plugin}" | |
| fi | |
| name="$(sanitize_field "$name")" | |
| [ -z "$name" ] && name="$base_name" | |
| enc=$(printf '%s' "$name" | jq -sRr @uri) | |
| proxy_prefix="" | |
| if [ -n "$proxy_base" ]; then | |
| host_part="$(printf '%s' "$source_url" | awk -F/ 'NF>2 {print $3}')" | |
| host_part="$(sanitize_field "$host_part")" | |
| host_lc="$(printf '%s' "$host_part" | tr 'A-Z' 'a-z')" | |
| case "$host_lc" in | |
| kelee.one|*.kelee.one) | |
| proxy_prefix="$proxy_base" | |
| ;; | |
| esac | |
| fi | |
| base_url="http://${script_hub_host}:${script_hub_port}/file/_start_/${proxy_prefix}${source_url}/_end_/${enc}.sgmodule" | |
| query="type=loon-plugin&target=surge-module&category=$enc_cat&headers=$header_value" | |
| url="${base_url}?${query}" | |
| echo "↓ $url" | |
| if curl -fSLo "Chores/sgmodule/$name.sgmodule" \ | |
| -A "Surge Mac/2985" \ | |
| --connect-timeout 30 \ | |
| --max-time 120 \ | |
| --retry 3 \ | |
| --retry-delay 5 \ | |
| --retry-all-errors \ | |
| "$url"; then | |
| echo "✓ Downloaded $name.sgmodule" | |
| else | |
| echo "::warning ::Failed to download $name.sgmodule" | |
| rm -f "Chores/sgmodule/$name.sgmodule" | |
| fi | |
| done < plugin_urls.txt | |
| # ---------- 3. Mirror external JS files and rewrite links ---------- | |
| - name: Mirror external JS & rewrite links | |
| shell: bash | |
| run: | | |
| set -e | |
| mkdir -p Chores/js | |
| # Abort early if no sgmodule files were generated | |
| if [ ! -d "Chores/sgmodule" ] \ | |
| || [ -z "$(ls -A Chores/sgmodule 2>/dev/null)" ]; then | |
| echo "No sgmodule files found to process" | |
| exit 0 | |
| fi | |
| echo "Generated sgmodule files:" | |
| ls -la Chores/sgmodule/ | |
| # Extract every JavaScript URL referenced by the sgmodules | |
| echo "Searching for JS URLs..." | |
| # Use a precise regex to capture URLs that follow script-path | |
| script_regex='script-path\s*=\s*https?://[^[:space:],"]+\.js[^[:space:],"]*' | |
| find Chores/sgmodule -name "*.sgmodule" \ | |
| -exec grep -hoE "$script_regex" {} \; 2>/dev/null \ | |
| | sed -E 's/^script-path\s*=\s*//' \ | |
| | sort -u > external-js-raw.txt || touch external-js-raw.txt | |
| echo "Found JavaScript URLs:" | |
| cat external-js-raw.txt || echo "No JS URLs found" | |
| # Filter out URLs that already point at the mirror | |
| if [ -s external-js-raw.txt ]; then | |
| grep -v "$MIRROR_KEY" external-js-raw.txt \ | |
| | grep -v "$MIRROR_RAW" > external-js.txt \ | |
| || touch external-js.txt | |
| else | |
| touch external-js.txt | |
| fi | |
| echo "External JS URLs to mirror:" | |
| cat external-js.txt || echo "No external JS URLs to mirror" | |
| # Nothing to mirror? Clean up and exit early | |
| if [ ! -s external-js.txt ]; then | |
| echo "No external JS links found to mirror." | |
| rm -f external-js.txt external-js-raw.txt | |
| exit 0 | |
| fi | |
| # Download and mirror the JS files | |
| echo "Processing external JS files:" | |
| failed_downloads=0 | |
| while IFS= read -r url; do | |
| [ -z "$url" ] && continue | |
| # Extract a filename while stripping any query string | |
| filename=$(basename "${url%%[\?#]*}") | |
| file_path="Chores/js/$filename" | |
| # Ensure the filename ends with .js | |
| if [[ ! "$filename" =~ \.js$ ]]; then | |
| filename="${filename}.js" | |
| file_path="Chores/js/$filename" | |
| fi | |
| echo "Mirroring: $url → Chores/js/$filename" | |
| if curl -Ls -A "Surge Mac/3272" \ | |
| --connect-timeout 30 \ | |
| --max-time 60 \ | |
| "$url" \ | |
| -o "$file_path"; then | |
| # Ensure the mirrored file has meaningful content | |
| if [ -s "$file_path" ]; then | |
| file_size=$(stat -c%s "$file_path") | |
| if [ "$file_size" -gt 10 ]; then | |
| echo "✓ Successfully downloaded $filename" | |
| # Rewrite every sgmodule reference with the mirrored URL | |
| mirror_url="$MIRROR_RAW/$filename" | |
| perl_expr="s,\\Q${url}\\E,${mirror_url},g" | |
| find Chores/sgmodule -name "*.sgmodule" -type f \ | |
| -exec perl -pi -e "$perl_expr" {} \; | |
| echo "✓ Updated links: $url → $mirror_url" | |
| else | |
| echo "✗ Downloaded file is empty or too small: $filename" | |
| rm -f "$file_path" | |
| failed_downloads=$((failed_downloads + 1)) | |
| fi | |
| else | |
| echo "✗ Downloaded file is empty or too small: $filename" | |
| rm -f "$file_path" | |
| failed_downloads=$((failed_downloads + 1)) | |
| fi | |
| else | |
| echo "✗ Failed to download: $url" | |
| failed_downloads=$((failed_downloads + 1)) | |
| fi | |
| done < external-js.txt | |
| echo "Processing complete. Failed downloads: $failed_downloads" | |
| # Remove temporary files | |
| rm -f external-js.txt external-js-raw.txt | |
| - name: Commit & push (if changed) | |
| shell: bash | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -e | |
| git config user.name "github-actions[bot]" | |
| git config user.email \ | |
| "github-actions[bot]@users.noreply.github.com" | |
| # Skip committing when nothing was generated | |
| if [ ! -d "Chores/sgmodule" ] && [ ! -d "Chores/js" ]; then | |
| echo "No files to commit." | |
| exit 0 | |
| fi | |
| # Stage managed artifacts | |
| git add Chores/sgmodule Chores/js 2>/dev/null || true | |
| # Bail out if staging area is empty | |
| if git diff --cached --quiet; then | |
| echo "Nothing to commit." | |
| exit 0 | |
| fi | |
| # Show the staged changes | |
| echo "Changes to be committed:" | |
| git diff --cached --name-status | |
| # Sync with the latest upstream history | |
| git fetch origin | |
| if git diff --quiet HEAD origin/main; then | |
| echo "Local branch is up to date with origin/main" | |
| else | |
| echo "Pulling latest changes from origin..." | |
| git pull --rebase origin main | |
| # Re-stage files after a successful rebase | |
| git add Chores/sgmodule Chores/js 2>/dev/null || true | |
| # Abort if nothing remains staged after rebase | |
| if git diff --cached --quiet; then | |
| echo "Nothing to commit after rebase." | |
| exit 0 | |
| fi | |
| fi | |
| # Commit and push the new artifacts | |
| timestamp=$(date -u '+%Y-%m-%d %H:%M:%S UTC') | |
| commit_msg="Update sgmodule & mirror JS files - ${timestamp}" | |
| git commit -m "$commit_msg" | |
| echo "Pushing changes..." | |
| git push origin main | |
| echo "✓ Successfully committed and pushed changes" |