Updated setup for the search bar

Author: AlBovo

.env.example

@@ -7,6 +7,8 @@ TYPESENSE_COLLECTION_NAME=training-wiki
 
 TYPESENSE_HOST=localhost
 TYPESENSE_PORT=8108
-TYPESENSE_PROTOCOL=https
+TYPESENSE_PROTOCOL=http
 
-TYPESENSE_SEARCH_API_KEY=replace-with-a-search-only-key
+TYPESENSE_ADMIN_API_KEY=xyz
+
+TYPESENSE_SEARCH_API_KEY=xyz

.github/workflows/ci-deploy.yml

@@ -12,38 +12,26 @@ on:
       - docker-compose.yml
       - docker-entrypoint.sh
       - .dockerignore
+      - docsearch.config.json
+      - .env.example
+      - README.md
+      - LICENSE
 
 concurrency:
   group: deploy-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  changes:
-    name: Detect changes
-    runs-on: ubuntu-latest
-    outputs:
-      docker_changed: ${{ steps.filter.outputs.docker }}
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Paths filter
-        id: filter
-        uses: dorny/paths-filter@v3
-        with:
-          filters: |
-            docker:
-              - 'Dockerfile'
-              - 'docker-compose.yml'
-              - 'docker-entrypoint.sh'
-              - '.dockerignore'
-
   build:
-    needs: changes
-    if: ${{ needs.changes.outputs.docker_changed != 'true' }}
     runs-on: ubuntu-latest
 
+    env:
+      TYPESENSE_COLLECTION_NAME: ${{ secrets.TYPESENSE_COLLECTION_NAME }}
+      TYPESENSE_HOST: ${{ secrets.TYPESENSE_HOST }}
+      TYPESENSE_PORT: ${{ secrets.TYPESENSE_PORT }}
+      TYPESENSE_PROTOCOL: ${{ secrets.TYPESENSE_PROTOCOL }}
+      TYPESENSE_SEARCH_API_KEY: ${{ secrets.TYPESENSE_SEARCH_API_KEY }}
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -65,7 +53,6 @@ jobs:
 
   deploy:
     needs: build
-    if: ${{ needs.build.result == 'success' }}
     runs-on: ubuntu-latest
 
     steps:
@@ -86,15 +73,24 @@ jobs:
 
       - name: Build on server (npm)
         uses: appleboy/ssh-action@v1.2.0
+        env:
+          TYPESENSE_COLLECTION_NAME: ${{ secrets.TYPESENSE_COLLECTION_NAME }}
+          TYPESENSE_HOST: ${{ secrets.TYPESENSE_HOST }}
+          TYPESENSE_PORT: ${{ secrets.TYPESENSE_PORT }}
+          TYPESENSE_PROTOCOL: ${{ secrets.TYPESENSE_PROTOCOL }}
+          TYPESENSE_SEARCH_API_KEY: ${{ secrets.TYPESENSE_SEARCH_API_KEY }}
         with:
           host: ${{ secrets.SSH_HOST }}
           port: 6007
           username: root
           password: ${{ secrets.ROOT_PASSWORD }}
+          envs: TYPESENSE_COLLECTION_NAME,TYPESENSE_HOST,TYPESENSE_PORT,TYPESENSE_PROTOCOL,TYPESENSE_SEARCH_API_KEY
           script_stop: true
           command_timeout: 30m
           script: |
             set -euo pipefail
             cd /var/www/html
             npm ci
             npm run build
+            touch /trigger/run
+            echo "Scraper trigger set — re-index will start within seconds"

README.md

@@ -6,8 +6,6 @@
 
 Wiki/knowledge base built with [Docusaurus](https://docusaurus.io/) with search powered by Typesense DocSearch.
 
-Prerequisites: Node.js >= 20, npm. For deployment: Docker + Docker Compose.
-
 ## Local setup
 
 ```bash
@@ -25,4 +23,25 @@ npm run start
 docker compose up -d --build
 ```
 
-Note: `TYPESENSE_*` variables are read **at build time**; if you change them, you must rebuild (`--build`).
+3) Create the search-only API key (once):
+
+```bash
+docker compose --profile init run --rm typesense-init
+```
+
+4) Index the docs into Typesense (required for getting results):
+
+```bash
+docker compose --profile scraper run --rm scraper
+```
+
+Optional: keep a watcher running and trigger re-index when needed:
+
+```bash
+docker compose up -d scraper-watcher
+
+# trigger a re-index
+docker compose exec wiki sh -lc 'touch /trigger/run'
+```
+
+Note: `TYPESENSE_*` variables are read **at build time**; if you change them, rebuild (`--build`).

docker-compose.yml

@@ -1,16 +1,48 @@
+x-scraper-common: &scraper-common
+  image: typesense/docsearch-scraper:latest
+  depends_on:
+    - typesense
+    - wiki
+  environment:
+    - TYPESENSE_API_KEY=${TYPESENSE_ADMIN_API_KEY:-xyz}
+    - TYPESENSE_HOST=typesense
+    - TYPESENSE_PORT=8108
+    - TYPESENSE_PROTOCOL=http
+    - CONFIG=/tmp/docsearch/docsearch.config.json
+  volumes:
+    - ./docsearch.config.json:/tmp/docsearch/docsearch.config.json:ro
+
 services:
   typesense:
-    image: typesense/typesense:0.25.2
+    image: typesense/typesense:30.2
     container_name: typesense
     ports:
       - "127.0.0.1:8108:8108"
     volumes:
       - typesense-data:/data
     command: >-
       --data-dir=/data
-      --api-key=${TYPESENSE_SEARCH_API_KEY:-xyz}
+      --api-key=${TYPESENSE_ADMIN_API_KEY:-xyz}
       --enable-cors
 
+  typesense-init:
+    image: curlimages/curl:8.8.0
+    profiles: ["init"]
+    depends_on:
+      - typesense
+    entrypoint: ["/bin/sh", "-c"]
+    environment:
+      - TYPESENSE_ADMIN_API_KEY=${TYPESENSE_ADMIN_API_KEY:-xyz}
+      - TYPESENSE_SEARCH_API_KEY=${TYPESENSE_SEARCH_API_KEY:-xyz}
+      - TYPESENSE_COLLECTION_NAME=${TYPESENSE_COLLECTION_NAME:-training-wiki}
+    command: >-
+      until curl -sf http://typesense:8108/health >/dev/null; do sleep 1; done;
+      curl -sf -X POST http://typesense:8108/keys \
+        -H "X-TYPESENSE-API-KEY: $$TYPESENSE_ADMIN_API_KEY" \
+        -H "Content-Type: application/json" \
+        -d "{\"description\":\"training-wiki search key\",\"actions\":[\"documents:search\"],\"collections\":[\"$$TYPESENSE_COLLECTION_NAME.*\"],\"value\":\"$$TYPESENSE_SEARCH_API_KEY\"}" \
+        >/dev/null || true
+
   wiki:
     build:
       context: .
@@ -26,6 +58,8 @@ services:
     ports:
       - "127.0.0.1:1337:3000"
       - "6007:22"
+    volumes:
+      - scraper-trigger:/trigger
     environment:
       - ROOT_PASSWORD=${ROOT_PASSWORD}
       - TYPESENSE_COLLECTION_NAME=${TYPESENSE_COLLECTION_NAME:-training-wiki}
@@ -34,5 +68,31 @@ services:
       - TYPESENSE_PROTOCOL=${TYPESENSE_PROTOCOL:-http}
       - TYPESENSE_SEARCH_API_KEY=${TYPESENSE_SEARCH_API_KEY:-xyz}
 
+  scraper:
+    <<: *scraper-common
+    profiles: ["scraper"]
+
+  scraper-watcher:
+    <<: *scraper-common
+    restart: unless-stopped
+    volumes:
+      - ./docsearch.config.json:/tmp/docsearch/docsearch.config.json:ro
+      - scraper-trigger:/trigger
+    entrypoint: ["/bin/sh", "-c"]
+    command: >-
+      set -eu;
+      trigger="/trigger/run";
+      echo "[scraper-watcher] waiting for trigger...";
+      while true; do
+        if [ -f "$$trigger" ]; then
+          rm -f "$$trigger";
+          echo "[scraper-watcher] trigger detected, scraping...";
+          pipenv run python -m src.index || echo "[scraper-watcher] scrape failed";
+          echo "[scraper-watcher] done, waiting for next trigger...";
+        fi;
+        sleep 5;
+      done
+
 volumes:
   typesense-data:
+  scraper-trigger:

docsearch.config.json

@@ -0,0 +1,40 @@
+{
+  "index_name": "training-wiki",
+  "start_urls": [
+    {
+      "url": "http://wiki:3000/"
+    }
+  ],
+  "sitemap_urls": [
+    "http://wiki:3000/sitemap.xml"
+  ],
+  "sitemap_alternate_links": true,
+  "selectors": {
+    "lvl0": {
+      "selector": "(//ul[contains(@class,'menu__list')]//a[contains(@class, 'menu__link menu__link--sublist menu__link--active')]/text() | //nav[contains(@class, 'navbar')]//a[contains(@class,'navbar__link--active')]/text())[last()]",
+      "type": "xpath",
+      "global": true,
+      "default_value": "Documentation"
+    },
+    "lvl1": "article h1, header h1",
+    "lvl2": "article h2",
+    "lvl3": "article h3",
+    "lvl4": "article h4",
+    "lvl5": "article h5",
+    "lvl6": "article h6",
+    "text": "article p, article li, article td:last-child"
+  },
+  "strip_chars": " .,;:#",
+  "custom_settings": {
+    "separatorsToIndex": "_",
+    "attributesForFaceting": ["language", "version", "type", "docusaurus_tag"],
+    "attributesToRetrieve": [
+      "hierarchy",
+      "content",
+      "anchor",
+      "url",
+      "url_without_anchor",
+      "type"
+    ]
+  }
+}

docusaurus.config.ts

@@ -4,12 +4,16 @@ import type * as Preset from '@docusaurus/preset-classic';
 
 // This runs in Node.js - Don't use client-side code here (browser APIs, JSX...)
 
+const typesenseEnabled = Boolean(
+  process.env.TYPESENSE_HOST && process.env.TYPESENSE_SEARCH_API_KEY,
+);
+
 const config: Config = {
   title: 'PascalCTF Wiki',
   tagline: 'A wiki made for beginners to learn CTF and cybersecurity',
   favicon: 'img/favicon.ico',
 
-  themes: ['docusaurus-theme-search-typesense'],
+  themes: typesenseEnabled ? ['docusaurus-theme-search-typesense'] : [],
 
   // Future flags, see https://docusaurus.io/docs/api/docusaurus-config#future
   future: {
@@ -62,26 +66,30 @@ const config: Config = {
     colorMode: {
       respectPrefersColorScheme: true,
     },
-    typesense: {
-      // Must match `index_name` in the scraper config.
-      typesenseCollectionName:
-        process.env.TYPESENSE_COLLECTION_NAME ?? 'training-wiki',
+    ...(typesenseEnabled
+      ? {
+          typesense: {
+            // Must match `index_name` in the scraper config.
+            typesenseCollectionName:
+              process.env.TYPESENSE_COLLECTION_NAME ?? 'training-wiki',
 
-      typesenseServerConfig: {
-        nodes: [
-          {
-            host: process.env.TYPESENSE_HOST ?? 'localhost',
-            port: Number(process.env.TYPESENSE_PORT ?? '8108'),
-            protocol: (process.env.TYPESENSE_PROTOCOL ?? 'http') as
-              | 'http'
-              | 'https',
+            typesenseServerConfig: {
+              nodes: [
+                {
+                  host: process.env.TYPESENSE_HOST!,
+                  port: Number(process.env.TYPESENSE_PORT ?? '8108'),
+                  protocol: (process.env.TYPESENSE_PROTOCOL ?? 'http') as
+                    | 'http'
+                    | 'https',
+                },
+              ],
+              apiKey: process.env.TYPESENSE_SEARCH_API_KEY!,
+            },
+            typesenseSearchParameters: {},
+            contextualSearch: true,
           },
-        ],
-        apiKey: process.env.TYPESENSE_SEARCH_API_KEY ?? 'xyz',
-      },
-      typesenseSearchParameters: {},
-      contextualSearch: true,
-    },
+        }
+      : {}),
     navbar: {
       title: 'Training Wiki',
       logo: {
@@ -179,7 +187,7 @@ const config: Config = {
       darkTheme: prismThemes.dracula,
     },
   } satisfies (Preset.ThemeConfig & {
-    typesense: {
+    typesense?: {
       typesenseCollectionName: string;
       typesenseServerConfig: {
         nodes: Array<{host: string; port: number; protocol: 'http' | 'https'}>;