diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.md index 74295f2..0b1b494 100644 --- a/docs/RELIABILITY.md +++ b/docs/RELIABILITY.md @@ -19,6 +19,8 @@ review_cycle_days: 21 ## Service Expectations - `kb-server` readiness requires database and Git-backed vault access. +- `/admin` should surface the same readiness blockers and recent operational signals visible through logs and DB-backed job/event tables. +- The Streamlit dashboard should degrade cleanly when the FastAPI backend is offline and still allow operators to launch configured start commands. - Autosave worker should tolerate transient Git/network failures. - `vault-sync` should converge after temporary API outages. @@ -31,6 +33,7 @@ review_cycle_days: 21 ## Reliability Signals - API health and readiness checks. +- Admin dashboard state for config source, readiness blockers, git state, pending batch paths, and PR visibility. - Job/event tables for write and publish operations. - Sync logs for pull/push loop success and retries. @@ -39,15 +42,35 @@ review_cycle_days: 21 ### DB outage (`kb-server`) - Signal: `GET /health` stays 200 while `GET /ready` returns non-200 with DB failure detail. +- Signal: `/admin` shows DB not ready and surfaces the same readiness error. - Signal: API logs show database connection failures from readiness and write paths. - Recovery check: when DB is restored, `GET /ready` returns 200 without process restart. ### Git remote / GitHub outage (`kb-server`) - Signal: autosave or batcher logs show push/PR failures while local commits continue. +- Signal: `/admin` PR summary shows GitHub/API errors or empty PR visibility despite queued/pushed work. - Signal: pending API changes remain on `kb-api/*` branch until push/PR succeeds. - Recovery check: retry loop or next batch cycle pushes and re-establishes PR state. +### Config drift / restart-needed state (`kb-server`) + +- Signal: `/admin` shows updated `.env` values, but runtime behavior still reflects old DB/auth/process settings. +- Cause: some settings are effectively startup-bound because the API process and worker initialize long-lived config or connections at startup. +- Recovery check: restart `kb-api` and `kb-worker`, then confirm `/admin` and `/ready` reflect the expected state. + +### API offline but local dashboard available (`kb-server`) + +- Signal: Streamlit dashboard reports backend offline instead of crashing. +- Cause: `kb-api` is down, misbound, or unreachable at the configured backend URL. +- Recovery check: launch the derived tmux start command via the dashboard, rerun the dashboard, and confirm `/admin/api/state` responds again. + +### Autosave worker offline (`kb-server`) + +- Signal: Streamlit dashboard shows worker runtime config but autosave activity stops advancing. +- Cause: `kb-worker` tmux session is down or was never started. +- Recovery check: launch or restart the worker from the dashboard and confirm vault events resume. + ### API outage (`vault-sync`) - Signal: sync loop logs pull/push request failures and keeps retrying on interval. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 8f1de51..d110487 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -18,13 +18,16 @@ review_cycle_days: 21 ## Auth Boundary - API key auth is enforced by server middleware when configured (`KB_API_KEY` non-empty). -- With auth enabled, all HTTP routes require `X-API-Key`, including `/health`, `/ready`, `/docs`, and `/openapi.json`. +- With auth enabled, non-admin HTTP routes require `X-API-Key`, including `/health`, `/ready`, `/docs`, and `/openapi.json`. +- `/admin` and `/admin/api/*` are intentionally exempt from `X-API-Key` so the local operator dashboard can bootstrap and operate the instance. - With auth disabled (`KB_API_KEY` empty), requests are accepted without API-key checks. - `KB_API_KEY` must never be committed in docs examples with live values. ## Secret Handling -- Secrets remain in local `.env` files or deployment secret stores. +- Preferred model: secrets such as `KB_API_KEY` and `GITHUB_TOKEN` live in deployment secret stores or process environment variables. +- Supported local/dev model: `/admin` can write `KB_API_KEY` and `GITHUB_TOKEN` into local `.env`. +- Admin responses must not echo stored secret values back to the browser after save. - Docs should only reference secret names, not values. - Generated docs must redact secrets by default. @@ -33,12 +36,16 @@ review_cycle_days: 21 - Path traversal and absolute-path writes are denied. - Only approved file types are writable. - Writes from `source=api` remain review-gated through PR workflow. +- Admin config writes change local instance configuration only; they do not authorize content writes outside existing approval boundaries. +- Admin start/restart actions only launch derived tmux commands for the configured local checkout and sessions; they do not elevate privileges or infer a broader process manager. ## Security Review Triggers Update this document when changing: - auth middleware/dependency behavior +- admin route exposure or operator trust boundary - request validation and path sanitization +- secret storage or secret presentation behavior in admin/config flows - external webhook/publish execution semantics - GitHub token scope or PR automation behavior diff --git a/docs/exec-plans/active/admin-ui-bootstrap.md b/docs/exec-plans/active/admin-ui-bootstrap.md new file mode 100644 index 0000000..e03e0d6 --- /dev/null +++ b/docs/exec-plans/active/admin-ui-bootstrap.md @@ -0,0 +1,55 @@ +--- +owner: platform +status: draft +last_verified: 2026-03-12 +source_of_truth: + - ../../../kb-server/AGENTS.md + - ../../../kb-server/app/main.py + - ../../../kb-server/app/core/config.py +related_code: + - ../../../kb-server/app/api/routes + - ../../../kb-server/app/services +related_tests: + - ../../../kb-server/tests +review_cycle_days: 14 +--- + +# Admin UI Bootstrap + +## Objective + +Add an initial admin surface to `kb-server` for setup, configuration, and operational visibility without changing the core note-writing workflow. + +## First Slice + +Ship a minimal but real `/admin` experience that provides: + +- current configuration visibility for key settings +- write support for local `.env` configuration updates +- write-only secret update inputs for `KB_API_KEY` and `GITHUB_TOKEN` +- readiness and vault/git status summary +- recent jobs, vault events, and publish runs +- visibility into pending `kb-api/*` PR state when GitHub is configured + +## Non-Goals + +- no note editing UI +- no browser-triggered host provisioning +- no direct mutation of PR approval semantics +- no requirement to store secrets in `.env` + +## Design Constraints + +- The admin UI is an operator surface layered on top of existing API and worker behavior. +- Existing API-key middleware remains in force when `KB_API_KEY` is configured. +- Secret values are never returned in API responses after save. +- Config writes update `.env`, but operators should be told that process restart may be required for full effect. +- The UI should degrade cleanly when GitHub is unconfigured or the vault/db is unavailable. + +## Delivery Plan + +1. Add admin service helpers for status aggregation and `.env` persistence. +2. Add `/admin`, `/admin/api/state`, and `/admin/api/config`. +3. Render a lightweight in-app dashboard with setup and status sections. +4. Add focused tests for config persistence and admin endpoints. +5. Update README with usage and restart expectations. diff --git a/docs/generated/api-surface.md b/docs/generated/api-surface.md index 0e7664f..7d37181 100644 --- a/docs/generated/api-surface.md +++ b/docs/generated/api-surface.md @@ -1,11 +1,12 @@ --- owner: platform status: generated -last_verified: 2026-03-07 +last_verified: 2026-03-12 source_of_truth: - ../../kb-server/app/api/routes/health.py - ../../kb-server/app/api/routes/notes.py - ../../kb-server/app/api/routes/publish.py + - ../../kb-server/app/api/routes/admin.py related_code: - ../../scripts/generate_context_artifacts.py related_tests: @@ -15,7 +16,7 @@ review_cycle_days: 7 # API Surface (Generated) -Generated on `2026-03-07` from route handlers. +Generated on `2026-03-12` from route handlers. | Method | Path | | --- | --- | @@ -26,5 +27,12 @@ Generated on `2026-03-07` from route handlers. | `PUT` | `/{path:path}` | | `DELETE` | `/{path:path}` | | `POST` | `/publish` | +| `GET` | `/admin` | +| `GET` | `/admin/api/state` | +| `POST` | `/admin/api/config` | +| `POST` | `/admin/api/start` | +| `POST` | `/admin/api/restart` | +| `POST` | `/admin/api/start-worker` | +| `POST` | `/admin/api/restart-worker` | Do not edit manually. Regenerate with `python3 scripts/generate_context_artifacts.py`. \ No newline at end of file diff --git a/docs/generated/env-catalog.md b/docs/generated/env-catalog.md index e09f48d..d8a6921 100644 --- a/docs/generated/env-catalog.md +++ b/docs/generated/env-catalog.md @@ -1,7 +1,7 @@ --- owner: platform status: generated -last_verified: 2026-03-07 +last_verified: 2026-03-12 source_of_truth: - ../../kb-server/.env.example - ../../kb-server/app/core/config.py @@ -16,7 +16,7 @@ review_cycle_days: 7 # Environment Catalog (Generated) -Generated on `2026-03-07` from settings and env sources. +Generated on `2026-03-12` from settings and env sources. ## kb-server `.env.example` @@ -36,6 +36,9 @@ Generated on `2026-03-07` from settings and env sources. | `GITHUB_REPO` | `owner/repo` | | `QUARTZ_BUILD_COMMAND` | `` | | `QUARTZ_WEBHOOK_URL` | `` | +| `ADMIN_TMUX_SESSION` | `kb-api` | +| `ADMIN_TMUX_WORKER_SESSION` | `kb-worker` | +| `ADMIN_TMUX_WORKDIR` | `/absolute/path/to/flight-deck/kb-server` | | `API_HOST` | `0.0.0.0` | | `API_PORT` | `8000` | @@ -57,6 +60,9 @@ Generated on `2026-03-07` from settings and env sources. | `git_pull_interval_seconds` | `60` | | `quartz_build_command` | `""` | | `quartz_webhook_url` | `""` | +| `admin_tmux_session` | `"kb-api"` | +| `admin_tmux_worker_session` | `"kb-worker"` | +| `admin_tmux_workdir` | `Path("/srv/flightdeck/kb-server")` | | `api_host` | `"0.0.0.0"` | | `api_port` | `8000` | diff --git a/docs/generated/stale-docs-report.md b/docs/generated/stale-docs-report.md index ef74303..7217f11 100644 --- a/docs/generated/stale-docs-report.md +++ b/docs/generated/stale-docs-report.md @@ -1,7 +1,7 @@ --- owner: platform status: generated -last_verified: 2026-03-07 +last_verified: 2026-03-12 source_of_truth: - ../../scripts/docs_garden.py related_code: @@ -14,7 +14,7 @@ review_cycle_days: 7 # Stale Documentation Report -Generated: `2026-03-07` +Generated: `2026-03-12` ## Ownership Summary diff --git a/docs/product-specs/kb-server.md b/docs/product-specs/kb-server.md index ce86721..65f2cf4 100644 --- a/docs/product-specs/kb-server.md +++ b/docs/product-specs/kb-server.md @@ -4,13 +4,16 @@ status: verified last_verified: 2026-03-07 source_of_truth: - ../../kb-server/app/api/routes/notes.py + - ../../kb-server/app/api/routes/admin.py - ../../kb-server/app/services/current_view_service.py - ../../kb-server/app/core/config.py related_code: - ../../kb-server/app/services/git_batcher.py + - ../../kb-server/app/services/admin_service.py - ../../kb-server/app/workers/autosave.py related_tests: - ../../kb-server/tests/test_current_view.py + - ../../kb-server/tests/test_admin.py - ../../kb-server/tests/test_source_and_delete.py review_cycle_days: 14 --- @@ -30,18 +33,31 @@ Provide a file-first API over a Git-backed vault with explicit approval boundari - `source=api` for PR-based pending writes - `source=human` for direct approved writes - Writes to `view=current` are rejected. +- `GET /admin` exposes an operator-facing setup and status surface. +- `GET /admin/api/state` returns admin config/status state for the local instance. +- `POST /admin/api/config` writes local configuration updates to `kb-server/.env`. +- `POST /admin/api/start` launches the derived tmux start command for the API. +- `POST /admin/api/restart` launches the derived tmux restart command for the API. +- `POST /admin/api/start-worker` launches the derived tmux start command for the autosave worker. +- `POST /admin/api/restart-worker` launches the derived tmux restart command for the autosave worker. +- `app/streamlit_admin.py` provides an operator dashboard backed by the admin API. ## Approval Model - API-origin changes are batched to daily `kb-api/*` branches and PRs. - Human-origin changes go to base branch directly. - Mainline approval remains controlled by maintainers. +- The admin UI is not a note-editing surface and does not bypass PR-based approval for content changes. ## Guardrails - Allowed file extensions: `.md`, `.markdown`, `.txt`. - No absolute paths and no traversal outside vault root. -- API key auth enforced when configured. +- API key auth is enforced on non-admin API routes when configured. +- Admin routes are intentionally available without `X-API-Key` so the local dashboard can bootstrap and operate the instance. +- Admin config writes are local instance management actions only; operators must still provision the DB, vault repo, and host runtime outside the browser. +- Admin runtime control derives the API and worker tmux commands from `ADMIN_TMUX_SESSION`, `ADMIN_TMUX_WORKER_SESSION`, `ADMIN_TMUX_WORKDIR`, `API_HOST`, and `API_PORT`. +- Process environment variables override `.env`, including when `.env` is edited through `/admin`. ## Related Operational Docs @@ -49,4 +65,3 @@ Provide a file-first API over a Git-backed vault with explicit approval boundari - `../../kb-server/BRANCHING_AND_CURRENT_VIEW.md` - `../SECURITY.md` - `../RELIABILITY.md` - diff --git a/kb-server/.env.example b/kb-server/.env.example index 2dc703a..34834de 100644 --- a/kb-server/.env.example +++ b/kb-server/.env.example @@ -5,6 +5,7 @@ VAULT_PATH=/Users/yashturkar/Downloads/flightdeck/vault DATABASE_URL=postgresql://kb:kb@localhost:5432/kb # API key — every request must include X-API-Key header when set. +# Recommended: leave blank here and export KB_API_KEY in your shell or service env. # Leave blank to disable auth (development only). KB_API_KEY= @@ -22,6 +23,7 @@ GIT_BATCH_DEBOUNCE_SECONDS=10 GIT_BATCH_BRANCH_PREFIX=kb-api # GitHub API for PR creation (required for API write workflow) +# Recommended: leave blank here and export GITHUB_TOKEN in your shell or service env. # Create a token at https://github.com/settings/tokens with 'repo' scope GITHUB_TOKEN= GITHUB_REPO=owner/repo @@ -30,6 +32,12 @@ GITHUB_REPO=owner/repo QUARTZ_BUILD_COMMAND= QUARTZ_WEBHOOK_URL= +# Streamlit dashboard tmux control +# The dashboard derives API/worker start/restart commands from these values. +ADMIN_TMUX_SESSION=kb-api +ADMIN_TMUX_WORKER_SESSION=kb-worker +ADMIN_TMUX_WORKDIR=/absolute/path/to/flight-deck/kb-server + # API API_HOST=0.0.0.0 API_PORT=8000 diff --git a/kb-server/AGENTS.md b/kb-server/AGENTS.md index e941a3c..e10b98f 100644 --- a/kb-server/AGENTS.md +++ b/kb-server/AGENTS.md @@ -134,6 +134,81 @@ python3 -m pytest tests/test_git_service.py tests/test_git_batcher.py 2. Document in `.env.example` 3. Update `README.md` environment variables table +## Logged Proposals + +### Admin / Management Web UI + +Requested scope: +- Initial setup flow for required configuration +- UI for viewing and editing relevant configuration values +- Vault and Git/repo configuration visibility +- Status dashboard for API, worker, health, jobs, and recent failures +- Visibility into pending PR workflow state and recent automation activity + +Intent: +- This is an admin surface for setup, operations, and debugging +- This is not a content editing UI for notes +- Content-affecting actions must continue to respect the PR-based workflow where applicable + +Recommended rollout: +1. Read-only admin dashboard + - Health/readiness + - Config validation results + - Vault/Git status summary + - Recent jobs, errors, publish runs, and worker activity +2. Setup + config management + - Guided first-run setup for `VAULT_PATH`, `DATABASE_URL`, `GITHUB_REPO`, branch settings, and optional auth config + - Editable non-secret config with validation +3. Workflow visibility + - Pending batched changes + - Open `kb-api/*` branches / PR state + - Recent sync/autosave/publish outcomes +4. Secret management + - Prefer write-only secret updates + - Do not casually echo `KB_API_KEY` or `GITHUB_TOKEN` back to the browser + - Preserve support for process environment variables overriding `.env` + +Implementation constraints: +- Treat the UI as an admin plane layered on top of existing API/worker behavior +- Do not silently provision system services or host-level dependencies from the browser +- Be explicit about which values are stored in `.env` versus provided by environment variables +- Add strong admin authentication before exposing any state-changing operational actions +- Update docs with the trust boundary and secret-handling model when implemented + +Current bootstrap behavior: +- `/admin` is now available as an initial admin surface +- it shows config, readiness, vault/db/git status, recent jobs/events, and PR summary +- it can write `.env` updates for visible config fields +- it exposes write-only inputs for `KB_API_KEY` and `GITHUB_TOKEN` +- `app/streamlit_admin.py` provides a prettier operator dashboard over the same admin API +- the Streamlit dashboard can derive tmux-based `kb-api` and `kb-worker` start/restart commands from `.env` + +Current operator flow: +1. Start the Streamlit dashboard. +2. If `kb-api` is offline, use the configured start command from the dashboard. +3. Open `/admin` or use the Streamlit dashboard against the running backend. +4. Enter non-secret instance config such as `VAULT_PATH`, `DATABASE_URL`, and `GITHUB_REPO`. +5. Save config, which writes `kb-server/.env`. +6. Restart `kb-api` and `kb-worker`. +7. Reopen `/admin` or rerun Streamlit and verify readiness and integration state. + +Current limitations: +- `/admin` does not provision PostgreSQL, create DB roles, or create databases +- `/admin` does not create the notes repo or GitHub repo +- `/admin` and Streamlit assume `tmux` is installed and only manage the API/worker processes for the configured workdir/session names +- host-level setup still happens outside the browser + +Secret model: +- Preferred: establish `KB_API_KEY` and `GITHUB_TOKEN` as process environment variables +- Supported for local/dev use: save `KB_API_KEY` and `GITHUB_TOKEN` through the admin UI into `.env` +- Do not treat `GITHUB_REPO` as a secret; it should normally live in `.env` +- Process environment still overrides `.env` + +Admin auth model: +- `KB_API_KEY` still protects the note and publish APIs +- `/admin` and `/admin/api/*` are intentionally exempt from `X-API-Key` so the local dashboard can bootstrap and operate the instance +- Treat the dashboard as a local operator surface, not an internet-facing control plane + ## Dependencies - Python 3.10+ diff --git a/kb-server/README.md b/kb-server/README.md index a8afee8..c69f4de 100644 --- a/kb-server/README.md +++ b/kb-server/README.md @@ -8,37 +8,40 @@ File-first knowledge base server. Watches a Markdown vault, auto-commits to Git, - Git 2.34+ - PostgreSQL 15+ + ## Quick start (development) ```bash # Clone and enter the project cd kb-server -# Create virtualenv and install python -m venv .venv source .venv/bin/activate pip install -e ".[dev]" -# Copy and edit environment -cp .env.example .env -# Edit .env: set VAULT_PATH, DATABASE_URL, etc. +cp .env.example .env # edit .env -# Create the database -createdb kb +# Optional secrets: prefer exporting these in your shell instead of storing them in .env +# Non-secret repo config like GITHUB_REPO should stay in .env +export KB_API_KEY=dev-key +export GITHUB_TOKEN=ghp_your_token -# Run migrations -alembic upgrade head +psql postgres +CREATE ROLE kb WITH LOGIN PASSWORD 'kb'; +CREATE DATABASE kb OWNER kb; # or ALTER DATABASE kb OWNER TO kb; +\q # to quit -# Start the API +python -m alembic upgrade head python -m uvicorn app.main:app --reload - -# In another terminal, start the autosave worker +# in another terminal: python -m app.workers.autosave ``` ## Vault setup -The vault must be an existing Git repository: +`kb-server` does not create your notes repository for you. Create the notes repo manually first, then set `VAULT_PATH` in `.env` to that existing local Git repository. + +Example: ```bash mkdir -p /srv/flightdeck/vault @@ -47,7 +50,7 @@ git init git remote add origin git@github.com:you/your-vault.git ``` -Create the directory structure you want (these are conventions, not enforced): +After that, point `VAULT_PATH` at this repo. The directory structure inside the repo is up to you. These folders are common conventions, but not required by the server: ```text vault/ @@ -77,10 +80,14 @@ vault/ | `GITHUB_REPO` | (empty) | GitHub repository in `owner/repo` format | | `QUARTZ_BUILD_COMMAND` | (empty) | Shell command to build Quartz site | | `QUARTZ_WEBHOOK_URL` | (empty) | URL to POST after push to trigger rebuild | +| `ADMIN_TMUX_SESSION` | `kb-api` | tmux session name used by the Streamlit dashboard to manage the API | +| `ADMIN_TMUX_WORKER_SESSION` | `kb-worker` | tmux session name used by the Streamlit dashboard to manage the autosave worker | +| `ADMIN_TMUX_WORKDIR` | `/srv/flightdeck/kb-server` | Absolute `kb-server` path used to build the tmux start/restart commands | | `API_HOST` | `0.0.0.0` | API bind address | | `API_PORT` | `8000` | API bind port | > **Note:** Unknown env keys in `.env` are silently ignored, so extra variables won't break startup. +> **Note:** Process environment variables override values from `.env`. Keep long-lived machine config in `.env`, including non-secret values like `GITHUB_REPO`, and prefer exporting secrets like `KB_API_KEY` and `GITHUB_TOKEN` from your shell, `tmux`, or service manager. > **Note:** `GITHUB_TOKEN` is used for GitHub API PR calls, not for `git push/pull` auth. > Git CLI operations run non-interactively and require preconfigured credentials (SSH key or PAT-backed credential helper). @@ -136,16 +143,147 @@ FastAPI also exposes interactive docs: - Swagger UI: `GET /docs` - OpenAPI JSON: `GET /openapi.json` +- Admin UI: `GET /admin` + +Streamlit dashboard: + +```bash +cd kb-server +./.venv/bin/streamlit run app/streamlit_admin.py +``` + +## Admin UI + +`/admin` is a lightweight management surface for setup and operations. The initial version includes: + +- current config visibility for the main `.env` fields +- write support for updating `.env` from the browser +- write-only secret update fields for `KB_API_KEY` and `GITHUB_TOKEN` +- readiness, vault, database, Git, and pending PR workflow status +- recent jobs, vault events, and publish runs +- `kb-api` and `kb-worker` start/restart support through derived tmux commands based on `ADMIN_TMUX_SESSION`, `ADMIN_TMUX_WORKER_SESSION`, and `ADMIN_TMUX_WORKDIR` + +Important behavior: + +- The admin UI is not a note editor. +- `/admin` and `/admin/api/*` are intentionally available without `X-API-Key` so the local dashboard can bootstrap and manage the instance. +- Process environment variables still override `.env`. +- Saving config writes to `.env`, but you should restart `kb-api` and `kb-worker` after changing database or auth settings. + +### Streamlit Dashboard + +The repo also includes a Streamlit dashboard backed by the same admin API: + +```bash +cd kb-server +./.venv/bin/streamlit run app/streamlit_admin.py +``` + +The Streamlit dashboard can: + +- view prettified readiness, vault, database, Git, batcher, autosave, jobs, events, publish, and PR status +- update config values, including `GITHUB_TOKEN` +- start and restart `kb-api` if `ADMIN_TMUX_WORKDIR` points at a valid `kb-server` checkout +- start and restart `kb-worker` in its configured tmux session + +The Streamlit start/restart buttons derive the tmux commands locally and asynchronously. Users only need to set `ADMIN_TMUX_SESSION`, `ADMIN_TMUX_WORKER_SESSION`, and `ADMIN_TMUX_WORKDIR`; the dashboard builds the standard `uvicorn` and autosave commands automatically from those values plus `API_HOST` and `API_PORT`. + +If the FastAPI backend is offline, the Streamlit dashboard shows that state instead of crashing. You can then use the sidebar start button to launch the server and rerun the page. + +The dashboard also exposes dedicated autosave feedback: + +- whether the configured worker tmux session is currently running +- the latest autosave job status and timestamps +- the latest autosave commit and push SHA +- the files included in the latest autosave run + +### Using Admin UI For First-Time Setup + +The current admin UI helps configure and validate an instance, but it does not provision host dependencies for you. + +Before using `/admin`, you still need to create or provide: + +- a running PostgreSQL instance +- a database and DB credentials referenced by `DATABASE_URL` +- an existing local notes repository for `VAULT_PATH` +- a configured Git remote if you want push/PR workflows + +First-time setup flow: + +1. Fill in the minimum required `.env` values: + - `DATABASE_URL` + - `VAULT_PATH` + - `GITHUB_REPO` if you want PR workflows + - `ADMIN_TMUX_WORKDIR` pointing at this `kb-server` checkout + - optionally `ADMIN_TMUX_SESSION` if you do not want `kb-api` + - optionally `ADMIN_TMUX_WORKER_SESSION` if you do not want `kb-worker` +2. Run migrations: + ```bash + ./.venv/bin/python -m alembic upgrade head + ``` +3. Start the Streamlit dashboard: + ```bash + cd kb-server + ./.venv/bin/streamlit run app/streamlit_admin.py + ``` +4. If `kb-api` or `kb-worker` is offline, use the dashboard sidebar start buttons. +5. Open `GET /admin` or use the Streamlit dashboard against the running backend. +6. Fill in the remaining non-secret instance config: + - `VAULT_PATH` + - `DATABASE_URL` + - `GITHUB_REPO` + - Git branch / remote settings + - optional Quartz settings +7. Save the form. This writes the values to `kb-server/.env`. +8. Restart `kb-api` and `kb-worker` from the dashboard or your tmux session. +9. Reopen `/admin` or rerun the Streamlit dashboard and verify readiness, vault, database, Git, runtime, and PR status. + +What `/admin` does not do yet: + +- it does not create the Postgres server, role, or database +- it does not create the notes repo for you +- it does not create the GitHub repo or remote +- it assumes `tmux` is installed and derives the API and worker start/restart commands from your configured workdir/session names + +### Secret Handling + +There are two supported ways to establish secrets: + +1. Preferred: process environment variables + - set `KB_API_KEY` and `GITHUB_TOKEN` outside `.env` + - use shell exports, `tmux` startup commands, or service-manager environment config + - process environment values override `.env` + +2. Optional: save through `/admin` + - the admin UI provides write-only fields for `KB_API_KEY` and `GITHUB_TOKEN` + - saving writes them into `kb-server/.env` + - the UI does not read them back after save + +Recommended split: + +- Keep in `.env`: `VAULT_PATH`, `DATABASE_URL`, `GITHUB_REPO`, and other non-secret machine config +- Keep in process env when possible: `KB_API_KEY`, `GITHUB_TOKEN` + +Example: + +```bash +export KB_API_KEY=your_api_key +export GITHUB_TOKEN=your_github_token +python -m uvicorn app.main:app --reload +``` ### Authentication -When `KB_API_KEY` is set, **every** request (including `/health`, `/ready`, -`/docs`, and `/openapi.json`) must include the key: +When `KB_API_KEY` is set, non-admin API requests must include the key. That includes `/health`, `/ready`, `/docs`, `/openapi.json`, `/notes/*`, and `/publish`. + +Example: ```bash curl -H "X-API-Key: YOUR_KEY" http://localhost:8000/health ``` +`/admin` and `/admin/api/*` are intentionally exempt from `X-API-Key` so the local setup and Streamlit dashboard can bootstrap the instance. + Requests without a valid key receive a `401` response. When `KB_API_KEY` is left blank (development mode), no authentication is diff --git a/kb-server/alembic/env.py b/kb-server/alembic/env.py index a9611c3..1b83b4f 100644 --- a/kb-server/alembic/env.py +++ b/kb-server/alembic/env.py @@ -1,8 +1,14 @@ +from pathlib import Path +import sys from logging.config import fileConfig from alembic import context from sqlalchemy import engine_from_config, pool +PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + from app.core.config import settings from app.models.db import Base diff --git a/kb-server/app/api/routes/admin.py b/kb-server/app/api/routes/admin.py new file mode 100644 index 0000000..d30fe01 --- /dev/null +++ b/kb-server/app/api/routes/admin.py @@ -0,0 +1,315 @@ +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import HTMLResponse, JSONResponse +from sqlalchemy.orm import Session + +from app.models.db import get_session +from app.services import admin_service + +router = APIRouter(tags=["admin"]) + +ADMIN_HTML = """ + + + + + KB Server Admin + + + +
+
+
+

KB Server Admin

+

Setup, config, and runtime visibility for the local KB server instance.

+
+
+ +
+
+ +
+
+

System Status

+
+
+
+
+

Setup Notes

+
    +
  • Point VAULT_PATH at an existing local Git repo.
    • +
  • Process env still overrides .env values.
    • +
  • Changing database or auth config usually requires restarting kb-api and kb-worker.
    • +
+
+
+ +
+
+

Configuration

+
+
+
+ + +
+
+
+
+

Vault and Git

+
+
+
+ +
+
+

Pending PR Workflow

+
+
+
+

Recent Jobs

+
+
+
+ +
+
+

Recent Vault Events

+
+
+
+

Publish Runs

+
+
+
+
+ + + +""" + + +@router.get("/admin", response_class=HTMLResponse) +def admin_page() -> HTMLResponse: + return HTMLResponse(ADMIN_HTML) + + +@router.get("/admin/api/state") +def admin_state(session: Session = Depends(get_session)) -> dict[str, Any]: + return { + "config": admin_service.current_config_view(), + "state": admin_service.system_state(session), + } + + +@router.post("/admin/api/config") +async def admin_update_config(request: Request) -> JSONResponse: + payload = await request.json() + config = payload.get("config", {}) + if not isinstance(config, dict): + raise HTTPException(status_code=400, detail="config must be an object") + + allowed = set(admin_service.VISIBLE_CONFIG_KEYS) | set(admin_service.SECRET_CONFIG_KEYS) + updates: dict[str, str] = {} + + for key, value in config.items(): + if key not in allowed: + continue + if key in admin_service.SECRET_CONFIG_KEYS and not value: + continue + updates[key] = str(value) + + result = admin_service.update_env_file(updates) + return JSONResponse( + { + "message": "Saved to .env. Restart kb-api and kb-worker to apply auth/database changes reliably.", + **result, + } + ) + + +@router.post("/admin/api/start") +def admin_start() -> JSONResponse: + runtime = admin_service.runtime_control_state()["api"] + if not runtime["workdir_exists"]: + raise HTTPException(status_code=501, detail="ADMIN_TMUX_WORKDIR does not exist") + if not runtime["venv_python_exists"]: + raise HTTPException(status_code=501, detail="Configured workdir is missing .venv/bin/python") + + admin_service.launch_command(admin_service.backend_start_command()) + return JSONResponse({"message": "Start command launched"}) + + +@router.post("/admin/api/restart") +def admin_restart() -> JSONResponse: + runtime = admin_service.runtime_control_state()["api"] + if not runtime["workdir_exists"]: + raise HTTPException(status_code=501, detail="ADMIN_TMUX_WORKDIR does not exist") + if not runtime["venv_python_exists"]: + raise HTTPException(status_code=501, detail="Configured workdir is missing .venv/bin/python") + + admin_service.launch_command(admin_service.backend_restart_command()) + return JSONResponse({"message": "Restart command launched"}) + + +@router.post("/admin/api/start-worker") +def admin_start_worker() -> JSONResponse: + runtime = admin_service.runtime_control_state()["worker"] + if not runtime["workdir_exists"]: + raise HTTPException(status_code=501, detail="ADMIN_TMUX_WORKDIR does not exist") + if not runtime["venv_python_exists"]: + raise HTTPException(status_code=501, detail="Configured workdir is missing .venv/bin/python") + + admin_service.launch_command(admin_service.worker_start_command()) + return JSONResponse({"message": "Worker start command launched"}) + + +@router.post("/admin/api/restart-worker") +def admin_restart_worker() -> JSONResponse: + runtime = admin_service.runtime_control_state()["worker"] + if not runtime["workdir_exists"]: + raise HTTPException(status_code=501, detail="ADMIN_TMUX_WORKDIR does not exist") + if not runtime["venv_python_exists"]: + raise HTTPException(status_code=501, detail="Configured workdir is missing .venv/bin/python") + + admin_service.launch_command(admin_service.worker_restart_command()) + return JSONResponse({"message": "Worker restart command launched"}) diff --git a/kb-server/app/core/auth.py b/kb-server/app/core/auth.py index 58b56b0..17c194c 100644 --- a/kb-server/app/core/auth.py +++ b/kb-server/app/core/auth.py @@ -28,7 +28,11 @@ async def dispatch( if not expected: return await call_next(request) + if request.url.path.startswith("/admin"): + return await call_next(request) + provided = request.headers.get("X-API-Key", "") + if not provided or not hmac.compare_digest(provided, expected): log.warning( "rejected request %s %s — invalid or missing API key", diff --git a/kb-server/app/core/config.py b/kb-server/app/core/config.py index 733378f..0d2f669 100644 --- a/kb-server/app/core/config.py +++ b/kb-server/app/core/config.py @@ -33,6 +33,9 @@ class Settings(BaseSettings): quartz_build_command: str = "" quartz_webhook_url: str = "" + admin_tmux_session: str = "kb-api" + admin_tmux_worker_session: str = "kb-worker" + admin_tmux_workdir: Path = Path("/srv/flightdeck/kb-server") api_host: str = "0.0.0.0" api_port: int = 8000 diff --git a/kb-server/app/main.py b/kb-server/app/main.py index 30f9a56..89b325f 100644 --- a/kb-server/app/main.py +++ b/kb-server/app/main.py @@ -41,10 +41,12 @@ def create_app() -> FastAPI: app.add_middleware(APIKeyMiddleware) from app.api.routes.health import router as health_router + from app.api.routes.admin import router as admin_router from app.api.routes.notes import router as notes_router from app.api.routes.publish import router as publish_router app.include_router(health_router) + app.include_router(admin_router) app.include_router(notes_router) app.include_router(publish_router) diff --git a/kb-server/app/services/admin_service.py b/kb-server/app/services/admin_service.py new file mode 100644 index 0000000..c8251c1 --- /dev/null +++ b/kb-server/app/services/admin_service.py @@ -0,0 +1,392 @@ +from __future__ import annotations + +import os +import re +import shlex +import subprocess +from pathlib import Path +from typing import Any + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.core.config import Settings, settings +from app.models.db import Job, PublishRun, VaultEvent +from app.services import git_service, github_service +from app.services.git_batcher import batcher + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +ENV_FILE_PATH = PROJECT_ROOT / ".env" + +VISIBLE_CONFIG_KEYS = ( + "VAULT_PATH", + "DATABASE_URL", + "GIT_REMOTE", + "GIT_BRANCH", + "GIT_PUSH_ENABLED", + "GIT_BATCH_DEBOUNCE_SECONDS", + "GIT_BATCH_BRANCH_PREFIX", + "GITHUB_REPO", + "AUTOSAVE_DEBOUNCE_SECONDS", + "GIT_PULL_INTERVAL_SECONDS", + "QUARTZ_BUILD_COMMAND", + "QUARTZ_WEBHOOK_URL", + "ADMIN_TMUX_SESSION", + "ADMIN_TMUX_WORKER_SESSION", + "ADMIN_TMUX_WORKDIR", + "API_HOST", + "API_PORT", +) +SECRET_CONFIG_KEYS = ("KB_API_KEY", "GITHUB_TOKEN") +CONFIG_KEY_PATTERN = re.compile(r"^\s*([A-Z0-9_]+)\s*=(.*)$") + + +def _load_env_file_map() -> dict[str, str]: + if not ENV_FILE_PATH.exists(): + return {} + + values: dict[str, str] = {} + for line in ENV_FILE_PATH.read_text(encoding="utf-8").splitlines(): + match = CONFIG_KEY_PATTERN.match(line) + if not match: + continue + key, value = match.groups() + values[key] = value.strip() + return values + + +def _stringify(value: Any) -> str: + if isinstance(value, bool): + return str(value).lower() + return str(value) + + +def _setting_value(key: str) -> str: + attr = key.lower() + value = getattr(settings, attr) + return _stringify(value) + + +def _coerce_setting_value(key: str, value: str) -> Any: + field = Settings.model_fields[key.lower()] + annotation = field.annotation + + if annotation is Path: + return Path(value) + if annotation is bool: + return value.strip().lower() in {"1", "true", "yes", "on"} + if annotation is int: + return int(value) + return value + + +def _value_source(key: str, env_file_values: dict[str, str]) -> str: + if key in os.environ: + return "environment" + if key in env_file_values: + return ".env" + return "default" + + +def current_config_view() -> list[dict[str, Any]]: + env_file_values = _load_env_file_map() + + rows = [ + { + "key": key, + "value": _setting_value(key), + "source": _value_source(key, env_file_values), + "secret": False, + } + for key in VISIBLE_CONFIG_KEYS + ] + + for key in SECRET_CONFIG_KEYS: + rows.append( + { + "key": key, + "value": "", + "source": _value_source(key, env_file_values), + "secret": True, + "configured": bool(getattr(settings, key.lower())), + } + ) + + return rows + + +def update_env_file(updates: dict[str, str]) -> dict[str, Any]: + ENV_FILE_PATH.parent.mkdir(parents=True, exist_ok=True) + existing_lines = [] + if ENV_FILE_PATH.exists(): + existing_lines = ENV_FILE_PATH.read_text(encoding="utf-8").splitlines() + + pending = {key: value for key, value in updates.items() if value is not None} + written_keys: list[str] = [] + new_lines: list[str] = [] + + for line in existing_lines: + match = CONFIG_KEY_PATTERN.match(line) + if not match: + new_lines.append(line) + continue + + key = match.group(1) + if key not in pending: + new_lines.append(line) + continue + + new_lines.append(f"{key}={pending[key]}") + written_keys.append(key) + del pending[key] + + for key, value in pending.items(): + if new_lines and new_lines[-1] != "": + new_lines.append("") + new_lines.append(f"{key}={value}") + written_keys.append(key) + + ENV_FILE_PATH.write_text("\n".join(new_lines).rstrip() + "\n", encoding="utf-8") + + for key, value in updates.items(): + if value is None: + continue + setattr(settings, key.lower(), _coerce_setting_value(key, value)) + + return { + "written_keys": written_keys, + "env_file": str(ENV_FILE_PATH), + "restart_required": True, + } + + +def launch_command(command: str) -> None: + subprocess.Popen( + command, + shell=True, + start_new_session=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + +def _tmux_session_exists(session_name: str) -> bool | None: + try: + result = subprocess.run( + ["tmux", "has-session", "-t", session_name], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except FileNotFoundError: + return None + return result.returncode == 0 + + +def _tmux_api_command() -> str: + workdir = settings.admin_tmux_workdir + return ( + f"cd {shlex.quote(str(workdir))} && " + "source .venv/bin/activate && " + f"python -m uvicorn app.main:app --host {settings.api_host} --port {settings.api_port}" + ) + + +def _tmux_worker_command() -> str: + workdir = settings.admin_tmux_workdir + return ( + f"cd {shlex.quote(str(workdir))} && " + "source .venv/bin/activate && " + "python -m app.workers.autosave" + ) + + +def _tmux_process_state(session_name: str, command: str) -> dict[str, Any]: + workdir = settings.admin_tmux_workdir + venv_python = workdir / ".venv" / "bin" / "python" + return { + "tmux_session": session_name, + "session_running": _tmux_session_exists(session_name), + "workdir": str(workdir), + "workdir_exists": workdir.is_dir(), + "venv_python_exists": venv_python.exists(), + "start_command": ( + f"tmux new-session -d -s {shlex.quote(session_name)} '{command}'" + ), + "restart_command": ( + f"tmux respawn-pane -k -t {shlex.quote(session_name)}:0.0 '{command}'" + ), + } + + +def runtime_control_state() -> dict[str, Any]: + return { + "api": _tmux_process_state(settings.admin_tmux_session, _tmux_api_command()), + "worker": _tmux_process_state(settings.admin_tmux_worker_session, _tmux_worker_command()), + } + + +def backend_start_command() -> str: + return runtime_control_state()["api"]["start_command"] + + +def backend_restart_command() -> str: + return runtime_control_state()["api"]["restart_command"] + + +def worker_start_command() -> str: + return runtime_control_state()["worker"]["start_command"] + + +def worker_restart_command() -> str: + return runtime_control_state()["worker"]["restart_command"] + + +def system_state(session: Session) -> dict[str, Any]: + ready_errors: list[str] = [] + runtime = runtime_control_state() + + try: + session.execute(text("SELECT 1")) + db_status = "ok" + except Exception as exc: + db_status = "error" + ready_errors.append(f"db: {exc}") + + vault_path = settings.vault_path + vault_exists = vault_path.is_dir() + vault_is_git = (vault_path / ".git").is_dir() + if not vault_exists: + ready_errors.append(f"vault directory missing: {vault_path}") + elif not vault_is_git: + ready_errors.append(f"vault is not a git repo: {vault_path}") + + git_summary: dict[str, Any] + try: + git_summary = { + "branch": git_service.current_branch(), + "has_changes": git_service.has_changes(), + "current_sha": git_service.current_sha(), + } + except Exception as exc: + git_summary = {"error": str(exc)} + + pr_summary: dict[str, Any] + try: + prs = github_service.list_open_kb_api_prs() + pr_summary = { + "count": len(prs), + "items": [ + { + "number": pr.get("number"), + "title": pr.get("title"), + "url": pr.get("html_url"), + "head": pr.get("head", {}).get("ref"), + } + for pr in prs[:10] + ], + } + except Exception as exc: + pr_summary = {"error": str(exc), "count": 0, "items": []} + + jobs = ( + session.query(Job) + .order_by(Job.created_at.desc()) + .limit(10) + .all() + ) + events = ( + session.query(VaultEvent) + .order_by(VaultEvent.created_at.desc()) + .limit(10) + .all() + ) + publish_runs = ( + session.query(PublishRun) + .order_by(PublishRun.started_at.desc()) + .limit(10) + .all() + ) + latest_autosave_job = ( + session.query(Job) + .filter(Job.job_type == "autosave") + .order_by(Job.created_at.desc()) + .first() + ) + latest_autosave_commit = ( + session.query(VaultEvent) + .filter(VaultEvent.event_type == "autosave_commit") + .order_by(VaultEvent.created_at.desc()) + .first() + ) + latest_autosave_push = ( + session.query(VaultEvent) + .filter(VaultEvent.event_type == "autosave_push") + .order_by(VaultEvent.created_at.desc()) + .first() + ) + + return { + "ready": not ready_errors, + "ready_errors": ready_errors, + "vault": { + "path": str(vault_path), + "exists": vault_exists, + "is_git_repo": vault_is_git, + }, + "database": {"status": db_status}, + "git": git_summary, + "prs": pr_summary, + "batcher": { + "pending_count": batcher.pending_count(), + "pending_paths": batcher.pending_paths(), + }, + "runtime": runtime, + "autosave": { + "worker_session_running": runtime["worker"]["session_running"], + "latest_job_status": latest_autosave_job.status if latest_autosave_job else None, + "latest_job_created_at": latest_autosave_job.created_at.isoformat() if latest_autosave_job and latest_autosave_job.created_at else None, + "latest_job_completed_at": latest_autosave_job.completed_at.isoformat() if latest_autosave_job and latest_autosave_job.completed_at else None, + "latest_job_error": latest_autosave_job.error if latest_autosave_job else None, + "latest_job_files": ( + latest_autosave_job.meta.get("files", []) + if latest_autosave_job and isinstance(latest_autosave_job.meta, dict) + else [] + ), + "latest_commit_sha": latest_autosave_commit.commit_sha if latest_autosave_commit else None, + "latest_commit_at": latest_autosave_commit.created_at.isoformat() if latest_autosave_commit and latest_autosave_commit.created_at else None, + "latest_push_sha": latest_autosave_push.commit_sha if latest_autosave_push else None, + "latest_push_at": latest_autosave_push.created_at.isoformat() if latest_autosave_push and latest_autosave_push.created_at else None, + }, + "jobs": [ + { + "id": job.id, + "type": job.job_type, + "status": job.status, + "created_at": job.created_at.isoformat() if job.created_at else None, + "error": job.error, + } + for job in jobs + ], + "events": [ + { + "id": event.id, + "type": event.event_type, + "file_path": event.file_path, + "commit_sha": event.commit_sha, + "created_at": event.created_at.isoformat() if event.created_at else None, + } + for event in events + ], + "publish_runs": [ + { + "id": run.id, + "trigger": run.trigger, + "status": run.status, + "started_at": run.started_at.isoformat() if run.started_at else None, + "completed_at": run.completed_at.isoformat() if run.completed_at else None, + "error": run.error, + } + for run in publish_runs + ], + } diff --git a/kb-server/app/services/git_batcher.py b/kb-server/app/services/git_batcher.py index 1a99f06..82b039a 100644 --- a/kb-server/app/services/git_batcher.py +++ b/kb-server/app/services/git_batcher.py @@ -57,6 +57,14 @@ def is_api_owned(self, path: str) -> bool: with self._lock: return path in self._api_owned + def pending_count(self) -> int: + with self._lock: + return len(self._pending) + + def pending_paths(self) -> list[str]: + with self._lock: + return sorted(self._pending) + def _reset_timer(self) -> None: if self._timer is not None: self._timer.cancel() diff --git a/kb-server/app/streamlit_admin.py b/kb-server/app/streamlit_admin.py new file mode 100644 index 0000000..1333069 --- /dev/null +++ b/kb-server/app/streamlit_admin.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +import httpx +import streamlit as st + +from app.services import admin_service + + +def backend_url() -> str: + return st.session_state.get("backend_url", "http://localhost:8000").rstrip("/") + + +def api_request(method: str, path: str, *, json: dict | None = None) -> httpx.Response: + with httpx.Client(timeout=10) as client: + return client.request(method, f"{backend_url()}{path}", json=json) + + +def try_api_request(method: str, path: str, *, json: dict | None = None) -> tuple[httpx.Response | None, str | None]: + try: + return api_request(method, path, json=json), None + except httpx.HTTPError as exc: + return None, str(exc) + + +def run_local_command(command: str, missing_message: str) -> None: + if not command: + st.error(missing_message) + return + admin_service.launch_command(command) + st.success("Command launched") + + +def format_value(value: object) -> str: + if value is None: + return "-" + if isinstance(value, bool): + return "Yes" if value else "No" + if isinstance(value, list): + return ", ".join(str(item) for item in value) if value else "-" + if value == "": + return "-" + return str(value) + + +def render_kv(title: str, values: dict) -> None: + st.subheader(title) + rows = [{"Field": key.replace("_", " ").title(), "Value": format_value(value)} for key, value in values.items()] + st.dataframe(rows, use_container_width=True, hide_index=True) + + +def render_table(title: str, rows: list[dict], *, empty_message: str) -> None: + st.subheader(title) + if not rows: + st.caption(empty_message) + return + formatted_rows = [] + for row in rows: + formatted_rows.append( + {key.replace("_", " ").title(): format_value(value) for key, value in row.items()} + ) + st.dataframe(formatted_rows, use_container_width=True, hide_index=True) + + +def normalize_runtime_state(runtime: dict | None) -> tuple[dict, dict]: + if not runtime: + current = admin_service.runtime_control_state() + return current["api"], current["worker"] + + if "api" in runtime and "worker" in runtime: + return runtime["api"], runtime["worker"] + + fallback = admin_service.runtime_control_state() + return runtime, fallback["worker"] + + +st.set_page_config(page_title="KB Server Dashboard", layout="wide") +st.title("KB Server Dashboard") + +st.sidebar.text_input("Backend URL", value="http://localhost:8000", key="backend_url") +runtime_state = admin_service.runtime_control_state() +api_runtime = runtime_state["api"] +worker_runtime = runtime_state["worker"] + +with st.sidebar: + st.subheader("API Control") + api_ready = api_runtime["workdir_exists"] and api_runtime["venv_python_exists"] + if st.button("Start kb-api", disabled=not api_ready): + run_local_command( + admin_service.backend_start_command(), + "Set ADMIN_TMUX_WORKDIR and ADMIN_TMUX_SESSION in .env first", + ) + if st.button("Restart kb-api", disabled=not api_ready): + run_local_command( + admin_service.backend_restart_command(), + "Set ADMIN_TMUX_WORKDIR and ADMIN_TMUX_SESSION in .env first", + ) + st.caption(f"session: `{api_runtime['tmux_session']}`") + + st.subheader("Worker Control") + worker_ready = worker_runtime["workdir_exists"] and worker_runtime["venv_python_exists"] + if st.button("Start kb-worker", disabled=not worker_ready): + run_local_command( + admin_service.worker_start_command(), + "Set ADMIN_TMUX_WORKDIR and ADMIN_TMUX_WORKER_SESSION in .env first", + ) + if st.button("Restart kb-worker", disabled=not worker_ready): + run_local_command( + admin_service.worker_restart_command(), + "Set ADMIN_TMUX_WORKDIR and ADMIN_TMUX_WORKER_SESSION in .env first", + ) + st.caption(f"session: `{worker_runtime['tmux_session']}`") + st.caption(f"workdir: `{api_runtime['workdir']}`") + + if not api_runtime["workdir_exists"]: + st.warning("Configured tmux workdir does not exist yet.") + elif not api_runtime["venv_python_exists"]: + st.warning("Configured workdir is missing `.venv/bin/python`.") + +state_response, state_error = try_api_request("GET", "/admin/api/state") +if state_response is None: + st.warning(f"Backend offline: {state_error}") + st.info("Use the sidebar controls to start the server, then click Rerun in Streamlit.") + st.stop() + +if not state_response.is_success: + st.error(state_response.text) + st.stop() + +payload = state_response.json() +state = payload["state"] +config_rows = payload["config"] +runtime_state = state["runtime"] +api_runtime, worker_runtime = normalize_runtime_state(runtime_state) + +col1, col2, col3, col4, col5 = st.columns(5) +col1.metric("Ready", "yes" if state["ready"] else "no") +col2.metric("DB", state["database"]["status"]) +col3.metric("Pending Batch", state["batcher"]["pending_count"]) +col4.metric("Open KB PRs", state["prs"]["count"]) +col5.metric("Autosave", format_value(state["autosave"]["latest_job_status"])) + +if state["ready_errors"]: + st.warning("\n".join(state["ready_errors"])) + +st.subheader("Configuration") +with st.form("config"): + updates: dict[str, str] = {} + left, right = st.columns(2) + for idx, row in enumerate(config_rows): + target = left if idx % 2 == 0 else right + label = f"{row['key']}" + help_text = f"Source: {row['source']}" + with target: + if row["secret"]: + updates[row["key"]] = st.text_input(label, type="password", help=help_text) + else: + updates[row["key"]] = st.text_input( + label, + value=row["value"], + help=help_text, + ) + if st.form_submit_button("Save Config"): + save_response, save_error = try_api_request("POST", "/admin/api/config", json={"config": updates}) + if save_response is None: + st.error(save_error) + st.stop() + if save_response.is_success: + st.success(save_response.json()["message"]) + else: + st.error(save_response.text) + +info_left, info_right = st.columns(2) +with info_left: + render_kv("Vault", state["vault"]) +with info_right: + render_kv("Git", state["git"]) + +ops_left, ops_right = st.columns(2) +with ops_left: + render_kv("Database", state["database"]) +with ops_right: + render_kv("Batcher", state["batcher"]) + +runtime_left, runtime_right = st.columns(2) +with runtime_left: + render_kv("API Runtime", api_runtime) +with runtime_right: + render_kv("Worker Runtime", worker_runtime) + +render_kv("Autosave Status", state["autosave"]) + +with st.expander("Pending PRs", expanded=True): + render_table( + "Open Pull Requests", + state["prs"].get("items", []), + empty_message=state["prs"].get("error", "No open kb-api PRs."), + ) + +with st.expander("Recent Jobs", expanded=True): + render_table("Job Activity", state["jobs"], empty_message="No jobs yet.") + +with st.expander("Recent Events", expanded=False): + render_table("Vault Events", state["events"], empty_message="No events yet.") + +with st.expander("Publish Runs", expanded=False): + render_table("Publish History", state["publish_runs"], empty_message="No publish runs yet.") diff --git a/kb-server/kb_server.egg-info/SOURCES.txt b/kb-server/kb_server.egg-info/SOURCES.txt index 6d27c3a..a2ed103 100644 --- a/kb-server/kb_server.egg-info/SOURCES.txt +++ b/kb-server/kb_server.egg-info/SOURCES.txt @@ -3,6 +3,7 @@ pyproject.toml app/__init__.py app/main.py app/api/__init__.py +app/api/deps.py app/api/routes/__init__.py app/api/routes/health.py app/api/routes/notes.py @@ -16,8 +17,10 @@ app/models/db.py app/schemas/__init__.py app/schemas/notes.py app/services/__init__.py +app/services/current_view_service.py app/services/git_batcher.py app/services/git_service.py +app/services/github_service.py app/services/publish_service.py app/services/vault_service.py app/workers/__init__.py @@ -28,7 +31,9 @@ kb_server.egg-info/dependency_links.txt kb_server.egg-info/requires.txt kb_server.egg-info/top_level.txt tests/test_autosave.py +tests/test_current_view.py tests/test_git_batcher.py tests/test_git_service.py tests/test_notes_api.py +tests/test_source_and_delete.py tests/test_vault_service.py \ No newline at end of file diff --git a/kb-server/tests/test_admin.py b/kb-server/tests/test_admin.py new file mode 100644 index 0000000..436802f --- /dev/null +++ b/kb-server/tests/test_admin.py @@ -0,0 +1,226 @@ +from pathlib import Path + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool +from starlette.requests import Request +from starlette.responses import JSONResponse + +from app.core.auth import APIKeyMiddleware +from app.core.config import settings +from app.models.db import Base, Job, VaultEvent +from app.services import admin_service + + +def test_update_env_file_preserves_existing_content(tmp_path, monkeypatch): + previous_vault = settings.vault_path + previous_key = settings.kb_api_key + previous_push_enabled = settings.git_push_enabled + previous_api_port = settings.api_port + previous_tmux_workdir = settings.admin_tmux_workdir + previous_tmux_worker_session = settings.admin_tmux_worker_session + env_file = tmp_path / ".env" + env_file.write_text( + "# comment\nVAULT_PATH=/tmp/vault\nKB_API_KEY=old-key\n", + encoding="utf-8", + ) + monkeypatch.setattr(admin_service, "ENV_FILE_PATH", env_file) + + try: + result = admin_service.update_env_file( + { + "VAULT_PATH": "/srv/new-vault", + "KB_API_KEY": "new-key", + "GITHUB_REPO": "owner/repo", + "GIT_PUSH_ENABLED": "false", + "API_PORT": "9000", + "ADMIN_TMUX_WORKDIR": "/srv/kb-server", + "ADMIN_TMUX_WORKER_SESSION": "kb-worker-test", + } + ) + + updated = env_file.read_text(encoding="utf-8") + assert "# comment" in updated + assert "VAULT_PATH=/srv/new-vault" in updated + assert "KB_API_KEY=new-key" in updated + assert "GITHUB_REPO=owner/repo" in updated + assert "GIT_PUSH_ENABLED=false" in updated + assert "API_PORT=9000" in updated + assert "ADMIN_TMUX_WORKDIR=/srv/kb-server" in updated + assert "ADMIN_TMUX_WORKER_SESSION=kb-worker-test" in updated + assert result["restart_required"] is True + + assert settings.vault_path == Path("/srv/new-vault") + assert settings.kb_api_key == "new-key" + assert settings.git_push_enabled is False + assert settings.api_port == 9000 + assert settings.admin_tmux_workdir == Path("/srv/kb-server") + assert settings.admin_tmux_worker_session == "kb-worker-test" + finally: + settings.vault_path = previous_vault + settings.kb_api_key = previous_key + settings.git_push_enabled = previous_push_enabled + settings.api_port = previous_api_port + settings.admin_tmux_workdir = previous_tmux_workdir + settings.admin_tmux_worker_session = previous_tmux_worker_session + + +def test_admin_state_hides_secret_values(monkeypatch, tmp_vault: Path): + previous_vault = settings.vault_path + previous_key = settings.kb_api_key + try: + settings.vault_path = tmp_vault + settings.kb_api_key = "secret-key" + monkeypatch.setattr( + admin_service.github_service, + "list_open_kb_api_prs", + lambda: [], + ) + env_file = tmp_vault.parent / ".env" + env_file.write_text("KB_API_KEY=from-file\n", encoding="utf-8") + monkeypatch.setattr(admin_service, "ENV_FILE_PATH", env_file) + + engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + TestSession = sessionmaker(bind=engine) + Base.metadata.create_all(bind=engine) + + with TestSession() as session: + state = admin_service.system_state(session) + config = admin_service.current_config_view() + + kb_key_rows = [row for row in config if row["key"] == "KB_API_KEY"] + assert kb_key_rows[0]["value"] == "" + assert kb_key_rows[0]["configured"] is True + assert state["vault"]["exists"] is True + assert state["vault"]["is_git_repo"] is True + finally: + settings.vault_path = previous_vault + settings.kb_api_key = previous_key + + +def test_admin_state_exposes_autosave_summary(monkeypatch, tmp_vault: Path): + previous_vault = settings.vault_path + previous_key = settings.kb_api_key + try: + settings.vault_path = tmp_vault + settings.kb_api_key = "" + monkeypatch.setattr( + admin_service.github_service, + "list_open_kb_api_prs", + lambda: [], + ) + monkeypatch.setattr( + admin_service, + "_tmux_session_exists", + lambda session_name: session_name == settings.admin_tmux_worker_session, + ) + + engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + TestSession = sessionmaker(bind=engine) + Base.metadata.create_all(bind=engine) + + with TestSession() as session: + session.add( + Job( + job_type="autosave", + status="completed", + meta={"files": ["notes/hello.md"]}, + ) + ) + session.add( + VaultEvent( + event_type="autosave_commit", + commit_sha="abc123", + ) + ) + session.add( + VaultEvent( + event_type="autosave_push", + commit_sha="abc123", + ) + ) + session.commit() + + state = admin_service.system_state(session) + + assert state["autosave"]["worker_session_running"] is True + assert state["autosave"]["latest_job_status"] == "completed" + assert state["autosave"]["latest_job_files"] == ["notes/hello.md"] + assert state["autosave"]["latest_commit_sha"] == "abc123" + assert state["autosave"]["latest_push_sha"] == "abc123" + finally: + settings.vault_path = previous_vault + settings.kb_api_key = previous_key + + +async def test_admin_routes_bypass_api_key(monkeypatch, tmp_vault: Path): + previous_key = settings.kb_api_key + try: + settings.kb_api_key = "secret-key" + middleware = APIKeyMiddleware(app=lambda scope, receive, send: None) + request = Request( + { + "type": "http", + "method": "GET", + "path": "/admin/api/state", + "headers": [], + "query_string": b"", + "scheme": "http", + "server": ("testserver", 80), + "client": ("testclient", 123), + "root_path": "", + "http_version": "1.1", + } + ) + + async def call_next(_request: Request): + return JSONResponse({"ok": True}) + + response = await middleware.dispatch(request, call_next) + + assert response.status_code == 200 + finally: + settings.kb_api_key = previous_key + + +def test_runtime_control_state_uses_tmux_settings(tmp_path): + previous_workdir = settings.admin_tmux_workdir + previous_session = settings.admin_tmux_session + previous_worker_session = settings.admin_tmux_worker_session + previous_host = settings.api_host + previous_port = settings.api_port + try: + settings.admin_tmux_workdir = tmp_path + settings.admin_tmux_session = "kb-api-test" + settings.admin_tmux_worker_session = "kb-worker-test" + settings.api_host = "127.0.0.1" + settings.api_port = 8123 + + runtime = admin_service.runtime_control_state() + api_runtime = runtime["api"] + worker_runtime = runtime["worker"] + + assert api_runtime["tmux_session"] == "kb-api-test" + assert api_runtime["workdir"] == str(tmp_path) + assert "tmux new-session -d -s kb-api-test" in api_runtime["start_command"] + assert "--host 127.0.0.1 --port 8123" in api_runtime["start_command"] + assert "tmux respawn-pane -k -t kb-api-test:0.0" in api_runtime["restart_command"] + assert worker_runtime["tmux_session"] == "kb-worker-test" + assert worker_runtime["workdir"] == str(tmp_path) + assert "tmux new-session -d -s kb-worker-test" in worker_runtime["start_command"] + assert "python -m app.workers.autosave" in worker_runtime["start_command"] + assert "tmux respawn-pane -k -t kb-worker-test:0.0" in worker_runtime["restart_command"] + finally: + settings.admin_tmux_workdir = previous_workdir + settings.admin_tmux_session = previous_session + settings.admin_tmux_worker_session = previous_worker_session + settings.api_host = previous_host + settings.api_port = previous_port diff --git a/kb-server/tests/test_config.py b/kb-server/tests/test_config.py new file mode 100644 index 0000000..c220fb6 --- /dev/null +++ b/kb-server/tests/test_config.py @@ -0,0 +1,28 @@ +from pathlib import Path + +from app.core.config import Settings + + +def test_environment_variables_override_env_file(tmp_path, monkeypatch): + env_file = tmp_path / ".env" + env_file.write_text( + "\n".join( + [ + "KB_API_KEY=from-env-file", + "GITHUB_TOKEN=from-env-file", + "VAULT_PATH=/tmp/from-env-file", + ] + ) + + "\n", + encoding="utf-8", + ) + + monkeypatch.setenv("KB_API_KEY", "from-environment") + monkeypatch.setenv("GITHUB_TOKEN", "from-environment") + monkeypatch.setenv("VAULT_PATH", "/tmp/from-environment") + + settings = Settings(_env_file=env_file) + + assert settings.kb_api_key == "from-environment" + assert settings.github_token == "from-environment" + assert settings.vault_path == Path("/tmp/from-environment") diff --git a/scripts/generate_context_artifacts.py b/scripts/generate_context_artifacts.py index 6d02d12..e61d7c7 100755 --- a/scripts/generate_context_artifacts.py +++ b/scripts/generate_context_artifacts.py @@ -3,8 +3,8 @@ from __future__ import annotations -import datetime as dt import re +import subprocess from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent @@ -53,12 +53,29 @@ def _parse_routes(path: Path) -> list[tuple[str, str]]: return routes +def _last_commit_date(paths: list[Path]) -> str: + rel_paths = [str(path.relative_to(REPO_ROOT)) for path in paths] + result = subprocess.run( + ["git", "log", "-1", "--format=%cs", "--", *rel_paths], + cwd=REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + def _write_api_surface() -> None: - health_routes = _parse_routes(REPO_ROOT / "kb-server" / "app" / "api" / "routes" / "health.py") - notes_routes = _parse_routes(REPO_ROOT / "kb-server" / "app" / "api" / "routes" / "notes.py") - publish_routes = _parse_routes(REPO_ROOT / "kb-server" / "app" / "api" / "routes" / "publish.py") - all_routes = health_routes + notes_routes + publish_routes - date = dt.date.today().isoformat() + route_paths = [ + REPO_ROOT / "kb-server" / "app" / "api" / "routes" / "health.py", + REPO_ROOT / "kb-server" / "app" / "api" / "routes" / "notes.py", + REPO_ROOT / "kb-server" / "app" / "api" / "routes" / "publish.py", + REPO_ROOT / "kb-server" / "app" / "api" / "routes" / "admin.py", + ] + all_routes: list[tuple[str, str]] = [] + for route_path in route_paths: + all_routes.extend(_parse_routes(route_path)) + date = _last_commit_date(route_paths) content = [ "---", @@ -69,6 +86,7 @@ def _write_api_surface() -> None: " - ../../kb-server/app/api/routes/health.py", " - ../../kb-server/app/api/routes/notes.py", " - ../../kb-server/app/api/routes/publish.py", + " - ../../kb-server/app/api/routes/admin.py", "related_code:", " - ../../scripts/generate_context_artifacts.py", "related_tests:", @@ -91,10 +109,13 @@ def _write_api_surface() -> None: def _write_env_catalog() -> None: - date = dt.date.today().isoformat() - kb_env = _parse_env_example(REPO_ROOT / "kb-server" / ".env.example") - kb_defaults = _parse_settings_defaults(REPO_ROOT / "kb-server" / "app" / "core" / "config.py") - vs_defaults = _parse_settings_defaults(REPO_ROOT / "vault-sync" / "vault_sync" / "config.py") + kb_env_path = REPO_ROOT / "kb-server" / ".env.example" + kb_settings_path = REPO_ROOT / "kb-server" / "app" / "core" / "config.py" + vs_settings_path = REPO_ROOT / "vault-sync" / "vault_sync" / "config.py" + date = _last_commit_date([kb_env_path, kb_settings_path, vs_settings_path]) + kb_env = _parse_env_example(kb_env_path) + kb_defaults = _parse_settings_defaults(kb_settings_path) + vs_defaults = _parse_settings_defaults(vs_settings_path) content = [ "---", @@ -165,4 +186,3 @@ def main() -> int: if __name__ == "__main__": raise SystemExit(main()) -