Dockerfile

# Wauldo Trust Score Leaderboard — reproducible benchmark harness.
#
# The image is NOT published to a public registry. You must build it
# yourself from a repo checkout. The 2-step workflow:
#
#   # 1. Build from the repo root:
#   docker build -t wauldo/leaderboard-bench .
#
#   # 2. Run (needs OPENROUTER_API_KEY for every framework except Wauldo).
#   #    The host `results/` dir must be writable by UID 999 (or the
#   #    container user you override with `--user`).
#   docker run --rm \
#       -e OPENROUTER_API_KEY=$OPENROUTER_API_KEY \
#       -v $(pwd)/results:/app/results \
#       wauldo/leaderboard-bench \
#       --frameworks all
#
# Everything is pinned: Python patch version, pip version, every
# framework dep, and the dataset is content-hashed at build time so the
# image refuses to build on a mutated dataset. 79 scorer + 16 pricing
# tests run at build time — a regression in the judge blocks the image.

FROM python:3.11.11-slim-bookworm

LABEL org.opencontainers.image.source="https://github.com/wauldo/wauldo-leaderboard"
LABEL org.opencontainers.image.description="Wauldo Trust Score Leaderboard — adversarial bench harness for 6 RAG frameworks"
LABEL org.opencontainers.image.licenses="MIT"

# Run as non-root for the actual bench execution. UID 999 is hard-coded
# (not left to the distro's next-free-system-UID) so bind-mount chown
# instructions are deterministic: on the host, `chown 999:999 results/`
# Just Works regardless of what other system users the base image has.
RUN groupadd --system --gid 999 bench \
    && useradd --system --uid 999 --gid 999 --create-home --home-dir /home/bench bench

WORKDIR /app

# System deps needed by faiss-cpu, tokenizers, onnxruntime
RUN apt-get update && apt-get install -y --no-install-recommends \
      build-essential \
      libgomp1 \
    && rm -rf /var/lib/apt/lists/*

# Install deps in a dedicated layer so a code edit doesn't invalidate them.
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt

# Copy the package payload. .dockerignore excludes results/, __pycache__/,
# and every *.pyc so the image stays lean.
COPY wauldo_leaderboard /app/wauldo_leaderboard
COPY datasets /app/datasets

# Sanity: the dataset must parse to exactly 70 tests, the sha256 must be a
# proper hex digest, and every module must import. If any of this fails the
# image build aborts — no silent drift into production.
RUN python -c "\
from wauldo_leaderboard.dataset import dataset_sha256, load_dataset; \
from wauldo_leaderboard.scorer import evaluate; \
from wauldo_leaderboard.aggregate import build_leaderboard; \
tests = load_dataset(); \
assert len(tests) == 70, f'expected 70 tests, got {len(tests)}'; \
h = dataset_sha256(); \
assert len(h) == 64 and all(c in '0123456789abcdef' for c in h), 'bad sha256'; \
print(f'[build] dataset: {len(tests)} tests, sha256={h[:16]}...'); \
print('[build] imports ok')"

# Run the scorer + pricing test suite at build time so a regression in
# scorer.py or pricing.py blocks the image from ever being tagged.
RUN pip install --no-cache-dir pytest==9.0.2 \
    && python -m pytest /app/wauldo_leaderboard/tests/ -q \
    && pip uninstall -y pytest

# Give the non-root user write access to the results directory. When a
# host volume is mounted over it, the host path must be chowned to UID
# 999 (`chown 999:999 results/` on the host). The default path is
# created empty so the runtime `run_<timestamp>` subdirs can land
# without a chown dance when no volume is mounted.
RUN mkdir -p /app/results \
    && chown -R bench:bench /app

# Smoke-test that the non-root user can actually write where it needs to.
# If this assertion fails the build aborts rather than shipping an image
# that fails mysteriously at runtime on the first invocation.
RUN su bench -s /bin/sh -c "test -w /app/results && touch /app/results/.perm_test && rm /app/results/.perm_test"

USER bench
ENV PYTHONUNBUFFERED=1 \
    PYTHONDONTWRITEBYTECODE=1

ENTRYPOINT ["python", "-m", "wauldo_leaderboard.harness"]
CMD ["--help"]