Initial v0.1.0: sample size, CUPED, ratio metrics (delta method), mSPRT, CLI

Author: wavde

.github/workflows/ci.yml

@@ -0,0 +1,28 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - name: Install
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+      - name: Lint
+        run: ruff check .
+      - name: Test
+        run: pytest

.gitignore

@@ -0,0 +1,16 @@
+__pycache__/
+*.py[cod]
+.Python
+.venv/
+venv/
+.env
+*.egg-info/
+build/
+dist/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.coverage
+.DS_Store
+.idea/
+.vscode/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Tejas Wavde
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,70 @@
+# experiment-toolkit
+
+> Small, well-tested utilities for online controlled experiments.
+
+![CI](https://github.com/wavde/experiment-toolkit/actions/workflows/ci.yml/badge.svg)
+![Python](https://img.shields.io/badge/python-3.10+-blue.svg)
+![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)
+
+## What's inside
+
+| Module | Purpose |
+|--------|---------|
+| `sample_size` | Required per-arm sample size for a given MDE, and the inverse |
+| `cuped` | Deng et al. (2013) CUPED variance reduction |
+| `ratio` | Delta-method variance for ratio metrics (revenue/session, etc.) |
+| `sequential` | mSPRT always-valid p-values for peeking-robust experiments |
+
+Every function is tested, typed, and has a reference to the paper it implements.
+
+## Install
+
+```bash
+# from source (PyPI release TBD)
+pip install git+https://github.com/wavde/experiment-toolkit.git
+```
+
+## Quick start
+
+```python
+from experiment_toolkit import sample_size_for_mde, apply_cuped, msprt_pvalue
+
+# How many users do I need per arm to detect a 2% lift (sd=1.0)?
+n = sample_size_for_mde(mde=0.02, std_dev=1.0, alpha=0.05, power=0.80)
+# ~39,000 per arm
+
+# Apply CUPED with a pre-experiment covariate
+y_adj = apply_cuped(y, pre_period_y)
+
+# Always-valid p-value — safe to peek
+p = msprt_pvalue(delta_hat=0.015, sigma=1.0, n_per_arm=5000, tau=0.05)
+```
+
+## CLI
+
+```bash
+experiment-toolkit sample-size --mde 0.02 --sd 1.0
+# Required per-arm sample size: 39,242
+
+experiment-toolkit mde --n 10000 --sd 1.0
+# Detectable effect (MDE): 0.0396
+```
+
+## Development
+
+```bash
+pip install -e ".[dev]"
+pytest
+ruff check .
+```
+
+## References
+
+- Deng, Xu, Kohavi, Walker (2013) — CUPED
+- Deng, Knoblich, Lu (2018) — Delta Method in Metric Analytics
+- Johari, Pekelis, Walsh (2015) — Always Valid Inference
+- Kohavi, Tang, Xu (2020) — *Trustworthy Online Controlled Experiments*
+
+## License
+
+MIT — see [LICENSE](LICENSE).

pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "experiment-toolkit"
+version = "0.1.0"
+description = "Small, well-tested utilities for online controlled experiments: sample size, CUPED, sequential testing, delta-method variance."
+readme = "README.md"
+license = { file = "LICENSE" }
+requires-python = ">=3.10"
+authors = [{ name = "Tejas Wavde" }]
+keywords = ["experimentation", "ab-testing", "causal-inference", "statistics"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Mathematics",
+]
+dependencies = [
+    "numpy>=1.24",
+    "scipy>=1.10",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=7", "ruff>=0.3", "mypy>=1.8"]
+
+[project.urls]
+Homepage = "https://github.com/wavde/experiment-toolkit"
+Issues = "https://github.com/wavde/experiment-toolkit/issues"
+
+[project.scripts]
+experiment-toolkit = "experiment_toolkit.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/experiment_toolkit"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "W", "UP", "B", "SIM"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-q"

src/experiment_toolkit/__init__.py

@@ -0,0 +1,17 @@
+"""experiment-toolkit: small utilities for online controlled experiments."""
+
+from experiment_toolkit.cuped import apply_cuped, compute_theta
+from experiment_toolkit.ratio import ratio_metric_variance
+from experiment_toolkit.sample_size import mde_for_n, sample_size_for_mde
+from experiment_toolkit.sequential import msprt_pvalue
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "apply_cuped",
+    "compute_theta",
+    "mde_for_n",
+    "msprt_pvalue",
+    "ratio_metric_variance",
+    "sample_size_for_mde",
+]

src/experiment_toolkit/cli.py

@@ -0,0 +1,44 @@
+"""Minimal CLI for experiment-toolkit.
+
+Example usage:
+    experiment-toolkit sample-size --mde 0.02 --sd 1.0
+    experiment-toolkit mde --n 10000 --sd 1.0
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from experiment_toolkit.sample_size import mde_for_n, sample_size_for_mde
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(prog="experiment-toolkit")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    p_ss = sub.add_parser("sample-size", help="compute required per-arm sample size")
+    p_ss.add_argument("--mde", type=float, required=True, help="minimum detectable effect")
+    p_ss.add_argument("--sd", type=float, required=True, help="outcome standard deviation")
+    p_ss.add_argument("--alpha", type=float, default=0.05)
+    p_ss.add_argument("--power", type=float, default=0.80)
+
+    p_mde = sub.add_parser("mde", help="compute detectable effect given sample size")
+    p_mde.add_argument("--n", type=int, required=True, help="per-arm sample size")
+    p_mde.add_argument("--sd", type=float, required=True)
+    p_mde.add_argument("--alpha", type=float, default=0.05)
+    p_mde.add_argument("--power", type=float, default=0.80)
+
+    args = parser.parse_args(argv)
+
+    if args.cmd == "sample-size":
+        n = sample_size_for_mde(args.mde, args.sd, args.alpha, args.power)
+        print(f"Required per-arm sample size: {n:,}")
+    elif args.cmd == "mde":
+        mde = mde_for_n(args.n, args.sd, args.alpha, args.power)
+        print(f"Detectable effect (MDE): {mde:.4f}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/experiment_toolkit/cuped.py

@@ -0,0 +1,29 @@
+"""CUPED variance reduction — see Deng et al. (2013)."""
+
+from __future__ import annotations
+
+import numpy as np
+from numpy.typing import ArrayLike
+
+
+def compute_theta(y: ArrayLike, x: ArrayLike) -> float:
+    """Optimal CUPED coefficient theta = Cov(Y, X) / Var(X)."""
+    y_arr = np.asarray(y, dtype=float)
+    x_arr = np.asarray(x, dtype=float)
+    var_x = np.var(x_arr, ddof=1)
+    if var_x == 0:
+        return 0.0
+    return float(np.cov(y_arr, x_arr, ddof=1)[0, 1] / var_x)
+
+
+def apply_cuped(
+    y: ArrayLike,
+    x: ArrayLike,
+    theta: float | None = None,
+) -> np.ndarray:
+    """Return CUPED-adjusted outcome: Y - theta * (X - mean(X))."""
+    y_arr = np.asarray(y, dtype=float)
+    x_arr = np.asarray(x, dtype=float)
+    if theta is None:
+        theta = compute_theta(y_arr, x_arr)
+    return y_arr - theta * (x_arr - x_arr.mean())

src/experiment_toolkit/ratio.py

@@ -0,0 +1,57 @@
+"""
+Variance of ratio metrics via the delta method.
+
+Real product metrics are often ratios (revenue / session, clicks / impression,
+minutes / DAU). A naive two-sample t-test on per-user ratios is usually wrong
+because the numerator and denominator are correlated.
+
+Delta-method approximation for R = E[N] / E[D] with per-user (n_i, d_i):
+
+    Var(R) ~= (1/mean(d))^2 * ( Var(N) - 2*R*Cov(N,D) + R^2 * Var(D) )
+
+Reference: Deng, Knoblich, Lu (2018), "Applying the Delta Method in Metric
+Analytics: A Practical Guide with Novel Ideas."
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from numpy.typing import ArrayLike
+
+
+def ratio_metric_variance(numerator: ArrayLike, denominator: ArrayLike) -> tuple[float, float]:
+    """
+    Return (ratio_estimate, standard_error) for the per-unit ratio metric.
+
+    Parameters
+    ----------
+    numerator : per-unit numerator values (e.g., minutes watched per user)
+    denominator : per-unit denominator values (e.g., sessions per user)
+
+    Returns
+    -------
+    (ratio, standard_error)
+    """
+    n = np.asarray(numerator, dtype=float)
+    d = np.asarray(denominator, dtype=float)
+    if n.shape != d.shape:
+        raise ValueError("numerator and denominator must have the same shape")
+    if len(n) < 2:
+        raise ValueError("need at least 2 observations")
+
+    mean_n = n.mean()
+    mean_d = d.mean()
+    if mean_d == 0:
+        raise ValueError("mean of denominator is zero; ratio undefined")
+
+    ratio = mean_n / mean_d
+
+    # Delta-method variance of the sample ratio.
+    var_n = np.var(n, ddof=1)
+    var_d = np.var(d, ddof=1)
+    cov_nd = np.cov(n, d, ddof=1)[0, 1]
+
+    var_ratio = (var_n - 2 * ratio * cov_nd + ratio**2 * var_d) / (len(n) * mean_d**2)
+    se = float(np.sqrt(max(var_ratio, 0.0)))
+
+    return float(ratio), se