Weekly retrain pipeline + hourly accuracy tracking

.github/workflows/daily_predict.yml

@@ -0,0 +1,65 @@
+name: Daily Predict and Log
+
+# Runs every day at 00:30 UTC (06:00 IST) — light-weight only.
+# Does NOT retrain models (that's done locally on Fridays via train_weekly.bat).
+# This workflow only:
+#   - Scrapes the last 2 days of SLDC demand (catches up)
+#   - Refreshes the 7-day weather forecast
+#   - Refreshes AQI
+#   - Generates the next 7 days of hourly predictions using the existing
+#     committed models, and logs each hour to prediction_log_hourly
+#   - Fills actuals + error for past hourly predictions
+#
+# Total runtime: ~2-3 min. Fits comfortably in GitHub Actions free tier.
+
+on:
+  schedule:
+    - cron: "30 0 * * *"          # 00:30 UTC daily
+  workflow_dispatch:               # manual "Run workflow" button
+
+jobs:
+  predict-and-log:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Create .env
+        run: |
+          echo "DATABASE_URL=${{ secrets.DATABASE_URL }}" > .env
+          echo "JWT_SECRET=${{ secrets.JWT_SECRET }}" >> .env
+
+      - name: Run predict-only pipeline
+        env:
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+        run: python scripts/run_full_pipeline.py --no-train
+
+      - name: Summary
+        if: always()
+        env:
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+        run: |
+          python - <<'PY'
+          from src.data.db.session import get_session
+          from src.data.db.models import DemandRecord, PredictionLogHourly
+          from sqlalchemy import func
+          from datetime import date, timedelta
+          with get_session() as s:
+              d = s.query(func.max(DemandRecord.timestamp)).scalar()
+              p_total = s.query(func.count(PredictionLogHourly.id)).scalar()
+              cutoff = date.today() - timedelta(days=7)
+              p_recent = s.query(func.count(PredictionLogHourly.id)).filter(
+                  PredictionLogHourly.target_date >= cutoff
+              ).scalar()
+              print(f"::notice::Demand latest: {d}")
+              print(f"::notice::Hourly predictions total: {p_total}, last 7d: {p_recent}")
+          PY

.github/workflows/scheduler.yml

@@ -1,10 +1,10 @@
 name: Data Collection & Prediction
 
 on:
-  schedule:
-    # Run every 6 hours: midnight, 6am, noon, 6pm IST
-    - cron: "30 18,0,6,12 * * *"
-  workflow_dispatch: # Allow manual trigger
+  # DEPRECATED — superseded by daily_predict.yml.
+  # Kept for manual fallback only. The schedule has been removed so it
+  # no longer fires automatically; trigger via workflow_dispatch if needed.
+  workflow_dispatch:
 
 jobs:
   collect-and-predict:

.gitignore

@@ -54,3 +54,7 @@ desktop.ini
 .pytest_cache/
 .coverage
 htmlcov/
+
+# Pipeline run artifacts (regenerated each Friday)
+pipeline.log
+bash.exe.stackdump

frontend/src/app/(dashboard)/admin/page.tsx

@@ -16,6 +16,7 @@ import {
   getScraperStatus,
   getSchedulerJobs,
   getMe,
+  triggerPipeline,
 } from "@/lib/api";
 
 // ---------------------------------------------------------------------------
@@ -107,6 +108,21 @@ export default function AdminPage() {
     }
   };
 
+  const [pipelineLoading, setPipelineLoading] = useState(false);
+  const handleTriggerPipeline = async () => {
+    setPipelineLoading(true);
+    const { toast } = await import("sonner");
+    try {
+      const res = await triggerPipeline("daily_predict.yml");
+      toast.success("Daily pipeline triggered — opening Actions in a new tab");
+      window.open(res.actions_url, "_blank");
+    } catch (err: any) {
+      toast.error(`Trigger failed: ${err.message}`);
+    } finally {
+      setPipelineLoading(false);
+    }
+  };
+
   // Scraper rows
   const scraperRows: { source: string; latest: string; rows: number }[] = [];
   if (scraper) {
@@ -151,14 +167,28 @@ export default function AdminPage() {
   return (
     <div className="space-y-6">
       {/* Header */}
-      <motion.div initial={{ opacity: 0, y: -10 }} animate={{ opacity: 1, y: 0 }}>
-        <div className="flex items-center gap-2">
-          <Settings className="w-5 h-5 text-muted-foreground" />
-          <h1 className="text-2xl font-bold tracking-tight">Admin Panel</h1>
+      <motion.div initial={{ opacity: 0, y: -10 }} animate={{ opacity: 1, y: 0 }}
+        className="flex items-start justify-between gap-4">
+        <div>
+          <div className="flex items-center gap-2">
+            <Settings className="w-5 h-5 text-muted-foreground" />
+            <h1 className="text-2xl font-bold tracking-tight">Admin Panel</h1>
+          </div>
+          <p className="text-sm text-muted-foreground mt-0.5">
+            Model management, retraining, and pipeline monitoring
+          </p>
         </div>
-        <p className="text-sm text-muted-foreground mt-0.5">
-          Model management, retraining, and pipeline monitoring
-        </p>
+        <Button
+          onClick={handleTriggerPipeline}
+          disabled={pipelineLoading}
+          size="sm"
+          className="gap-2 shrink-0"
+        >
+          {pipelineLoading
+            ? <Loader2 className="w-4 h-4 animate-spin" />
+            : <Play className="w-4 h-4" />}
+          Run Daily Pipeline
+        </Button>
       </motion.div>
 
       {/* ----------------------------------------------------------------- */}

frontend/src/lib/api.ts

@@ -151,3 +151,14 @@ export const getRetrainStatus = () =>
 export const getScraperStatus = () => fetcher<any>("/api/v1/admin/scraper-status");
 
 export const getSchedulerJobs = () => fetcher<any[]>("/api/v1/admin/scheduler-jobs");
+
+export const triggerPipeline = (workflow: string = "daily_predict.yml") =>
+  fetcher<{ status: string; workflow: string; actions_url: string }>(
+    "/api/v1/admin/trigger-pipeline",
+    { method: "POST", body: JSON.stringify({ workflow }) }
+  );
+
+export const getHourlyAccuracy = (days: number = 30) =>
+  fetcher<{ entries: any[]; by_hour: any[]; summary: any }>(
+    `/api/v1/dashboard/hourly-accuracy?days=${days}`
+  );