From a252418f99b49f7f0ade397bb14e195416c7900b Mon Sep 17 00:00:00 2001 From: Jean Van Dyk Date: Sat, 2 Aug 2025 13:34:23 +0200 Subject: [PATCH 1/4] Adding the notebook --- notebooks/Kalman_Filter_Gradients.ipynb | 743 ++++++++++++++++++++++++ 1 file changed, 743 insertions(+) create mode 100644 notebooks/Kalman_Filter_Gradients.ipynb diff --git a/notebooks/Kalman_Filter_Gradients.ipynb b/notebooks/Kalman_Filter_Gradients.ipynb new file mode 100644 index 00000000..9f33d39e --- /dev/null +++ b/notebooks/Kalman_Filter_Gradients.ipynb @@ -0,0 +1,743 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "69ae14a1", + "metadata": {}, + "source": [ + "### Imports" + ] + }, + { + "cell_type": "code", + "execution_count": 127, + "id": "90979a41", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import pytensor\n", + "import pytensor.tensor as pt\n", + "import matplotlib.pyplot as plt\n", + "from pytensor.compile.builders import OpFromGraph\n", + "from time import perf_counter\n", + "from collections import defaultdict" + ] + }, + { + "cell_type": "markdown", + "id": "a0d008fc", + "metadata": {}, + "source": [ + "### Generate dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 128, + "id": "f75a72e8", + "metadata": {}, + "outputs": [], + "source": [ + "def generate_kalman_dataset(n, N=100, seed=0):\n", + " rng = np.random.default_rng(seed)\n", + "\n", + " # 0. Initial state and cov\n", + " A0 = rng.normal(loc=0.0, scale=1.0, size=(n,))\n", + " # 3. Random process noise covariance Q (PSD)\n", + " P0_base = rng.normal(0, 1, size=(n, n))\n", + " P0 = P0_base @ P0_base.T + np.eye(n) * 1e-3 # ensure positive definite\n", + "\n", + " # 1. T stable random transition matrix T\n", + " T = rng.normal(0, 1, size=(n, n))\n", + " eigvals = np.linalg.eigvals(T)\n", + " spectral_radius = max(abs(eigvals))\n", + " T = T / (1.1 * spectral_radius) # shrink to ensure stability\n", + "\n", + " # 2. Random observation matrix Z\n", + " Z = rng.normal(0, 1, size=(n, n)) # full observations (m = n)\n", + "\n", + " # 3. Random process noise covariance Q (PSD)\n", + " Q_base = rng.normal(0, 1, size=(n, n))\n", + " Q = Q_base @ Q_base.T + np.eye(n) * 1e-3 # ensure positive definite\n", + "\n", + " # 4. Random observation noise covariance H (PSD)\n", + " H_base = rng.normal(0, 1, size=(n, n))\n", + " H = H_base @ H_base.T + np.eye(n) * 1e-3\n", + "\n", + " # 5. Initial state\n", + " x = np.zeros((N, n))\n", + " y = np.zeros((N, n))\n", + " x[0] = A0\n", + "\n", + " # 6. Simulate the system\n", + " for t in range(1, N):\n", + " w_t = rng.multivariate_normal(mean=np.zeros(n), cov=Q)\n", + " x[t] = T @ x[t-1] + w_t\n", + "\n", + " for t in range(N):\n", + " v_t = rng.multivariate_normal(mean=np.zeros(n), cov=H)\n", + " y[t] = Z @ x[t] + v_t\n", + "\n", + " return {\n", + " \"T\": T, \"Z\": Z, \"Q\": Q, \"H\": H,\n", + " \"x\": x, \"y\": y, \"A0\": A0, \"P0\": P0\n", + " }" + ] + }, + { + "cell_type": "markdown", + "id": "51b7e885", + "metadata": {}, + "source": [ + "### Symbolic variable" + ] + }, + { + "cell_type": "code", + "execution_count": 129, + "id": "3661408d", + "metadata": {}, + "outputs": [], + "source": [ + "# Paramètres symboliques\n", + "A_sym = pt.matrix(\"A\") # (n, n)\n", + "H_sym = pt.matrix(\"H\") # (n, n)\n", + "Q_sym = pt.matrix(\"Q\") # (n, n)\n", + "R_sym = pt.matrix(\"R\") # (n, n)\n", + "T_sym = pt.matrix(\"T\") # (n, n)\n", + "Z_sym = pt.matrix(\"Z\") # (n, n)\n", + "\n", + "x0_sym = pt.vector(\"x0\") # (n,)\n", + "y_sym = pt.matrix(\"y\") # (T, n) : observations\n", + "\n", + "a0_sym = pt.vector(\"a0\") \n", + "P0_sym = pt.matrix(\"P0\") \n", + "\n", + "data_sym = pt.matrix('data_sym') # [T, obs_dim]" + ] + }, + { + "cell_type": "markdown", + "id": "19e6a32d", + "metadata": {}, + "source": [ + "### Kalman filter with classic gradient" + ] + }, + { + "cell_type": "code", + "execution_count": 130, + "id": "35351096", + "metadata": {}, + "outputs": [], + "source": [ + "def predict(a, P, T, Q):\n", + " a_hat = T @ a # x_n|n-1\n", + " P_hat = T @ P @ T.T + Q # P_n|n-1\n", + " return a_hat, P_hat\n", + "\n", + "def update(y, a, P, Z, H):\n", + " v = y - Z.dot(a) # z_n\n", + " PZT = P.dot(Z.T) \n", + "\n", + " F = Z.dot(PZT) + H # S_n\n", + " F_inv = pt.linalg.inv(F) # S_n^(-1)\n", + " K = PZT.dot(F_inv) # K_n\n", + "\n", + " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", + " a_filtered = a + K.dot(v) # x_n|n\n", + " P_filtered = I_KZ @ P # P_n|n\n", + "\n", + " inner_term = v.T @ F_inv @ v\n", + " _, F_logdet = pt.linalg.slogdet(F) # log det S_n\n", + " ll = (F_logdet + inner_term).ravel()[0] # Loss\n", + "\n", + " return [a_filtered, P_filtered, Z.dot(a), F, ll]\n", + "\n", + "def kalman_step(y, a, P, T, Z, H, Q):\n", + " a_filtered, P_filtered, obs_mu, obs_cov, ll = update(y=y, a=a, P=P, Z=Z, H=H)\n", + " a_hat, P_hat = predict(a=a_filtered, P=P_filtered, T=T, Q=Q)\n", + " return [a_filtered, a_hat, obs_mu, P_filtered, P_hat, obs_cov, ll]\n", + "\n", + "\n", + "outputs_info = [None, a0_sym, None, None, P0_sym, None, None]\n", + "\n", + "results_seq, updates = pytensor.scan(\n", + " kalman_step,\n", + " sequences=[data_sym],\n", + " outputs_info=outputs_info,\n", + " non_sequences=[T_sym, Z_sym, H_sym, Q_sym],\n", + " strict=False,\n", + ")\n", + "# --- Loss ---\n", + "a_upd_seq, a_pred_seq, y_hat_seq, P_upd_seq, P_pred_seq, obs_cov, ll_seq = results_seq\n", + "loss = pt.sum(ll_seq)" + ] + }, + { + "cell_type": "markdown", + "id": "ece2f47e", + "metadata": {}, + "source": [ + "### Custom gradient" + ] + }, + { + "cell_type": "code", + "execution_count": 131, + "id": "afb362e5", + "metadata": {}, + "outputs": [], + "source": [ + "def custom_grad(inp, out, out_grad):\n", + " y, a, P, T, Z, H, Q = inp\n", + " a_filtered, P_filtered, y_hat = out\n", + " a_hat_grad, P_hat_grad, y_grad = out_grad\n", + "\n", + " PZT = P.dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + "\n", + " y_hat = Z.dot(a)\n", + " v = y - y_hat\n", + "\n", + " H_inv = pt.linalg.inv(H)\n", + " F_inv = pt.linalg.inv(F)\n", + "\n", + " K = PZT.dot(F_inv)\n", + " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", + " \n", + " grad_a_pred = I_KZ.T @ T.T @ a_hat_grad - 2 * Z.T @ F_inv @ v\n", + " grad_y = K.T @ T.T @ a_hat_grad + 2 * F_inv @ v\n", + "\n", + "\n", + " a_hat_grad = a_hat_grad.dimshuffle(0, 'x')\n", + " v = v.dimshuffle(0, 'x')\n", + " \n", + " P_filtered_grad = T.T @ P_hat_grad @ T\n", + " a_filtered_grad = T.T @ a_hat_grad \n", + "\n", + " grad_P_hat = I_KZ.T @ ( P_filtered_grad + 0.5 * a_filtered_grad @ v.T @ H_inv @ Z + 0.5 * Z.T @ H_inv @ v @ a_filtered_grad.T ) @ I_KZ + Z.T @ F_inv @ Z - Z.T @ F_inv @ v @ v.T @ F_inv @ Z\n", + " grad_Z = None\n", + " grad_T = None\n", + " grad_Q = P_hat_grad\n", + " grad_H = K.T @ P_filtered_grad @ K - 0.5 * K.T @ a_filtered_grad @ v.T @ F_inv - 0.5 * F_inv @ v @ a_filtered_grad.T @ K + F_inv - F_inv @ v @ v.T @ F_inv\n", + "\n", + " return [grad_P_hat,\n", + " grad_a_pred,\n", + " grad_y,\n", + " grad_Z,\n", + " grad_T,\n", + " grad_Q,\n", + " grad_H]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 132, + "id": "ee21ef4e", + "metadata": {}, + "outputs": [], + "source": [ + "def grad_a_hat(inp, out, out_grad):\n", + " y, a, P, T, Z, H, Q = inp\n", + " a_hat_grad, _, _ = out_grad\n", + "\n", + " v = y - Z.dot(a) \n", + "\n", + " PZT = P.dot(Z.T)\n", + " F = Z.dot(PZT) + H \n", + " F_inv = pt.linalg.inv(F)\n", + " \n", + " K = PZT.dot(F_inv) \n", + " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", + "\n", + " grad_a_pred = I_KZ.T @ T.T @ a_hat_grad - 2 * Z.T @ F_inv @ v\n", + "\n", + " return grad_a_pred" + ] + }, + { + "cell_type": "code", + "execution_count": 133, + "id": "8c89b018", + "metadata": {}, + "outputs": [], + "source": [ + "def grad_P_hat(inp, out, out_grad):\n", + " y, a, P, T, Z, H, Q = inp\n", + " a_hat_grad, P_hat_grad, ll_grad = out_grad\n", + "\n", + " v = y - Z.dot(a)\n", + " v = v.dimshuffle(0, 'x')\n", + " a_hat_grad = a_hat_grad.dimshuffle(0, 'x') \n", + "\n", + " P_filtered_grad = T.T @ P_hat_grad @ T\n", + " a_filtered_grad = T.T @ a_hat_grad \n", + "\n", + " PZT = P.dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + "\n", + " H_inv = pt.linalg.inv(H) \n", + " F_inv = pt.linalg.inv(F)\n", + " \n", + " K = PZT.dot(F_inv) \n", + " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", + "\n", + " grad_P_hat = I_KZ.T @ ( P_filtered_grad + 0.5 * a_filtered_grad @ v.T @ H_inv @ Z + 0.5 * Z.T @ H_inv @ v @ a_filtered_grad.T ) @ I_KZ + Z.T @ F_inv @ Z - Z.T @ F_inv @ v @ v.T @ F_inv @ Z\n", + "\n", + " return grad_P_hat" + ] + }, + { + "cell_type": "code", + "execution_count": 134, + "id": "bba53a26", + "metadata": {}, + "outputs": [], + "source": [ + "def grad_y(inp, out, out_grad):\n", + " y, a, P, T, Z, H, Q = inp\n", + " a_hat_grad, P_h_grad, y_grad = out_grad\n", + "\n", + " y_hat = Z.dot(a)\n", + " v = y - y_hat\n", + "\n", + " PZT = P.dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + " F_inv = pt.linalg.inv(F)\n", + "\n", + " K = PZT.dot(F_inv) \n", + " \n", + " return K.T @ T.T @ a_hat_grad + 2 * F_inv @ v" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "c17949b7", + "metadata": {}, + "outputs": [], + "source": [ + "def grad_Q(inp, out, out_grad):\n", + " _, P_h_grad, _ = out_grad\n", + " return P_h_grad" + ] + }, + { + "cell_type": "code", + "execution_count": 135, + "id": "84cb6867", + "metadata": {}, + "outputs": [], + "source": [ + "def grad_H(inp, out, out_grad):\n", + " y, a, P, T, Z, H, Q = inp\n", + " a_hat_grad, P_h_grad, y_grad = out_grad\n", + " \n", + " y_hat = Z.dot(a)\n", + " v = y - y_hat\n", + "\n", + " PZT = P.dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + " F_inv = pt.linalg.inv(F)\n", + "\n", + " K = PZT.dot(F_inv)\n", + "\n", + " v = v.dimshuffle(0, 'x')\n", + " a_hat_grad = a_hat_grad.dimshuffle(0, 'x') \n", + "\n", + " a_filtered_grad = T.T @ a_hat_grad\n", + " P_filtered_grad = T.T @ P_h_grad @ T\n", + "\n", + " return K.T @ P_filtered_grad @ K - 0.5 * K.T @ a_filtered_grad @ v.T @ F_inv - 0.5 * F_inv @ v @ a_filtered_grad.T @ K + F_inv - F_inv @ v @ v.T @ F_inv" + ] + }, + { + "cell_type": "markdown", + "id": "607753a1", + "metadata": {}, + "source": [ + "### Custom Kalman Filter" + ] + }, + { + "cell_type": "code", + "execution_count": 136, + "id": "7cead2c1", + "metadata": {}, + "outputs": [], + "source": [ + "y_sym = pt.vector(\"y\")\n", + "\n", + "kalman_step_op = OpFromGraph(\n", + " inputs=[y_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", + " outputs=kalman_step(y_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym),\n", + " lop_overrides=[grad_y, grad_a_hat, grad_P_hat, None, None, grad_H, grad_Q],\n", + " inline=True\n", + ")\n", + "\n", + "outputs_info = [None, a0_sym, None, None, P0_sym, None, None]\n", + "\n", + "results_op, updates = pytensor.scan(\n", + " kalman_step_op,\n", + " sequences=[data_sym],\n", + " outputs_info=outputs_info,\n", + " non_sequences=[T_sym, Z_sym, H_sym, Q_sym],\n", + " strict=False,\n", + ")\n", + "# --- Loss ---\n", + "a_upd_op, a_pred_op, y_hat_op, P_upd_op, P_pred_op, obs_cov, ll_op = results_op\n", + "loss_op = pt.sum(ll_op)" + ] + }, + { + "cell_type": "markdown", + "id": "f0575c2c", + "metadata": {}, + "source": [ + "### Speed observation" + ] + }, + { + "cell_type": "code", + "execution_count": 137, + "id": "07c3879e", + "metadata": {}, + "outputs": [], + "source": [ + "states = [1, 5, 10, 20, 35, 50, 75, 90, 100]" + ] + }, + { + "cell_type": "code", + "execution_count": 140, + "id": "f90f682d", + "metadata": {}, + "outputs": [], + "source": [ + "def benchmark_kalman_gradients(loss, state_dims, N=30):\n", + " results = defaultdict(dict)\n", + " for _ in range(10):\n", + " for n in state_dims:\n", + " data = generate_kalman_dataset(n, N=N, seed=42 + n)\n", + "\n", + " # --- gradients symboliques ---\n", + " t0 = perf_counter()\n", + " grad_list = pt.grad(loss, [a0_sym])\n", + " t1 = perf_counter()\n", + " grad_symbolic_time = t1 - t0\n", + " results[n][\"grad_symbolic_time\"] = grad_symbolic_time/10\n", + "\n", + " # --- compilation ---\n", + " t0 = perf_counter()\n", + " f_grad = pytensor.function(\n", + " inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", + " outputs=grad_list,\n", + " )\n", + " t1 = perf_counter()\n", + " compile_time = t1 - t0\n", + " results[n][\"compile_time\"] = compile_time/10\n", + "\n", + " # --- exécution ---\n", + " t0 = perf_counter()\n", + " _ = f_grad(\n", + " data[\"y\"],\n", + " data[\"A0\"],\n", + " data[\"P0\"],\n", + " data[\"T\"],\n", + " data[\"Z\"],\n", + " data[\"H\"],\n", + " data[\"Q\"],\n", + " )\n", + " t1 = perf_counter()\n", + " exec_time = t1 - t0\n", + " results[n][\"exec_time\"] = exec_time/10\n", + "\n", + " return results" + ] + }, + { + "cell_type": "code", + "execution_count": 141, + "id": "27a60fb3", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", + " r = _umath_linalg.det(a, signature=signature)\n", + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", + " r = _umath_linalg.det(a, signature=signature)\n", + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", + " r = _umath_linalg.det(a, signature=signature)\n", + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", + " r = _umath_linalg.det(a, signature=signature)\n", + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", + " r = _umath_linalg.det(a, signature=signature)\n", + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", + " r = _umath_linalg.det(a, signature=signature)\n", + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", + " r = _umath_linalg.det(a, signature=signature)\n", + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", + " r = _umath_linalg.det(a, signature=signature)\n", + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", + " r = _umath_linalg.det(a, signature=signature)\n", + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", + " r = _umath_linalg.det(a, signature=signature)\n" + ] + } + ], + "source": [ + "results = benchmark_kalman_gradients(loss, states, N=5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3f6d314a", + "metadata": {}, + "outputs": [], + "source": [ + "results_op = benchmark_kalman_gradients(loss_op, states, N=5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "382e90ef", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 123, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj4AAAGdCAYAAAASUnlxAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAX+JJREFUeJzt3XlcVdX+//HXAQ6HQUBlRmVKTVMrQTMts+li6K0sb9mgaXX9ZTkRtzK1blbm0K1ut5wazDIbrCyzspK+BlmSqakZmpmhKIqIA6DMh/X74xRFoAkBh+H9fDzOQ87aa+/9OTvlvNvDWhZjjEFERESkBXBxdgEiIiIiDUXBR0RERFoMBR8RERFpMRR8REREpMVQ8BEREZEWQ8FHREREWgwFHxEREWkxFHxERESkxXBzdgGNSXl5Ofv378fHxweLxeLsckREROQ0GGPIz88nLCwMF5dTn9NR8Pmd/fv306FDB2eXISIiIrWwd+9e2rdvf8o+Cj6/4+PjAzgOnK+vr5OrERERkdORl5dHhw4dKr7HT0XB53d+vbzl6+ur4CMiItLEnM5tKrq5WURERFoMBR8RERFpMRR8REREpMVQ8BEREZEWQ8FHREREWgwFHxEREWkxFHxERESkxVDwERERkRZDwUdERERaDAUfERERaTFqFXzmzZtHVFQUHh4exMbGsmbNmlP2T0lJITY2Fg8PD6Kjo1mwYEGl5WlpaQwdOpTIyEgsFgtPP/10lW38uuyPr7Fjx1b0GTVqVJXl559/fm0+ooiIiDRDNQ4+S5cuJSEhgalTp7Jp0yb69+9PfHw8GRkZ1fZPT09n0KBB9O/fn02bNjFlyhQmTJjAsmXLKvoUFBQQHR3NrFmzCAkJqXY769ev58CBAxWvpKQkAK677rpK/a644opK/VauXFnTjygiIiLNlMUYY2qyQp8+fYiJiWH+/PkVbV27dmXIkCHMnDmzSv9JkyaxYsUKtm/fXtE2ZswYtmzZQmpqapX+kZGRJCQkkJCQcMo6EhIS+PDDD9m5c2fFpGSjRo3i2LFjLF++vCYfqUJeXh5+fn7k5uZqklIREZE6VG7KeSbzGQKsAQwPHl6n267J93eNzviUlJSwceNG4uLiKrXHxcWxdu3aatdJTU2t0n/gwIFs2LCB0tLSmuy+Uh1LlizhtttuqzITa3JyMkFBQXTu3JnRo0eTnZ190u0UFxeTl5dX6SUiIiJ1q9SUMm3PNF7NfpWnM58mvTDdabXUKPjk5ORgt9sJDg6u1B4cHExWVla162RlZVXbv6ysjJycnBqW67B8+XKOHTvGqFGjKrXHx8fz2muvsXr1ap588knWr1/PpZdeSnFxcbXbmTlzJn5+fhWvDh061KoeERERqV6hvZB/7foXHx35CFdcmRYxjSjPKKfV41ablf54lsUYU6Xtz/pX1366Fi5cSHx8PGFhYZXahw0bVvFz9+7d6dWrFxEREXz00Udce+21VbYzefJkEhMTK97n5eUp/IiIiNSRY2XHSNiVwNYTW7FZbMyOnk1/v/5OralGwScgIABXV9cqZ3eys7OrnNX5VUhISLX93dzc8Pf3r2G5sGfPHj777DPefffdP+0bGhpKREQEO3furHa5zWbDZrPVuAYRERE5taySLMb9NI70onR8XX353xn/4+xWZzu7rJpd6nJ3dyc2NrbiiapfJSUl0a9fv2rX6du3b5X+q1atolevXlit1hqWC4sWLSIoKIjBgwf/ad/Dhw+zd+9eQkNDa7wfERERqZ30wnRu23Eb6UXpBFmDeLHzi40i9EAtHmdPTEzkxRdf5KWXXmL79u3cfffdZGRkMGbMGMBx+eiWW26p6D9mzBj27NlDYmIi27dv56WXXmLhwoXcc889FX1KSkrYvHkzmzdvpqSkhMzMTDZv3sxPP/1Uad/l5eUsWrSIkSNH4uZW+WTV8ePHueeee0hNTWX37t0kJydz5ZVXEhAQwDXXXFPTjykiIiK1sPXEVm7/8XYOlh4k0hbJS2e+xBmeZzi7rN+YWpg7d66JiIgw7u7uJiYmxqSkpFQsGzlypBkwYECl/snJyaZnz57G3d3dREZGmvnz51danp6eboAqrz9u59NPPzWA2bFjR5WaCgoKTFxcnAkMDDRWq9WEh4ebkSNHmoyMjNP+XLm5uQYwubm5p72OiIiIOHx57EvTb1M/E7Mxxtyy/RZzpPRIg+y3Jt/fNR7HpznTOD4iIiK1s/LwSqbtmYYdO/18+/F41ON4uno2yL5r8v1dq6e6RERERH615OAS/pv5XwDi28TzUORDWC01v4+3ISj4iIiISK0YY3h2/7O8cvAVAG4OupmEdgm4WBrvHOgKPiIiIlJjZaaM6Xum88GRDwCYEDaBW4JvqfUYfQ1FwUdERERqpLC8kMk/T2ZN3hpcceWBiAe4yv8qZ5d1WhR8RERE5LTlluVy96672XJiCzaLjVlRs7io9UXOLuu0KfiIiIjIaTlYcpDxP41nV9EufFx9+O8Z/6Vnq57OLqtGFHxERETkT6UXpTPup3FklWQRaA1kTsc5dPTs6OyyakzBR0RERE7p+xPfM+GnCeTac4mwRTC341xCbU1zOqjG+7yZiIiION3avLXcsfMOcu25dPPqxsLOC5ts6AEFHxERETmJj498TMJPCRSVF3G+z/ks6LSANtY2zi7rL9GlLhEREani9ezXeXLfkwAMbDOQhyMexurSOEdjrgkFHxEREalgjGHu/rksOrgIgBsCb+Bf7f/VqEdjrgkFHxEREQEcozHPyJjB+4ffB2Bs2FhuDb610Y/GXBMKPiIiIkJReRFT0qeQkpuCCy5MCZ/CNQHXOLusOqfgIyIi0sLlleVx96672XxiM+4Wd2ZEzeCS1pc4u6x6oeAjIiLSgh0qOcTYn8ayq2gXrVxb8d/o/xLjE+PssuqNgo+IiEgLtadoD2N/GsuBkgMEWAOYc8YcOnl1cnZZ9UrBR0REpAVKO5HGhF0TOFZ2jHBbOHM6zqGdrZ2zy6p3zePZNBERETltX+d9zR077+BY2TG6enVlYeeFLSL0gIKPiIhIi/LpkU+ZuGsiheWFnOdzHs91eo621rbOLqvB6FKXiIhIC/Fm9ps8se8JDIa4NnE8HPEw7i7uzi6rQSn4iIiINHPGGOYfmM/CrIUADAscxj3t72k2ozHXhIKPiIhIM1ZmypiVMYv3Dr8HwJ2hd3J7yO3NajTmmlDwERERaaaKy4uZkj6F5NxkXHBhcvhkrg241tllOZWCj4iISDOUX5ZP4s+JfHv8W9wt7jwW+RiXtrnU2WU5nYKPiIhIM3Oo9BDjfxrPzsKdeLt489QZT9HLp5ezy2oUFHxERESakYyiDMb9NI7Mkkz83fx5tuOznOl1prPLajRa3u3cIiIizdT2gu3c9uNtZJZk0t7WnpfOfKlxhZ7DuyB9jVNL0BkfERGRZuCbvG/418//oqC8gDM9z+TZjs/ib/V3dlkORXnwxX9g3QLwCoDxG8Dd2ymlKPiIiIg0cUlHk3hw94OUmlJ6+/TmiegnaOXaytllQbkdNr8G//cInDjkaAvq6ghCCj4iIiJSU28deovH9z6OwXBZ68uYHjm9cYzGvGctfDwJsr5zvPfvCANnQKc4cOIYQgo+IiIiTZAxhucOPMcLWS8A8I+Af3Bfh/twtbg6t7BjGZD0b0hzDJiIzQ8ungS9R4Ob8wOZgo+IiEgTYzd2Zu+dzbKcZQDcEXoHo0NGO3c05uLj8NXTsPZZKCsCiwvEjoJLpoJ3gPPq+gMFHxERkSaitLyUT45+wpKDS/ip6CcsWLi/w/38I/AfziuqvBy2vgWfTYP8A462yP5wxSwI6e68uk5CwUdERKSRyyvLY1nOMpYeWsqhUsdNwl4uXjwU8RCXt7nceYXtXQ+f3A+ZGxzv20RC3HTo8nen3sdzKgo+IiIijVRmcSavZ7/O+4ffp7C8EIAAawA3BN7A0ICh+Lr5OqewvP2OMzzfLXW8d28FF90Dfe4Eq4dzajpNCj4iIiKNzNYTW1lycAmrj62mnHIAOnp0ZETwCAa2GYjVxeqcwkoLHffwfPlfKC0ALNDzZrj03+AT7JyaakjBR0REpBGwGztf5H7BqwdfZcuJLRXt5/ucz4jgEfTx6eO8m5eNgbR3IekhyN3raOtwPsTPgrCezqmplhR8REREnKiwvJAPD3/Ia9mvsbfYESrcLG5c0eYKhgcNp5NXJ+cWuH8TfDIZMlId733bQ9wj0O3aRnsfz6ko+IiIiDjB4dLDvHXoLd4+9Da59lwAfFx9GBowlBsCbyDQPdC5BeYfhNWPwKbXAANWL7ggAfqNB3cv59b2F9RqktJ58+YRFRWFh4cHsbGxrFlz6gnHUlJSiI2NxcPDg+joaBYsWFBpeVpaGkOHDiUyMhKLxcLTTz9dZRvTpk3DYrFUeoWEhFTqY4xh2rRphIWF4enpycUXX0xaWlptPqKIiEi9SC9M59E9j/L37//Oi1kvkmvPpZ17O+5tfy8ru69kfLvxzg09ZcWOe3iejYVNSwADPa6HcRscAxE24dADtTjjs3TpUhISEpg3bx4XXHABzz33HPHx8Wzbto3w8PAq/dPT0xk0aBCjR49myZIlfPXVV9x1110EBgYydOhQAAoKCoiOjua6667j7rvvPum+u3XrxmeffVbx3tW18uiUjz/+OE899RQvv/wynTt3Zvr06fztb39jx44d+Pj41PSjioiI1AljDBuOb2DJwSV8mfdlRXt3r+6MCB7BJa0vcf6Iy8bADx/BqqlwdLejrV0sXDEbOvR2aml1yWKMMTVZoU+fPsTExDB//vyKtq5duzJkyBBmzpxZpf+kSZNYsWIF27dvr2gbM2YMW7ZsITU1tUr/yMhIEhISSEhIqNQ+bdo0li9fzubNm6utyxhDWFgYCQkJTJo0CYDi4mKCg4OZPXs2d9xxx59+try8PPz8/MjNzcXX10mPCIqISLNRakr57OhnLDm4hB8KfwDAgoWL/S5mePBwzvE+x7mjLf/qYJpjPJ70LxzvW4XA5dPg7GHgUquLQw2qJt/fNTrjU1JSwsaNG7n//vsrtcfFxbF27dpq10lNTSUuLq5S28CBA1m4cCGlpaVYraf/SN7OnTsJCwvDZrPRp08fZsyYQXR0NOA4s5SVlVVpXzabjQEDBrB27dpqg09xcTHFxcUV7/Py8k67FhERkZPJt+ezPGc5b2S/wcHSgwDYLDau8r+Km4JuItyj6hUSpziRA58/BhtfBlMOrjbHPTwX3g22RjC7ez2oUfDJycnBbrcTHFz5Wf3g4GCysrKqXScrK6va/mVlZeTk5BAaGnpa++7Tpw+LFy+mc+fOHDx4kOnTp9OvXz/S0tLw9/ev2H91+9qzZ0+125w5cyYPP/zwae1fRETkzxwoOcAb2W+wPGc5J8pPANDWrS3DAocxNHAobdzaOLnCX9hL4ZsXIHkWFDturOasIfC3hx2jLzdjtXqq64+n5YwxpzxVV13/6tpPJT4+vuLnHj160LdvX8444wxeeeUVEhMTa1Xb5MmTK62bl5dHhw4dTrsmERERgO0F23n14Kt8dvQz7NgBiPKIYnjQcOLbxmNzsTm5wt/5cRV8OgUO73S8D+nhuI8n8gLn1tVAahR8AgICcHV1rXJ2Jzs7u8qZll+FhIRU29/NzQ1/f/8alvsbb29vevTowc6dOyv2A44zTL8/i3Sq2mw2GzZbI/rLKCIiTUa5KeervK949eCrbDy+saK9t09vhgcNp59vP1wsjej+mEM7HIHnp18eEvIOhEsfhJ7DwcXJN1Y3oBoFH3d3d2JjY0lKSuKaa66paE9KSuLqq6+udp2+ffvywQcfVGpbtWoVvXr1qtH9PX9UXFzM9u3b6d+/PwBRUVGEhISQlJREz56OUSRLSkpISUlh9uzZtd6PiIjI7xWXF7PyyEqWHFzC7uLdALjiSlybOG4OvpmuXl2dW+AfFR51XNL65gUwdnCxwvlj4KJ7wcPP2dU1uBpf6kpMTGTEiBH06tWLvn378vzzz5ORkcGYMWMAx+WjzMxMFi9eDDie4JozZw6JiYmMHj2a1NRUFi5cyBtvvFGxzZKSErZt21bxc2ZmJps3b6ZVq1Z07NgRgHvuuYcrr7yS8PBwsrOzmT59Onl5eYwcORJwXOJKSEhgxowZdOrUiU6dOjFjxgy8vLy46aab/tpREhGRFu9o2VHePvQ2bx96myNlRwDwdvHm2oBruSHoBkLcQ/5kCw3MXgYbFzluXi486mg7c5Bj9nT/M5xbmxPVOPgMGzaMw4cP88gjj3DgwAG6d+/OypUriYiIAODAgQNkZGRU9I+KimLlypXcfffdzJ07l7CwMJ555pmKMXwA9u/fX3GWBuCJJ57giSeeYMCAASQnJwOwb98+brzxRnJycggMDOT888/n66+/rtgvwH333UdhYSF33XUXR48epU+fPqxatUpj+IiISK3tKdrDa9mv8eHhDyk2jieBQ9xDuDHwRoYEDKGVayN8+mnX545pJg79MpRMYFe4YiaccYlz62oEajyOT3OmcXxERAQcD8ZsPrGZVw++yhe5X2BwfFV29erK8KDhXNbmMqwWJ82QfiqHd8GqB2DHSsd7zzZwyVSIvRVcm+8sVfU2jo+IiEhzVmbKWH1sNUsOLiGt4Lcpj/r79mdE8AhiWsU0jgEH/6goD774D3w9H8pLweIK542GAZPAq62zq2tUFHxERKTFO2E/wfuH3+eN7DfYX7IfAHeLO4PbDubmoJuJ8oxycoUnUW53zKe1+lE4ccjR1vFyGDgDAs90bm2NlIKPiIi0WNkl2bx56E3ezXmXfHs+AK3dWnNdwHVcH3g9ba2N+GzJ7q/gk0mQtdXx3r+TI/B0jjv1ei2cgo+IiLQ4Owt28mr2q3x69FPKTBkA4bZwhgcNZ7D/YDxcPJxc4UnYy+DHT2D9C/BzsqPN5ueYNb33aHBzd2p5TYGCj4iItAjGGFLzU1lycAnr8tdVtPds1ZMRQSPo79e/cQ04+HsncuDbV2DDIsjd62izuEDsKMfNy94BTi2vKVHwERGRZq2kvIRPjn7Cawdf46einwBwwYXLWl/G8ODhdPfu7uQKT2HfRvjmeUh7F+wljjbPthAzAnrd1uzn1aoPCj4iItIs5ZXl8U7OOyw9tJSc0hwAPF08GeI/hBuDbqSdrZ2TKzyJ0iJH0Pnmedi/6bf2sJ5w3v+DbteA1dN59TVxCj4iItKs7Cvex+vZr7Pi8AoKywsBCLQGcmPgjVwbcC0+bo10UNuje2DDS/DtYih0jAyNqzt0u9YReNrHOre+ZkLBR0REmoWtJ7by6sFX+fzY55RTDkAnz06MCBpBXJs4rC6NcMDB8nL4+XNY/yLs+Bh+GSgRvw7Q61aIGan7d+qYgo+IiDRZdmMnJTeFJQeXsOXElor2vr59GRE0gvN8zmucAw4WHoMtbzgmDj2y67f26IsdT2d1vqJZj7TsTDqqIiLS5BSWF/LB4Q94Pft19hY7nnJys7gxqO0gbg66mY6eHZ1c4UkcTHOEne+WQmmBo83dB869CXr/EwI7O7e+FkDBR0REmozDpYdZemgp7xx6h1x7LgC+rr78I+AfXB90PYHWQCdXWA17KWz/wHE5a89Xv7UHdnFMK3H2MLA10vuOmiEFHxERafTSi9J59eCrrDyyklJTCkA793bcHHQzV/lfhadrI3zKKT8LNr7seOUfcLRZXKHr3x2XsyIvhMZ4Ga6ZU/AREZFG62DJQZ478BwfHP6g4oblHt49GBE0gotbX4yrxdXJFf6BMZDxteNR9O0roNwxKjTeQY7BBmNHgV8jfYy+hVDwERGRRie/LJ+XD77MG9lvUGyKARjgN4CRwSM5p9U5Tq6uGiUn4Lu3HJezDn7/W3uHPo5H0btepekkGgkFHxERaTSKy4t5+9DbvJT1UsU9POd6n8vEdhM5u9XZTq6uGod3OcLOpteg2FEvbp5w9nWOy1mhjbDmFk7BR0REnM5u7Hxy5BPmHZhHVkkWANEe0YwPG09/v/6N65H0cjvsTHJcztr1f7+1t4l0hJ2eN4NnG6eVJ6em4CMiIk5jjCE1L5Vn9j/DzsKdAARZgxgTOobB/oNxszSir6mCI7DpVccZnmMZvzRaoNPfHJezzrgMXBrpJKdSoRH9jRIRkZZk24lt/C/zf2w4vgGAVq6tuDX4VoYFDcPTpRE9pbV/E3zzInz/DpQVOdo8WkPP4dD7dmgb7dTypGYUfEREpEHtLdrL3P1zSTqWBIDVYmVY4DBuC7kNPzc/J1f3i7JiSFsO61+Afet/aw/pAefdAd2HgruX08qT2lPwERGRBnGk9AgvZL3AskPLsGPHgoVBbQdxZ+idhNpCnV2eQ+4+x0ShG1+BAseM7rhYodsQx/07Hc7T2DtNnIKPiIjUqwJ7AUuyl/DqwVcpKHdM03CB7wWMCxtHZ69GMEWDMZCe4phKYsdKMI7xgvAJg163QexIaBXk3Bqlzij4iIhIvSg1pbyX8x4vHHiBI2VHAOjm1Y3x7cbT26e3k6sDivJgy5uOm5VzdvzWHtnfMZXEmYM1UWgzpP+iIiJSp4wxfHbsM+bun1sxgWgHWwfGho3l8taXO//R9OwfHPfubHkTSo472txbwTk3OCYKDerq3PqkXin4iIhIndmQv4FnMp8hrSANgLZubRkdOpprAq7BarE6rzB7meMy1jfPw+41v7UHdHbcu3PODeDh67z6pMEo+IiIyF+2s2Anz+5/lq/yHLOPe7p4ckvwLdwcdDPert7OK+x4Nnz7CmxYBHmZjjaLC5w5yHE5K2qAblZuYRR8RESk1g6UHGDB/gV8dOQjDAZXXBkaOJR/hvwTf6u/c4oyxvEI+jcvQNp7UO6YzR2vAMeNyrG3QusOzqlNnE7BR0REaiy3LJeXsl7irUNvUWJKAPhb679xV9hdhHuEO6eo0kLY+o7jclbWd7+1t+vlGFm52xBwszmnNmk0FHxEROS0FZUX8Wb2myw6uIjjdseNwbGtYpnYbiLdvLs5p6gj6bBhIXz7KhQdc7S52qDHdXDePyGsp3PqkkZJwUdERP6U3dj58PCHLDiwgOzSbAA6enRkQrsJ9PPt1/BPapWXOyYI/eYF2LkKMI721uGOJ7N6jgCvtg1bkzQJCj4iInJSxhjW5K5hzv457CraBUCIewh3hd7FFW2vwNXi2rAFFR6FTa85zvAc+fm39jMuc1zO6vQ3cGngmqRJUfAREZFqfXf8O57Z/wybjm8CwNfVl9tCbuP6wOuxuTTwvTIHt8G6+fDd21BW6Giz+UHPm6HX7RDQsWHrkSZLwUdERCrZXbSbufvnsvrYagBsFhs3BN3ArcG34uPm07DF5OyEz2dA2ru/tQV3d1zOOvt6cHfio/LSJCn4iIgIAIdKD/HCgRdYnrMcO3ZccOFK/yu5I/QOgt2DG7aYYxmQPBu2vP7b3Fldr4Lz74Twvhp7R2pNwUdEpIU7bj/O4oOLeS37NYrKiwAY4DeAsWFjOcPzjIYtJj8LvngCNr782/g7nePh0qkQ0qNha5FmScFHRKSFKikv4Z2cd1iYtZBjZccAONv7bCa0m0DPVg38CHjBEfjyv46ntH69hydqAFz6IHRoBBOaSrOh4CMi0sKUm3I+Pfop8/fPJ7PEMY1DhC2Cce3GcYnfJQ37aHpRHqTOdbxK8h1t7c+Dyx6EqIsarg5pMRR8RERakK/zvuaZzGfYUbgDgABrAHeE3sFV/lfhZmnAr4SSAscIy1897XhEHRyXsi59EDrF6R4eqTcKPiIiLcD2gu08m/ks6/LXAeDt4s3I4JHcFHQTnq6eDVdIWTFsfAXWPAHHDzraAjrDJVOg69Xg4tJwtUiLVKu/YfPmzSMqKgoPDw9iY2NZs2bNKfunpKQQGxuLh4cH0dHRLFiwoNLytLQ0hg4dSmRkJBaLhaeffrrKNmbOnEnv3r3x8fEhKCiIIUOGsGPHjkp9Ro0ahcViqfQ6//zza/MRRUSahcziTKamT2X4D8NZl78ON4sbNwbeyPvd3+f20NsbLvTYy+DbxfBsLHx8ryP0tI6AIQvgrq+h2zUKPdIganzGZ+nSpSQkJDBv3jwuuOACnnvuOeLj49m2bRvh4VUnpktPT2fQoEGMHj2aJUuW8NVXX3HXXXcRGBjI0KFDASgoKCA6OprrrruOu+++u9r9pqSkMHbsWHr37k1ZWRlTp04lLi6Obdu24e392zgOV1xxBYsWLap47+7uXtOPKCLS5B0tPcrCrIW8nfM2ZaYMgPg28dwZdiftbO0arpDycscYPJ/PgCOOkZ/xCYWL7nVMK+Gm39HSsCzGGFOTFfr06UNMTAzz58+vaOvatStDhgxh5syZVfpPmjSJFStWsH379oq2MWPGsGXLFlJTU6v0j4yMJCEhgYSEhFPWcejQIYKCgkhJSeGiixw3wI0aNYpjx46xfPnymnykCnl5efj5+ZGbm4uvr2+ttiEi4kyF9kJez36dVw6+wonyEwCc73M+49uNp4tXl4YrxBjYsRJWPwbZaY42L3+4MBF63w7WBry8Js1eTb6/a3TGp6SkhI0bN3L//fdXao+Li2Pt2rXVrpOamkpcXFyltoEDB7Jw4UJKS0uxWq01KaFCbm4uAG3bVp6ELjk5maCgIFq3bs2AAQN47LHHCAoKqnYbxcXFFBcXV7zPy8urVS0iIs5WZsp4P+d9njvwHIfLDgNwpueZTGw3kT6+fRquEGPg589h9XTI3Ohos/lBv/Fw/hiwNfDIzyJ/UKPgk5OTg91uJzi48giewcHBZGVlVbtOVlZWtf3LysrIyckhNDS0hiU7Js1LTEzkwgsvpHv37hXt8fHxXHfddURERJCens6DDz7IpZdeysaNG7HZqs4rM3PmTB5++OEa719EpLEwxvB57ufMyZzDnuI9ALRzb8ddYXcR1yYOF0sD3jeT8TX836Ow50vHe6sX9BnjCD2aKV0aiVo91fXHMR6MMacc96G6/tW1n65x48bx3Xff8eWXX1ZqHzZsWMXP3bt3p1evXkRERPDRRx9x7bXXVtnO5MmTSUxMrHifl5dHhw4dalWTiEhD23R8E89kPsN3J74DoLVba/4Z8k+GBgzF3aUB753Zv9lxhuenJMd7V3fHxKH9E6FV9WfcRZylRsEnICAAV1fXKmd3srOzq5zV+VVISEi1/d3c3PD3969huTB+/HhWrFjBF198Qfv27U/ZNzQ0lIiICHbu3FntcpvNVu2ZIBGRxmxX4S7m7J/DF7lfAODh4sHNQTdzS/AttHJt1XCFZP8Anz8G21c43ltcoedwGHAf+J3697OIs9Qo+Li7uxMbG0tSUhLXXHNNRXtSUhJXX311tev07duXDz74oFLbqlWr6NWrV43u7zHGMH78eN577z2Sk5OJior603UOHz7M3r17a3U5TUSksTlYcpDnDjzHB4c/oJxyXHFlSMAQRoeOJtAa2HCFHPnZMYHod0sBA1igx3Vw8f3g38Bze4nUUI0vdSUmJjJixAh69epF3759ef7558nIyGDMmDGA4/JRZmYmixcvBhxPcM2ZM4fExERGjx5NamoqCxcu5I033qjYZklJCdu2bav4OTMzk82bN9OqVSs6duwIwNixY3n99dd5//338fHxqTiL5Ofnh6enJ8ePH2fatGkMHTqU0NBQdu/ezZQpUwgICKgU0kREmpr8snwWHVzEm9lvUmwcD2Rc2vpSxoaNJdIjsuEKyc2ELx6HTUug3PGIPF3+DpdMheCzGq4Okb/C1MLcuXNNRESEcXd3NzExMSYlJaVi2ciRI82AAQMq9U9OTjY9e/Y07u7uJjIy0syfP7/S8vT0dIPjfxsqvX6/neqWA2bRokXGGGMKCgpMXFycCQwMNFar1YSHh5uRI0eajIyM0/5cubm5BjC5ubk1PiYiInWtyF5kFmctNhdvvtjEbIwxMRtjzO07bjffHf+uYQvJzzbm4/uNeSTQmId8Ha/F1xizb2PD1iFyEjX5/q7xOD7NmcbxEZHGwG7sfHzkY+YfmE9WiePs9hkeZzCu3Tj6+/ZvuElEC4/C2mfh6wVQ6hgTiPB+jglEI/o1TA0ip6HexvEREZH6Y4xhbd5ansl8hp+KfgIg2BrMmLAxDG47GFeLa8MUUnwc1s13hJ4ix5hphPWESx+AMy7TBKLSpCn4iIg0Amkn0vhf5v/YeNwx6F8r11bcFnwbw4KG4eHi0TBFlBbBhoWw5ikoyHG0BZ3luIeny2AFHmkWFHxERJwooyiDufvn8tmxzwBwt7gzLHAYt4bcip+bX8MUYS+FTa9Cyn8gf7+jrW00XDwFul8LLg10pkmkASj4iIg4weHSw7xw4AXezXkXO3YsWBjcdjBjwsYQ6t5AQ3CU22Hr25A8E47udrT5tneMw3PuTeBauymFRBozBR8RkQZ0wn6CJQeX8Gr2qxSWFwJwge8FjA8bTyevTg1TRHm5Y9DBz2dAzg5Hm3cg9L8HYkeBtYEurYk4gYKPiEgDKDWlvJfzHi8ceIEjZUcA6ObVjQntJtDLp1fDFGEM7EyC1Y9ClmOaCzxawwUToc8d4O7dMHWIOJGCj4hIPTLGkHQsiXn757G3eC8A4bZwxoaN5bLWlzXco+npaxzzae392vHevRWcfxf0HQuerRumBpFGQMFHRKSerM9fzzOZz7CtwDEyvb+bP6NDRzMkYAhWSwPdP7NvI6x+BH5Odrx384De/4QL7wbvgIapQaQRUfAREaljOwt28sz+Z1ibtxYALxcvRgSPYHjQcLxcvRqmiKzvHROI7ljpeO9ihZhb4KJ7wVfzF0rLpeAjIlJHDhQfYP6B+aw8shKDwRVX/hH4D24PuR1/q3/DFJHzEyTPgO/fBQxYXOCcGx1ParWJbJgaRBoxBR8Rkb/oWNkxXsp6ibcOvUWpKQUgrk0cd4XeRQePDg1URAakzIbNb4CxO9q6XeMYiyewc8PUINIEKPiIiNRSUXkRb2S/wcsHX+a4/TgAvVr1YkK7CXTz7tYwReRnwZonYcMiKHeELjpf4RhtOfTshqlBpAlR8BERqaEyU8aHhz/kuQPPkV2aDUAnz05MCJtAX9++DfOkVsER+OppWPc8lDnGAyLqIrj0QehwXv3vX6SJUvARETlNxhi+yP2COfvn8HPRzwCEuIdwV+hdxLeNx8XiUv9FFOXB1/MgdS4U5zna2vd2BJ7oAfW/f5EmTsFHROQ0bDm+hWcyn2Hzic0A+Ln6cXvI7fwj8B/YXGz1X0BJAax/Ab58GgodAyAS3MMxY3rngZpAVOQ0KfiIiJxCelE6czLnkJybDIDNYuPGoBsZFTwKHzef+i+grBi+XQxf/AeOH3S0+XeCS6bAWUPApQHOMok0Iwo+IiLVOFRyiOeznuf9nPexY8cFF670v5IxoWMIcg+q/wLsZbDlDUh5HHIzHG2tw2HA/XD2MHDVr2+R2tC/HBGR38m35/PqwVdZcnAJxaYYgAF+AxgXNo5oz+j6L6C8HNLedcyYfvgnR1urEBhwL/S8Bdzc678GkWZMwUdEBCgpL+GdnHd48cCL5NpzATjH+xzGtxtPz1Y9678AY2DHx47Rlg9+72jzbAv9Ex1TTFg9678GkRZAwUdEWrRyU84nRz9h/v757C/ZD0CkLZJx7cZxsd/F9f9oujGOebRWT4fMDY42my/0Gw/n3wm2BriPSKQFUfARkRYrNS+VZzOfZUfhDgACrAHcEXoHV/lfhZulAX49ZqyD1Y/C7jWO91Yv6HMH9JsAXm3rf/8iLZCCj4i0ONsLtvNs5rOsy18HgLeLN6NCRnFj0I14ujTAJaUDWxxneHaucrx3dYdet8GFieATXP/7F2nBFHxEpMXYV7yPefvn8enRTwFws7hxfeD13BZyG23c2tR/AYd2OO7h2fa+473FFXreDBfdB60baE4vkRZOwUdEmr2jpUd5MetF3sl5hzJTBkB8m3juDLuTdrZ29V/AkXTHBKLfLQVTDligxz/g4sngf0b9719EKij4iEiztjxnOU/te4oT5ScA6Ovbl3Fh4+ji1aX+d5633zEOz6ZXodwRuOjyd8fgg8ENNImpiFSi4CMizdY3ed8wPWM6BkMXzy5MbDeR83wbYALPEzmw5ilY/yLYHWMBccaljukl2sXW//5F5KQUfESkWTpUcoipu6diMFzlfxUPhj9Y/5OIFh6Dtc/C1/Oh1HGGifC+jglEIy+o332LyGlR8BGRZqfMlDF592SOlB2hs2dnJnWYVL+hp/g4rFsAa5+BIsfgh4Se6wg8HS/TBKIijYiCj4g0O/P2z2PT8U14u3gzO2o2Hi4e9bOj0iLY8BJ8+RScOORoC+wKl0513MujwCPS6Cj4iEizknIshVcOvgLAQxEPEe4RXvc7sZfCpiWOGdPzMh1tbaIcNy13HwournW/TxGpEwo+ItJsZBZn8tCehwC4MfBGLmtzWd3uoNwOW99xTCB6NN3R5tsOBtwH594Mrta63Z+I1DkFHxFpForLi5mUPol8ez49vHswsd3Eutu4MbD9A8fgg4d+cLR5B0L/f0HsrWCtp0tpIlLnFHxEpFl4at9TbC/Yjp+rHzOjZmJ1qYOzL8bAT5855tM6sMXR5uEHF0yE8+4AW6u/vg8RaVAKPiLS5H185GPeyXkHCxamR04n1D30r29095eO+bQyUh3v3Vs5ZkvvOw48W//17YuIUyj4iEiTll6YzmMZjwFwW8ht9PPr99c2mLkR/u9R+Plzx3tXG5w3Gi68G7wD/mK1IuJsCj4i0mQV2gu5L/0+CssL6e3TmztC76j9xg6mwerHYMdHjvcubhBzC1x0L/iG1U3BIuJ0Cj4i0iQZY5i5dyY/F/1MgDWAxyIfw9VSi8fID++Cz2fA98sAAxYXOPsGuHgStIms67JFxMkUfESkSXrv8Ht8dOQjXHFlZuRM/K3+NdvAsb2OGdM3vw7G7mg7a4hjLJ7AM+u8XhFpHBR8RKTJ+aHgB/6z9z8A3BV2FzE+Mae/cv5BWPMkbFwE9hJHW6eBjtGWQ8+ph2pFpDFR8BGRJiXfns+k9EmUmBL6+/bnluBbTm/FgiPw1f9g3XNQVuhoi+zvmE8rvE/9FSwijUqtZu2bN28eUVFReHh4EBsby5o1a07ZPyUlhdjYWDw8PIiOjmbBggWVlqelpTF06FAiIyOxWCw8/fTTtdqvMYZp06YRFhaGp6cnF198MWlpabX5iCLSCBljeHj3w+wr3keoeygPRz7855OPFudD8mz43znw1dOO0NOuF9zyPoz6UKFHpIWpcfBZunQpCQkJTJ06lU2bNtG/f3/i4+PJyMiotn96ejqDBg2if//+bNq0iSlTpjBhwgSWLVtW0aegoIDo6GhmzZpFSEhIrff7+OOP89RTTzFnzhzWr19PSEgIf/vb38jPz6/pxxSRRuj17Nf5PPdzrBYrs6Nm4+fmd+oVcjPh+UsgeQYU50Fwd7jxTfjnZxB9cYPULCKNjKmh8847z4wZM6ZSW5cuXcz9999fbf/77rvPdOnSpVLbHXfcYc4///xq+0dERJj//ve/Nd5veXm5CQkJMbNmzapYXlRUZPz8/MyCBQv+9HMZY0xubq4BTG5u7mn1F5GGszl/s+m9sbeJ2RhjlmYv/fMVjuw25r89jHnI15gnuxqz9R1j7Pb6L1REGlxNvr9rdManpKSEjRs3EhcXV6k9Li6OtWvXVrtOampqlf4DBw5kw4YNlJaW1tl+09PTycrKqtTHZrMxYMCAk9ZWXFxMXl5epZeIND5HS49yf/r92LEzsM1Argu47tQrHN4FiwbBsT2OWdNv++SXWdNrdXVfRJqRGv0WyMnJwW63ExwcXKk9ODiYrKysatfJysqqtn9ZWRk5OTl1tt9f/6xJbTNnzsTPz6/i1aFDh9OqR0Qajt3YeWD3A2SXZhNhi2Bq+FQsFsvJV8j+ARbFQ94+COgMt34MrcMbrmARadRq9b8/f/ylY4w55S+i6vpX114X+61JbZMnTyY3N7fitXfv3hrVIyL1b2HWQr7O/xqbxcbj0Y/j7ep98s5ZW+HlQXD8IAR1g1ErwbcO5u0SkWajRo+zBwQE4OrqWuUMSnZ2dpUzLb8KCQmptr+bmxv+/qc34Njp7PfXm6KzsrIIDQ2tts8f2Ww2bDbbadUgIg1vXd46nj/wPABTwqfQ0bPjyTtnboRXr4WiYxB6Lox4D7zaNkidItJ01OiMj7u7O7GxsSQlJVVqT0pKol+/6icG7Nu3b5X+q1atolevXlit1jrbb1RUFCEhIZX6lJSUkJKSctLaRKTxyi7JZuruqRgMQ/yH8Hf/v5+8c8bX8MrVjtDT/jwYuUKhR0SqVeMBDBMTExkxYgS9evWib9++PP/882RkZDBmzBjAcfkoMzOTxYsXAzBmzBjmzJlDYmIio0ePJjU1lYULF/LGG29UbLOkpIRt27ZV/JyZmcnmzZtp1aoVHTt2PK39WiwWEhISmDFjBp06daJTp07MmDEDLy8vbrrppr92lESkQZWaUianT+Zo2VHO9DyTezvce/LOP6fAGzdAaYFjQMIb3wRbq4YrVkSalto8NjZ37lwTERFh3N3dTUxMjElJSalYNnLkSDNgwIBK/ZOTk03Pnj2Nu7u7iYyMNPPnz6+0PD093QBVXn/czqn2a4zjkfaHHnrIhISEGJvNZi666CKzdevW0/5cepxdpHF4eu/TJmZjjOm/qb/JKMw4eccfk4x5NMjxyPriIcYUn2i4IkWk0ajJ97fFmF/uNBby8vLw8/MjNzcXX19fZ5cj0iKlHEsh8edEAP4T9R8ubXNp9R1/+AjeGgnlpdA5Hq5/Bdx0z55IS1ST728NaiEijca+4n08tOchAG4Kuunkoef7ZfDWLY7Qc9YQuH6xQo+InBYFHxFpFIrLi5n08yTy7fn08O7BhLAJ1Xfc/Dos+yeUl8HZN8DQheDm3rDFikiTpdnZRaRReHLfk/xQ+AN+rn7MipqF1aWapz43LIIPExw/x9wCf/+fRmMWkRrRbwwRcbqPj3zMspxlWLAwPWo6Ie7VTFb89fzfQs95dyj0iEit6LeGiDhVemE6j2U8BsDtIbfTz7eacbfWPAWf3O/4+YKJED9boUdEakWXukTEaQrthdyXfh+F5YWc53Me/y/0/1XuYAwkz4KUWY73A+6Hi++HGk53IyLyKwUfEXEKYwwz9s7g56KfCbAGMD1yOq4W1993gM8egq/+53h/+TS48G6n1CoizYeCj4g4xXuH32PlkZW44sqsqFn4W383d195uePS1jfPOd5fMQvOv9M5hYpIs6LgIyINbnvBdv6z9z8AjA0bS89WPX9bWF7uuIn521cAC/z9v9DrVqfUKSLNj4KPiDSo/LJ8Jv08iRJTwkV+FzEieMRvC+1l8P5Y+O5NsLjA1fPg3BudV6yINDsKPiLSYIwxTNszjcySTMLcw3g44mFcLL88nWUvdQxMuG05WFxh6AvQfahT6xWR5kfBR0QazGvZr5Gcm4zVYmV21Gx83X6ZU6e0CN4eBT9+DK7ucN3L0GWwM0sVkWZKwUdEGsTm45t5JvMZABLbJ3KW91mOBSUFsPRm2LUa3Dxg2GvQ6XInVioizZmCj4jUu6OlR5mcPhk7dga2Gch1Adc5FhQfhzdugN1rwOoNN74B0QOcW6yINGsKPiJSr+zGztTdU8kuzSbSFskD4Q9gsVigKBeW/AP2fQPuPjD8HQg/39nlikgzp+AjIvVqYdZC1uWvw8PFg8ejH8fL1QsKjsCr18CBzeDRGka8C+1inV2qiLQACj4iUm++zvua5w88D8CUDlM4w/MMOH4IFl8N2Wng5Q+3vA8hPZxcqYi0FAo+IlIvskuyeWD3AxgM1/hfw2D/wZB3ABZfBTk/QqsQR+gJ6uLsUkWkBVHwEZE6V2pKuT/9fo6WHeVMzzO5t8O9cCwDXrkKjqaDb3sYuQL8z3B2qSLSwij4iEidm5s5ly0ntuDt4s3s6NnYjmU6Qk/uXmgdASM/gDYRzi5TRFogBR8RqVPJx5J5NftVAKZFTKNDXqHj8lb+AfDv6Ag9vmFOrlJEWioFHxGpM/uK9/HQnocAuDnoZi4tCYbFg+DEIQg6y3FPT6sgJ1cpIi2Zgo+I1Ini8mIm/TyJ4/bjnO19NuMt/eHlwVB4FELOhhHLwdvf2WWKSAun4CMideLJfU/yQ+EPtHZrzUzbMKyLr4XiPGjXC4YvA8/Wzi5RRETBR0T+upVHVrIsZxkWLEz3vJGQ126D0hMQcQHctBRsPs4uUUQEUPARkb/o58KfeSzjMQD+6X4ZfZc+AGWFEH0x3PAGuHs5t0ARkd9R8BGRWiuwF3Bf+n0UlRfRx+UMRr/3EthLoNNAuH4xWD2cXaKISCUuzi5ARJomYwwzMmaQXpROID5M/+RzXO0l0PUqGLZEoUdEGiUFHxGplXdz3uXjox/jioWZX/1A26IS6HEd/GMRuLk7uzwRkWop+IhIjW0v2M5/9v0HgHFpOfQ8XAg9h8M1z4GrrqCLSOOl4CMiNZJfls+knydRakoZcOA4I3Ydhd7/hCufBRdXZ5cnInJKCj4ictqMMUzbM43MkkzCTpQybXM2lr7jYNAT4KJfJyLS+OmctIictiUHl5Ccm4zVbnh8Qxa+ff8Fl0wFi8XZpYmInBYFHxE5LZvyN/Fs5v/AAvd8f4iusffBRfc4uywRkRpR8BGRP3Wk5DCTfxiL3dVwxb58hnaZBH3HObssEZEaU/ARkVOy20t5YMP1HPIoJiq/hKmhd2PpfaezyxIRqRXdjSgiJ1du58WUa1nncQyPsnIe9/knXgo9ItKEKfiISPXspaSuHMYLfpkATLVdTXTMRCcXJSLy1yj4iEhVZcUcfO9mHvD/CWOxcK1rDIPOmebsqkRE/jIFHxGprLSQ0jdvYnLr7zlmc+VMlzDu6THH2VWJiNSJWgWfefPmERUVhYeHB7GxsaxZs+aU/VNSUoiNjcXDw4Po6GgWLFhQpc+yZcs466yzsNlsnHXWWbz33nuVlkdGRmKxWKq8xo4dW9Fn1KhRVZaff/75tfmIIi1TyQl4/XrmuG9iS1tPWuHB7K7zsLnYnF2ZiEidqHHwWbp0KQkJCUydOpVNmzbRv39/4uPjycjIqLZ/eno6gwYNon///mzatIkpU6YwYcIEli1bVtEnNTWVYcOGMWLECLZs2cKIESO4/vrrWbduXUWf9evXc+DAgYpXUlISANddd12l/V1xxRWV+q1cubKmH1GkZSrKg1ev5fPCjSzp2AaAh6IfpYOtg5MLExGpOxZjjKnJCn369CEmJob58+dXtHXt2pUhQ4Ywc+bMKv0nTZrEihUr2L59e0XbmDFj2LJlC6mpqQAMGzaMvLw8Pv7444o+V1xxBW3atOGNN96oto6EhAQ+/PBDdu7cieWXUWNHjRrFsWPHWL58eU0+UoW8vDz8/PzIzc3F19e3VtsQaZIKjsCSoew79h03DwjnuNWFm4NuJrF9orMrExH5UzX5/q7RGZ+SkhI2btxIXFxcpfa4uDjWrl1b7TqpqalV+g8cOJANGzZQWlp6yj4n22ZJSQlLlizhtttuqwg9v0pOTiYoKIjOnTszevRosrOzT/p5iouLycvLq/QSaXFO5MArV1GctYn7zmvPcasL53ifw/h2451dmYhInatR8MnJycFutxMcHFypPTg4mKysrGrXycrKqrZ/WVkZOTk5p+xzsm0uX76cY8eOMWrUqErt8fHxvPbaa6xevZonn3yS9evXc+mll1JcXFztdmbOnImfn1/Fq0MHndKXFiY/C14eDAe38kTPDuzwdaO1W2tmRs3EarE6uzoRkTpXq5Gb/3iWxRhTpe3P+v+xvSbbXLhwIfHx8YSFhVVqHzZsWMXP3bt3p1evXkRERPDRRx9x7bXXVtnO5MmTSUz87VR+Xl6ewo+0HLn74JWr4MguVnbswLvt3LFg4bHIxwh2D/7z9UVEmqAaBZ+AgABcXV2rnInJzs6ucsbmVyEhIdX2d3Nzw9/f/5R9qtvmnj17+Oyzz3j33Xf/tN7Q0FAiIiLYuXNntcttNhs2m55WkRbo6G545Uo4lsGusAge69YKTDGjQ0Zzvq+ehBSR5qtGl7rc3d2JjY2teKLqV0lJSfTr16/adfr27Vul/6pVq+jVqxdWq/WUfarb5qJFiwgKCmLw4MF/Wu/hw4fZu3cvoaGhf9pXpMXI+QleiodjGRQEnMGkvlEUmWL6+PThn6H/dHZ1IiL1qsaPsycmJvLiiy/y0ksvsX37du6++24yMjIYM2YM4Lh8dMstt1T0HzNmDHv27CExMZHt27fz0ksvsXDhQu65556KPhMnTmTVqlXMnj2bH374gdmzZ/PZZ5+RkJBQad/l5eUsWrSIkSNH4uZW+WTV8ePHueeee0hNTWX37t0kJydz5ZVXEhAQwDXXXFPTjynSPB3cBoviIX8/JvBMHht4Ceml+wi0BjI9cjquFldnVygiUr9MLcydO9dEREQYd3d3ExMTY1JSUiqWjRw50gwYMKBS/+TkZNOzZ0/j7u5uIiMjzfz586ts8+233zZnnnmmsVqtpkuXLmbZsmVV+nz66acGMDt27KiyrKCgwMTFxZnAwEBjtVpNeHi4GTlypMnIyDjtz5Wbm2sAk5ube9rriDQZ+zcbMyvSmId8jZl/gXl73yITszHG9N7Y23yb/62zqxMRqbWafH/XeByf5kzj+EiztW8DLLkWinIhLIbt/5jFrXsmUmpKmdhuIrcE3/Ln2xARaaRq8v1dq6e6RKQJ2bMWXrsOSo5DeF/yhi1kUvqdlJpSBvgNYETQCGdXKCLSYDRJqUhz9nMyLBnqCD1RF2FufodpWU+SWZJJO/d2PBzx8CmHohARaW4UfESaqx9XwWvXQ2kBdPwb3PQWrx57l5TcFKwWK7OjZ+Pj5uPsKkVEGpSCj0hztP0DePMmsBdDl7/DDa+xqfgH5mTOAeDe9vfS1aurk4sUEWl4Cj4izc3Wd+CtkVBeCt2HwnUvc8ScYHL6ZOzYiW8Tz7UBVUcyFxFpCRR8RJqTvevh3dFg7HDuzXDtC9hdXJi6eyqHSg8R5RHFlPApuq9HRFosBR+R5qKsGN4fC6Ycul0LV80BF1deOPAC3+R/g4eLB49HPY6Xq5ezKxURcRoFH5Hm4ov/QM4O8A6CwU+Ciwupeam8mPUiAFPDpxLtGe3kIkVEnEvBR6Q5yNoKX/7X8fPgJ8CrLQdLDvLA7gcwGIYGDGVQ20HOrVFEpBFQ8BFp6uylsPwuKC+DrlfBWVdTakqZnD6ZY2XH6OLZhX+1/5ezqxQRaRQUfESaurXPQNZ34NEaBj0BwLOZz7LlxBZaubZidvRsbC4259YoItJIKPiINGWHfoTk2Y6f42eDTzCrj63mtezXAJgWMY32tvZOLFBEpHFR8BFpqsrtsGKcY5DCjn+Ds4exv3g/D+95GIDhQcO5pPUlTi5SRKRxUfARaaq+eQH2rgN3H7jyaeyU8+DuBzluP87Z3mczrt04Z1coItLoKPiINEVHd8P/Oc7s8LeHwa89iw8uZvOJzXi7eDM9cjpWi9WpJYqINEYKPiJNjTGwYoJj8tGICyH2VrYXbGf+/vkA3NPhHtrZ2jm5SBGRxknBR6Sp+XYxpKeAmydc9QyFFPNA+gPYsXNZ68u4su2Vzq5QRKTRUvARaUry9sOqBxw/XzoV/M/g2cxn2V28mwBrgObhEhH5Ewo+Ik2FMfBhIhTnQbtYOP8u1uauZemhpYDj0fXWbq2dW6OISCOn4CPSVHy/DH78GFyscPVcjpbnMW3PNABuCLyBvr59nVufiEgT4ObsAkTkNJzIgY/vc/w84D5MYBem/3wPh8sOE+0Rzfh2451bn4hIE6EzPiJNwcf3QcFhCO4OF97NisMrSM5Nxs3ixqORj+Lh4uHsCkVEmgQFH5HG7oePHJe5LK5w9Rz2lmXxxD7HnFx3hd5FF68uTi5QRKTp0KUukcas8JjjhmaAfuMpC+3Bv38cTUF5ATGtYhgePNyp5YmINDU64yPSmK2aCsezwL8jXHw/L2e9zHcnvsPbxZtHIh7B1eLq7ApFRJoUBR+RxmrXati0BLDA1XP5vmQXzx94HoD7O9xPqC3UufWJiDRButQl0hgVH4cVEx0/n/f/KGx3Dg/+cBN27MS1iSO+bbxz6xMRaaJ0xkekMfq/RyA3A1qHw2X/5r+Z/yWjOINgazCTO0zW6MwiIrWk4CPS2OxJhW8cl7S48n98UfQty3KWAY7RmX3dfJ1YnIhI06ZLXSKNSWkhrBgHGOg5nCPh5/Lo9mEA3Bx0M+f5nufc+kREmjid8RFpTJJnweGfoFUI5m/TeTTjUY6UHaGjR0fGho11dnUiIk2ego9IY7F/E6x91vHz3//LewWf80XuF1gtVqZHTcfmYnNufSIizYAudYk0BmUl8P44MHboPpSMyO48+cONAIwLG0cnz05OLlBEpHlQ8BFpDL56Gg5+D17+lF4xgwd230dReRG9fXpzU9BNzq5ORKTZ0KUuEWfL3g4pjzt+jn+chXnLSStIw8fVh2kR03Cx6J+piEhd0W9UEWcqt8P7Y6G8FDrH811UJ17KegmAKR2mEOIe4uQCRUSaF13qEnGmr+dB5kaw+XJi0HQe3PMv7NiJbxNPXNs4Z1cnItLs6IyPiLMc3gWrpzt+jpvOk7mvs694HyHuIUzqMMm5tYmINFMKPiLOUF4OKyZAWRFEDeDz6HDeP/w+Fiw8EvEIPm4+zq5QRKRZqlXwmTdvHlFRUXh4eBAbG8uaNWtO2T8lJYXY2Fg8PDyIjo5mwYIFVfosW7aMs846C5vNxllnncV7771Xafm0adOwWCyVXiEhle9/MMYwbdo0wsLC8PT05OKLLyYtLa02H1Gkfm1cBHu+BKsXhwY9zKMZjwJwS/AtxPrEOrk4EZHmq8bBZ+nSpSQkJDB16lQ2bdpE//79iY+PJyMjo9r+6enpDBo0iP79+7Np0yamTJnChAkTWLZsWUWf1NRUhg0bxogRI9iyZQsjRozg+uuvZ926dZW21a1bNw4cOFDx2rp1a6Xljz/+OE899RRz5sxh/fr1hISE8Le//Y38/PyafkyR+pO7D5IeAsBc9m8ezV1Erj2XMz3PZEzoGCcXJyLSvFmMMaYmK/Tp04eYmBjmz59f0da1a1eGDBnCzJkzq/SfNGkSK1asYPv27RVtY8aMYcuWLaSmpgIwbNgw8vLy+Pjjjyv6XHHFFbRp04Y33ngDcJzxWb58OZs3b662LmMMYWFhJCQkMGmS4/6I4uJigoODmT17Nnfccceffra8vDz8/PzIzc3F11cTQUo9MAZeuw5+SoIOfXjrytuZve9x3C3uLOmyhDM8z3B2hSIiTU5Nvr9rdManpKSEjRs3EhdX+WmTuLg41q5dW+06qampVfoPHDiQDRs2UFpaeso+f9zmzp07CQsLIyoqihtuuIGff/65Yll6ejpZWVmVtmOz2RgwYMBJaxNpcFvedIQeVxvp8ZN5OvN/AExoN0GhR0SkAdQo+OTk5GC32wkODq7UHhwcTFZWVrXrZGVlVdu/rKyMnJycU/b5/Tb79OnD4sWL+fTTT3nhhRfIysqiX79+HD58uGIbv653urUVFxeTl5dX6SVSb/IPwif3A1B68T08kPsixaaYPj59GBY4zMnFiYi0DLW6udlisVR6b4yp0vZn/f/Y/mfbjI+PZ+jQofTo0YPLL7+cjz76CIBXXnml1rXNnDkTPz+/ileHDh1O+hlE/rKV90DRMQg5m+ejPPih8Af8XP14OOJhjc4sItJAavTbNiAgAFdX1ypnULKzs6ucaflVSEhItf3d3Nzw9/c/ZZ+TbRPA29ubHj16sHPnzoptADXazuTJk8nNza147d2796T7E/lLtr0P21eAixub4ifycvarAEwJn0Kge6CTixMRaTlqFHzc3d2JjY0lKSmpUntSUhL9+vWrdp2+fftW6b9q1Sp69eqF1Wo9ZZ+TbRMcl6m2b99OaGgoAFFRUYSEhFTaTklJCSkpKSfdjs1mw9fXt9JLpM4VHIGP7gHg+IXj+Hf+K5RTzt/b/p3L21zu5OJERFqWGk9ZkZiYyIgRI+jVqxd9+/bl+eefJyMjgzFjHI/hTp48mczMTBYvXgw4nuCaM2cOiYmJjB49mtTUVBYuXFjxtBbAxIkTueiii5g9ezZXX30177//Pp999hlffvllRZ977rmHK6+8kvDwcLKzs5k+fTp5eXmMHDkScFziSkhIYMaMGXTq1IlOnToxY8YMvLy8uOkmzW4tTvTpFDiRDYFd+E9UOfuP7ifMPYx7O9zr7MpERFoeUwtz5841ERERxt3d3cTExJiUlJSKZSNHjjQDBgyo1D85Odn07NnTuLu7m8jISDN//vwq23z77bfNmWeeaaxWq+nSpYtZtmxZpeXDhg0zoaGhxmq1mrCwMHPttdeatLS0Sn3Ky8vNQw89ZEJCQozNZjMXXXSR2bp162l/rtzcXAOY3Nzc015H5JR2fGrMQ77GPORnknbONzEbY0yvjb3Mt/nfOrsyEZFmoybf3zUex6c50zg+UqeK8mDe+ZCXSXa/27kheDO59lxuC76Nse3GOrs6EZFmo97G8RGRGvjsIcjLpLxNJNMiCsi159LVqyv/L/T/ObsyEZEWS8FHpD6kr4ENLwGwNO461p1Yj81i49HIR7G6WJ1cnIhIy6XgI1LXSgpgxXgAdvW5nmeKHGNOJbRPIMojypmViYi0eAo+InXt88fgaDolfu14IDyXElPCBb4XcF3Adc6uTESkxVPwEalL+zbA1/MAWHD5QH4s3kVrt9b8O+LfpxzdXEREGoaCj0hdKSuG98eCKWfDeYNYXLoGgAfCHyDAGuDk4kREBBR8ROrOF0/AoR/I9wvk3+2PYjBc7X81l7S+xNmViYjILxR8ROpC1lb48ikAZl9yAQfLDtHe1p572t/j5MJEROT3FHxE/ip7Gbw/DsrL+LT3RXxc/j2uuDI9cjperl7Ork5ERH5HwUfkr0p9Fg5sJsuvLTPb5QFwW8ht9PDu4eTCRETkjxR8RP6KnJ3w+UzKgYcGnEN++XG6eXXj9tDbnV2ZiIhUQ8FHpLbKyx2XuOzFvHZebzawFw8XD6ZHTsdq0ejMIiKNkYKPSG2tfwH2fs3Otq2ZG5IPwL/a/4twj3AnFyYiIiej4CNSG0f3wGcPU+xiYeoFnSmljIv8LuIa/2ucXZmIiJyCgo9ITRkDH0yA0hPM7dODXZYjtHVry4PhD2p0ZhGRRk7BR6SmNi2Bn5P5JtiP1wILAHgw4kHaWts6uTAREfkzCj4iNZF3AD6dSq7VhYfOiwBgaMBQLvK7yMmFiYjI6VDwETldxsBHiZjiXGb26Uy2pYBwWzh3t7vb2ZWJiMhpUvAROV3fL4MdK/m4Q2uS2pZVjM7s6erp7MpEROQ0KfiInI4TOfDxfRzwdGPWuSEAjA4dTTfvbk4uTEREakLBR+R0fDwJe8Fh/t0nihOWMs72PptbQ251dlUiIlJDCj4if2bHx/D9O7zasQ3f+hq8XLx4JPIR3Cxuzq5MRERqSMFH5FQKj8GHd/ODrzvzuwYAcE/7e+hg6+DcukREpFYUfEROZdUDFJ3I4oHzOlBmMVzidwlX+V/l7KpERKSWFHxETmbX57DpVZ45y590Lwv+bv5MjZiq0ZlFRJowBR+R6hQfhw8msDbQi6XRrQGYFjGNNm5tnFuXiIj8JQo+ItVZ/ShHC/YxLSYUgGGBw+jn18/JRYmIyF+l4CPyRxlfY9Y9x4yzgzhssxBpi2R8u/HOrkpEROqAgo/I75UWwfvj+KBDK1aHtXKMzhw1HU8Xjc4sItIcKPiI/F7KbPYVpvOfHkEAjAkbQ1evrk4uSkRE6oqCj8iv9m+mbO3/+HdMMAVuFs71PpeRwSOdXZWIiNQhBR8RAHspvD+OV87wZUtbT7xdvHk08lFcLa7OrkxEROqQgo8IwJdPk1b8I8+d2RaA+zrcR5gtzMlFiYhIXVPwEcn+gcKv/sODMcHYXSxc3vpyBrcd7OyqRESkHij4SMtWbof3x/L0mb7saeVOoDWQKeFTNDqziEgzpeAjLdu6Bawp3cY7UX6AY3RmPzc/JxclIiL1RcFHWq7Duziy5jEe6el4dP2moJs43/d8JxclIiL1ScFHWqbycswHE5jezYcjNjfO8IhmXNg4Z1clIiL1TMFHWqZvX+a98u9ICW2FFTemRz6GzcXm7KpERKSeKfhIy5O7j4wvH+bJ7gEA3NVuLJ29Oju5KBERaQi1Cj7z5s0jKioKDw8PYmNjWbNmzSn7p6SkEBsbi4eHB9HR0SxYsKBKn2XLlnHWWWdhs9k466yzeO+99yotnzlzJr1798bHx4egoCCGDBnCjh07KvUZNWoUFoul0uv883XPhvyOMZR+mMADPVpR5OZCbKtYhgcNd3ZVIiLSQGocfJYuXUpCQgJTp05l06ZN9O/fn/j4eDIyMqrtn56ezqBBg+jfvz+bNm1iypQpTJgwgWXLllX0SU1NZdiwYYwYMYItW7YwYsQIrr/+etatW1fRJyUlhbFjx/L111+TlJREWVkZcXFxnDhxotL+rrjiCg4cOFDxWrlyZU0/ojRn373FSy4bSWvjQSuLFw9HPoyLRSc+RURaCosxxtRkhT59+hATE8P8+fMr2rp27cqQIUOYOXNmlf6TJk1ixYoVbN++vaJtzJgxbNmyhdTUVACGDRtGXl4eH3/8cUWfK664gjZt2vDGG29UW8ehQ4cICgoiJSWFiy66CHCc8Tl27BjLly+vyUeqkJeXh5+fH7m5ufj6+tZqG9KIHc9m65J+3N7LF7uLhemR04lvG+/sqkRE5C+qyfd3jf5Xt6SkhI0bNxIXF1epPS4ujrVr11a7TmpqapX+AwcOZMOGDZSWlp6yz8m2CZCbmwtA27ZtK7UnJycTFBRE586dGT16NNnZ2SfdRnFxMXl5eZVe0nwVfHw3D3TzxO5iYWDrOIUeEZEWqEbBJycnB7vdTnBwcKX24OBgsrKyql0nKyur2v5lZWXk5OScss/JtmmMITExkQsvvJDu3btXtMfHx/Paa6+xevVqnnzySdavX8+ll15KcXFxtduZOXMmfn5+Fa8OHTqc+gBI07VtBU+5rmOftzvBLm25P3yysysSEREncKvNSn8czt8Yc8oh/qvr/8f2mmxz3LhxfPfdd3z55ZeV2ocNG1bxc/fu3enVqxcRERF89NFHXHvttVW2M3nyZBITEyve5+XlKfw0RwVHSF43iffO8cNi4OEzZuDrpkuZIiItUY2CT0BAAK6urlXOxGRnZ1c5Y/OrkJCQavu7ubnh7+9/yj7VbXP8+PGsWLGCL774gvbt25+y3tDQUCIiIti5c2e1y202Gzabxm5p7nKS7uXRLo7/zsMDb6S3T28nVyQiIs5So0td7u7uxMbGkpSUVKk9KSmJfv36VbtO3759q/RftWoVvXr1wmq1nrLP77dpjGHcuHG8++67rF69mqioqD+t9/Dhw+zdu5fQ0NDT+nzS/Jgfk3jE/SuO2Vzp5NqOu9pPcHZJIiLiTKaG3nzzTWO1Ws3ChQvNtm3bTEJCgvH29ja7d+82xhhz//33mxEjRlT0//nnn42Xl5e5++67zbZt28zChQuN1Wo177zzTkWfr776yri6uppZs2aZ7du3m1mzZhk3Nzfz9ddfV/S58847jZ+fn0lOTjYHDhyoeBUUFBhjjMnPzzf/+te/zNq1a016err5/PPPTd++fU27du1MXl7eaX223NxcA5jc3NyaHhZpjApzzVtv9DAxG2PM+Rt6mR8LfnR2RSIiUg9q8v1d4+BjjDFz5841ERERxt3d3cTExJiUlJSKZSNHjjQDBgyo1D85Odn07NnTuLu7m8jISDN//vwq23z77bfNmWeeaaxWq+nSpYtZtmxZ5UKh2teiRYuMMcYUFBSYuLg4ExgYaKxWqwkPDzcjR440GRkZp/25FHyal/SVd5i+6841MRtjzGuZLzm7HBERqSc1+f6u8Tg+zZnG8Wk+StNTuDX9Lra39uA8t07M7fG6BioUEWmm6m0cH5EmoaSAF7bczfbWHviWuzGty/8UekREBFDwkWZo85f3suiXUQkmd5hCsHv1TxyKiEjLo+AjzcrxvV/yb+sayi0WBrudS1zw1c4uSUREGhEFH2k+yop5Ii2BTG8roaXu3NvtaWdXJCIijYyCjzQb/5eayAdBBosxPBo9Ex9XH2eXJCIijYyCjzQLhzLX8Jj1KwBGul5Az8CLnVuQiIg0Sgo+0uSVl5Uw7cd/kevuyplFNsb0eNLZJYmISCOl4CNN3lsbJvC1nx2b3TC989NYXd2dXZKIiDRStZqdXaRBGQPF+XDiEJzIgROHOHoig7SinXxfnsFiz3TAwkTXS4kOOM/Z1YqISCOm4CPOUVZcEWJ++/NQNe9zKCzM4QcfSGvtQVobG2mtPcj0toLXrxuz0Dffg+v7P+7MTyQiIk2Ago/UjfJyKDx6igBzqHLQKc6tdjNlFkj3cef71h6kBdpI6+zBLp922F0sVfpGlHrSvdyf7tZorurzbywuunIrIiKnpuAj1TMGSk6c1hkZThyCghww5TXbhYsbB/yD+D7Qz3E2p5Vhu62QIou9Sl9/t7Z09+5Bd+/udPPqxlleZ+HjpsfVRUSkZhR8BMrtkJ4CW5dB9rbfwkxZYc235dkGvAN/eQVU+vmYlzfb3E/wvUsOafYM0op+5GjZUaD0l5eDt4s3Xb260s27G929utPNuxtB1iAslqpnfURERGpCwaelMgYObIbv3oLvl8Hxg9X3c/N0BJdW1YWZP7z38gdXKwBF5UXsKNhBWkEa35/4nrSCVPYV74Piypt3xZXOXp3p5tWtIuhEeETganGt388vIiItkoJPS3MkHba+7Qg8h3f+1u7ZBrpdA2dcBq2Cfws07t7wJ2da7MZOelE6aUfXVQSdnwp/wk7VS1bhtvDfQo53dzp7dsbmYqvrTykiIlItBZ+W4EQOpL0H3y2Ffet/a3fzgDPj4exhjsDj9ufj3xhjyCrNIu1EmuNVkMa2gm0Ulle9LObv5k937+6c5XUW3bwd9+X4ufnV5ScTERGpEQWf5qrkBPywEra+BT/9H5hfzr5YXCBqAJx9PXT5O3j4nnIzuWW5bCvYVhFy0k6kcbjscJV+Xi5edPXqWnHzcTfvbgRbg3VfjoiINCoKPs2JvQx+TnaEne0fQumJ35aFnusIO92Hgk9ItasXlRexs3Cn456cX4JORnFGlX6uuNLJs1PF5apuXt2I9IjUfTkiItLoKfg0dcZA5reOsPP9MsfTWL9qHeEIOz2uh8DOFc1Hy46yu2g36UXp7C7aXfHaX7Ifg6myiw62DpVuPu7s1RkPF4+G+HQiIiJ1SsGnqTq8y3GD8ta34ciu39o920L3aynvcR0Hgtqzu3gP6UXfsHvPW46gU7ybY2XHTrrZNm5tKh4h//W+nNZurev944iIiDQEBZ+moLTI8QRW9g9waLvjclbmRgCKXSxktPEhvfN57O7Qkd1eLqQX72HP4USKc4pPuslQ91AiPSKJ8ogi0vbLnx6RtHFro/tyRESk2VLwaUzKiuHwT5C9HQ79gD17G3lHd3C4cC9H3F046u7KEZsrWX5u7G4fSnobP/a7l/1ycWoPlOyBkt82Z7VYCbeF/xZwfvkzwhaBp6unkz6kiIiI8yj4NLBCeyGHi7I4euR7jhxN40jeLo4W7OVIySGOmOMccXfhiM2Vo56uHOvkSrnFAnQ4ydbKAPBx9SHKI6oi3ER6RBJliyLMFqYbjkVERH5HwacB5B/fx00/3MgRCily+cPNw178bpbxVlXWtWDBz8WHtu4BtHFrQ1u3tgRYAyqdxWnr1laXp0RERE6Dgk8D8HLzIcty4pezN2ArK8e/xNDGuNPWxYe21kDaeLXD3+cM2rSKoq21LW3d2tLG2obWbq1xs+g/k4iISF3QN2oDcPXw4+Vj/fDzDKVtwDl4BZ0Dfu3/dCoIERERqVsKPg2k22VznF2CiIhIi+fi7AJEREREGoqCj4iIiLQYCj4iIiLSYij4iIiISIuh4CMiIiIthoKPiIiItBgKPiIiItJiKPiIiIhIi6HgIyIiIi2Ggo+IiIi0GAo+IiIi0mIo+IiIiEiLoeAjIiIiLYZmZ/8dYwwAeXl5Tq5ERERETtev39u/fo+fioLP7+Tn5wPQoUMHJ1ciIiIiNZWfn4+fn98p+1jM6cSjFqK8vJz9+/fj4+ODxWKp9Xby8vLo0KEDe/fuxdfXtw4rlOroeDccHeuGo2PdcHSsG059HWtjDPn5+YSFheHicuq7eHTG53dcXFxo3759nW3P19dX/4gakI53w9Gxbjg61g1Hx7rh1Mex/rMzPb/Szc0iIiLSYij4iIiISIuh4FMPbDYbDz30EDabzdmltAg63g1Hx7rh6Fg3HB3rhtMYjrVubhYREZEWQ2d8REREpMVQ8BEREZEWQ8FHREREWgwFHxEREWkxFHzqwbx584iKisLDw4PY2FjWrFnj7JKavJkzZ9K7d298fHwICgpiyJAh7Nixo1IfYwzTpk0jLCwMT09PLr74YtLS0pxUcfMwc+ZMLBYLCQkJFW06znUrMzOT4cOH4+/vj5eXF+eeey4bN26sWK7jXTfKysp44IEHiIqKwtPTk+joaB555BHKy8sr+uhY184XX3zBlVdeSVhYGBaLheXLl1dafjrHtbi4mPHjxxMQEIC3tzdXXXUV+/btq5+CjdSpN99801itVvPCCy+Ybdu2mYkTJxpvb2+zZ88eZ5fWpA0cONAsWrTIfP/992bz5s1m8ODBJjw83Bw/fryiz6xZs4yPj49ZtmyZ2bp1qxk2bJgJDQ01eXl5Tqy86frmm29MZGSkOfvss83EiRMr2nWc686RI0dMRESEGTVqlFm3bp1JT083n332mfnpp58q+uh4143p06cbf39/8+GHH5r09HTz9ttvm1atWpmnn366oo+Ode2sXLnSTJ061SxbtswA5r333qu0/HSO65gxY0y7du1MUlKS+fbbb80ll1xizjnnHFNWVlbn9Sr41LHzzjvPjBkzplJbly5dzP333++kipqn7OxsA5iUlBRjjDHl5eUmJCTEzJo1q6JPUVGR8fPzMwsWLHBWmU1Wfn6+6dSpk0lKSjIDBgyoCD46znVr0qRJ5sILLzzpch3vujN48GBz2223VWq79tprzfDhw40xOtZ15Y/B53SO67Fjx4zVajVvvvlmRZ/MzEzj4uJiPvnkkzqvUZe66lBJSQkbN24kLi6uUntcXBxr1651UlXNU25uLgBt27YFID09naysrErH3mazMWDAAB37Whg7diyDBw/m8ssvr9Su41y3VqxYQa9evbjuuusICgqiZ8+evPDCCxXLdbzrzoUXXsj//d//8eOPPwKwZcsWvvzySwYNGgToWNeX0zmuGzdupLS0tFKfsLAwunfvXi/HXpOU1qGcnBzsdjvBwcGV2oODg8nKynJSVc2PMYbExEQuvPBCunfvDlBxfKs79nv27GnwGpuyN998k2+//Zb169dXWabjXLd+/vln5s+fT2JiIlOmTOGbb75hwoQJ2Gw2brnlFh3vOjRp0iRyc3Pp0qULrq6u2O12HnvsMW688UZAf7fry+kc16ysLNzd3WnTpk2VPvXx3angUw8sFkul98aYKm1Se+PGjeO7777jyy+/rLJMx/6v2bt3LxMnTmTVqlV4eHictJ+Oc90oLy+nV69ezJgxA4CePXuSlpbG/PnzueWWWyr66Xj/dUuXLmXJkiW8/vrrdOvWjc2bN5OQkEBYWBgjR46s6KdjXT9qc1zr69jrUlcdCggIwNXVtUpCzc7OrpJ2pXbGjx/PihUr+Pzzz2nfvn1Fe0hICICO/V+0ceNGsrOziY2Nxc3NDTc3N1JSUnjmmWdwc3OrOJY6znUjNDSUs846q1Jb165dycjIAPT3ui7de++93H///dxwww306NGDESNGcPfddzNz5kxAx7q+nM5xDQkJoaSkhKNHj560T11S8KlD7u7uxMbGkpSUVKk9KSmJfv36Oamq5sEYw7hx43j33XdZvXo1UVFRlZZHRUUREhJS6diXlJSQkpKiY18Dl112GVu3bmXz5s0Vr169enHzzTezefNmoqOjdZzr0AUXXFBlWIYff/yRiIgIQH+v61JBQQEuLpW/8lxdXSseZ9exrh+nc1xjY2OxWq2V+hw4cIDvv/++fo59nd8u3cL9+jj7woULzbZt20xCQoLx9vY2u3fvdnZpTdqdd95p/Pz8THJysjlw4EDFq6CgoKLPrFmzjJ+fn3n33XfN1q1bzY033qhHUevA75/qMkbHuS598803xs3NzTz22GNm586d5rXXXjNeXl5myZIlFX10vOvGyJEjTbt27SoeZ3/33XdNQECAue+++yr66FjXTn5+vtm0aZPZtGmTAcxTTz1lNm3aVDGMy+kc1zFjxpj27dubzz77zHz77bfm0ksv1ePsTcncuXNNRESEcXd3NzExMRWPXEvtAdW+Fi1aVNGnvLzcPPTQQyYkJMTYbDZz0UUXma1btzqv6Gbij8FHx7luffDBB6Z79+7GZrOZLl26mOeff77Sch3vupGXl2cmTpxowsPDjYeHh4mOjjZTp041xcXFFX10rGvn888/r/b388iRI40xp3dcCwsLzbhx40zbtm2Np6en+fvf/24yMjLqpV6LMcbU/XkkERERkcZH9/iIiIhIi6HgIyIiIi2Ggo+IiIi0GAo+IiIi0mIo+IiIiEiLoeAjIiIiLYaCj4iIiLQYCj4iIiLSYij4iIiISIuh4CMiIiIthoKPiIiItBgKPiIiItJi/H9Wd9osbAGjygAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "tps_execution = [results[i][\"exec_time\"] for i in states]\n", + "tps_execution_op = [results_op[i][\"exec_time\"] for i in states]\n", + "plt.plot(states, tps_execution, color=\"C1\")\n", + "plt.plot(states, tps_execution_op, color=\"limegreen\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "287cb6ef", + "metadata": {}, + "source": [ + "### Handmade Numpy Backpropagation " + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "id": "e1469fb0", + "metadata": {}, + "outputs": [], + "source": [ + "def compute_grad(observations, a0, P0, a_pred_seq, P_pred_seq, Z, H, T):\n", + " # Constant\n", + " SHAPE_a0 = a0.shape[0]\n", + " NB_obs = len(observations)\n", + "\n", + " # Initialisation for the backprop\n", + " PZT = P_pred_seq[-2].dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + " F_inv = np.linalg.solve(F, np.eye(F.shape[0]))\n", + " \n", + " grad = [0 for _ in range(NB_obs)]\n", + " grad[-1] = - 2 * Z.T @ F_inv @ (observations[-1] - Z @ a_pred_seq[-2])\n", + "\n", + " # Backprop\n", + " for i in range(3, NB_obs+1):\n", + "\n", + " PZT = P_pred_seq[-i].dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + " F_inv = np.linalg.solve(F, np.eye(F.shape[0]))\n", + "\n", + " K = PZT.dot(F_inv)\n", + " I_KZ = np.eye(SHAPE_a0) - K.dot(Z)\n", + "\n", + " grad[1-i] = I_KZ.T @ T.T @ grad[2-i] - (2 * Z.T @ F_inv @ (observations[1-i] - Z @ a_pred_seq[-i])).T \n", + "\n", + " # Last iter with a0/P0\n", + " PZT = P0.dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + " F_inv = np.linalg.solve(F, np.eye(F.shape[0]))\n", + "\n", + " K = PZT.dot(F_inv)\n", + " I_KZ = np.eye(SHAPE_a0) - K.dot(Z)\n", + "\n", + " grad[0] = I_KZ.T @ T.T @ grad[1] - (2 * Z.T @ F_inv @ (observations[0] - Z @ a0)).T\n", + "\n", + " return grad" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "id": "479b8832", + "metadata": {}, + "outputs": [], + "source": [ + "def benchmark_kalman_gradients_np(loss, a_pred_seq, P_pred_seq, state_dims, N=3):\n", + " results = defaultdict(dict)\n", + " kalman_fn = pytensor.function(inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", + " outputs=(a_pred_seq, P_pred_seq))\n", + " \n", + " for _ in range(10):\n", + " for n in state_dims:\n", + " data = generate_kalman_dataset(n, N=N, seed=42 + n)\n", + "\n", + " # --- forward pass ---\n", + " t0 = perf_counter()\n", + " a_pred, P_pred = kalman_fn(data[\"y\"],\n", + " data[\"A0\"],\n", + " data[\"P0\"],\n", + " data[\"T\"],\n", + " data[\"Z\"],\n", + " data[\"H\"],\n", + " data[\"Q\"],)\n", + " t1 = perf_counter()\n", + " forward_pass = t1 - t0\n", + " results[n][\"Forward pass\"] = forward_pass/10\n", + "\n", + " # --- Backprop ---\n", + " t0 = perf_counter()\n", + " grad = compute_grad(data[\"y\"],\n", + " data[\"A0\"],\n", + " data[\"P0\"],\n", + " a_pred,\n", + " P_pred,\n", + " data[\"Z\"],\n", + " data[\"H\"],\n", + " data[\"T\"],)\n", + " t1 = perf_counter()\n", + " compile_time = t1 - t0\n", + " results[n][\"Backprop\"] = compile_time/10\n", + "\n", + " return results" + ] + }, + { + "cell_type": "code", + "execution_count": 121, + "id": "1e633e75", + "metadata": {}, + "outputs": [], + "source": [ + "results_np = benchmark_kalman_gradients_np(loss, a_pred_seq, P_pred_seq, states, N=30)" + ] + }, + { + "cell_type": "code", + "execution_count": 122, + "id": "7109d7bf", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 122, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAGdCAYAAADqsoKGAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAUUxJREFUeJzt3Xl8VPW9//FXkkkmIZAAiSRQtgRRQFRIYjEg4NYgixXLVdoqYnsvv6ZVIaRVNq11waBWy7UIFEu9WqpQRSsoKHEhgsSFEBYBQQUMAjGEZYYlZP3+/jgwMGQhExJOMnk/+5gH53zne8585jyQefd7lm+AMcYgIiIi0sQF2l2AiIiISH1QqBERERG/oFAjIiIifkGhRkRERPyCQo2IiIj4BYUaERER8QsKNSIiIuIXFGpERETELzjsLuBCqqioYO/evbRq1YqAgAC7yxEREZFaMMZw5MgROnToQGBg9eMxzSrU7N27l06dOtldhoiIiNTB7t276dixY7XvN6tQ06pVK8A6KBERETZXIyIiIrXhdrvp1KmT53e8Os0q1Jw65RQREaFQIyIi0sSc69IRXSgsIiIifkGhRkRERPyCQo2IiIj4hTqFmtmzZxMXF0doaCiJiYmsWrWqxv5ZWVkkJiYSGhpKfHw8c+fO9Xp/8+bNjBo1iq5duxIQEMDMmTOr3M+ePXu48847iYqKokWLFvTp04ecnJy6fAURERHxMz6HmkWLFpGWlsa0adPIzc1l4MCBDB06lLy8vCr779y5k2HDhjFw4EByc3OZOnUq48ePZ/HixZ4+x48fJz4+nhkzZhAbG1vlfg4dOsSAAQMIDg5m+fLlbNmyhWeeeYbWrVv7+hVERETEDwUYY4wvG/Tr14+EhATmzJnjaevZsycjR44kIyOjUv9JkyaxZMkStm7d6mlLTU1lw4YNZGdnV+rftWtX0tLSSEtL82qfPHkyn3zyyTlHhWridruJjIzE5XLp7icREZEmora/3z6N1JSUlJCTk0NKSopXe0pKCmvWrKlym+zs7Er9hwwZwtq1ayktLa31Zy9ZsoSkpCRuu+022rVrR9++fXnhhRd8KV9ERET8mE+hprCwkPLycmJiYrzaY2JiyM/Pr3Kb/Pz8KvuXlZVRWFhY68/esWMHc+bMoXv37rz33nukpqYyfvx4Xn755Wq3KS4uxu12e71ERETEP9Xp4XtnP/zGGFPjA3Gq6l9Ve00qKipISkriiSeeAKBv375s3ryZOXPmcNddd1W5TUZGBo888kitP0NERESaLp9GaqKjowkKCqo0KlNQUFBpNOaU2NjYKvs7HA6ioqJq/dnt27enV69eXm09e/as9gJlgClTpuByuTyv3bt31/rzREREpGnxKdSEhISQmJhIZmamV3tmZib9+/evcpvk5ORK/VesWEFSUhLBwcG1/uwBAwawbds2r7bt27fTpUuXardxOp2eKRE0NYKIiIh/8/mW7vT0dP7+97/zj3/8g61btzJx4kTy8vJITU0FrNGRM08Hpaam8t1335Gens7WrVv5xz/+wfz58/nDH/7g6VNSUsL69etZv349JSUl7Nmzh/Xr1/PNN994+kycOJFPP/2UJ554gm+++YZXXnmFefPmcc8995zP9xcRERF/Yerg+eefN126dDEhISEmISHBZGVled4bO3asGTx4sFf/lStXmr59+5qQkBDTtWtXM2fOHK/3d+7caYBKr7P3s3TpUtO7d2/jdDpNjx49zLx583yq2+VyGcC4XC6fthMREZEalJcZ8/nfjfnPPQ2y+9r+fvv8nJqmTM+pERERqWd7c+HtdNi7zlofuxTiBtXrR9T297tOdz+JiIhIM1d0GD6aDl/8HUwFOCPg+oegywDbSlKoERERkdozBja9Bu9Ng2MFVtvlt0HK49Cq6qmOLhSFGhEREamd/dvhnXTYdXLKoqjuMPzPEH+trWWdolAjIiIiNSs5Dqv+DJ88BxWl4AiFQX+A/uPB4bS7Og+FGhEREanetndh+f1w+OTDbrsPgWFPQZuutpZVFYUaERERqexwHiyfDNvesdYjOsLQJ6HHcPBhmqMLSaFGRERETisrgU+fh6ynoPQ4BDog+R4YPAlCwu2urkYKNSIiImLZtRre+T3s/8pa7zIAhj8D7XraW1ctKdSIiIg0d0cLYMVDsHGhtd4i2rpF+8qfN9pTTVVRqBEREWmuKsoh50X44FE44QICIOlXcMMfIayN3dX5TKFGRESkOTp7eoP2V8Lwv0DHRHvrOg8KNSIiIs1J0WH48HFregPM6ekNrvpvCAyyu7rzolAjIiLSHFQ7vcF0aBVjb231RKFGRETE3+3fZt3V5DW9wTMQP9jeuuqZQo2IiIi/KjkOHz8Na/56xvQG90P/+xrV9Ab1RaFGRETEH21bDsseANfJ6Q0uucl6InAjnN6gvijUiIiI+JPDebB8EmxbZq1HdrLCzKXDmtQzZ+pCoUZERMQflJVA9ixreoOyopPTG9wLgx9o9NMb1BeFGhERkaZu5yrrQuDCbdZ6l2tOTm/Qw966LjCFGhERkabqaAGseBA2LrLWwy+ypje4YrTfn2qqikKNiIhIU1NRDmv/AR88BsUnpze46r/h+geb5PQG9UWhRkREpCnZsw7eSbemOQBo3wdGPAs/arrTG9QXhRoREZGmoOgwfPgYfDEfa3qDSLjhIUj6dZOf3qC+KNSIiIg0ZsbAxn/DimlwbL/VdsVo+MljfjO9QX1RqBEREWmszp7eIPoS666muEH21tVIKdSIiIg0NiXH4eOnYM2sk9MbhMHg+yH5PnCE2F1do6VQIyIi0liUnoBN/4asp8+Y3mDoyekNuthbWxOgUCMiImK3o/th7Xz4/AU4Xmi1RXaCoU9Bj2H21taEKNSIiIjYpeAr+HQ2bFgI5cVWW0RHuDrVuqupmUxvUF8UakRERC4kY2DHSsh+Hr7JPN3eIQH63ws9b4Eg/TzXhY6aiIjIhVBWDJtet8JMweaTjQHQYzj0vw869WuWUxvUJ4UaERGRhnT84OnrZY7+YLUFh0PfO63TTG3j7a3PjyjUiIiINITCr63rZda/CmVFVlurDtDvN5A4tlnP0dRQAuuy0ezZs4mLiyM0NJTExERWrVpVY/+srCwSExMJDQ0lPj6euXPner2/efNmRo0aRdeuXQkICGDmzJk17i8jI4OAgADS0tLqUr6IiEjDMAZ2roJXRsOsJGvSybIiiL0CfvYCpG2Ea9IUaBqIz6Fm0aJFpKWlMW3aNHJzcxk4cCBDhw4lLy+vyv47d+5k2LBhDBw4kNzcXKZOncr48eNZvHixp8/x48eJj49nxowZxMbG1vj5X3zxBfPmzeOKK67wtXQREZGGUVYCGxbB3wbBSyNg+7tW+yVD4e534DcfwxW3Q1CwvXX6uQBjjPFlg379+pGQkMCcOXM8bT179mTkyJFkZGRU6j9p0iSWLFnC1q1bPW2pqals2LCB7OzsSv27du1KWlpalaMwR48eJSEhgdmzZ/P444/Tp0+fc47qnMntdhMZGYnL5SIiIqLW24mIiFSp6BDk/B98Ng+O7LXaHGHQ55dw9e8g+mJby/MXtf399mmkpqSkhJycHFJSUrzaU1JSWLNmTZXbZGdnV+o/ZMgQ1q5dS2lpqS8fzz333MPw4cO58cYba9W/uLgYt9vt9RIRETlvB76FZffDs73g/T9ZgaZlDFz/EKRvgRHPKtDYwKcLhQsLCykvLycmxntW0JiYGPLz86vcJj8/v8r+ZWVlFBYW0r59+1p99sKFC1m3bh1ffPFFrevNyMjgkUceqXV/ERGRahkDeZ9C9iz46h3g5ImOmN6QfA/0HgUOp60lNnd1uvsp4Kz76I0xldrO1b+q9urs3r2bCRMmsGLFCkJDQ2td55QpU0hPT/esu91uOnXqVOvtRUREKC+FLW9Zz5fZu+50e/cUK8zEDdbzZRoJn0JNdHQ0QUFBlUZlCgoKKo3GnBIbG1tlf4fDQVRUVK0+Nycnh4KCAhITEz1t5eXlfPzxx8yaNYvi4mKCgoIqbed0OnE6lZpFRKQOTrgg5yX47G/g/t5qC3LClT+3wsxFl9pbn1TiU6gJCQkhMTGRzMxMbr31Vk97ZmYmt9xyS5XbJCcns3TpUq+2FStWkJSURHBw7a4Cv+GGG9i0aZNX269+9St69OjBpEmTqgw0IiIidXJolxVk1r0MJUettvCL4Kpx1nxMLS+ytTypns+nn9LT0xkzZgxJSUkkJyczb9488vLySE1NBaxTPnv27OHll18GrDudZs2aRXp6OuPGjSM7O5v58+fz6quvevZZUlLCli1bPMt79uxh/fr1tGzZkosvvphWrVrRu3dvrzrCw8OJioqq1C4iIlInu7+A7L/C1qVgKqy2i3pYozKX3w7Btb/8Qezhc6gZPXo0Bw4c4NFHH2Xfvn307t2bZcuW0aVLFwD27dvn9cyauLg4li1bxsSJE3n++efp0KEDzz33HKNGjfL02bt3L3379vWs//nPf+bPf/4zgwcPZuXKlefx9URERGpQXgZfvW1dL/P956fbu11vhZluN+h6mSbE5+fUNGV6To2IiABQfATW/RM+mwOHT/4f8aAQ6wF5V/8OYi6ztz7xUtvfb839JCIizcfh3fD536wLgItPPrssrC1c9T/Wq1XVN71I06BQIyIi/m9PjnWKafN/wJRbbVHdIfl3cMXPIaSFreVJ/VCoERER/1RRDtuWWw/LyztjWp64QZB8L1z8Ewis07zO0kgp1IiIiH8pOQa5/4JPZ8OhnVZbYDBc/l/W9TLtNSGyv1KoERER/+DeC5/Pg7UvwonDVltoa+vZMj8eBxEd7KxOLgCFGhERadr2bbCul/lyMVSUWW1t461RmT6/hJBwe+uTC0ahRkREmp6KCvh6hXW9zK5Vp9u7DLCeL3PJTRCop803Nwo1IiLSdJQchw2vWtfLHPjGagsIgt4/s0ZmfpRgb31iK4UaERFp/I78cPJ6mflQdMhqc0ZC4ljo9xuI7GhvfdIoKNSIiEjjlf+lNSqz6TUoL7HaWnexRmX63gHOVvbWJ42KQo2IiDQuxsA3H1iTS+5Yebq9Uz/repkeI3S9jFRJoUZERBqH0hOwcZE1MrP/K6stIBB63QJX3wOdrrK3Pmn0FGpERMRexwrh8xfgi7/D8UKrLaQVJNxlXS/Tpou99UmToVAjIiL2KDlujcqsngklR6y2yE7QLxUSxkBopK3lSdOjUCMiIhdWRTlsWAgfPg5H9lptsVfANWnQ8xYI0k+T1I3+5oiIyIXz7Yew4o/wwyZrPbIT3PAw9B6lySXlvCnUiIhIw8v/EjL/CN9+YK07I2HQ7+HHv4HgUHtrE7+hUCMiIg3HvRc+nA7r/wUYa7bsH4+DQfdDi7Z2Vyd+RqFGRETqX/ER+OR/Yc0sKCuy2nqNhBsftiabFGkACjUiIlJ/ystg3f/ByhlwbL/V1qkfpDwOnX5sa2ni/xRqRETk/BkD25bD+w9D4XarrW083PgI9LwZAgLsrU+aBYUaERE5P3vWwYqH4LvV1npYW7h2MiT+Chwh9tYmzYpCjYiI1M2h7+CDR+HL1611Ryhc/Vu4ZqIenCe2UKgRERHfFB2CVc/AZ387OXN2AFwxGq5/EFp3srs6acYUakREpHbKiq35mbKeghOHrba4wZDyGLS/0tbSREChRkREzsUY2PwmvP8nOPyd1XZRTyvMXHyjLgKWRkOhRkREqvddNqx4EPastdZbxsJ1U6HPHZqjSRod/Y0UEZHKCr+xbs/+6m1rPTgcBkyA/vdCSLi9tYlUQ6FGREROO1ZoPTgv50WoKIOAQEi4C66dCq1i7K5OpEYKNSIiAiXH4dPZsHomlByx2i65yXp4XrsetpYmUlsKNSIizVlFBWxcCB8+Du49Vlv7K61pDeIG2VubiI8UakREmqtvP4LMhyB/k7Ue2Qlu+CP0/i8IDLS3NpE6qNPf2tmzZxMXF0doaCiJiYmsWrWqxv5ZWVkkJiYSGhpKfHw8c+fO9Xp/8+bNjBo1iq5duxIQEMDMmTMr7SMjI4OrrrqKVq1a0a5dO0aOHMm2bdvqUr6ISPP2w2ZYMAr+OdIKNM5I6zTTvWvhitsVaKTJ8vlv7qJFi0hLS2PatGnk5uYycOBAhg4dSl5eXpX9d+7cybBhwxg4cCC5ublMnTqV8ePHs3jxYk+f48ePEx8fz4wZM4iNja1yP1lZWdxzzz18+umnZGZmUlZWRkpKCseOHfP1K4iINE/uffDWvTD3GvjmfQh0QL/fwvhcuCYNgkPtrlDkvAQYY4wvG/Tr14+EhATmzJnjaevZsycjR44kIyOjUv9JkyaxZMkStm7d6mlLTU1lw4YNZGdnV+rftWtX0tLSSEtLq7GO/fv3065dO7Kyshg0qHbnfd1uN5GRkbhcLiIiImq1jYhIk1d8BD55DrJnQelxq63XLXDDwxDVzd7aRGqhtr/fPl1TU1JSQk5ODpMnT/ZqT0lJYc2aNVVuk52dTUpKilfbkCFDmD9/PqWlpQQHB/tSgofL5QKgbdu21fYpLi6muLjYs+52u+v0WSIiTVJ5GeS+DB9lwLECq61TP+si4E4/trc2kQbgU6gpLCykvLycmBjvZxXExMSQn59f5Tb5+flV9i8rK6OwsJD27dv7WDIYY0hPT+eaa66hd+/e1fbLyMjgkUce8Xn/IiJNmjGw/V3IfBgKT1572DYebvwT9PyppjUQv1Wnu58CzvoPwhhTqe1c/atqr617772XjRs3snr16hr7TZkyhfT0dM+62+2mUyfNICsifmzPOljxEHx38t/HsLZw7WRI/BU4QuytTaSB+RRqoqOjCQoKqjQqU1BQUGk05pTY2Ngq+zscDqKionwsF+677z6WLFnCxx9/TMeOHWvs63Q6cTqdPn+GiEiTc+g7+PAx2PSatR7khKt/CwPTITTS3tpELhCf7n4KCQkhMTGRzMxMr/bMzEz69+9f5TbJycmV+q9YsYKkpCSfrqcxxnDvvffyxhtv8OGHHxIXF+dL6SIi/qnosDUyMyvpdKC54udwXw785BEFGmlWfD79lJ6ezpgxY0hKSiI5OZl58+aRl5dHamoqYJ3y2bNnDy+//DJg3ek0a9Ys0tPTGTduHNnZ2cyfP59XX33Vs8+SkhK2bNniWd6zZw/r16+nZcuWXHzxxQDcc889vPLKK7z11lu0atXKM/oTGRlJWFjY+R0FEZGmpqwEvvg7fPwUFB2y2uIGwU8egw59bC1NxC4+39IN1sP3nnrqKfbt20fv3r35y1/+4rmt+u6772bXrl2sXLnS0z8rK4uJEyeyefNmOnTowKRJkzwhCGDXrl1VjrwMHjzYs5/qrr958cUXufvuu2tVt27pFpEmzxjY8h94/09waJfVdlEPK8x0/4kuAha/VNvf7zqFmqZKoUZEmrS8T2HFg/D9F9Z6yxi4bir0uROCNOuN+K8GeU6NiIjYoPAbeP9h+Optaz24BQyYAMn3grOlvbWJNCIKNSIijdWxQsh6Etb+AyrKICAQ+o6xRmdaVT2ljEhzplAjItLYlBbBp7Nh9UwoPvkk9O5DrLuZ2vW0tTSRxkyhRkSksaiogI2L4MPHwf291RZ7hTWtQfxge2sTaQIUakREGoNvP4LMhyB/k7Ue2Qmufwguvw0CfXqkmEizpVAjImKnH7ZA5h/hm5MPKXVGWE8B7pcKwXoGl4gvFGpEROxwJB8+mg65C8BUQKADrvofGPQAhPs+hYyIKNSIiFxYxUdhzXOw5q9Qetxq6/lTawbtqG62libS1CnUiIhcCOVlkPtP+OgJOFZgtXX8sXURcOd+9tYm4icUakREGpIxsP0967qZwm1WW5s4a2Sm1y2a1kCkHinUiIg0lL251gzau1ZZ62FtYfAkSPo1OELsrU3EDynUiIjUt8N58MFjsOnf1nqQE65OhWvSIay1raWJ+DOFGhGR+lJ0GFY9A5/9DcqLrbYrRsP1D0LrzraWJtIcKNSIiJyvshJYO9+ap6nokNXWdSCkPAYd+tpbm0gzolAjIlJXxsCW/8D7j8ChnVbbRT3gJ49C9xRdBCxygSnUiIjURd5nsOJB+P5zaz28HVw/DfrcCUH6p1XEDvovT0TEFwe+hff/BFuXWOvBLaD/eOh/Hzhb2lqaSHOnUCMiUhvHDljXzKydDxVlEBAIfe+Ea6dCRHu7qxMRFGpERGpWWgSfzYVVz0Kx22rrngI3PgIxveytTUS8KNSIiFSlosJ6zswHj4H7e6st9nJrWoP4a20tTUSqplAjInK2HSutJwHnb7TWIzrCDQ/B5bdDYKCtpYlI9RRqREROKdhqzdH09Qpr3RkB10yEq38LwWH21iYi56RQIyJyJB8+mg65C8BUQKADkv4bBj8A4dF2VycitaRQIyLNV/FRWPNXWPMclB632nrebF0EHNXN3tpExGcKNSLS/JSXwfoF8NETcPQHq63jVdZFwJ2vtrc2EakzhRoRaT6Msa6Xyfwj7P/KamvT1RqZ6XWLpjUQaeIUakSkedi73prWYNcqaz2sDQyeZF074wixtTQRqR8KNSLi3w7vhg8fg42LrPUgJ/T7DQz8PYS1trU0EalfCjUi4p+KDsPqZ+HTuVBebLVdfrv1vJnWnW0tTUQahkKNiPiXshJY+w9rnqaig1Zb14Hwk0fhRwn21iYiDUqhRkT8gzGw5S344BE4uMNqi77UCjOXDNFFwCLNgEKNiDR9eZ9ZFwF//7m1Ht4OrpsCfe+CIP0zJ9Jc1GkSk9mzZxMXF0doaCiJiYmsWrWqxv5ZWVkkJiYSGhpKfHw8c+fO9Xp/8+bNjBo1iq5duxIQEMDMmTPr5XNFxM8d3g3/vgv+kWIFmuAW1h1N49dB0q8VaESaGZ9DzaJFi0hLS2PatGnk5uYycOBAhg4dSl5eXpX9d+7cybBhwxg4cCC5ublMnTqV8ePHs3jxYk+f48ePEx8fz4wZM4iNja2XzxURP3dwJ/xjiHXKKSAQEu6C+9bBdVPB2cru6kTEBgHGGOPLBv369SMhIYE5c+Z42nr27MnIkSPJyMio1H/SpEksWbKErVu3etpSU1PZsGED2dnZlfp37dqVtLQ00tLSzutzq+J2u4mMjMTlchEREVGrbUSkETqcBy8OB1eedd3Mbf8HMb3srkpEGkhtf799GqkpKSkhJyeHlJQUr/aUlBTWrFlT5TbZ2dmV+g8ZMoS1a9dSWlraYJ8LUFxcjNvt9nqJSBPn3gsv3WwFmqiLYewSBRoRAXwMNYWFhZSXlxMTE+PVHhMTQ35+fpXb5OfnV9m/rKyMwsLCBvtcgIyMDCIjIz2vTp061erzRKSROpJvBZpDu6zpDcYuhVZVn7IWkeanThcKB5x1a6QxplLbufpX1V7fnztlyhRcLpfntXv3bp8+T0QakaP74aWfwoFvILKzFWgiOthdlYg0Ij7dGhAdHU1QUFCl0ZGCgoJKoyinxMbGVtnf4XAQFRXVYJ8L4HQ6cTqdtfoMEWnEjh+El2+Bwm0Q8SPrlJOeCiwiZ/FppCYkJITExEQyMzO92jMzM+nfv3+V2yQnJ1fqv2LFCpKSkggODm6wzxURP1F0yAo0BZuhZaw1QtM2zu6qRKQR8vkhDunp6YwZM4akpCSSk5OZN28eeXl5pKamAtYpnz179vDyyy8D1p1Os2bNIj09nXHjxpGdnc38+fN59dVXPfssKSlhy5YtnuU9e/awfv16WrZsycUXX1yrzxURP3TCBf/8GeRvhPCLrEAT1c3uqkSksTJ18Pzzz5suXbqYkJAQk5CQYLKysjzvjR071gwePNir/8qVK03fvn1NSEiI6dq1q5kzZ47X+zt37jRApdfZ+6npc2vD5XIZwLhcLp+2ExEbnHAb8/efGPNwhDEzuhqTv9nuikTEJrX9/fb5OTVNmZ5TI9JElByDf90G330Coa2tEZr2V9hdlYjYpEGeUyMi0uBKi+DVn1uBxhkBY95UoBGRWlGoEZHGo/QELLwDdn4MIS3hzjfgRwl2VyUiTYRCjYg0DmUl8NpY+PYDa2LKO16HTlfZXZWINCEKNSJiv/JSeP1XsP1dcITCLxdBl2S7qxKRJkahRkTsVV4Gb4yDr96GICf8/BWIG2R3VSLSBCnUiIh9Ksrhrd/B5jchMBhGL4CLb7C7KhFpohRqRMQeFRWwZDxsXASBDrj9Jbgkxe6qRKQJU6gRkQvPGHhnIqxfAAFBMGo+9Bhud1Ui0sQp1IjIhWUMLH8Acv4PAgLh1r/BZSPtrkpE/IBCjYhcOMbAigfh83lAANzyPFxxm91ViYifUKgRkQvDGPjgEcieZa3fPBP6/NLWkkTEvyjUiMiFsXIGrP6LtTzsz5B4t63liIj/UagRkYb38Z8ha4a1PCQDfjzO3npExC8p1IhIw/rkOfjwMWv5xkcg+Xf21iMifkuhRkQazqdzIfMha/m6B+GaNFvLERH/plAjIg3ji/nw7iRredADMPh+e+sREb+nUCMi9W/dP+GddGt5wAS4bqq99YhIs6BQIyL1a8NCWHKftXz176zraAIC7K1JRJoFhRoRqT9fLob//BYwcNX/wJAnFGhE5IJRqBGR+rFlCSweB6YCEu6CoU8r0IjIBaVQIyLnb9tyeP1XYMrhyl/CiP+FQP3zIiIXlv7VEZHz8/X78O+7oKIMLr8NbpmlQCMittC/PCJSd99+BAt/CeUl0OsWGDkXAoPsrkpEmimFGhGpm12r4dVfQHkxXDocRs2HIIfdVYlIM6ZQIyK+y/sU/nU7lBVB9xS47UUICra7KhFp5hRqRMQ33+fAgv+C0mMQfx3c/k9wOO2uSkREoUZEfLB3PfzzVig5Al0Hws9fgeBQu6sSEQEUakSktvI3wT9HQrELOifDLxZCSAu7qxIR8VCoEZFzK9gKL98CRYeg41Vwx2vgbGl3VSIiXhRqRKRmhV/DSz+F4wegfR+443VwtrK7KhGRShRqRKR6B76Fl26GYwUQczmMeRPCWttdlYhIlRRqRKRqh76zRmiO7IN2veCut6BFW7urEhGpVp1CzezZs4mLiyM0NJTExERWrVpVY/+srCwSExMJDQ0lPj6euXPnVuqzePFievXqhdPppFevXrz55pte75eVlfHggw8SFxdHWFgY8fHxPProo1RUVNTlK4hITVzfw0sjwP09RF9iBZrwKLurEhGpkc+hZtGiRaSlpTFt2jRyc3MZOHAgQ4cOJS8vr8r+O3fuZNiwYQwcOJDc3FymTp3K+PHjWbx4sadPdnY2o0ePZsyYMWzYsIExY8Zw++2389lnn3n6PPnkk8ydO5dZs2axdetWnnrqKZ5++mn++te/1uFri0i13Hvh/0bA4TxoGw93LYGW7eyuSkTknAKMMcaXDfr160dCQgJz5szxtPXs2ZORI0eSkZFRqf+kSZNYsmQJW7du9bSlpqayYcMGsrOzARg9ejRut5vly5d7+tx00020adOGV199FYARI0YQExPD/PnzPX1GjRpFixYt+Oc//1mr2t1uN5GRkbhcLiIiInz52iLNw5Ef4P+Gw4GvoXUX+NUyiOxod1Ui0szV9vfbp5GakpIScnJySElJ8WpPSUlhzZo1VW6TnZ1dqf+QIUNYu3YtpaWlNfY5c5/XXHMNH3zwAdu3bwdgw4YNrF69mmHDhlVbb3FxMW632+slItU4Vggv/9QKNJGdYOxSBRoRaVJ8mn2usLCQ8vJyYmJivNpjYmLIz8+vcpv8/Pwq+5eVlVFYWEj79u2r7XPmPidNmoTL5aJHjx4EBQVRXl7O9OnT+cUvflFtvRkZGTzyyCO+fEWR5un4Qes5NPu/glbtYewSaNPF7qpERHxSpwuFAwICvNaNMZXaztX/7PZz7XPRokUsWLCAV155hXXr1vHSSy/x5z//mZdeeqnaz50yZQoul8vz2r1797m/nEhzU3TYelLwD19CyxhrhKZtvN1ViYj4zKeRmujoaIKCgiqNyhQUFFQaaTklNja2yv4Oh4OoqKga+5y5z/vvv5/Jkyfz85//HIDLL7+c7777joyMDMaOHVvlZzudTpxOTbQnUq0TbljwM9i3AVpEWxcFR3e3uyoRkTrxaaQmJCSExMREMjMzvdozMzPp379/ldskJydX6r9ixQqSkpIIDg6usc+Z+zx+/DiBgd7lBgUF6ZZukboqPgr/ug325EBYG+u27XY97K5KRKTOfBqpAUhPT2fMmDEkJSWRnJzMvHnzyMvLIzU1FbBO+ezZs4eXX34ZsO50mjVrFunp6YwbN47s7Gzmz5/vuasJYMKECQwaNIgnn3ySW265hbfeeov333+f1atXe/rcfPPNTJ8+nc6dO3PZZZeRm5vLs88+y69//evzPQYizU/JcXhlNOz+FEIjYcx/ILa33VWJiJwfUwfPP/+86dKliwkJCTEJCQkmKyvL897YsWPN4MGDvfqvXLnS9O3b14SEhJiuXbuaOXPmVNrna6+9Zi699FITHBxsevToYRYvXuz1vtvtNhMmTDCdO3c2oaGhJj4+3kybNs0UFxfXum6Xy2UA43K5fPvCIv6kpMiYl35qzMMRxkz/kTG719pdkYhIjWr7++3zc2qaMj2nRpq9smJYeAd8kwnB4dZcTp372V2ViEiNGuQ5NSLShJWVwL/HWoHGEQZ3vKZAIyJ+RaFGpDkoL4XFv4bty8ERCr9cCF0H2F2ViEi9UqgR8XcV5fDmb2DrUggKgdH/gvhr7a5KRKTeKdSI+LOKcvjP7+DLxRAYDLf/E7rfaHdVIiINQqFGxF9VVMDSCbBxIQQEwW0vwqU32V2ViEiDUagR8UfGwLI/QO4/ISAQRv0det5sd1UiIg1KoUbE3xgD706BtfOBABg5F3r/zO6qREQanEKNiD8xBjIfgs/mWOu3zIIrR9tbk4jIBaJQI+IvjIEPH4M1f7XWR/wF+t5pb00iIheQQo2Iv8h6ClY9Yy0PfQqSNC+aiDQvCjUi/mDVs7DyCWs5ZTr0+4299YiI2EChRqSpWzMLPnjEWr7hj9D/XnvrERGxiUKNSFP22TxYMc1avnYKDPy9vfWIiNhIoUakqVr7Iiy/31oe+HsYPMneekREbKZQI9IU5S6At9Os5eR74fqHICDA1pJEROymUCPS1Gz8N7x18rqZH/8GUh5XoBERQaFGpGnZ/KY14zbGumV76JMKNCIiJynUiDQVhV/DG/8PTIX1UL1hzyjQiIicQaFGpCkwBt5Jh/IS6HY93PwcBOo/XxGRM+lfRZGmYNPrsPNjcITC8GchMMjuikREGh2FGpHGrugwvDfVWh74B2gbZ2s5IiKNlUKNSGP34WNwrACiusOA8XZXIyLSaCnUiDRme3Lgi/nW8vBnwOG0tx4RkUZMoUaksaooh7fTAQOX3w7xg+2uSESkUVOoEWmsvpgP+9aDMxKGTLe7GhGRRk+hRqQxOpJvXUsDcOMfoWU7e+sREWkCFGpEGqP3pkKxGzokQOKv7K5GRKRJUKgRaWy+/RC+XAwBgTDiL3omjYhILSnUiDQmpSfgnd9byz/+f9Chj63liIg0JQo1Io3JJzPh4A5oGQvXTbO7GhGRJkWhRqSxOPAtrHrWWr7pCQiNsLceEZEmRqFGpDEwBpb9AcqLIf46uOxndlckItLk1CnUzJ49m7i4OEJDQ0lMTGTVqlU19s/KyiIxMZHQ0FDi4+OZO3dupT6LFy+mV69eOJ1OevXqxZtvvlmpz549e7jzzjuJioqiRYsW9OnTh5ycnLp8BZHGZfMb1gXCQU7rycEBAXZXJCLS5PgcahYtWkRaWhrTpk0jNzeXgQMHMnToUPLy8qrsv3PnToYNG8bAgQPJzc1l6tSpjB8/nsWLF3v6ZGdnM3r0aMaMGcOGDRsYM2YMt99+O5999pmnz6FDhxgwYADBwcEsX76cLVu28Mwzz9C6dWvfv7VIY3LCDe+emrAyHaK62VuPiEgTFWCMMb5s0K9fPxISEpgzZ46nrWfPnowcOZKMjIxK/SdNmsSSJUvYunWrpy01NZUNGzaQnZ0NwOjRo3G73SxfvtzT56abbqJNmza8+uqrAEyePJlPPvnknKNCNXG73URGRuJyuYiI0PUK0kgsnwSfzYW28fDbbAgOtbsiEZFGpba/3z6N1JSUlJCTk0NKSopXe0pKCmvWrKlym+zs7Er9hwwZwtq1ayktLa2xz5n7XLJkCUlJSdx22220a9eOvn378sILL9RYb3FxMW632+sl0qjsXQ+fz7OWhz+jQCMich58CjWFhYWUl5cTExPj1R4TE0N+fn6V2+Tn51fZv6ysjMLCwhr7nLnPHTt2MGfOHLp37857771Hamoq48eP5+WXX6623oyMDCIjIz2vTp06+fJ1RRpWRTm8PRFMBfQeBd2ut7siEZEmrU4XCgecdRGjMaZS27n6n91+rn1WVFSQkJDAE088Qd++ffnNb37DuHHjvE6DnW3KlCm4XC7Pa/fu3ef+ciIXSs6LsHcdOCNgyBN2VyMi0uT5FGqio6MJCgqqNCpTUFBQaaTllNjY2Cr7OxwOoqKiauxz5j7bt29Pr169vPr07Nmz2guUAZxOJxEREV4vkUbhaAG8/6i1fP1D0CrW3npERPyAT6EmJCSExMREMjMzvdozMzPp379/ldskJydX6r9ixQqSkpIIDg6usc+Z+xwwYADbtm3z6rN9+3a6dOniy1cQaRzemwbFLmjfB676b7urERHxD8ZHCxcuNMHBwWb+/Plmy5YtJi0tzYSHh5tdu3YZY4yZPHmyGTNmjKf/jh07TIsWLczEiRPNli1bzPz5801wcLB5/fXXPX0++eQTExQUZGbMmGG2bt1qZsyYYRwOh/n00089fT7//HPjcDjM9OnTzddff23+9a9/mRYtWpgFCxbUunaXy2UA43K5fP3aIvXn25XGPBxhzMORxnyfY3c1IiKNXm1/v30ONcYY8/zzz5suXbqYkJAQk5CQYLKysjzvjR071gwePNir/8qVK03fvn1NSEiI6dq1q5kzZ06lfb722mvm0ksvNcHBwaZHjx5m8eLFlfosXbrU9O7d2zidTtOjRw8zb948n+pWqBHblZ4w5rlEK9S8nW53NSIiTUJtf799fk5NU6bn1Ijtsp6Gjx6H8HZw7xcQ1truikREGr0GeU6NiJyHgzth1Z+t5SFPKNCIiNQzhRqRC8EYWHY/lJ2AuMFw+X/ZXZGIiN9RqBG5ELYugW8yIShEE1aKiDQQhRqRhlZ8BJZPtpYHpEF0d1vLERHxVwo1Ig3toww4shfadLVm4RYRkQahUCPSkPZttGbgBhj2DASH2VuPiIgfU6gRaSgVFfBOOphy6DUSut9od0UiIn5NoUakoax7Cb7/AkJawk0ZdlcjIuL3FGpEGsLR/fD+n6zl66ZBRAdbyxERaQ4UakQaQuYf4cRhiL0cfvz/7K5GRKRZUKgRqW+7VsOGV4AAGDETghx2VyQi0iwo1IjUp7ISePvkbduJd0PHJFvLERFpThRqROpT9iwo3AYtouHGh+2uRkSkWVGoEakvh3ZB1lPW8pDpENbG1nJERJobhRqR+mAMLJ8EZUXQdSBcMdruikREmh2FGpH68NU7sP1dCAzWhJUiIjZRqBE5X8VHrVEagAHj4aJL7a1HRKSZUqgROV9ZM8D9PbTuDAP/YHc1IiLNlkKNyPn4YTNkz7aWh/0ZQlrYW4+ISDOmUCNSVxUV1jNpTDn0vBkuGWJ3RSIizZpCjUhdrV8Auz+F4HC4aYbd1YiINHsKNSJ1ceyANb8TwHVTILKjvfWIiIhCjUidvP9HKDoE7S6Dfql2VyMiIijUiPjuu2zIXWAtj/gLBAXbW4+IiAAKNSK+KS+Fd05OWJlwF3TuZ289IiLioVAj4otPZ0PBFmgRBTc+Ync1IiJyBoUakdo6vBtWnrzL6SePQYu29tYjIiJeFGpEamv5JCg9Dp37Q59f2l2NiIicRaFGpDa2LYdt70CgA0Y8qwkrRUQaIYUakXMpOQbLHrCWk++Fdj3trUdERKqkUCNyLh8/Da48iOwMgx+wuxoREamGQo1ITQq2wpq/WstDn4SQcHvrERGRatUp1MyePZu4uDhCQ0NJTExk1apVNfbPysoiMTGR0NBQ4uPjmTt3bqU+ixcvplevXjidTnr16sWbb75Z7f4yMjIICAggLS2tLuWL1I4x8M7voaIMLh0GPYbZXZGIiNTA51CzaNEi0tLSmDZtGrm5uQwcOJChQ4eSl5dXZf+dO3cybNgwBg4cSG5uLlOnTmX8+PEsXrzY0yc7O5vRo0czZswYNmzYwJgxY7j99tv57LPPKu3viy++YN68eVxxxRW+li7imw2vwnefQHALa5RGREQatQBjjPFlg379+pGQkMCcOXM8bT179mTkyJFkZGRU6j9p0iSWLFnC1q1bPW2pqals2LCB7OxsAEaPHo3b7Wb58uWePjfddBNt2rTh1Vdf9bQdPXqUhIQEZs+ezeOPP06fPn2YOXNmrWt3u91ERkbicrmIiIjw5WtLc3P8IMxKguMHrIfsXZNmd0UiIs1WbX+/fRqpKSkpIScnh5SUFK/2lJQU1qxZU+U22dnZlfoPGTKEtWvXUlpaWmOfs/d5zz33MHz4cG688cZa1VtcXIzb7fZ6idTK+3+yAs1FPSH5HrurERGRWvAp1BQWFlJeXk5MTIxXe0xMDPn5+VVuk5+fX2X/srIyCgsLa+xz5j4XLlzIunXrqhwNqk5GRgaRkZGeV6dOnWq9rTRjuz+HdS9ZyyOe1YSVIiJNRJ0uFA4468FjxphKbefqf3Z7TfvcvXs3EyZMYMGCBYSGhta6zilTpuByuTyv3bt313pbaabKy+DtidZynzuhS3976xERkVpz+NI5OjqaoKCgSqMyBQUFlUZaTomNja2yv8PhICoqqsY+p/aZk5NDQUEBiYmJnvfLy8v5+OOPmTVrFsXFxQQFBVX6bKfTidPp9OUrSnP32Vz44UsIawM/edTuakRExAc+jdSEhISQmJhIZmamV3tmZib9+1f9/2iTk5Mr9V+xYgVJSUkEBwfX2OfUPm+44QY2bdrE+vXrPa+kpCTuuOMO1q9fX2WgEfGZaw+sPHl688ZHIDzK3npERMQnPo3UAKSnpzNmzBiSkpJITk5m3rx55OXlkZqaClinfPbs2cPLL78MWHc6zZo1i/T0dMaNG0d2djbz58/3uqtpwoQJDBo0iCeffJJbbrmFt956i/fff5/Vq1cD0KpVK3r37u1VR3h4OFFRUZXaRers3clQchQ69YO+Y+yuRkREfORzqBk9ejQHDhzg0UcfZd++ffTu3Ztly5bRpUsXAPbt2+f1zJq4uDiWLVvGxIkTef755+nQoQPPPfcco0aN8vTp378/Cxcu5MEHH+Shhx6iW7duLFq0iH79+tXDVxSphe0rYOsSCAiC4c9CoB62LSLS1Pj8nJqmTM+pkSqVHIfZV8Ph76wJK4dMt7siERE5Q4M8p0bEL616xgo0ET+Ca6fYXY2IiNSRQo00b/u3wyf/ay0PfRKcLe2tR0RE6kyhRpovY+CddKgohe5DoMcIuysSEZHzoFAjzdfGf8OuVeAIg2FPQQ0PkBQRkcZPoUaap6JDsGKatTz4fmjT1dZyRETk/CnUSPP0waNwbD9EXwrJ99ldjYiI1AOFGml+vs+BtS9ay8OfAUeIvfWIiEi9UKiR5qW8DN5OAwxc+QuIG2h3RSIiUk8UaqR5+eLvkL8RQlvDTx6zuxoREb/hLnPz5bEvba3B52kSRJos9z748HFr+caHoeVF9tYjItIEHSk7wo4TO/j2xLfsKNrhWS4sLQRg1ZWraBHUwpbaFGqk+XhvCpQcgY5XQcLddlcjItKoHSk/ws6inVZ4ObGDb4u+ZeeJnRSUFlS7TUxwDPtL99MlqMsFrPQ0hRppHr55Hza/CQGBmrBSROQMR8uPsvPETnYU7fAEmB1FO/ih9Idqt4kJjiEuNI5uYd2ID42nW1g34kLjaBlk71PZFWrE/5UWwTt/sJb7pUL7K+ytR0TEBsfLj1uB5eSoy6k/awovFwVf5Akt8aHxdAvtRlxYHK2CWl3AymtPoUb83+q/wKGd0Ko9XDfV7mpERBrU8fLj7Dyx0+ualx0ndrCvZF+120QHR9Mt1Aou8WHx1p+h8UQ4qp8RuzFSqBH/VviNFWoAbpoBzsb5/y5ERHxVVF5knTY6a/Rlb8neareJckR5Qku30G6e5UhH5AWsvOEo1Ij/MgaW/R7KS+DiG6HXLXZXJCLis6KKInad2HX6mpeToy97S/ZiMFVu09bR1vu00clrXlo7Wl/Y4i8whRrxX18uhh0rwREKw57WhJUi0iiVVJRwoPQAhWWF7C/ZT2FZIfkl+Z5RmD3Fe6oNL20cbbzCy6nTR20cbS7wt2gcFGrEP51wwXsnr58Z+AdoG29vPSLS7JyoOEFhaSGFpYXsL91f5XJhaSGuctc59xUZFOk16nLq9FGb4OYZXqqjUCP+6cPH4egPENUdBoy3uxoR8SPHy49XGU72l1qjLKeWj5YfrfU+gwOCiQ6Otl6OaC4KuYiuzq7Eh50ML442BGi0+ZwUasT/7FkHn79gLQ9/BhxOe+sRkUbPGMPRiqPVjqac2X684nit9+sMcJ4OK8HRXBR8UZXrkUGRCi31QKFG/EtFObw9ETBw+e0QP9juikTERsYY3OXu6kdWSgs917IUm+Ja7zcsMMwroHiWHdFEh0RzkcNabxnUUmHlAlKoEf+y9h+wbz04I2HIdLurEZEGUmEqcJW5Kp32qSq8lJiSWu+3ZVBLK5jUMKoSHRxNeFB4A347qSuFGvEfR36ADx61lm94CFq2s7ceEfFZuSnnUNmhc15cW1haSDnltd5vZFCkV0Cp7nRQWGBYA347aWgKNeI/3psKxW7o0BeSfm13NSIClFaUcrj8MIfLDuMqc3n9ebj89PLBsoMUlhZysPSgT2GljaONd0CpYpQlKjgKZ6CurWsOFGrEP3z7EXz5ujVh5Yi/QGCQ3RWJ+J2SihIrhJwMKV5BpdxVqe1w2WGOVRzz+XMCCfSElZpOAUU5oggODG6AbypNlUKNNH2lJ+Cd31vLV42zRmpEpEaegFJ2mEPlhyqPolQRVOoSUMAKKZGOSCKDImntaE1rR2siHaeXT71OjbS0CW6DI0A/T+I7/a2Rpu+T/4WD30LLGLh+mt3ViFxwxRXFXqd0Ko2inDG6cqrNl9uSzxREEBGOiNPhpKqgEtTaq61VUCsCAwLr+VuLVKZQI03bgW9h1TPW8k0ZEOofk7JJ81VcUVxpxORcp3uKKorq9FlBBHmCSKQj0hNGzhxFiXRE0sbRhtZB1nLLoJYKKNJoKdRI02UMLLsfyosh/jq47Gd2VyTi5UTFCe/TOWddj3J2u6vMdV4B5ewRk2pP95wML+FB4Qoo4lcUaqTp2vIf+PYDCHJaTw7WA66kARVVFJ3zuhOvoFLu4kTFiTp91pkBxSuUnDWScmZQaRmoh7yJKNRI03TCDcsnW8vXTISobvbWI01KUUVR5XBSxW3GZ758edrsmRwBDq8RE6+gUsVISqQjUgFFpI4UaqRp+ugJOJpvzb59zUS7qxEbnQooVd1OXN1txucTUE6dvqnq7p0z20/9GR4YroAicoHUKdTMnj2bp59+mn379nHZZZcxc+ZMBg4cWG3/rKws0tPT2bx5Mx06dOCBBx4gNTXVq8/ixYt56KGH+Pbbb+nWrRvTp0/n1ltv9byfkZHBG2+8wVdffUVYWBj9+/fnySef5NJLL63LV5CmbN8G+Pxv1vLwZyA41N565IJzlbl479B7vH3gbTYf31ynfQQHBFcaManqupMz21sEtlBAEWnEfA41ixYtIi0tjdmzZzNgwAD+9re/MXToULZs2ULnzp0r9d+5cyfDhg1j3LhxLFiwgE8++YTf/e53XHTRRYwaNQqA7OxsRo8ezWOPPcatt97Km2++ye23387q1avp168fYAWje+65h6uuuoqysjKmTZtGSkoKW7ZsITxcc3A0G6cmrDQV1oXB3a63uyK5QMpMGdnubJYeWMrHro8pNaWe90ICQirdUlzVdSdnBpWwwDAFFBE/E2CMMb5s0K9fPxISEpgzZ46nrWfPnowcOZKMjIxK/SdNmsSSJUvYunWrpy01NZUNGzaQnZ0NwOjRo3G73SxfvtzT56abbqJNmza8+uqrVdaxf/9+2rVrR1ZWFoMGDapV7W63m8jISFwuFxEREbXaRhqZL+bDO+ngjIB7PoeI9nZXJA3s26JvWXpgKcsOLuNA2QFP+6VhlzIiagQpbVKIckQpoIj4sdr+fvs0UlNSUkJOTg6TJ0/2ak9JSWHNmjVVbpOdnU1KSopX25AhQ5g/fz6lpaUEBweTnZ3NxIkTK/WZOXNmtbW4XC4A2rZt68tXkKbsaAF88Ii1fP2DCjR+7NTppaUHlrLl+BZPextHG4a2HcqItiO4tIVOPYuIN59CTWFhIeXl5cTExHi1x8TEkJ+fX+U2+fn5VfYvKyujsLCQ9u3bV9unun0aY0hPT+eaa66hd+/e1dZbXFxMcfHpCwLdbneN308auRUPwQkXtL8Srvofu6uRelbd6aUgghgUOYibo26mf2R/ggM014+IVK1OFwqfPcxrjKlx6Leq/me3+7LPe++9l40bN7J69eoa68zIyOCRRx6psY80ETs/ho0LgQBNWOlnvin6hrcPvF3l6aWbo27mpjY30Sa4jY0VikhT4VOoiY6OJigoqNIISkFBQaWRllNiY2Or7O9wOIiKiqqxT1X7vO+++1iyZAkff/wxHTt2rLHeKVOmkJ6e7ll3u9106tSpxm2kESorOWPCyv+GHyXaW4+ct8Nlh3nv4HssPbiUrcdPX2/XxtGGYW2HMaLtCC5pcYmNFYpIU+RTqAkJCSExMZHMzEyv260zMzO55ZZbqtwmOTmZpUuXerWtWLGCpKQkgoODPX0yMzO9rqtZsWIF/fv396wbY7jvvvt48803WblyJXFxcees1+l04nQ6ffmK0hiteQ4Kt0N4O7j+IburkToqM2Wsca/h7QNvk+XKosyUATq9JCL1x+fTT+np6YwZM4akpCSSk5OZN28eeXl5nufOTJkyhT179vDyyy8D1p1Os2bNIj09nXHjxpGdnc38+fO97mqaMGECgwYN4sknn+SWW27hrbfe4v333/c6vXTPPffwyiuv8NZbb9GqVSvPyE5kZCRhYWHndRCkETu4Ez5+2loeMh3CWttajvju66KvefvA2yw/uNzr9FKPsB6MiBqh00siUm98DjWjR4/mwIEDPProo+zbt4/evXuzbNkyunTpAsC+ffvIy8vz9I+Li2PZsmVMnDiR559/ng4dOvDcc895nlED0L9/fxYuXMiDDz7IQw89RLdu3Vi0aJHnGTWA5xbya6+91queF198kbvvvtvXryGNXUUFFB2yJqwsOwFxg+Dy2+yuSmqputNLbR1tGdp2KDe3vZnuLbrbWKGI+COfn1PTlOk5NTYzxgoqRwvgWIH1Z3XLx/ZDhXV6gqAQ+O0aiNaPYGNWakrJdmWz9KB199Kp00uOAId1eqntzSRHJuv0koj4rEGeUyNSiTFw4nDNAeXomUGl9Jy79NIiGm58WIGmEfu66GuWHljK8oPLOVh20NPeI6wHN0fdzJC2Q2jj0OklEWl4CjVSmTHW82CO7YejP5wOJKeWPYFlv/VneYlv+w+NhJYx1oW/LU++wi+y2jzrJ9scIQ3zHeW8HCo7ZJ1eOrCUr4q+8rTr9JKI2EmhprkwBordp4PI0R+s5aM/nA4oR384GV4KoNzHWYydkVUElIvOCC+nli8Ch+5Ia4pqOr00OHIwI6JGkByh00siYh+FGn/n3gsbXoX1r8CBb3zb1hnhHVDC21UdVsLbaaZsP/b18a9ZerDy6aWeLXpyc1vr9FJrR2v7ChQROUmhxh+VnoBt70Duv2DHR9aM1qeEtDojoJx6nRxBOXs5WLfKN1eHyg7x7sF3WXpgKduKtnna2zraWg/HixpB9zCdXhKRxkWhxl8YA/vWW0Fm02vWxbundO4Pfe+Anjdb17OIVKHUlLLGtYalB5ayyr1Kp5dEpMlRqGnqju6HTf+2wkzB5tPtET+CK38BfX4JUd3sq08avVOnl5YdXMahskOedp1eEpGmRqGmKSovha9XWEHm6/fOeJ6LE3qOgD53QPy1mvRRqlXd6aUoR5Tn9NLFYRfbWKGIiO8UapqSH7bA+n/BxkXWXUqndEiwTi/1HgVheh6IVK2600vBAcGeuZeSI5JxBOifBRFpmvSvV2NXdAg2vW6Fmb25p9vDL4IrRlujMjG97KtPGr3tx7d77l468/RSrxa9uDnqZlLapOj0koj4BYWaxqii3LprKfdf8NU7p58ZE+iAS26ygkz3n0CQLtiUqh0qPcTyQ8t5+8DbOr0kIs2GQk1jceyAdaHvjpWwYSG495x+r91l1umly2+3bscWqUKpKeUT1yfW6SXXKsopB3R6SUSaD/3rdqGVFUPhdvhhs/fraL53v9DW1qzUfe+A9n0gIMCOaqUJ2HZ8G0sPLOXdQ+96nV66rMVljIgawZA2Q4h06FZ+EfF/CjUNxRhrtOXs8HLg69N3K52tTVeIvRwu+xlcOkxP6ZVqHSw9yLuHrLuXthdt97RHOaIY3nY4I6JG0C1Mt/KLSPOiUFMfio9CwVb44UsruBRssZZPuKruHxppnVKKOePVric4W13YuqVJ2V+yn3VH1/HeofdY7VrtdXppcORgbo66masjrtbpJRFptvSv3/kqL4On4queADLQAVHdvcNLzGXWg/F0OklqYIxhV/Eu1h9dT+7RXNYfXc+ekj1efXR6SUTEm0LN+QpyQPQl1nNjzg4v0ZdoRmqplTJTxrbj206HmGPrva6PAQgkkO5h3bk64mqGtx2u00siImdRqKkP/70CQlrYXYU0IUUVRWw6ton1R9ez/uh6Nh7bSFFFkVefkIAQeof3pm/LvvRp2Ycrwq+gZVBLmyoWEWn8FGrqgwKNnMPhssOeAJN7NJetx7d6rok5pVVQK64Mv5K+LfvSt2VferboSUhgiE0Vi4g0PQo1Ig1gX/E+z2mk3KO57Dixo1KfdsHtPKMwfVv2pVtoNwIDAm2oVkTEPyjUiJynClPBjhM7yD2a67mo94fSHyr1iwuNs0JMuBVi2oe0J0AXjIuI1BuFGhEflVaUsvX4Vk+I2XBsA+5yt1efIILo0aKHZySmT8s+tHFoslERkYakUCNyDsfKj7Hx2EbPKMyXx76k2Hjfwh8aGMoV4Vd4RmIuD7+csKAwmyoWEWmeFGpEznKg9IDngt7co7lsL9pOBRVefVo7WntOI/Vt2ZdLWlxCcIAmGBURsZNCjTRrxhi+L/6e3GO5niCTV5xXqV+HkA5eF/V2dXbV9TAiIo2MQo00K+WmnK+LvvZ6yF1haaFXnwAC6BbazTMK06dlH2JCYmyqWEREakuhRvxacUUxm49t9txaveHoBo5VHPPqExwQTK8WvTyjMFeGX0mEI8KmikVEpK4UasSvHCk7wvpjpx9yt+X4FkpNqVef8MBwrmx5peeamF7hvQgN1IzoIiJNnUKNNGkFJQVeD7n7pugbDMarT5QjyutUUvew7gQFBNlUsYiINBSFGmkyTs1cferW6qpmrgbo7Ox8+qLe8L50dHbURb0iIs2AQo3YqtyUU2JKKKsoo8SUUGpKKa0opdSUUmJKKKooYvOxzZ7RmMNlh722DySQS8Iu8YzEXNnySqKDo+35MiIiYiuFGj9XYSooM2WUmTJPUPAKECdDRFWBoqb3ykwZJRVn7MOUVrleZqoOK6eWz37+y7k4A5z0Du/tuaj38vDLNXO1iIgACjX14om8JzhafhSwbgcODAgkgIA6rQOeEHIqiJSZMk8IOLPN6/2z/6yw/jx7JujGLjgg+PQrMBhngJP40HjPSEyPFj00c7WIiFSpTqFm9uzZPP300+zbt4/LLruMmTNnMnDgwGr7Z2VlkZ6ezubNm+nQoQMPPPAAqampXn0WL17MQw89xLfffku3bt2YPn06t95663l97oWy8vBKDpQdsLuMWgkggJCAEIIDreAQEhCCI8BBSGCIV6Coat0R4PDa9tT2Xutn9jsZTDz9zrHuCHDo2hcREakzn0PNokWLSEtLY/bs2QwYMIC//e1vDB06lC1bttC5c+dK/Xfu3MmwYcMYN24cCxYs4JNPPuF3v/sdF110EaNGjQIgOzub0aNH89hjj3Hrrbfy5ptvcvvtt7N69Wr69etXp8+9kH7b4bcUVRRhjMHrf2etg3U66Mz1U33OPA3jCHDgCHAQHBBc6c9Ty47Ayn2q6n+qr2c5QINzIiLinwKMMebc3U7r168fCQkJzJkzx9PWs2dPRo4cSUZGRqX+kyZNYsmSJWzdutXTlpqayoYNG8jOzgZg9OjRuN1uli9f7ulz00030aZNG1599dU6fW5V3G43kZGRuFwuIiL0cDUREZGmoLa/34G+7LSkpIScnBxSUlK82lNSUlizZk2V22RnZ1fqP2TIENauXUtpaWmNfU7tsy6fC1BcXIzb7fZ6iYiIiH/yKdQUFhZSXl5OTIz3PDgxMTHk5+dXuU1+fn6V/cvKyigsLKyxz6l91uVzATIyMoiMjPS8OnXqVLsvKiIiIk2OT6HmlLMv5jTG1HiBZ1X9z26vzT59/dwpU6bgcrk8r927d1fbV0RERJo2n64ajY6OJigoqNLoSEFBQaVRlFNiY2Or7O9wOIiKiqqxz6l91uVzAZxOJ06ns3ZfTkRERJo0n0ZqQkJCSExMJDMz06s9MzOT/v37V7lNcnJypf4rVqwgKSmJ4ODgGvuc2mddPldERESaGeOjhQsXmuDgYDN//nyzZcsWk5aWZsLDw82uXbuMMcZMnjzZjBkzxtN/x44dpkWLFmbixIlmy5YtZv78+SY4ONi8/vrrnj6ffPKJCQoKMjNmzDBbt241M2bMMA6Hw3z66ae1/tzacLlcBjAul8vXry0iIiI2qe3vt8+hxhhjnn/+edOlSxcTEhJiEhISTFZWlue9sWPHmsGDB3v1X7lypenbt68JCQkxXbt2NXPmzKm0z9dee81ceumlJjg42PTo0cMsXrzYp8+tDYUaERGRpqe2v98+P6emKdNzakRERJqeBnlOjYiIiEhjpVAjIiIifkGhRkRERPyCQo2IiIj4BYUaERER8Qs+PVG4qTt1o5cmthQREWk6Tv1un+uG7WYVao4cOQKgiS1FRESaoCNHjhAZGVnt+83qOTUVFRXs3buXVq1a1TgR5rm43W46derE7t279bybBqZjfeHoWF84OtYXjo71hdOQx9oYw5EjR+jQoQOBgdVfOdOsRmoCAwPp2LFjve0vIiJC/5FcIDrWF46O9YWjY33h6FhfOA11rGsaoTlFFwqLiIiIX1CoEREREb+gUFMHTqeThx9+GKfTaXcpfk/H+sLRsb5wdKwvHB3rC6cxHOtmdaGwiIiI+C+N1IiIiIhfUKgRERERv6BQIyIiIn5BoUZERET8gkKNj2bPnk1cXByhoaEkJiayatUqu0tq8jIyMrjqqqto1aoV7dq1Y+TIkWzbts2rjzGGP/3pT3To0IGwsDCuvfZaNm/ebFPF/iMjI4OAgADS0tI8bTrW9WfPnj3ceeedREVF0aJFC/r06UNOTo7nfR3r+lFWVsaDDz5IXFwcYWFhxMfH8+ijj1JRUeHpo2NdNx9//DE333wzHTp0ICAggP/85z9e79fmuBYXF3PfffcRHR1NeHg4P/3pT/n+++8bpmAjtbZw4UITHBxsXnjhBbNlyxYzYcIEEx4ebr777ju7S2vShgwZYl588UXz5ZdfmvXr15vhw4ebzp07m6NHj3r6zJgxw7Rq1cosXrzYbNq0yYwePdq0b9/euN1uGytv2j7//HPTtWtXc8UVV5gJEyZ42nWs68fBgwdNly5dzN13320+++wzs3PnTvP++++bb775xtNHx7p+PP744yYqKsq8/fbbZufOnea1114zLVu2NDNnzvT00bGum2XLlplp06aZxYsXG8C8+eabXu/X5rimpqaaH/3oRyYzM9OsW7fOXHfddebKK680ZWVl9V6vQo0PfvzjH5vU1FSvth49epjJkyfbVJF/KigoMIDJysoyxhhTUVFhYmNjzYwZMzx9Tpw4YSIjI83cuXPtKrNJO3LkiOnevbvJzMw0gwcP9oQaHev6M2nSJHPNNddU+76Odf0ZPny4+fWvf+3V9rOf/czceeedxhgd6/pydqipzXE9fPiwCQ4ONgsXLvT02bNnjwkMDDTvvvtuvdeo00+1VFJSQk5ODikpKV7tKSkprFmzxqaq/JPL5QKgbdu2AOzcuZP8/HyvY+90Ohk8eLCOfR3dc889DB8+nBtvvNGrXce6/ixZsoSkpCRuu+022rVrR9++fXnhhRc87+tY159rrrmGDz74gO3btwOwYcMGVq9ezbBhwwAd64ZSm+Oak5NDaWmpV58OHTrQu3fvBjn2zWpCy/NRWFhIeXk5MTExXu0xMTHk5+fbVJX/McaQnp7ONddcQ+/evQE8x7eqY//dd99d8BqbuoULF7Ju3Tq++OKLSu/pWNefHTt2MGfOHNLT05k6dSqff/4548ePx+l0ctddd+lY16NJkybhcrno0aMHQUFBlJeXM336dH7xi18A+nvdUGpzXPPz8wkJCaFNmzaV+jTEb6dCjY8CAgK81o0xldqk7u699142btzI6tWrK72nY3/+du/ezYQJE1ixYgWhoaHV9tOxPn8VFRUkJSXxxBNPANC3b182b97MnDlzuOuuuzz9dKzP36JFi1iwYAGvvPIKl112GevXryctLY0OHTowduxYTz8d64ZRl+PaUMdep59qKTo6mqCgoErJsqCgoFJKlbq57777WLJkCR999BEdO3b0tMfGxgLo2NeDnJwcCgoKSExMxOFw4HA4yMrK4rnnnsPhcHiOp471+Wvfvj29evXyauvZsyd5eXmA/l7Xp/vvv5/Jkyfz85//nMsvv5wxY8YwceJEMjIyAB3rhlKb4xobG0tJSQmHDh2qtk99UqippZCQEBITE8nMzPRqz8zMpH///jZV5R+MMdx777288cYbfPjhh8TFxXm9HxcXR2xsrNexLykpISsrS8feRzfccAObNm1i/fr1nldSUhJ33HEH69evJz4+Xse6ngwYMKDSowm2b99Oly5dAP29rk/Hjx8nMND75ywoKMhzS7eOdcOozXFNTEwkODjYq8++ffv48ssvG+bY1/ulx37s1C3d8+fPN1u2bDFpaWkmPDzc7Nq1y+7SmrTf/va3JjIy0qxcudLs27fP8zp+/Linz4wZM0xkZKR54403zKZNm8wvfvEL3Y5ZT868+8kYHev68vnnnxuHw2GmT59uvv76a/Ovf/3LtGjRwixYsMDTR8e6fowdO9b86Ec/8tzS/cYbb5jo6GjzwAMPeProWNfNkSNHTG5ursnNzTWAefbZZ01ubq7nUSa1Oa6pqammY8eO5v333zfr1q0z119/vW7pbiyef/5506VLFxMSEmISEhI8tx1L3QFVvl588UVPn4qKCvPwww+b2NhY43Q6zaBBg8ymTZvsK9qPnB1qdKzrz9KlS03v3r2N0+k0PXr0MPPmzfN6X8e6frjdbjNhwgTTuXNnExoaauLj4820adNMcXGxp4+Odd189NFHVf77PHbsWGNM7Y5rUVGRuffee03btm1NWFiYGTFihMnLy2uQegOMMab+x39ERERELixdUyMiIiJ+QaFGRERE/IJCjYiIiPgFhRoRERHxCwo1IiIi4hcUakRERMQvKNSIiIiIX1CoEREREb+gUCMiIiJ+QaFGRERE/IJCjYiIiPgFhRoRERHxC/8fLd2ekRekri0AAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "tps_execution = [results[i][\"exec_time\"] for i in states]\n", + "tps_execution_np = [results_np[i][\"Backprop\"] + results_np[i][\"Forward pass\"] for i in states]\n", + "plt.plot(states, tps_execution, color=\"C1\")\n", + "plt.plot(states, tps_execution_np, color=\"limegreen\")" + ] + }, + { + "cell_type": "code", + "execution_count": 124, + "id": "f2dbe75f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.0017101200064644217\n" + ] + } + ], + "source": [ + "print(results[20][\"exec_time\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 126, + "id": "614d5f94", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.00022822999162599444\n" + ] + } + ], + "source": [ + "print(results_np[20][\"Backprop\"] + results_np[20][\"Forward pass\"])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "CausalPy", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 644b51f855c2c21ba01d3e90b53ba597c0ec579e Mon Sep 17 00:00:00 2001 From: Jean Van Dyk Date: Mon, 4 Aug 2025 17:03:30 +0200 Subject: [PATCH 2/4] Updating the notebook --- notebooks/Kalman_Filter_Gradient.ipynb | 1002 +++++++++++++++++++++++ notebooks/Kalman_Filter_Gradients.ipynb | 743 ----------------- 2 files changed, 1002 insertions(+), 743 deletions(-) create mode 100644 notebooks/Kalman_Filter_Gradient.ipynb delete mode 100644 notebooks/Kalman_Filter_Gradients.ipynb diff --git a/notebooks/Kalman_Filter_Gradient.ipynb b/notebooks/Kalman_Filter_Gradient.ipynb new file mode 100644 index 00000000..7bea8141 --- /dev/null +++ b/notebooks/Kalman_Filter_Gradient.ipynb @@ -0,0 +1,1002 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "69ae14a1", + "metadata": {}, + "source": [ + "### Imports" + ] + }, + { + "cell_type": "code", + "execution_count": 214, + "id": "90979a41", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import pytensor\n", + "import pytensor.tensor as pt\n", + "import matplotlib.pyplot as plt\n", + "from pytensor.compile.builders import OpFromGraph\n", + "from time import perf_counter\n", + "from collections import defaultdict\n", + "import pymc_extras as pmx\n", + "from pymc_extras.statespace import structural as sts\n", + "import pytensor\n", + "from pytensor.graph.basic import explicit_graph_inputs\n", + "import numpy as np" + ] + }, + { + "cell_type": "markdown", + "id": "a0d008fc", + "metadata": {}, + "source": [ + "### Generate a random dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fdb156d6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + ".f at 0x000001820DC942C0>\n" + ] + } + ], + "source": [ + "mod = (\n", + " sts.LevelTrendComponent(order=2, innovations_order=[0, 1], name='level') +\n", + " sts.AutoregressiveComponent(order=1, name='ar') +\n", + " sts.MeasurementError(name='obs_error')\n", + ").build(verbose = False)\n", + "\n", + "param_values = {\n", + " 'initial_level': np.array([10, 0.1]),\n", + " 'sigma_level': np.array([1e-2]),\n", + " 'params_ar': np.array([0.95]),\n", + " 'sigma_ar': np.array(1e-2),\n", + " 'sigma_obs_error': np.array(1e-2),\n", + "}\n", + "\n", + "data_fn = pmx.statespace.compile_statespace(mod, steps=100)\n", + "hidden_state_data, obs_data = data_fn(**param_values)\n", + "\n", + "matrices = mod._unpack_statespace_with_placeholders()\n", + "\n", + "matrix_fn = pytensor.function(list(explicit_graph_inputs(matrices)),\n", + " matrices)\n", + "a0, P0, c, d, T, Z, R, H, Q = matrix_fn(**param_values, initial_state_cov=np.eye(mod.k_states))" + ] + }, + { + "cell_type": "markdown", + "id": "51b7e885", + "metadata": {}, + "source": [ + "### Symbolic variable" + ] + }, + { + "cell_type": "code", + "execution_count": 218, + "id": "3661408d", + "metadata": {}, + "outputs": [], + "source": [ + "# Paramètres symboliques\n", + "A_sym = pt.matrix(\"A\") # (n, n)\n", + "H_sym = pt.matrix(\"H\") # (n, n)\n", + "Q_sym = pt.matrix(\"Q\") # (n, n)\n", + "R_sym = pt.matrix(\"R\") # (n, n)\n", + "T_sym = pt.matrix(\"T\") # (n, n)\n", + "Z_sym = pt.matrix(\"Z\") # (n, n)\n", + "y_sym = pt.matrix(\"y\") # (T, n) : observations\n", + "\n", + "a0_sym = pt.vector(\"a0\") # (n,) \n", + "P0_sym = pt.matrix(\"P0\") # (n, n)\n", + "\n", + "data_sym = pt.matrix('data_sym') # [T, obs_dim]" + ] + }, + { + "cell_type": "markdown", + "id": "19e6a32d", + "metadata": {}, + "source": [ + "## Kalman filter with classic gradient" + ] + }, + { + "cell_type": "markdown", + "id": "4fb4cef1", + "metadata": {}, + "source": [ + "### The Loss\n", + "\n", + "The Negative Log-Likelihood loss os given in the paper as the following expression :\n", + "\n", + "$$\n", + "L_{NLL} = \\sum l_{n|n} + l_{n|n-1}\n", + "$$\n", + "\n", + "Where :\n", + "\n", + "$$\n", + "\\begin{align}\n", + "&l_{n|n} = 0 \\\\\n", + "&l_{n|n-1} = log det(F) + v_n^TFv_n\n", + "\\end{align}\n", + "$$" + ] + }, + { + "cell_type": "code", + "execution_count": 219, + "id": "35351096", + "metadata": {}, + "outputs": [], + "source": [ + "def predict(a, P, T, Q):\n", + " a_hat = T @ a # x_n|n-1\n", + " P_hat = T @ P @ T.T + Q # P_n|n-1\n", + " return a_hat, P_hat\n", + "\n", + "def update(y, a, P, Z, H):\n", + " v = y - Z.dot(a) # z_n\n", + " PZT = P.dot(Z.T) \n", + "\n", + " F = Z.dot(PZT) + H # S_n\n", + " F_inv = pt.linalg.inv(F) # S_n^(-1)\n", + " K = PZT.dot(F_inv) # K_n\n", + "\n", + " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", + " a_filtered = a + K.dot(v) # x_n|n\n", + " P_filtered = I_KZ @ P # P_n|n\n", + "\n", + " inner_term = v.T @ F_inv @ v\n", + " _, F_logdet = pt.linalg.slogdet(F) # log det S_n\n", + " ll = (F_logdet + inner_term).ravel()[0] # Loss\n", + "\n", + " return [a_filtered, P_filtered, Z.dot(a), F, ll]\n", + "\n", + "def kalman_step(y, a, P, T, Z, H, Q):\n", + " a_filtered, P_filtered, obs_mu, obs_cov, ll = update(y=y, a=a, P=P, Z=Z, H=H)\n", + " a_hat, P_hat = predict(a=a_filtered, P=P_filtered, T=T, Q=Q)\n", + " return [a_filtered, a_hat, obs_mu, P_filtered, P_hat, obs_cov, ll]\n", + "\n", + "\n", + "outputs_info = [None, a0_sym, None, None, P0_sym, None, None]\n", + "\n", + "results_seq, updates = pytensor.scan(\n", + " kalman_step,\n", + " sequences=[data_sym],\n", + " outputs_info=outputs_info,\n", + " non_sequences=[T_sym, Z_sym, H_sym, Q_sym],\n", + " strict=False,\n", + ")\n", + "\n", + "# --- Loss ---\n", + "a_upd_seq, a_pred_seq, y_hat_seq, P_upd_seq, P_pred_seq, obs_cov, ll_seq = results_seq\n", + "loss = pt.sum(ll_seq)" + ] + }, + { + "cell_type": "markdown", + "id": "ece2f47e", + "metadata": {}, + "source": [ + "## Custom gradient" + ] + }, + { + "cell_type": "markdown", + "id": "5dc91ae7", + "metadata": {}, + "source": [ + "### Gradient with respect to **$a_{n-1|n-1}$**\n", + "\n", + "From the article we have :\n", + "\n", + "$$\n", + "\\begin{align}\n", + "&\\frac{dL}{da_{n-1|n-1}} = T_n^T \\frac{dL}{da_{n|n-1}} \n", + "+ \\frac{dl_{n-1|n-1}}{da_{n-1|n-1}} \\quad &\\text{(equation 22)} \\\\\n", + "&\\frac{dl_{n|n}}{da_{n|n}} = 0 \\quad &\\text{(equation 28)}\n", + "\\end{align}\n", + "$$\n", + "\n", + "Givent this two equations, we now have :\n", + "$$\n", + "\\begin{align}\n", + "&\\frac{dL}{da_{n-1|n-1}} = T_n^T \\frac{dL}{da_{n|n-1}} \n", + "\\end{align}\n", + "$$\n", + "\n", + "### Gradient with respect to **$P_{n-1|n-1}$**\n", + "\n", + "From the article we have :\n", + "\n", + "$$\n", + "\\begin{align}\n", + "&\\frac{dL}{dP_{n-1|n-1}} = T_n^T \\frac{dL}{dP_{n|n-1}} T_n\n", + "+ \\frac{dl_{n-1|n-1}}{dP_{n-1|n-1}} \\quad &\\text{(equation 23)} \\\\\n", + "&\\frac{dl_{n|n}}{dP_{n|n}} = 0 \\quad &\\text{(equation 28)}\n", + "\\end{align}\n", + "$$\n", + "\n", + "Givent this two equations, we now have :\n", + "$$\n", + "\\begin{align}\n", + "&\\frac{dL}{dP_{n-1|n-1}} = T_n^T \\frac{dL}{dP_{n|n-1}} T_n\n", + "\\end{align}\n", + "$$\n" + ] + }, + { + "cell_type": "markdown", + "id": "22a3560b", + "metadata": {}, + "source": [ + "### Gradient with respect to **$a_{n|n-1}$**\n", + "\n", + "From the article we have :\n", + "\n", + "$$\n", + "\\begin{align}\n", + "&\\frac{dL}{da_{n|n-1}} = (I - K_n Z_n)^T \\frac{dL}{da_{n|n}} + \\frac{dl_{n|n-1}}{da_{n|n-1}} \\quad &\\text{(equation 20)} \\\\\n", + "&\\frac{dl_{n|n-1}}{da_{n|n-1}} = -2 Z_n^{T}F_n^{-1} v_n \\quad &\\text{(equation 30)} \\\\\n", + "&\\frac{dL}{da_{n|n}} = T_n^T \\frac{dL}{da_{n+1|n}} \\quad &\\text{see gradient with respect to} \\quad a_{n|n}\n", + "\\end{align}\n", + "$$\n", + "\n", + "Givent this two equations, we now have :\n", + "$$\n", + "\\begin{align}\n", + "&\\frac{dL}{da_{n|n-1}} = (I - K_n Z_n)^T T_n^T \\frac{dL}{da_{n+1|n}} - 2 Z_n^{T}F^{-1} v_n\n", + "\\end{align}\n", + "$$\n" + ] + }, + { + "cell_type": "code", + "execution_count": 132, + "id": "ee21ef4e", + "metadata": {}, + "outputs": [], + "source": [ + "def grad_a_hat(inp, out, out_grad):\n", + " y, a, P, T, Z, H, Q = inp\n", + " a_hat_grad, _, _ = out_grad\n", + "\n", + " v = y - Z.dot(a) \n", + "\n", + " PZT = P.dot(Z.T)\n", + " F = Z.dot(PZT) + H \n", + " F_inv = pt.linalg.inv(F)\n", + " \n", + " K = PZT.dot(F_inv) \n", + " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", + "\n", + " grad_a_pred = I_KZ.T @ T.T @ a_hat_grad - 2 * Z.T @ F_inv @ v\n", + "\n", + " return grad_a_pred" + ] + }, + { + "cell_type": "markdown", + "id": "293d8d65", + "metadata": {}, + "source": [ + "### Gradient with respect to **$P_{n|n-1}$**\n", + "\n", + "From the article we have :\n", + "\n", + "$$\n", + "\\begin{align}\n", + "&\\frac{dL}{dP_{n|n-1}} = (I - K_n Z_n)^T [\n", + " \\frac{dL}{dP_{n|n}}\n", + " + \\frac{1}{2} \\frac{dL}{da_{n|n}} v_n^T H_n^-1 Z_n\n", + " + \\frac{1}{2} Z_n^T R_n^{-1} v_n (\\frac{dL}{da_{n|n}})^T\n", + " ](I - K_n Z_n) \n", + " + \\frac{dl{n|n-1}}{dP_{n|n-1}} \\quad &\\text{(equation 21)} \\\\\n", + "&\\frac{dl_{n|n-1}}{dP_{n|n-1}} = Z_n^T F_n^{-1} Z_n - Z_n^T F_n^-1 v_n v_n^T F_n^{-1} Z_n \\quad &\\text{(equation 29)} \\\\\n", + "&\\frac{dL}{da_{n|n}} = T_n^T \\frac{dL}{da_{n+1|n}} \\quad &\\text{see gradient with respect to} \\quad a_{n|n} \\\\\n", + "&\\frac{dL}{dP_{n|n}} = T_n^T \\frac{dL}{dP_{n+1|n}} T_n \\quad &\\text{see gradient with respect to} \\quad P_{n|n}\n", + "\\end{align}\n", + "$$\n", + "\n", + "Givent this two equations, we now have :\n", + "$$\n", + "\\begin{align}\n", + "&\\frac{dL}{dP_{n|n-1}} = (I - K_n Z_n)^T [\n", + " T_n^T \\frac{dL}{dP_{n+1|n}} T_n\n", + " + \\frac{1}{2} T_n^T \\frac{dL}{da_{n+1|n}} v_n^T H_n^{-1} Z_n\n", + " + \\frac{1}{2} Z_n^T H_n^{-1} v_n (T_n^T \\frac{dL}{da_{n+1|n}})^T\n", + " ](I - K_n Z_n) \n", + " + Z_n^T F_n^{-1} Z_n \n", + " - Z_n^T F_n^{-1} v_n v_n^T F_n^{-1} Z_n\n", + "\\end{align}\n", + "$$" + ] + }, + { + "cell_type": "code", + "execution_count": 133, + "id": "8c89b018", + "metadata": {}, + "outputs": [], + "source": [ + "def grad_P_hat(inp, out, out_grad):\n", + " y, a, P, T, Z, H, Q = inp\n", + " a_hat_grad, P_hat_grad, ll_grad = out_grad\n", + "\n", + " v = y - Z.dot(a)\n", + " v = v.dimshuffle(0, 'x')\n", + " a_hat_grad = a_hat_grad.dimshuffle(0, 'x') \n", + "\n", + " P_filtered_grad = T.T @ P_hat_grad @ T\n", + " a_filtered_grad = T.T @ a_hat_grad \n", + "\n", + " PZT = P.dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + "\n", + " H_inv = pt.linalg.inv(H) \n", + " F_inv = pt.linalg.inv(F)\n", + " \n", + " K = PZT.dot(F_inv) \n", + " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", + "\n", + " grad_P_hat = I_KZ.T @ ( P_filtered_grad + 0.5 * a_filtered_grad @ v.T @ H_inv @ Z + 0.5 * Z.T @ H_inv @ v @ a_filtered_grad.T ) @ I_KZ + Z.T @ F_inv @ Z - Z.T @ F_inv @ v @ v.T @ F_inv @ Z\n", + "\n", + " return grad_P_hat" + ] + }, + { + "cell_type": "markdown", + "id": "f0f2dce4", + "metadata": {}, + "source": [ + "### Gradient with respect to **y**\n", + "\n", + "From the article we have :\n", + "\n", + "$$\n", + "\\begin{align}\n", + "&\\frac{dL}{dy_n} = K_n^T\\frac{dL}{da_{n|n}} + \\frac{dl_{n|n-1}}{dy_n} \\quad &\\text{(equation 24)} \\\\\n", + "&\\frac{dl_{n|n-1}}{dy_n} = 2F^{-1}v_n \\quad &\\text{(equation 31)} \\\\\n", + "&\\frac{dL}{da_{n|n}} = T_n^T \\frac{dL}{da_{n+1|n}} \\quad &\\text{see gradient with respect to} \\quad a_{n|n} \\\\\n", + "\\end{align}\n", + "$$\n", + "\n", + "Givent this two equations, we now have :\n", + "$$\n", + "\\begin{align}\n", + "&\\frac{dL}{dy_n} = K_n^TT_n^T\\frac{dL}{da_{n+1|n}} + 2F^{-1}v_n\n", + "\\end{align}\n", + "$$\n" + ] + }, + { + "cell_type": "code", + "execution_count": 134, + "id": "bba53a26", + "metadata": {}, + "outputs": [], + "source": [ + "def grad_y(inp, out, out_grad):\n", + " y, a, P, T, Z, H, Q = inp\n", + " a_hat_grad, P_h_grad, y_grad = out_grad\n", + "\n", + " y_hat = Z.dot(a)\n", + " v = y - y_hat\n", + "\n", + " PZT = P.dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + " F_inv = pt.linalg.inv(F)\n", + "\n", + " K = PZT.dot(F_inv) \n", + " \n", + " return K.T @ T.T @ a_hat_grad + 2 * F_inv @ v" + ] + }, + { + "cell_type": "markdown", + "id": "d6b48789", + "metadata": {}, + "source": [ + "### Gradient with respect to Q\n", + "\n", + "From the article we have :\n", + "\n", + "$$\n", + "\\begin{align}\n", + "\\frac{dL}{dQ_n} = \\frac{dL}{dP_{n|n-1}} & \\quad \\text{(equation 25)}\n", + "\\end{align}\n", + "$$" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "c17949b7", + "metadata": {}, + "outputs": [], + "source": [ + "def grad_Q(inp, out, out_grad):\n", + " _, P_h_grad, _ = out_grad\n", + " return P_h_grad" + ] + }, + { + "cell_type": "markdown", + "id": "f0bc0287", + "metadata": {}, + "source": [ + "## Gradient with respect to **H**\n", + "\n", + "From the article we have :\n", + "\n", + "$$\n", + "\\begin{align}\n", + "&\\frac{dL}{dH_n} = K_n^T\\frac{dL}{dP_{n|n}}K_n \n", + "- \\frac{1}{2} K_n^T \\frac{dL}{da_{n|n}} v_n^T F^{-1}\n", + "- \\frac{1}{2} S_n^{-1} v_n (\\frac{dL}{da_{n|n}})^T K_n\n", + "+ \\frac{dl_{n|n-1}}{dH_n} \n", + "\\quad &\\text{(equation 26)} \\\\\n", + "&\\frac{dl_{n|n-1}}{dH_n} = F^{-1} - F_n^{-1} v_n v_n^T F_n^{-1} \n", + "\\quad &\\text{(equation 31)} \\\\\n", + "&\\frac{dL}{da_{n|n}} = T_n^T \\frac{dL}{da_{n+1|n}} \\quad &\\text{see gradient with respect to} \\quad a_{n|n} \\\\\n", + "&\\frac{dL}{dP_{n|n}} = T_n^T \\frac{dL}{dP_{n+1|n}} T_n \\quad &\\text{see gradient with respect to} \\quad P_{n|n}\n", + "\\end{align}\n", + "$$\n", + "\n", + "Givent this two equations, we now have :\n", + "$$\n", + "\\begin{align}\n", + "&\\frac{dL}{dH_n} = K_n^T T_n^T \\frac{dL}{dP_{n+1|n}} T_n K_n \n", + "- \\frac{1}{2} K_n^T T_n^T \\frac{dL}{da_{n+1|n}} v_n^T F^{-1}\n", + "- \\frac{1}{2} F_n^{-1} v_n (T_n^T \\frac{dL}{da_{n+1|n}})^T K_n\n", + "+ F^{-1} - F_n^{-1} v_n v_n^T F_n^{-1}\n", + "\\end{align}\n", + "$$\n" + ] + }, + { + "cell_type": "code", + "execution_count": 135, + "id": "84cb6867", + "metadata": {}, + "outputs": [], + "source": [ + "def grad_H(inp, out, out_grad):\n", + " y, a, P, T, Z, H, Q = inp\n", + " a_hat_grad, P_h_grad, y_grad = out_grad\n", + " \n", + " y_hat = Z.dot(a)\n", + " v = y - y_hat\n", + "\n", + " PZT = P.dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + " F_inv = pt.linalg.inv(F)\n", + "\n", + " K = PZT.dot(F_inv)\n", + "\n", + " v = v.dimshuffle(0, 'x')\n", + " a_hat_grad = a_hat_grad.dimshuffle(0, 'x') \n", + "\n", + " a_filtered_grad = T.T @ a_hat_grad\n", + " P_filtered_grad = T.T @ P_h_grad @ T\n", + "\n", + " return K.T @ P_filtered_grad @ K - 0.5 * K.T @ a_filtered_grad @ v.T @ F_inv - 0.5 * F_inv @ v @ a_filtered_grad.T @ K + F_inv - F_inv @ v @ v.T @ F_inv" + ] + }, + { + "cell_type": "markdown", + "id": "bd458dee", + "metadata": {}, + "source": [ + "### Total grad" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "afb362e5", + "metadata": {}, + "outputs": [], + "source": [ + "def custom_grad(inp, out, out_grad):\n", + " y, a, P, T, Z, H, Q = inp\n", + " a_filtered, P_filtered, y_hat = out\n", + " a_hat_grad, P_hat_grad, y_grad = out_grad\n", + "\n", + " PZT = P.dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + "\n", + " y_hat = Z.dot(a)\n", + " v = y - y_hat\n", + "\n", + " H_inv = pt.linalg.inv(H)\n", + " F_inv = pt.linalg.inv(F)\n", + "\n", + " K = PZT.dot(F_inv)\n", + " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", + " \n", + " grad_a_pred = I_KZ.T @ T.T @ a_hat_grad - 2 * Z.T @ F_inv @ v\n", + " grad_y = K.T @ T.T @ a_hat_grad + 2 * F_inv @ v\n", + "\n", + "\n", + " a_hat_grad = a_hat_grad.dimshuffle(0, 'x')\n", + " v = v.dimshuffle(0, 'x')\n", + " \n", + " P_filtered_grad = T.T @ P_hat_grad @ T\n", + " a_filtered_grad = T.T @ a_hat_grad \n", + "\n", + " grad_P_hat = I_KZ.T @ ( P_filtered_grad + 0.5 * a_filtered_grad @ v.T @ H_inv @ Z + 0.5 * Z.T @ H_inv @ v @ a_filtered_grad.T ) @ I_KZ + Z.T @ F_inv @ Z - Z.T @ F_inv @ v @ v.T @ F_inv @ Z\n", + " grad_Z = None\n", + " grad_T = None\n", + " grad_Q = P_hat_grad\n", + " grad_H = K.T @ P_filtered_grad @ K - 0.5 * K.T @ a_filtered_grad @ v.T @ F_inv - 0.5 * F_inv @ v @ a_filtered_grad.T @ K + F_inv - F_inv @ v @ v.T @ F_inv\n", + "\n", + " return [grad_P_hat,\n", + " grad_a_pred,\n", + " grad_y,\n", + " grad_Z,\n", + " grad_T,\n", + " grad_Q,\n", + " grad_H]\n" + ] + }, + { + "cell_type": "markdown", + "id": "607753a1", + "metadata": {}, + "source": [ + "## Custom Kalman Filter" + ] + }, + { + "cell_type": "code", + "execution_count": 259, + "id": "7cead2c1", + "metadata": {}, + "outputs": [], + "source": [ + "y_sym = pt.vector(\"y\")\n", + "\n", + "kalman_step_op = OpFromGraph(\n", + " inputs=[y_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", + " outputs=kalman_step(y_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym),\n", + " lop_overrides=[grad_y, grad_a_hat, grad_P_hat, None, None, grad_H, grad_Q],\n", + " inline=True\n", + ")\n", + "\n", + "outputs_info = [None, a0_sym, None, None, P0_sym, None, None]\n", + "\n", + "results_op, updates = pytensor.scan(\n", + " kalman_step_op,\n", + " sequences=[data_sym],\n", + " outputs_info=outputs_info,\n", + " non_sequences=[T_sym, Z_sym, H_sym, Q_sym],\n", + " strict=False,\n", + ")\n", + "# --- Loss ---\n", + "a_upd_op, a_pred_op, y_hat_op, P_upd_op, P_pred_op, obs_cov, ll_op = results_op\n", + "loss_op = pt.sum(ll_op)" + ] + }, + { + "cell_type": "markdown", + "id": "3f79c5c6", + "metadata": {}, + "source": [ + "## Handmade Numpy Backpropagation " + ] + }, + { + "cell_type": "code", + "execution_count": 264, + "id": "b6eb5d48", + "metadata": {}, + "outputs": [], + "source": [ + "def compute_grad_a0(observations, a0, P0, a_pred_seq, P_pred_seq, Z, H, T):\n", + " # Constant\n", + " SHAPE_a0 = a0.shape[0]\n", + " NB_obs = len(observations)\n", + "\n", + " # Initialisation for the backprop\n", + " PZT = P_pred_seq[-2].dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + " F_inv = np.linalg.solve(F, np.eye(F.shape[0]))\n", + " \n", + " grad = [0 for _ in range(NB_obs)]\n", + " grad[-1] = - 2 * Z.T @ F_inv @ (observations[-1] - Z @ a_pred_seq[-2])\n", + "\n", + " # Backprop\n", + " for i in range(3, NB_obs+1):\n", + "\n", + " PZT = P_pred_seq[-i].dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + " F_inv = np.linalg.solve(F, np.eye(F.shape[0]))\n", + "\n", + " K = PZT.dot(F_inv)\n", + " I_KZ = np.eye(SHAPE_a0) - K.dot(Z)\n", + "\n", + " grad[1-i] = I_KZ.T @ T.T @ grad[2-i] - (2 * Z.T @ F_inv @ (observations[1-i] - Z @ a_pred_seq[-i])).T \n", + "\n", + " # Last iter with a0/P0\n", + " PZT = P0.dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + " F_inv = np.linalg.solve(F, np.eye(F.shape[0]))\n", + "\n", + " K = PZT.dot(F_inv)\n", + " I_KZ = np.eye(SHAPE_a0) - K.dot(Z)\n", + "\n", + " grad[0] = I_KZ.T @ T.T @ grad[1] - (2 * Z.T @ F_inv @ (observations[0] - Z @ a0)).T\n", + "\n", + " return grad" + ] + }, + { + "cell_type": "markdown", + "id": "f0575c2c", + "metadata": {}, + "source": [ + "## Speed observation" + ] + }, + { + "cell_type": "markdown", + "id": "c99fddf9", + "metadata": {}, + "source": [ + "### Benchmark for pytensor computed gradients" + ] + }, + { + "cell_type": "code", + "execution_count": 253, + "id": "908946b0", + "metadata": {}, + "outputs": [], + "source": [ + "def benchmark_kalman_gradients(loss, obs_data, a0, P0, T, Z, R, H, Q):\n", + " results = defaultdict(dict)\n", + " exec_time = 0\n", + "\n", + " grad_list = pt.grad(loss, [a0_sym])\n", + " f_grad = pytensor.function(\n", + " inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", + " outputs=grad_list,\n", + " )\n", + "\n", + " for _ in range(20):\n", + " \n", + " # --- exécution ---\n", + " t0 = perf_counter()\n", + " _ = f_grad(\n", + " obs_data[:, np.newaxis],\n", + " a0,\n", + " P0,\n", + " T,\n", + " Z,\n", + " H,\n", + " R @ Q @ R.T,\n", + " )\n", + " t1 = perf_counter()\n", + " exec_time += (t1 - t0)/20\n", + " \n", + " \n", + " results[\"exec_time\"] = exec_time\n", + "\n", + " return results" + ] + }, + { + "cell_type": "markdown", + "id": "f03a7555", + "metadata": {}, + "source": [ + "### Benchmark for numpy computed gradient" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a85fe92e", + "metadata": {}, + "outputs": [], + "source": [ + "def benchmark_kalman_gradients_np(a_pred_seq, P_pred_seq, obs_data, a0, P0, T, Z, R, H, Q):\n", + " results = defaultdict(dict)\n", + " forward_pass = 0\n", + " backprop = 0\n", + " kalman_fn = pytensor.function(inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", + " outputs=(a_pred_seq, P_pred_seq))\n", + "\n", + " for _ in range(20):\n", + "\n", + " # --- forward pass ---\n", + " t0 = perf_counter()\n", + " a_pred, P_pred = kalman_fn(obs_data[:, np.newaxis],\n", + " a0,\n", + " P0,\n", + " T,\n", + " Z,\n", + " H,\n", + " R@Q@R.T,)\n", + " t1 = perf_counter()\n", + " forward_pass += (t1 - t0)/20\n", + " \n", + "\n", + " # --- Backprop ---\n", + " t0 = perf_counter()\n", + " _ = compute_grad_a0(\n", + " obs_data,\n", + " a0,\n", + " P0,\n", + " a_pred,\n", + " P_pred,\n", + " Z,\n", + " H,\n", + " T,)\n", + " t1 = perf_counter()\n", + " backprop += (t1 - t0)/20\n", + "\n", + " results[\"Forward pass\"] = forward_pass \n", + " results[\"Backprop\"] = backprop\n", + "\n", + " return results" + ] + }, + { + "cell_type": "markdown", + "id": "b413f411", + "metadata": {}, + "source": [ + "### Comparison" + ] + }, + { + "cell_type": "code", + "execution_count": 254, + "id": "27a60fb3", + "metadata": {}, + "outputs": [], + "source": [ + "results = benchmark_kalman_gradients(loss, obs_data, a0, P0, T, Z, R, H, Q)" + ] + }, + { + "cell_type": "code", + "execution_count": 257, + "id": "a413c8e9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "defaultdict(, {'exec_time': 0.016296579997288063})\n" + ] + } + ], + "source": [ + "print(results)" + ] + }, + { + "cell_type": "code", + "execution_count": 260, + "id": "d35b98d6", + "metadata": {}, + "outputs": [], + "source": [ + "results_op = benchmark_kalman_gradients(loss_op, obs_data, a0, P0, T, Z, R, H, Q)" + ] + }, + { + "cell_type": "code", + "execution_count": 261, + "id": "539c18c2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "defaultdict(, {'exec_time': 0.015451419999590144})\n" + ] + } + ], + "source": [ + "print(results_op)" + ] + }, + { + "cell_type": "code", + "execution_count": 267, + "id": "1e633e75", + "metadata": {}, + "outputs": [], + "source": [ + "results_np = benchmark_kalman_gradients_np(a_pred_seq, P_pred_seq, obs_data, a0, P0, T, Z, R, H, Q)" + ] + }, + { + "cell_type": "code", + "execution_count": 268, + "id": "7118dfec", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "defaultdict(, {'Forward pass': 0.00269013000652194, 'Backprop': 0.002321220003068447})\n" + ] + } + ], + "source": [ + "print(results_np)" + ] + }, + { + "cell_type": "markdown", + "id": "d77cf70b", + "metadata": {}, + "source": [ + "## Error observation" + ] + }, + { + "cell_type": "markdown", + "id": "90fabd6f", + "metadata": {}, + "source": [ + "### Comparing the gradient with respect to a0" + ] + }, + { + "cell_type": "code", + "execution_count": 274, + "id": "fbae0189", + "metadata": {}, + "outputs": [], + "source": [ + "# First the classic way with autodiff\n", + "\n", + "grad_list = pt.grad(loss, [a0_sym])\n", + "f_grad = pytensor.function(\n", + " inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", + " outputs=grad_list,\n", + ")\n", + "\n", + "grad_a0 = f_grad(obs_data[:, np.newaxis], a0, P0, T, Z, H, R @ Q @ R.T)\n", + "\n", + "# Now using our OpFromGraph custom gradient\n", + "\n", + "grad_list_op = pt.grad(loss_op, [a0_sym])\n", + "f_grad = pytensor.function(\n", + " inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", + " outputs=grad_list_op,\n", + ")\n", + "\n", + "grad_a0_op = f_grad(obs_data[:, np.newaxis], a0, P0, T, Z, H, R @ Q @ R.T)\n", + "\n", + "# And here using our handmaid numpy backprop\n", + "\n", + "kalman_fn = pytensor.function(inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", + " outputs=(a_pred_seq, P_pred_seq))\n", + "a_pred, P_pred = kalman_fn(obs_data[:, np.newaxis], a0, P0, T, Z, H, R@Q@R.T)\n", + "\n", + "grad_a0_np = compute_grad_a0(obs_data, a0, P0, a_pred, P_pred, Z, H, T)[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 278, + "id": "c3a114b2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Comparison between classic a0 gradient and our custom OpFromGraph : True\n", + "Comparison between classic a0 gradient and our handmaid NumPy backprop : True\n" + ] + } + ], + "source": [ + "print(\"Comparison between classic a0 gradient and our custom OpFromGraph :\", np.allclose(grad_a0, grad_a0_op))\n", + "print(\"Comparison between classic a0 gradient and our handmaid NumPy backprop :\", np.allclose(grad_a0, grad_a0_np))" + ] + }, + { + "cell_type": "code", + "execution_count": 279, + "id": "867d5e2f", + "metadata": {}, + "outputs": [], + "source": [ + "# First the classic way with autodiff\n", + "\n", + "grad_list = pt.grad(loss, [data_sym, a0_sym, P0_sym, H_sym, Q_sym])\n", + "f_grad = pytensor.function(\n", + " inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", + " outputs=grad_list,\n", + ")\n", + "\n", + "grad_a0 = f_grad(obs_data[:, np.newaxis], a0, P0, T, Z, H, R @ Q @ R.T)\n", + "\n", + "# Now using our OpFromGraph custom gradient\n", + "\n", + "grad_list_op = pt.grad(loss_op, [data_sym, a0_sym, P0_sym, H_sym, Q_sym])\n", + "f_grad = pytensor.function(\n", + " inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", + " outputs=grad_list_op,\n", + ")\n", + "\n", + "grad_a0_op = f_grad(obs_data[:, np.newaxis], a0, P0, T, Z, H, R @ Q @ R.T)" + ] + }, + { + "cell_type": "code", + "execution_count": 289, + "id": "25f0a57b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Comparison between classic y gradient and our custom OpFromGraph : True\n", + "Comparison between classic a0 gradient and our custom OpFromGraph : True\n", + "Comparison between classic P0 gradient and our custom OpFromGraph : True\n", + "Comparison between classic H gradient and our custom OpFromGraph : True\n", + "Comparison between classic Q gradient and our custom OpFromGraph : True\n" + ] + } + ], + "source": [ + "print(\"Comparison between classic y gradient and our custom OpFromGraph :\", np.allclose(grad_a0[0], grad_a0_op[0]))\n", + "print(\"Comparison between classic a0 gradient and our custom OpFromGraph :\", np.allclose(grad_a0[1], grad_a0_op[1]))\n", + "print(\"Comparison between classic P0 gradient and our custom OpFromGraph :\", np.allclose((grad_a0[2] + grad_a0[2].T)/2, grad_a0_op[2]))\n", + "print(\"Comparison between classic H gradient and our custom OpFromGraph :\", np.allclose(grad_a0[3], grad_a0_op[3]))\n", + "print(\"Comparison between classic Q gradient and our custom OpFromGraph :\", np.allclose((grad_a0[4] + grad_a0[4].T)/2, grad_a0_op[4]))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "CausalPy", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/Kalman_Filter_Gradients.ipynb b/notebooks/Kalman_Filter_Gradients.ipynb deleted file mode 100644 index 9f33d39e..00000000 --- a/notebooks/Kalman_Filter_Gradients.ipynb +++ /dev/null @@ -1,743 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "69ae14a1", - "metadata": {}, - "source": [ - "### Imports" - ] - }, - { - "cell_type": "code", - "execution_count": 127, - "id": "90979a41", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "import pytensor\n", - "import pytensor.tensor as pt\n", - "import matplotlib.pyplot as plt\n", - "from pytensor.compile.builders import OpFromGraph\n", - "from time import perf_counter\n", - "from collections import defaultdict" - ] - }, - { - "cell_type": "markdown", - "id": "a0d008fc", - "metadata": {}, - "source": [ - "### Generate dataset" - ] - }, - { - "cell_type": "code", - "execution_count": 128, - "id": "f75a72e8", - "metadata": {}, - "outputs": [], - "source": [ - "def generate_kalman_dataset(n, N=100, seed=0):\n", - " rng = np.random.default_rng(seed)\n", - "\n", - " # 0. Initial state and cov\n", - " A0 = rng.normal(loc=0.0, scale=1.0, size=(n,))\n", - " # 3. Random process noise covariance Q (PSD)\n", - " P0_base = rng.normal(0, 1, size=(n, n))\n", - " P0 = P0_base @ P0_base.T + np.eye(n) * 1e-3 # ensure positive definite\n", - "\n", - " # 1. T stable random transition matrix T\n", - " T = rng.normal(0, 1, size=(n, n))\n", - " eigvals = np.linalg.eigvals(T)\n", - " spectral_radius = max(abs(eigvals))\n", - " T = T / (1.1 * spectral_radius) # shrink to ensure stability\n", - "\n", - " # 2. Random observation matrix Z\n", - " Z = rng.normal(0, 1, size=(n, n)) # full observations (m = n)\n", - "\n", - " # 3. Random process noise covariance Q (PSD)\n", - " Q_base = rng.normal(0, 1, size=(n, n))\n", - " Q = Q_base @ Q_base.T + np.eye(n) * 1e-3 # ensure positive definite\n", - "\n", - " # 4. Random observation noise covariance H (PSD)\n", - " H_base = rng.normal(0, 1, size=(n, n))\n", - " H = H_base @ H_base.T + np.eye(n) * 1e-3\n", - "\n", - " # 5. Initial state\n", - " x = np.zeros((N, n))\n", - " y = np.zeros((N, n))\n", - " x[0] = A0\n", - "\n", - " # 6. Simulate the system\n", - " for t in range(1, N):\n", - " w_t = rng.multivariate_normal(mean=np.zeros(n), cov=Q)\n", - " x[t] = T @ x[t-1] + w_t\n", - "\n", - " for t in range(N):\n", - " v_t = rng.multivariate_normal(mean=np.zeros(n), cov=H)\n", - " y[t] = Z @ x[t] + v_t\n", - "\n", - " return {\n", - " \"T\": T, \"Z\": Z, \"Q\": Q, \"H\": H,\n", - " \"x\": x, \"y\": y, \"A0\": A0, \"P0\": P0\n", - " }" - ] - }, - { - "cell_type": "markdown", - "id": "51b7e885", - "metadata": {}, - "source": [ - "### Symbolic variable" - ] - }, - { - "cell_type": "code", - "execution_count": 129, - "id": "3661408d", - "metadata": {}, - "outputs": [], - "source": [ - "# Paramètres symboliques\n", - "A_sym = pt.matrix(\"A\") # (n, n)\n", - "H_sym = pt.matrix(\"H\") # (n, n)\n", - "Q_sym = pt.matrix(\"Q\") # (n, n)\n", - "R_sym = pt.matrix(\"R\") # (n, n)\n", - "T_sym = pt.matrix(\"T\") # (n, n)\n", - "Z_sym = pt.matrix(\"Z\") # (n, n)\n", - "\n", - "x0_sym = pt.vector(\"x0\") # (n,)\n", - "y_sym = pt.matrix(\"y\") # (T, n) : observations\n", - "\n", - "a0_sym = pt.vector(\"a0\") \n", - "P0_sym = pt.matrix(\"P0\") \n", - "\n", - "data_sym = pt.matrix('data_sym') # [T, obs_dim]" - ] - }, - { - "cell_type": "markdown", - "id": "19e6a32d", - "metadata": {}, - "source": [ - "### Kalman filter with classic gradient" - ] - }, - { - "cell_type": "code", - "execution_count": 130, - "id": "35351096", - "metadata": {}, - "outputs": [], - "source": [ - "def predict(a, P, T, Q):\n", - " a_hat = T @ a # x_n|n-1\n", - " P_hat = T @ P @ T.T + Q # P_n|n-1\n", - " return a_hat, P_hat\n", - "\n", - "def update(y, a, P, Z, H):\n", - " v = y - Z.dot(a) # z_n\n", - " PZT = P.dot(Z.T) \n", - "\n", - " F = Z.dot(PZT) + H # S_n\n", - " F_inv = pt.linalg.inv(F) # S_n^(-1)\n", - " K = PZT.dot(F_inv) # K_n\n", - "\n", - " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", - " a_filtered = a + K.dot(v) # x_n|n\n", - " P_filtered = I_KZ @ P # P_n|n\n", - "\n", - " inner_term = v.T @ F_inv @ v\n", - " _, F_logdet = pt.linalg.slogdet(F) # log det S_n\n", - " ll = (F_logdet + inner_term).ravel()[0] # Loss\n", - "\n", - " return [a_filtered, P_filtered, Z.dot(a), F, ll]\n", - "\n", - "def kalman_step(y, a, P, T, Z, H, Q):\n", - " a_filtered, P_filtered, obs_mu, obs_cov, ll = update(y=y, a=a, P=P, Z=Z, H=H)\n", - " a_hat, P_hat = predict(a=a_filtered, P=P_filtered, T=T, Q=Q)\n", - " return [a_filtered, a_hat, obs_mu, P_filtered, P_hat, obs_cov, ll]\n", - "\n", - "\n", - "outputs_info = [None, a0_sym, None, None, P0_sym, None, None]\n", - "\n", - "results_seq, updates = pytensor.scan(\n", - " kalman_step,\n", - " sequences=[data_sym],\n", - " outputs_info=outputs_info,\n", - " non_sequences=[T_sym, Z_sym, H_sym, Q_sym],\n", - " strict=False,\n", - ")\n", - "# --- Loss ---\n", - "a_upd_seq, a_pred_seq, y_hat_seq, P_upd_seq, P_pred_seq, obs_cov, ll_seq = results_seq\n", - "loss = pt.sum(ll_seq)" - ] - }, - { - "cell_type": "markdown", - "id": "ece2f47e", - "metadata": {}, - "source": [ - "### Custom gradient" - ] - }, - { - "cell_type": "code", - "execution_count": 131, - "id": "afb362e5", - "metadata": {}, - "outputs": [], - "source": [ - "def custom_grad(inp, out, out_grad):\n", - " y, a, P, T, Z, H, Q = inp\n", - " a_filtered, P_filtered, y_hat = out\n", - " a_hat_grad, P_hat_grad, y_grad = out_grad\n", - "\n", - " PZT = P.dot(Z.T)\n", - " F = Z.dot(PZT) + H\n", - "\n", - " y_hat = Z.dot(a)\n", - " v = y - y_hat\n", - "\n", - " H_inv = pt.linalg.inv(H)\n", - " F_inv = pt.linalg.inv(F)\n", - "\n", - " K = PZT.dot(F_inv)\n", - " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", - " \n", - " grad_a_pred = I_KZ.T @ T.T @ a_hat_grad - 2 * Z.T @ F_inv @ v\n", - " grad_y = K.T @ T.T @ a_hat_grad + 2 * F_inv @ v\n", - "\n", - "\n", - " a_hat_grad = a_hat_grad.dimshuffle(0, 'x')\n", - " v = v.dimshuffle(0, 'x')\n", - " \n", - " P_filtered_grad = T.T @ P_hat_grad @ T\n", - " a_filtered_grad = T.T @ a_hat_grad \n", - "\n", - " grad_P_hat = I_KZ.T @ ( P_filtered_grad + 0.5 * a_filtered_grad @ v.T @ H_inv @ Z + 0.5 * Z.T @ H_inv @ v @ a_filtered_grad.T ) @ I_KZ + Z.T @ F_inv @ Z - Z.T @ F_inv @ v @ v.T @ F_inv @ Z\n", - " grad_Z = None\n", - " grad_T = None\n", - " grad_Q = P_hat_grad\n", - " grad_H = K.T @ P_filtered_grad @ K - 0.5 * K.T @ a_filtered_grad @ v.T @ F_inv - 0.5 * F_inv @ v @ a_filtered_grad.T @ K + F_inv - F_inv @ v @ v.T @ F_inv\n", - "\n", - " return [grad_P_hat,\n", - " grad_a_pred,\n", - " grad_y,\n", - " grad_Z,\n", - " grad_T,\n", - " grad_Q,\n", - " grad_H]\n" - ] - }, - { - "cell_type": "code", - "execution_count": 132, - "id": "ee21ef4e", - "metadata": {}, - "outputs": [], - "source": [ - "def grad_a_hat(inp, out, out_grad):\n", - " y, a, P, T, Z, H, Q = inp\n", - " a_hat_grad, _, _ = out_grad\n", - "\n", - " v = y - Z.dot(a) \n", - "\n", - " PZT = P.dot(Z.T)\n", - " F = Z.dot(PZT) + H \n", - " F_inv = pt.linalg.inv(F)\n", - " \n", - " K = PZT.dot(F_inv) \n", - " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", - "\n", - " grad_a_pred = I_KZ.T @ T.T @ a_hat_grad - 2 * Z.T @ F_inv @ v\n", - "\n", - " return grad_a_pred" - ] - }, - { - "cell_type": "code", - "execution_count": 133, - "id": "8c89b018", - "metadata": {}, - "outputs": [], - "source": [ - "def grad_P_hat(inp, out, out_grad):\n", - " y, a, P, T, Z, H, Q = inp\n", - " a_hat_grad, P_hat_grad, ll_grad = out_grad\n", - "\n", - " v = y - Z.dot(a)\n", - " v = v.dimshuffle(0, 'x')\n", - " a_hat_grad = a_hat_grad.dimshuffle(0, 'x') \n", - "\n", - " P_filtered_grad = T.T @ P_hat_grad @ T\n", - " a_filtered_grad = T.T @ a_hat_grad \n", - "\n", - " PZT = P.dot(Z.T)\n", - " F = Z.dot(PZT) + H\n", - "\n", - " H_inv = pt.linalg.inv(H) \n", - " F_inv = pt.linalg.inv(F)\n", - " \n", - " K = PZT.dot(F_inv) \n", - " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", - "\n", - " grad_P_hat = I_KZ.T @ ( P_filtered_grad + 0.5 * a_filtered_grad @ v.T @ H_inv @ Z + 0.5 * Z.T @ H_inv @ v @ a_filtered_grad.T ) @ I_KZ + Z.T @ F_inv @ Z - Z.T @ F_inv @ v @ v.T @ F_inv @ Z\n", - "\n", - " return grad_P_hat" - ] - }, - { - "cell_type": "code", - "execution_count": 134, - "id": "bba53a26", - "metadata": {}, - "outputs": [], - "source": [ - "def grad_y(inp, out, out_grad):\n", - " y, a, P, T, Z, H, Q = inp\n", - " a_hat_grad, P_h_grad, y_grad = out_grad\n", - "\n", - " y_hat = Z.dot(a)\n", - " v = y - y_hat\n", - "\n", - " PZT = P.dot(Z.T)\n", - " F = Z.dot(PZT) + H\n", - " F_inv = pt.linalg.inv(F)\n", - "\n", - " K = PZT.dot(F_inv) \n", - " \n", - " return K.T @ T.T @ a_hat_grad + 2 * F_inv @ v" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "c17949b7", - "metadata": {}, - "outputs": [], - "source": [ - "def grad_Q(inp, out, out_grad):\n", - " _, P_h_grad, _ = out_grad\n", - " return P_h_grad" - ] - }, - { - "cell_type": "code", - "execution_count": 135, - "id": "84cb6867", - "metadata": {}, - "outputs": [], - "source": [ - "def grad_H(inp, out, out_grad):\n", - " y, a, P, T, Z, H, Q = inp\n", - " a_hat_grad, P_h_grad, y_grad = out_grad\n", - " \n", - " y_hat = Z.dot(a)\n", - " v = y - y_hat\n", - "\n", - " PZT = P.dot(Z.T)\n", - " F = Z.dot(PZT) + H\n", - " F_inv = pt.linalg.inv(F)\n", - "\n", - " K = PZT.dot(F_inv)\n", - "\n", - " v = v.dimshuffle(0, 'x')\n", - " a_hat_grad = a_hat_grad.dimshuffle(0, 'x') \n", - "\n", - " a_filtered_grad = T.T @ a_hat_grad\n", - " P_filtered_grad = T.T @ P_h_grad @ T\n", - "\n", - " return K.T @ P_filtered_grad @ K - 0.5 * K.T @ a_filtered_grad @ v.T @ F_inv - 0.5 * F_inv @ v @ a_filtered_grad.T @ K + F_inv - F_inv @ v @ v.T @ F_inv" - ] - }, - { - "cell_type": "markdown", - "id": "607753a1", - "metadata": {}, - "source": [ - "### Custom Kalman Filter" - ] - }, - { - "cell_type": "code", - "execution_count": 136, - "id": "7cead2c1", - "metadata": {}, - "outputs": [], - "source": [ - "y_sym = pt.vector(\"y\")\n", - "\n", - "kalman_step_op = OpFromGraph(\n", - " inputs=[y_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", - " outputs=kalman_step(y_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym),\n", - " lop_overrides=[grad_y, grad_a_hat, grad_P_hat, None, None, grad_H, grad_Q],\n", - " inline=True\n", - ")\n", - "\n", - "outputs_info = [None, a0_sym, None, None, P0_sym, None, None]\n", - "\n", - "results_op, updates = pytensor.scan(\n", - " kalman_step_op,\n", - " sequences=[data_sym],\n", - " outputs_info=outputs_info,\n", - " non_sequences=[T_sym, Z_sym, H_sym, Q_sym],\n", - " strict=False,\n", - ")\n", - "# --- Loss ---\n", - "a_upd_op, a_pred_op, y_hat_op, P_upd_op, P_pred_op, obs_cov, ll_op = results_op\n", - "loss_op = pt.sum(ll_op)" - ] - }, - { - "cell_type": "markdown", - "id": "f0575c2c", - "metadata": {}, - "source": [ - "### Speed observation" - ] - }, - { - "cell_type": "code", - "execution_count": 137, - "id": "07c3879e", - "metadata": {}, - "outputs": [], - "source": [ - "states = [1, 5, 10, 20, 35, 50, 75, 90, 100]" - ] - }, - { - "cell_type": "code", - "execution_count": 140, - "id": "f90f682d", - "metadata": {}, - "outputs": [], - "source": [ - "def benchmark_kalman_gradients(loss, state_dims, N=30):\n", - " results = defaultdict(dict)\n", - " for _ in range(10):\n", - " for n in state_dims:\n", - " data = generate_kalman_dataset(n, N=N, seed=42 + n)\n", - "\n", - " # --- gradients symboliques ---\n", - " t0 = perf_counter()\n", - " grad_list = pt.grad(loss, [a0_sym])\n", - " t1 = perf_counter()\n", - " grad_symbolic_time = t1 - t0\n", - " results[n][\"grad_symbolic_time\"] = grad_symbolic_time/10\n", - "\n", - " # --- compilation ---\n", - " t0 = perf_counter()\n", - " f_grad = pytensor.function(\n", - " inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", - " outputs=grad_list,\n", - " )\n", - " t1 = perf_counter()\n", - " compile_time = t1 - t0\n", - " results[n][\"compile_time\"] = compile_time/10\n", - "\n", - " # --- exécution ---\n", - " t0 = perf_counter()\n", - " _ = f_grad(\n", - " data[\"y\"],\n", - " data[\"A0\"],\n", - " data[\"P0\"],\n", - " data[\"T\"],\n", - " data[\"Z\"],\n", - " data[\"H\"],\n", - " data[\"Q\"],\n", - " )\n", - " t1 = perf_counter()\n", - " exec_time = t1 - t0\n", - " results[n][\"exec_time\"] = exec_time/10\n", - "\n", - " return results" - ] - }, - { - "cell_type": "code", - "execution_count": 141, - "id": "27a60fb3", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", - " r = _umath_linalg.det(a, signature=signature)\n", - "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", - " r = _umath_linalg.det(a, signature=signature)\n", - "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", - " r = _umath_linalg.det(a, signature=signature)\n", - "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", - " r = _umath_linalg.det(a, signature=signature)\n", - "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", - " r = _umath_linalg.det(a, signature=signature)\n", - "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", - " r = _umath_linalg.det(a, signature=signature)\n", - "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", - " r = _umath_linalg.det(a, signature=signature)\n", - "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", - " r = _umath_linalg.det(a, signature=signature)\n", - "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", - " r = _umath_linalg.det(a, signature=signature)\n", - "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\numpy\\linalg\\_linalg.py:2383: RuntimeWarning: overflow encountered in det\n", - " r = _umath_linalg.det(a, signature=signature)\n" - ] - } - ], - "source": [ - "results = benchmark_kalman_gradients(loss, states, N=5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3f6d314a", - "metadata": {}, - "outputs": [], - "source": [ - "results_op = benchmark_kalman_gradients(loss_op, states, N=5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "382e90ef", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 123, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj4AAAGdCAYAAAASUnlxAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAX+JJREFUeJzt3XlcVdX+//HXAQ6HQUBlRmVKTVMrQTMts+li6K0sb9mgaXX9ZTkRtzK1blbm0K1ut5wazDIbrCyzspK+BlmSqakZmpmhKIqIA6DMh/X74xRFoAkBh+H9fDzOQ87aa+/9OTvlvNvDWhZjjEFERESkBXBxdgEiIiIiDUXBR0RERFoMBR8RERFpMRR8REREpMVQ8BEREZEWQ8FHREREWgwFHxEREWkxFHxERESkxXBzdgGNSXl5Ofv378fHxweLxeLsckREROQ0GGPIz88nLCwMF5dTn9NR8Pmd/fv306FDB2eXISIiIrWwd+9e2rdvf8o+Cj6/4+PjAzgOnK+vr5OrERERkdORl5dHhw4dKr7HT0XB53d+vbzl6+ur4CMiItLEnM5tKrq5WURERFoMBR8RERFpMRR8REREpMVQ8BEREZEWQ8FHREREWgwFHxEREWkxFHxERESkxVDwERERkRZDwUdERERaDAUfERERaTFqFXzmzZtHVFQUHh4exMbGsmbNmlP2T0lJITY2Fg8PD6Kjo1mwYEGl5WlpaQwdOpTIyEgsFgtPP/10lW38uuyPr7Fjx1b0GTVqVJXl559/fm0+ooiIiDRDNQ4+S5cuJSEhgalTp7Jp0yb69+9PfHw8GRkZ1fZPT09n0KBB9O/fn02bNjFlyhQmTJjAsmXLKvoUFBQQHR3NrFmzCAkJqXY769ev58CBAxWvpKQkAK677rpK/a644opK/VauXFnTjygiIiLNlMUYY2qyQp8+fYiJiWH+/PkVbV27dmXIkCHMnDmzSv9JkyaxYsUKtm/fXtE2ZswYtmzZQmpqapX+kZGRJCQkkJCQcMo6EhIS+PDDD9m5c2fFpGSjRo3i2LFjLF++vCYfqUJeXh5+fn7k5uZqklIREZE6VG7KeSbzGQKsAQwPHl6n267J93eNzviUlJSwceNG4uLiKrXHxcWxdu3aatdJTU2t0n/gwIFs2LCB0tLSmuy+Uh1LlizhtttuqzITa3JyMkFBQXTu3JnRo0eTnZ190u0UFxeTl5dX6SUiIiJ1q9SUMm3PNF7NfpWnM58mvTDdabXUKPjk5ORgt9sJDg6u1B4cHExWVla162RlZVXbv6ysjJycnBqW67B8+XKOHTvGqFGjKrXHx8fz2muvsXr1ap588knWr1/PpZdeSnFxcbXbmTlzJn5+fhWvDh061KoeERERqV6hvZB/7foXHx35CFdcmRYxjSjPKKfV41ablf54lsUYU6Xtz/pX1366Fi5cSHx8PGFhYZXahw0bVvFz9+7d6dWrFxEREXz00Udce+21VbYzefJkEhMTK97n5eUp/IiIiNSRY2XHSNiVwNYTW7FZbMyOnk1/v/5OralGwScgIABXV9cqZ3eys7OrnNX5VUhISLX93dzc8Pf3r2G5sGfPHj777DPefffdP+0bGhpKREQEO3furHa5zWbDZrPVuAYRERE5taySLMb9NI70onR8XX353xn/4+xWZzu7rJpd6nJ3dyc2NrbiiapfJSUl0a9fv2rX6du3b5X+q1atolevXlit1hqWC4sWLSIoKIjBgwf/ad/Dhw+zd+9eQkNDa7wfERERqZ30wnRu23Eb6UXpBFmDeLHzi40i9EAtHmdPTEzkxRdf5KWXXmL79u3cfffdZGRkMGbMGMBx+eiWW26p6D9mzBj27NlDYmIi27dv56WXXmLhwoXcc889FX1KSkrYvHkzmzdvpqSkhMzMTDZv3sxPP/1Uad/l5eUsWrSIkSNH4uZW+WTV8ePHueeee0hNTWX37t0kJydz5ZVXEhAQwDXXXFPTjykiIiK1sPXEVm7/8XYOlh4k0hbJS2e+xBmeZzi7rN+YWpg7d66JiIgw7u7uJiYmxqSkpFQsGzlypBkwYECl/snJyaZnz57G3d3dREZGmvnz51danp6eboAqrz9u59NPPzWA2bFjR5WaCgoKTFxcnAkMDDRWq9WEh4ebkSNHmoyMjNP+XLm5uQYwubm5p72OiIiIOHx57EvTb1M/E7Mxxtyy/RZzpPRIg+y3Jt/fNR7HpznTOD4iIiK1s/LwSqbtmYYdO/18+/F41ON4uno2yL5r8v1dq6e6RERERH615OAS/pv5XwDi28TzUORDWC01v4+3ISj4iIiISK0YY3h2/7O8cvAVAG4OupmEdgm4WBrvHOgKPiIiIlJjZaaM6Xum88GRDwCYEDaBW4JvqfUYfQ1FwUdERERqpLC8kMk/T2ZN3hpcceWBiAe4yv8qZ5d1WhR8RERE5LTlluVy96672XJiCzaLjVlRs7io9UXOLuu0KfiIiIjIaTlYcpDxP41nV9EufFx9+O8Z/6Vnq57OLqtGFHxERETkT6UXpTPup3FklWQRaA1kTsc5dPTs6OyyakzBR0RERE7p+xPfM+GnCeTac4mwRTC341xCbU1zOqjG+7yZiIiION3avLXcsfMOcu25dPPqxsLOC5ts6AEFHxERETmJj498TMJPCRSVF3G+z/ks6LSANtY2zi7rL9GlLhEREani9ezXeXLfkwAMbDOQhyMexurSOEdjrgkFHxEREalgjGHu/rksOrgIgBsCb+Bf7f/VqEdjrgkFHxEREQEcozHPyJjB+4ffB2Bs2FhuDb610Y/GXBMKPiIiIkJReRFT0qeQkpuCCy5MCZ/CNQHXOLusOqfgIyIi0sLlleVx96672XxiM+4Wd2ZEzeCS1pc4u6x6oeAjIiLSgh0qOcTYn8ayq2gXrVxb8d/o/xLjE+PssuqNgo+IiEgLtadoD2N/GsuBkgMEWAOYc8YcOnl1cnZZ9UrBR0REpAVKO5HGhF0TOFZ2jHBbOHM6zqGdrZ2zy6p3zePZNBERETltX+d9zR077+BY2TG6enVlYeeFLSL0gIKPiIhIi/LpkU+ZuGsiheWFnOdzHs91eo621rbOLqvB6FKXiIhIC/Fm9ps8se8JDIa4NnE8HPEw7i7uzi6rQSn4iIiINHPGGOYfmM/CrIUADAscxj3t72k2ozHXhIKPiIhIM1ZmypiVMYv3Dr8HwJ2hd3J7yO3NajTmmlDwERERaaaKy4uZkj6F5NxkXHBhcvhkrg241tllOZWCj4iISDOUX5ZP4s+JfHv8W9wt7jwW+RiXtrnU2WU5nYKPiIhIM3Oo9BDjfxrPzsKdeLt489QZT9HLp5ezy2oUFHxERESakYyiDMb9NI7Mkkz83fx5tuOznOl1prPLajRa3u3cIiIizdT2gu3c9uNtZJZk0t7WnpfOfKlxhZ7DuyB9jVNL0BkfERGRZuCbvG/418//oqC8gDM9z+TZjs/ib/V3dlkORXnwxX9g3QLwCoDxG8Dd2ymlKPiIiIg0cUlHk3hw94OUmlJ6+/TmiegnaOXaytllQbkdNr8G//cInDjkaAvq6ghCCj4iIiJSU28deovH9z6OwXBZ68uYHjm9cYzGvGctfDwJsr5zvPfvCANnQKc4cOIYQgo+IiIiTZAxhucOPMcLWS8A8I+Af3Bfh/twtbg6t7BjGZD0b0hzDJiIzQ8ungS9R4Ob8wOZgo+IiEgTYzd2Zu+dzbKcZQDcEXoHo0NGO3c05uLj8NXTsPZZKCsCiwvEjoJLpoJ3gPPq+gMFHxERkSaitLyUT45+wpKDS/ip6CcsWLi/w/38I/AfziuqvBy2vgWfTYP8A462yP5wxSwI6e68uk5CwUdERKSRyyvLY1nOMpYeWsqhUsdNwl4uXjwU8RCXt7nceYXtXQ+f3A+ZGxzv20RC3HTo8nen3sdzKgo+IiIijVRmcSavZ7/O+4ffp7C8EIAAawA3BN7A0ICh+Lr5OqewvP2OMzzfLXW8d28FF90Dfe4Eq4dzajpNCj4iIiKNzNYTW1lycAmrj62mnHIAOnp0ZETwCAa2GYjVxeqcwkoLHffwfPlfKC0ALNDzZrj03+AT7JyaakjBR0REpBGwGztf5H7BqwdfZcuJLRXt5/ucz4jgEfTx6eO8m5eNgbR3IekhyN3raOtwPsTPgrCezqmplhR8REREnKiwvJAPD3/Ia9mvsbfYESrcLG5c0eYKhgcNp5NXJ+cWuH8TfDIZMlId733bQ9wj0O3aRnsfz6ko+IiIiDjB4dLDvHXoLd4+9Da59lwAfFx9GBowlBsCbyDQPdC5BeYfhNWPwKbXAANWL7ggAfqNB3cv59b2F9RqktJ58+YRFRWFh4cHsbGxrFlz6gnHUlJSiI2NxcPDg+joaBYsWFBpeVpaGkOHDiUyMhKLxcLTTz9dZRvTpk3DYrFUeoWEhFTqY4xh2rRphIWF4enpycUXX0xaWlptPqKIiEi9SC9M59E9j/L37//Oi1kvkmvPpZ17O+5tfy8ru69kfLvxzg09ZcWOe3iejYVNSwADPa6HcRscAxE24dADtTjjs3TpUhISEpg3bx4XXHABzz33HPHx8Wzbto3w8PAq/dPT0xk0aBCjR49myZIlfPXVV9x1110EBgYydOhQAAoKCoiOjua6667j7rvvPum+u3XrxmeffVbx3tW18uiUjz/+OE899RQvv/wynTt3Zvr06fztb39jx44d+Pj41PSjioiI1AljDBuOb2DJwSV8mfdlRXt3r+6MCB7BJa0vcf6Iy8bADx/BqqlwdLejrV0sXDEbOvR2aml1yWKMMTVZoU+fPsTExDB//vyKtq5duzJkyBBmzpxZpf+kSZNYsWIF27dvr2gbM2YMW7ZsITU1tUr/yMhIEhISSEhIqNQ+bdo0li9fzubNm6utyxhDWFgYCQkJTJo0CYDi4mKCg4OZPXs2d9xxx59+try8PPz8/MjNzcXX10mPCIqISLNRakr57OhnLDm4hB8KfwDAgoWL/S5mePBwzvE+x7mjLf/qYJpjPJ70LxzvW4XA5dPg7GHgUquLQw2qJt/fNTrjU1JSwsaNG7n//vsrtcfFxbF27dpq10lNTSUuLq5S28CBA1m4cCGlpaVYraf/SN7OnTsJCwvDZrPRp08fZsyYQXR0NOA4s5SVlVVpXzabjQEDBrB27dpqg09xcTHFxcUV7/Py8k67FhERkZPJt+ezPGc5b2S/wcHSgwDYLDau8r+Km4JuItyj6hUSpziRA58/BhtfBlMOrjbHPTwX3g22RjC7ez2oUfDJycnBbrcTHFz5Wf3g4GCysrKqXScrK6va/mVlZeTk5BAaGnpa++7Tpw+LFy+mc+fOHDx4kOnTp9OvXz/S0tLw9/ev2H91+9qzZ0+125w5cyYPP/zwae1fRETkzxwoOcAb2W+wPGc5J8pPANDWrS3DAocxNHAobdzaOLnCX9hL4ZsXIHkWFDturOasIfC3hx2jLzdjtXqq64+n5YwxpzxVV13/6tpPJT4+vuLnHj160LdvX8444wxeeeUVEhMTa1Xb5MmTK62bl5dHhw4dTrsmERERgO0F23n14Kt8dvQz7NgBiPKIYnjQcOLbxmNzsTm5wt/5cRV8OgUO73S8D+nhuI8n8gLn1tVAahR8AgICcHV1rXJ2Jzs7u8qZll+FhIRU29/NzQ1/f/8alvsbb29vevTowc6dOyv2A44zTL8/i3Sq2mw2GzZbI/rLKCIiTUa5KeervK949eCrbDy+saK9t09vhgcNp59vP1wsjej+mEM7HIHnp18eEvIOhEsfhJ7DwcXJN1Y3oBoFH3d3d2JjY0lKSuKaa66paE9KSuLqq6+udp2+ffvywQcfVGpbtWoVvXr1qtH9PX9UXFzM9u3b6d+/PwBRUVGEhISQlJREz56OUSRLSkpISUlh9uzZtd6PiIjI7xWXF7PyyEqWHFzC7uLdALjiSlybOG4OvpmuXl2dW+AfFR51XNL65gUwdnCxwvlj4KJ7wcPP2dU1uBpf6kpMTGTEiBH06tWLvn378vzzz5ORkcGYMWMAx+WjzMxMFi9eDDie4JozZw6JiYmMHj2a1NRUFi5cyBtvvFGxzZKSErZt21bxc2ZmJps3b6ZVq1Z07NgRgHvuuYcrr7yS8PBwsrOzmT59Onl5eYwcORJwXOJKSEhgxowZdOrUiU6dOjFjxgy8vLy46aab/tpREhGRFu9o2VHePvQ2bx96myNlRwDwdvHm2oBruSHoBkLcQ/5kCw3MXgYbFzluXi486mg7c5Bj9nT/M5xbmxPVOPgMGzaMw4cP88gjj3DgwAG6d+/OypUriYiIAODAgQNkZGRU9I+KimLlypXcfffdzJ07l7CwMJ555pmKMXwA9u/fX3GWBuCJJ57giSeeYMCAASQnJwOwb98+brzxRnJycggMDOT888/n66+/rtgvwH333UdhYSF33XUXR48epU+fPqxatUpj+IiISK3tKdrDa9mv8eHhDyk2jieBQ9xDuDHwRoYEDKGVayN8+mnX545pJg79MpRMYFe4YiaccYlz62oEajyOT3OmcXxERAQcD8ZsPrGZVw++yhe5X2BwfFV29erK8KDhXNbmMqwWJ82QfiqHd8GqB2DHSsd7zzZwyVSIvRVcm+8sVfU2jo+IiEhzVmbKWH1sNUsOLiGt4Lcpj/r79mdE8AhiWsU0jgEH/6goD774D3w9H8pLweIK542GAZPAq62zq2tUFHxERKTFO2E/wfuH3+eN7DfYX7IfAHeLO4PbDubmoJuJ8oxycoUnUW53zKe1+lE4ccjR1vFyGDgDAs90bm2NlIKPiIi0WNkl2bx56E3ezXmXfHs+AK3dWnNdwHVcH3g9ba2N+GzJ7q/gk0mQtdXx3r+TI/B0jjv1ei2cgo+IiLQ4Owt28mr2q3x69FPKTBkA4bZwhgcNZ7D/YDxcPJxc4UnYy+DHT2D9C/BzsqPN5ueYNb33aHBzd2p5TYGCj4iItAjGGFLzU1lycAnr8tdVtPds1ZMRQSPo79e/cQ04+HsncuDbV2DDIsjd62izuEDsKMfNy94BTi2vKVHwERGRZq2kvIRPjn7Cawdf46einwBwwYXLWl/G8ODhdPfu7uQKT2HfRvjmeUh7F+wljjbPthAzAnrd1uzn1aoPCj4iItIs5ZXl8U7OOyw9tJSc0hwAPF08GeI/hBuDbqSdrZ2TKzyJ0iJH0Pnmedi/6bf2sJ5w3v+DbteA1dN59TVxCj4iItKs7Cvex+vZr7Pi8AoKywsBCLQGcmPgjVwbcC0+bo10UNuje2DDS/DtYih0jAyNqzt0u9YReNrHOre+ZkLBR0REmoWtJ7by6sFX+fzY55RTDkAnz06MCBpBXJs4rC6NcMDB8nL4+XNY/yLs+Bh+GSgRvw7Q61aIGan7d+qYgo+IiDRZdmMnJTeFJQeXsOXElor2vr59GRE0gvN8zmucAw4WHoMtbzgmDj2y67f26IsdT2d1vqJZj7TsTDqqIiLS5BSWF/LB4Q94Pft19hY7nnJys7gxqO0gbg66mY6eHZ1c4UkcTHOEne+WQmmBo83dB869CXr/EwI7O7e+FkDBR0REmozDpYdZemgp7xx6h1x7LgC+rr78I+AfXB90PYHWQCdXWA17KWz/wHE5a89Xv7UHdnFMK3H2MLA10vuOmiEFHxERafTSi9J59eCrrDyyklJTCkA793bcHHQzV/lfhadrI3zKKT8LNr7seOUfcLRZXKHr3x2XsyIvhMZ4Ga6ZU/AREZFG62DJQZ478BwfHP6g4oblHt49GBE0gotbX4yrxdXJFf6BMZDxteNR9O0roNwxKjTeQY7BBmNHgV8jfYy+hVDwERGRRie/LJ+XD77MG9lvUGyKARjgN4CRwSM5p9U5Tq6uGiUn4Lu3HJezDn7/W3uHPo5H0btepekkGgkFHxERaTSKy4t5+9DbvJT1UsU9POd6n8vEdhM5u9XZTq6uGod3OcLOpteg2FEvbp5w9nWOy1mhjbDmFk7BR0REnM5u7Hxy5BPmHZhHVkkWANEe0YwPG09/v/6N65H0cjvsTHJcztr1f7+1t4l0hJ2eN4NnG6eVJ6em4CMiIk5jjCE1L5Vn9j/DzsKdAARZgxgTOobB/oNxszSir6mCI7DpVccZnmMZvzRaoNPfHJezzrgMXBrpJKdSoRH9jRIRkZZk24lt/C/zf2w4vgGAVq6tuDX4VoYFDcPTpRE9pbV/E3zzInz/DpQVOdo8WkPP4dD7dmgb7dTypGYUfEREpEHtLdrL3P1zSTqWBIDVYmVY4DBuC7kNPzc/J1f3i7JiSFsO61+Afet/aw/pAefdAd2HgruX08qT2lPwERGRBnGk9AgvZL3AskPLsGPHgoVBbQdxZ+idhNpCnV2eQ+4+x0ShG1+BAseM7rhYodsQx/07Hc7T2DtNnIKPiIjUqwJ7AUuyl/DqwVcpKHdM03CB7wWMCxtHZ69GMEWDMZCe4phKYsdKMI7xgvAJg163QexIaBXk3Bqlzij4iIhIvSg1pbyX8x4vHHiBI2VHAOjm1Y3x7cbT26e3k6sDivJgy5uOm5VzdvzWHtnfMZXEmYM1UWgzpP+iIiJSp4wxfHbsM+bun1sxgWgHWwfGho3l8taXO//R9OwfHPfubHkTSo472txbwTk3OCYKDerq3PqkXin4iIhIndmQv4FnMp8hrSANgLZubRkdOpprAq7BarE6rzB7meMy1jfPw+41v7UHdHbcu3PODeDh67z6pMEo+IiIyF+2s2Anz+5/lq/yHLOPe7p4ckvwLdwcdDPert7OK+x4Nnz7CmxYBHmZjjaLC5w5yHE5K2qAblZuYRR8RESk1g6UHGDB/gV8dOQjDAZXXBkaOJR/hvwTf6u/c4oyxvEI+jcvQNp7UO6YzR2vAMeNyrG3QusOzqlNnE7BR0REaiy3LJeXsl7irUNvUWJKAPhb679xV9hdhHuEO6eo0kLY+o7jclbWd7+1t+vlGFm52xBwszmnNmk0FHxEROS0FZUX8Wb2myw6uIjjdseNwbGtYpnYbiLdvLs5p6gj6bBhIXz7KhQdc7S52qDHdXDePyGsp3PqkkZJwUdERP6U3dj58PCHLDiwgOzSbAA6enRkQrsJ9PPt1/BPapWXOyYI/eYF2LkKMI721uGOJ7N6jgCvtg1bkzQJCj4iInJSxhjW5K5hzv457CraBUCIewh3hd7FFW2vwNXi2rAFFR6FTa85zvAc+fm39jMuc1zO6vQ3cGngmqRJUfAREZFqfXf8O57Z/wybjm8CwNfVl9tCbuP6wOuxuTTwvTIHt8G6+fDd21BW6Giz+UHPm6HX7RDQsWHrkSZLwUdERCrZXbSbufvnsvrYagBsFhs3BN3ArcG34uPm07DF5OyEz2dA2ru/tQV3d1zOOvt6cHfio/LSJCn4iIgIAIdKD/HCgRdYnrMcO3ZccOFK/yu5I/QOgt2DG7aYYxmQPBu2vP7b3Fldr4Lz74Twvhp7R2pNwUdEpIU7bj/O4oOLeS37NYrKiwAY4DeAsWFjOcPzjIYtJj8LvngCNr782/g7nePh0qkQ0qNha5FmScFHRKSFKikv4Z2cd1iYtZBjZccAONv7bCa0m0DPVg38CHjBEfjyv46ntH69hydqAFz6IHRoBBOaSrOh4CMi0sKUm3I+Pfop8/fPJ7PEMY1DhC2Cce3GcYnfJQ37aHpRHqTOdbxK8h1t7c+Dyx6EqIsarg5pMRR8RERakK/zvuaZzGfYUbgDgABrAHeE3sFV/lfhZmnAr4SSAscIy1897XhEHRyXsi59EDrF6R4eqTcKPiIiLcD2gu08m/ks6/LXAeDt4s3I4JHcFHQTnq6eDVdIWTFsfAXWPAHHDzraAjrDJVOg69Xg4tJwtUiLVKu/YfPmzSMqKgoPDw9iY2NZs2bNKfunpKQQGxuLh4cH0dHRLFiwoNLytLQ0hg4dSmRkJBaLhaeffrrKNmbOnEnv3r3x8fEhKCiIIUOGsGPHjkp9Ro0ahcViqfQ6//zza/MRRUSahcziTKamT2X4D8NZl78ON4sbNwbeyPvd3+f20NsbLvTYy+DbxfBsLHx8ryP0tI6AIQvgrq+h2zUKPdIganzGZ+nSpSQkJDBv3jwuuOACnnvuOeLj49m2bRvh4VUnpktPT2fQoEGMHj2aJUuW8NVXX3HXXXcRGBjI0KFDASgoKCA6OprrrruOu+++u9r9pqSkMHbsWHr37k1ZWRlTp04lLi6Obdu24e392zgOV1xxBYsWLap47+7uXtOPKCLS5B0tPcrCrIW8nfM2ZaYMgPg28dwZdiftbO0arpDycscYPJ/PgCOOkZ/xCYWL7nVMK+Gm39HSsCzGGFOTFfr06UNMTAzz58+vaOvatStDhgxh5syZVfpPmjSJFStWsH379oq2MWPGsGXLFlJTU6v0j4yMJCEhgYSEhFPWcejQIYKCgkhJSeGiixw3wI0aNYpjx46xfPnymnykCnl5efj5+ZGbm4uvr2+ttiEi4kyF9kJez36dVw6+wonyEwCc73M+49uNp4tXl4YrxBjYsRJWPwbZaY42L3+4MBF63w7WBry8Js1eTb6/a3TGp6SkhI0bN3L//fdXao+Li2Pt2rXVrpOamkpcXFyltoEDB7Jw4UJKS0uxWq01KaFCbm4uAG3bVp6ELjk5maCgIFq3bs2AAQN47LHHCAoKqnYbxcXFFBcXV7zPy8urVS0iIs5WZsp4P+d9njvwHIfLDgNwpueZTGw3kT6+fRquEGPg589h9XTI3Ohos/lBv/Fw/hiwNfDIzyJ/UKPgk5OTg91uJzi48giewcHBZGVlVbtOVlZWtf3LysrIyckhNDS0hiU7Js1LTEzkwgsvpHv37hXt8fHxXHfddURERJCens6DDz7IpZdeysaNG7HZqs4rM3PmTB5++OEa719EpLEwxvB57ufMyZzDnuI9ALRzb8ddYXcR1yYOF0sD3jeT8TX836Ow50vHe6sX9BnjCD2aKV0aiVo91fXHMR6MMacc96G6/tW1n65x48bx3Xff8eWXX1ZqHzZsWMXP3bt3p1evXkRERPDRRx9x7bXXVtnO5MmTSUxMrHifl5dHhw4dalWTiEhD23R8E89kPsN3J74DoLVba/4Z8k+GBgzF3aUB753Zv9lxhuenJMd7V3fHxKH9E6FV9WfcRZylRsEnICAAV1fXKmd3srOzq5zV+VVISEi1/d3c3PD3969huTB+/HhWrFjBF198Qfv27U/ZNzQ0lIiICHbu3FntcpvNVu2ZIBGRxmxX4S7m7J/DF7lfAODh4sHNQTdzS/AttHJt1XCFZP8Anz8G21c43ltcoedwGHAf+J3697OIs9Qo+Li7uxMbG0tSUhLXXHNNRXtSUhJXX311tev07duXDz74oFLbqlWr6NWrV43u7zHGMH78eN577z2Sk5OJior603UOHz7M3r17a3U5TUSksTlYcpDnDjzHB4c/oJxyXHFlSMAQRoeOJtAa2HCFHPnZMYHod0sBA1igx3Vw8f3g38Bze4nUUI0vdSUmJjJixAh69epF3759ef7558nIyGDMmDGA4/JRZmYmixcvBhxPcM2ZM4fExERGjx5NamoqCxcu5I033qjYZklJCdu2bav4OTMzk82bN9OqVSs6duwIwNixY3n99dd5//338fHxqTiL5Ofnh6enJ8ePH2fatGkMHTqU0NBQdu/ezZQpUwgICKgU0kREmpr8snwWHVzEm9lvUmwcD2Rc2vpSxoaNJdIjsuEKyc2ELx6HTUug3PGIPF3+DpdMheCzGq4Okb/C1MLcuXNNRESEcXd3NzExMSYlJaVi2ciRI82AAQMq9U9OTjY9e/Y07u7uJjIy0syfP7/S8vT0dIPjfxsqvX6/neqWA2bRokXGGGMKCgpMXFycCQwMNFar1YSHh5uRI0eajIyM0/5cubm5BjC5ubk1PiYiInWtyF5kFmctNhdvvtjEbIwxMRtjzO07bjffHf+uYQvJzzbm4/uNeSTQmId8Ha/F1xizb2PD1iFyEjX5/q7xOD7NmcbxEZHGwG7sfHzkY+YfmE9WiePs9hkeZzCu3Tj6+/ZvuElEC4/C2mfh6wVQ6hgTiPB+jglEI/o1TA0ip6HexvEREZH6Y4xhbd5ansl8hp+KfgIg2BrMmLAxDG47GFeLa8MUUnwc1s13hJ4ix5hphPWESx+AMy7TBKLSpCn4iIg0Amkn0vhf5v/YeNwx6F8r11bcFnwbw4KG4eHi0TBFlBbBhoWw5ikoyHG0BZ3luIeny2AFHmkWFHxERJwooyiDufvn8tmxzwBwt7gzLHAYt4bcip+bX8MUYS+FTa9Cyn8gf7+jrW00XDwFul8LLg10pkmkASj4iIg4weHSw7xw4AXezXkXO3YsWBjcdjBjwsYQ6t5AQ3CU22Hr25A8E47udrT5tneMw3PuTeBauymFRBozBR8RkQZ0wn6CJQeX8Gr2qxSWFwJwge8FjA8bTyevTg1TRHm5Y9DBz2dAzg5Hm3cg9L8HYkeBtYEurYk4gYKPiEgDKDWlvJfzHi8ceIEjZUcA6ObVjQntJtDLp1fDFGEM7EyC1Y9ClmOaCzxawwUToc8d4O7dMHWIOJGCj4hIPTLGkHQsiXn757G3eC8A4bZwxoaN5bLWlzXco+npaxzzae392vHevRWcfxf0HQuerRumBpFGQMFHRKSerM9fzzOZz7CtwDEyvb+bP6NDRzMkYAhWSwPdP7NvI6x+BH5Odrx384De/4QL7wbvgIapQaQRUfAREaljOwt28sz+Z1ibtxYALxcvRgSPYHjQcLxcvRqmiKzvHROI7ljpeO9ihZhb4KJ7wVfzF0rLpeAjIlJHDhQfYP6B+aw8shKDwRVX/hH4D24PuR1/q3/DFJHzEyTPgO/fBQxYXOCcGx1ParWJbJgaRBoxBR8Rkb/oWNkxXsp6ibcOvUWpKQUgrk0cd4XeRQePDg1URAakzIbNb4CxO9q6XeMYiyewc8PUINIEKPiIiNRSUXkRb2S/wcsHX+a4/TgAvVr1YkK7CXTz7tYwReRnwZonYcMiKHeELjpf4RhtOfTshqlBpAlR8BERqaEyU8aHhz/kuQPPkV2aDUAnz05MCJtAX9++DfOkVsER+OppWPc8lDnGAyLqIrj0QehwXv3vX6SJUvARETlNxhi+yP2COfvn8HPRzwCEuIdwV+hdxLeNx8XiUv9FFOXB1/MgdS4U5zna2vd2BJ7oAfW/f5EmTsFHROQ0bDm+hWcyn2Hzic0A+Ln6cXvI7fwj8B/YXGz1X0BJAax/Ab58GgodAyAS3MMxY3rngZpAVOQ0KfiIiJxCelE6czLnkJybDIDNYuPGoBsZFTwKHzef+i+grBi+XQxf/AeOH3S0+XeCS6bAWUPApQHOMok0Iwo+IiLVOFRyiOeznuf9nPexY8cFF670v5IxoWMIcg+q/wLsZbDlDUh5HHIzHG2tw2HA/XD2MHDVr2+R2tC/HBGR38m35/PqwVdZcnAJxaYYgAF+AxgXNo5oz+j6L6C8HNLedcyYfvgnR1urEBhwL/S8Bdzc678GkWZMwUdEBCgpL+GdnHd48cCL5NpzATjH+xzGtxtPz1Y9678AY2DHx47Rlg9+72jzbAv9Ex1TTFg9678GkRZAwUdEWrRyU84nRz9h/v757C/ZD0CkLZJx7cZxsd/F9f9oujGOebRWT4fMDY42my/0Gw/n3wm2BriPSKQFUfARkRYrNS+VZzOfZUfhDgACrAHcEXoHV/lfhZulAX49ZqyD1Y/C7jWO91Yv6HMH9JsAXm3rf/8iLZCCj4i0ONsLtvNs5rOsy18HgLeLN6NCRnFj0I14ujTAJaUDWxxneHaucrx3dYdet8GFieATXP/7F2nBFHxEpMXYV7yPefvn8enRTwFws7hxfeD13BZyG23c2tR/AYd2OO7h2fa+473FFXreDBfdB60baE4vkRZOwUdEmr2jpUd5MetF3sl5hzJTBkB8m3juDLuTdrZ29V/AkXTHBKLfLQVTDligxz/g4sngf0b9719EKij4iEiztjxnOU/te4oT5ScA6Ovbl3Fh4+ji1aX+d5633zEOz6ZXodwRuOjyd8fgg8ENNImpiFSi4CMizdY3ed8wPWM6BkMXzy5MbDeR83wbYALPEzmw5ilY/yLYHWMBccaljukl2sXW//5F5KQUfESkWTpUcoipu6diMFzlfxUPhj9Y/5OIFh6Dtc/C1/Oh1HGGifC+jglEIy+o332LyGlR8BGRZqfMlDF592SOlB2hs2dnJnWYVL+hp/g4rFsAa5+BIsfgh4Se6wg8HS/TBKIijYiCj4g0O/P2z2PT8U14u3gzO2o2Hi4e9bOj0iLY8BJ8+RScOORoC+wKl0513MujwCPS6Cj4iEizknIshVcOvgLAQxEPEe4RXvc7sZfCpiWOGdPzMh1tbaIcNy13HwournW/TxGpEwo+ItJsZBZn8tCehwC4MfBGLmtzWd3uoNwOW99xTCB6NN3R5tsOBtwH594Mrta63Z+I1DkFHxFpForLi5mUPol8ez49vHswsd3Eutu4MbD9A8fgg4d+cLR5B0L/f0HsrWCtp0tpIlLnFHxEpFl4at9TbC/Yjp+rHzOjZmJ1qYOzL8bAT5855tM6sMXR5uEHF0yE8+4AW6u/vg8RaVAKPiLS5H185GPeyXkHCxamR04n1D30r29095eO+bQyUh3v3Vs5ZkvvOw48W//17YuIUyj4iEiTll6YzmMZjwFwW8ht9PPr99c2mLkR/u9R+Plzx3tXG5w3Gi68G7wD/mK1IuJsCj4i0mQV2gu5L/0+CssL6e3TmztC76j9xg6mwerHYMdHjvcubhBzC1x0L/iG1U3BIuJ0Cj4i0iQZY5i5dyY/F/1MgDWAxyIfw9VSi8fID++Cz2fA98sAAxYXOPsGuHgStIms67JFxMkUfESkSXrv8Ht8dOQjXHFlZuRM/K3+NdvAsb2OGdM3vw7G7mg7a4hjLJ7AM+u8XhFpHBR8RKTJ+aHgB/6z9z8A3BV2FzE+Mae/cv5BWPMkbFwE9hJHW6eBjtGWQ8+ph2pFpDFR8BGRJiXfns+k9EmUmBL6+/bnluBbTm/FgiPw1f9g3XNQVuhoi+zvmE8rvE/9FSwijUqtZu2bN28eUVFReHh4EBsby5o1a07ZPyUlhdjYWDw8PIiOjmbBggWVlqelpTF06FAiIyOxWCw8/fTTtdqvMYZp06YRFhaGp6cnF198MWlpabX5iCLSCBljeHj3w+wr3keoeygPRz7855OPFudD8mz43znw1dOO0NOuF9zyPoz6UKFHpIWpcfBZunQpCQkJTJ06lU2bNtG/f3/i4+PJyMiotn96ejqDBg2if//+bNq0iSlTpjBhwgSWLVtW0aegoIDo6GhmzZpFSEhIrff7+OOP89RTTzFnzhzWr19PSEgIf/vb38jPz6/pxxSRRuj17Nf5PPdzrBYrs6Nm4+fmd+oVcjPh+UsgeQYU50Fwd7jxTfjnZxB9cYPULCKNjKmh8847z4wZM6ZSW5cuXcz9999fbf/77rvPdOnSpVLbHXfcYc4///xq+0dERJj//ve/Nd5veXm5CQkJMbNmzapYXlRUZPz8/MyCBQv+9HMZY0xubq4BTG5u7mn1F5GGszl/s+m9sbeJ2RhjlmYv/fMVjuw25r89jHnI15gnuxqz9R1j7Pb6L1REGlxNvr9rdManpKSEjRs3EhcXV6k9Li6OtWvXVrtOampqlf4DBw5kw4YNlJaW1tl+09PTycrKqtTHZrMxYMCAk9ZWXFxMXl5epZeIND5HS49yf/r92LEzsM1Argu47tQrHN4FiwbBsT2OWdNv++SXWdNrdXVfRJqRGv0WyMnJwW63ExwcXKk9ODiYrKysatfJysqqtn9ZWRk5OTl1tt9f/6xJbTNnzsTPz6/i1aFDh9OqR0Qajt3YeWD3A2SXZhNhi2Bq+FQsFsvJV8j+ARbFQ94+COgMt34MrcMbrmARadRq9b8/f/ylY4w55S+i6vpX114X+61JbZMnTyY3N7fitXfv3hrVIyL1b2HWQr7O/xqbxcbj0Y/j7ep98s5ZW+HlQXD8IAR1g1ErwbcO5u0SkWajRo+zBwQE4OrqWuUMSnZ2dpUzLb8KCQmptr+bmxv+/qc34Njp7PfXm6KzsrIIDQ2tts8f2Ww2bDbbadUgIg1vXd46nj/wPABTwqfQ0bPjyTtnboRXr4WiYxB6Lox4D7zaNkidItJ01OiMj7u7O7GxsSQlJVVqT0pKol+/6icG7Nu3b5X+q1atolevXlit1jrbb1RUFCEhIZX6lJSUkJKSctLaRKTxyi7JZuruqRgMQ/yH8Hf/v5+8c8bX8MrVjtDT/jwYuUKhR0SqVeMBDBMTExkxYgS9evWib9++PP/882RkZDBmzBjAcfkoMzOTxYsXAzBmzBjmzJlDYmIio0ePJjU1lYULF/LGG29UbLOkpIRt27ZV/JyZmcnmzZtp1aoVHTt2PK39WiwWEhISmDFjBp06daJTp07MmDEDLy8vbrrppr92lESkQZWaUianT+Zo2VHO9DyTezvce/LOP6fAGzdAaYFjQMIb3wRbq4YrVkSalto8NjZ37lwTERFh3N3dTUxMjElJSalYNnLkSDNgwIBK/ZOTk03Pnj2Nu7u7iYyMNPPnz6+0PD093QBVXn/czqn2a4zjkfaHHnrIhISEGJvNZi666CKzdevW0/5cepxdpHF4eu/TJmZjjOm/qb/JKMw4eccfk4x5NMjxyPriIcYUn2i4IkWk0ajJ97fFmF/uNBby8vLw8/MjNzcXX19fZ5cj0iKlHEsh8edEAP4T9R8ubXNp9R1/+AjeGgnlpdA5Hq5/Bdx0z55IS1ST728NaiEijca+4n08tOchAG4Kuunkoef7ZfDWLY7Qc9YQuH6xQo+InBYFHxFpFIrLi5n08yTy7fn08O7BhLAJ1Xfc/Dos+yeUl8HZN8DQheDm3rDFikiTpdnZRaRReHLfk/xQ+AN+rn7MipqF1aWapz43LIIPExw/x9wCf/+fRmMWkRrRbwwRcbqPj3zMspxlWLAwPWo6Ie7VTFb89fzfQs95dyj0iEit6LeGiDhVemE6j2U8BsDtIbfTz7eacbfWPAWf3O/4+YKJED9boUdEakWXukTEaQrthdyXfh+F5YWc53Me/y/0/1XuYAwkz4KUWY73A+6Hi++HGk53IyLyKwUfEXEKYwwz9s7g56KfCbAGMD1yOq4W1993gM8egq/+53h/+TS48G6n1CoizYeCj4g4xXuH32PlkZW44sqsqFn4W383d195uePS1jfPOd5fMQvOv9M5hYpIs6LgIyINbnvBdv6z9z8AjA0bS89WPX9bWF7uuIn521cAC/z9v9DrVqfUKSLNj4KPiDSo/LJ8Jv08iRJTwkV+FzEieMRvC+1l8P5Y+O5NsLjA1fPg3BudV6yINDsKPiLSYIwxTNszjcySTMLcw3g44mFcLL88nWUvdQxMuG05WFxh6AvQfahT6xWR5kfBR0QazGvZr5Gcm4zVYmV21Gx83X6ZU6e0CN4eBT9+DK7ucN3L0GWwM0sVkWZKwUdEGsTm45t5JvMZABLbJ3KW91mOBSUFsPRm2LUa3Dxg2GvQ6XInVioizZmCj4jUu6OlR5mcPhk7dga2Gch1Adc5FhQfhzdugN1rwOoNN74B0QOcW6yINGsKPiJSr+zGztTdU8kuzSbSFskD4Q9gsVigKBeW/AP2fQPuPjD8HQg/39nlikgzp+AjIvVqYdZC1uWvw8PFg8ejH8fL1QsKjsCr18CBzeDRGka8C+1inV2qiLQACj4iUm++zvua5w88D8CUDlM4w/MMOH4IFl8N2Wng5Q+3vA8hPZxcqYi0FAo+IlIvskuyeWD3AxgM1/hfw2D/wZB3ABZfBTk/QqsQR+gJ6uLsUkWkBVHwEZE6V2pKuT/9fo6WHeVMzzO5t8O9cCwDXrkKjqaDb3sYuQL8z3B2qSLSwij4iEidm5s5ly0ntuDt4s3s6NnYjmU6Qk/uXmgdASM/gDYRzi5TRFogBR8RqVPJx5J5NftVAKZFTKNDXqHj8lb+AfDv6Ag9vmFOrlJEWioFHxGpM/uK9/HQnocAuDnoZi4tCYbFg+DEIQg6y3FPT6sgJ1cpIi2Zgo+I1Ini8mIm/TyJ4/bjnO19NuMt/eHlwVB4FELOhhHLwdvf2WWKSAun4CMideLJfU/yQ+EPtHZrzUzbMKyLr4XiPGjXC4YvA8/Wzi5RRETBR0T+upVHVrIsZxkWLEz3vJGQ126D0hMQcQHctBRsPs4uUUQEUPARkb/o58KfeSzjMQD+6X4ZfZc+AGWFEH0x3PAGuHs5t0ARkd9R8BGRWiuwF3Bf+n0UlRfRx+UMRr/3EthLoNNAuH4xWD2cXaKISCUuzi5ARJomYwwzMmaQXpROID5M/+RzXO0l0PUqGLZEoUdEGiUFHxGplXdz3uXjox/jioWZX/1A26IS6HEd/GMRuLk7uzwRkWop+IhIjW0v2M5/9v0HgHFpOfQ8XAg9h8M1z4GrrqCLSOOl4CMiNZJfls+knydRakoZcOA4I3Ydhd7/hCufBRdXZ5cnInJKCj4ictqMMUzbM43MkkzCTpQybXM2lr7jYNAT4KJfJyLS+OmctIictiUHl5Ccm4zVbnh8Qxa+ff8Fl0wFi8XZpYmInBYFHxE5LZvyN/Fs5v/AAvd8f4iusffBRfc4uywRkRpR8BGRP3Wk5DCTfxiL3dVwxb58hnaZBH3HObssEZEaU/ARkVOy20t5YMP1HPIoJiq/hKmhd2PpfaezyxIRqRXdjSgiJ1du58WUa1nncQyPsnIe9/knXgo9ItKEKfiISPXspaSuHMYLfpkATLVdTXTMRCcXJSLy1yj4iEhVZcUcfO9mHvD/CWOxcK1rDIPOmebsqkRE/jIFHxGprLSQ0jdvYnLr7zlmc+VMlzDu6THH2VWJiNSJWgWfefPmERUVhYeHB7GxsaxZs+aU/VNSUoiNjcXDw4Po6GgWLFhQpc+yZcs466yzsNlsnHXWWbz33nuVlkdGRmKxWKq8xo4dW9Fn1KhRVZaff/75tfmIIi1TyQl4/XrmuG9iS1tPWuHB7K7zsLnYnF2ZiEidqHHwWbp0KQkJCUydOpVNmzbRv39/4uPjycjIqLZ/eno6gwYNon///mzatIkpU6YwYcIEli1bVtEnNTWVYcOGMWLECLZs2cKIESO4/vrrWbduXUWf9evXc+DAgYpXUlISANddd12l/V1xxRWV+q1cubKmH1GkZSrKg1ev5fPCjSzp2AaAh6IfpYOtg5MLExGpOxZjjKnJCn369CEmJob58+dXtHXt2pUhQ4Ywc+bMKv0nTZrEihUr2L59e0XbmDFj2LJlC6mpqQAMGzaMvLw8Pv7444o+V1xxBW3atOGNN96oto6EhAQ+/PBDdu7cieWXUWNHjRrFsWPHWL58eU0+UoW8vDz8/PzIzc3F19e3VtsQaZIKjsCSoew79h03DwjnuNWFm4NuJrF9orMrExH5UzX5/q7RGZ+SkhI2btxIXFxcpfa4uDjWrl1b7TqpqalV+g8cOJANGzZQWlp6yj4n22ZJSQlLlizhtttuqwg9v0pOTiYoKIjOnTszevRosrOzT/p5iouLycvLq/QSaXFO5MArV1GctYn7zmvPcasL53ifw/h2451dmYhInatR8MnJycFutxMcHFypPTg4mKysrGrXycrKqrZ/WVkZOTk5p+xzsm0uX76cY8eOMWrUqErt8fHxvPbaa6xevZonn3yS9evXc+mll1JcXFztdmbOnImfn1/Fq0MHndKXFiY/C14eDAe38kTPDuzwdaO1W2tmRs3EarE6uzoRkTpXq5Gb/3iWxRhTpe3P+v+xvSbbXLhwIfHx8YSFhVVqHzZsWMXP3bt3p1evXkRERPDRRx9x7bXXVtnO5MmTSUz87VR+Xl6ewo+0HLn74JWr4MguVnbswLvt3LFg4bHIxwh2D/7z9UVEmqAaBZ+AgABcXV2rnInJzs6ucsbmVyEhIdX2d3Nzw9/f/5R9qtvmnj17+Oyzz3j33Xf/tN7Q0FAiIiLYuXNntcttNhs2m55WkRbo6G545Uo4lsGusAge69YKTDGjQ0Zzvq+ehBSR5qtGl7rc3d2JjY2teKLqV0lJSfTr16/adfr27Vul/6pVq+jVqxdWq/WUfarb5qJFiwgKCmLw4MF/Wu/hw4fZu3cvoaGhf9pXpMXI+QleiodjGRQEnMGkvlEUmWL6+PThn6H/dHZ1IiL1qsaPsycmJvLiiy/y0ksvsX37du6++24yMjIYM2YM4Lh8dMstt1T0HzNmDHv27CExMZHt27fz0ksvsXDhQu65556KPhMnTmTVqlXMnj2bH374gdmzZ/PZZ5+RkJBQad/l5eUsWrSIkSNH4uZW+WTV8ePHueeee0hNTWX37t0kJydz5ZVXEhAQwDXXXFPTjynSPB3cBoviIX8/JvBMHht4Ceml+wi0BjI9cjquFldnVygiUr9MLcydO9dEREQYd3d3ExMTY1JSUiqWjRw50gwYMKBS/+TkZNOzZ0/j7u5uIiMjzfz586ts8+233zZnnnmmsVqtpkuXLmbZsmVV+nz66acGMDt27KiyrKCgwMTFxZnAwEBjtVpNeHi4GTlypMnIyDjtz5Wbm2sAk5ube9rriDQZ+zcbMyvSmId8jZl/gXl73yITszHG9N7Y23yb/62zqxMRqbWafH/XeByf5kzj+EiztW8DLLkWinIhLIbt/5jFrXsmUmpKmdhuIrcE3/Ln2xARaaRq8v1dq6e6RKQJ2bMWXrsOSo5DeF/yhi1kUvqdlJpSBvgNYETQCGdXKCLSYDRJqUhz9nMyLBnqCD1RF2FufodpWU+SWZJJO/d2PBzx8CmHohARaW4UfESaqx9XwWvXQ2kBdPwb3PQWrx57l5TcFKwWK7OjZ+Pj5uPsKkVEGpSCj0hztP0DePMmsBdDl7/DDa+xqfgH5mTOAeDe9vfS1aurk4sUEWl4Cj4izc3Wd+CtkVBeCt2HwnUvc8ScYHL6ZOzYiW8Tz7UBVUcyFxFpCRR8RJqTvevh3dFg7HDuzXDtC9hdXJi6eyqHSg8R5RHFlPApuq9HRFosBR+R5qKsGN4fC6Ycul0LV80BF1deOPAC3+R/g4eLB49HPY6Xq5ezKxURcRoFH5Hm4ov/QM4O8A6CwU+Ciwupeam8mPUiAFPDpxLtGe3kIkVEnEvBR6Q5yNoKX/7X8fPgJ8CrLQdLDvLA7gcwGIYGDGVQ20HOrVFEpBFQ8BFp6uylsPwuKC+DrlfBWVdTakqZnD6ZY2XH6OLZhX+1/5ezqxQRaRQUfESaurXPQNZ34NEaBj0BwLOZz7LlxBZaubZidvRsbC4259YoItJIKPiINGWHfoTk2Y6f42eDTzCrj63mtezXAJgWMY32tvZOLFBEpHFR8BFpqsrtsGKcY5DCjn+Ds4exv3g/D+95GIDhQcO5pPUlTi5SRKRxUfARaaq+eQH2rgN3H7jyaeyU8+DuBzluP87Z3mczrt04Z1coItLoKPiINEVHd8P/Oc7s8LeHwa89iw8uZvOJzXi7eDM9cjpWi9WpJYqINEYKPiJNjTGwYoJj8tGICyH2VrYXbGf+/vkA3NPhHtrZ2jm5SBGRxknBR6Sp+XYxpKeAmydc9QyFFPNA+gPYsXNZ68u4su2Vzq5QRKTRUvARaUry9sOqBxw/XzoV/M/g2cxn2V28mwBrgObhEhH5Ewo+Ik2FMfBhIhTnQbtYOP8u1uauZemhpYDj0fXWbq2dW6OISCOn4CPSVHy/DH78GFyscPVcjpbnMW3PNABuCLyBvr59nVufiEgT4ObsAkTkNJzIgY/vc/w84D5MYBem/3wPh8sOE+0Rzfh2451bn4hIE6EzPiJNwcf3QcFhCO4OF97NisMrSM5Nxs3ixqORj+Lh4uHsCkVEmgQFH5HG7oePHJe5LK5w9Rz2lmXxxD7HnFx3hd5FF68uTi5QRKTp0KUukcas8JjjhmaAfuMpC+3Bv38cTUF5ATGtYhgePNyp5YmINDU64yPSmK2aCsezwL8jXHw/L2e9zHcnvsPbxZtHIh7B1eLq7ApFRJoUBR+RxmrXati0BLDA1XP5vmQXzx94HoD7O9xPqC3UufWJiDRButQl0hgVH4cVEx0/n/f/KGx3Dg/+cBN27MS1iSO+bbxz6xMRaaJ0xkekMfq/RyA3A1qHw2X/5r+Z/yWjOINgazCTO0zW6MwiIrWk4CPS2OxJhW8cl7S48n98UfQty3KWAY7RmX3dfJ1YnIhI06ZLXSKNSWkhrBgHGOg5nCPh5/Lo9mEA3Bx0M+f5nufc+kREmjid8RFpTJJnweGfoFUI5m/TeTTjUY6UHaGjR0fGho11dnUiIk2ego9IY7F/E6x91vHz3//LewWf80XuF1gtVqZHTcfmYnNufSIizYAudYk0BmUl8P44MHboPpSMyO48+cONAIwLG0cnz05OLlBEpHlQ8BFpDL56Gg5+D17+lF4xgwd230dReRG9fXpzU9BNzq5ORKTZ0KUuEWfL3g4pjzt+jn+chXnLSStIw8fVh2kR03Cx6J+piEhd0W9UEWcqt8P7Y6G8FDrH811UJ17KegmAKR2mEOIe4uQCRUSaF13qEnGmr+dB5kaw+XJi0HQe3PMv7NiJbxNPXNs4Z1cnItLs6IyPiLMc3gWrpzt+jpvOk7mvs694HyHuIUzqMMm5tYmINFMKPiLOUF4OKyZAWRFEDeDz6HDeP/w+Fiw8EvEIPm4+zq5QRKRZqlXwmTdvHlFRUXh4eBAbG8uaNWtO2T8lJYXY2Fg8PDyIjo5mwYIFVfosW7aMs846C5vNxllnncV7771Xafm0adOwWCyVXiEhle9/MMYwbdo0wsLC8PT05OKLLyYtLa02H1Gkfm1cBHu+BKsXhwY9zKMZjwJwS/AtxPrEOrk4EZHmq8bBZ+nSpSQkJDB16lQ2bdpE//79iY+PJyMjo9r+6enpDBo0iP79+7Np0yamTJnChAkTWLZsWUWf1NRUhg0bxogRI9iyZQsjRozg+uuvZ926dZW21a1bNw4cOFDx2rp1a6Xljz/+OE899RRz5sxh/fr1hISE8Le//Y38/PyafkyR+pO7D5IeAsBc9m8ezV1Erj2XMz3PZEzoGCcXJyLSvFmMMaYmK/Tp04eYmBjmz59f0da1a1eGDBnCzJkzq/SfNGkSK1asYPv27RVtY8aMYcuWLaSmpgIwbNgw8vLy+Pjjjyv6XHHFFbRp04Y33ngDcJzxWb58OZs3b662LmMMYWFhJCQkMGmS4/6I4uJigoODmT17Nnfccceffra8vDz8/PzIzc3F11cTQUo9MAZeuw5+SoIOfXjrytuZve9x3C3uLOmyhDM8z3B2hSIiTU5Nvr9rdManpKSEjRs3EhdX+WmTuLg41q5dW+06qampVfoPHDiQDRs2UFpaeso+f9zmzp07CQsLIyoqihtuuIGff/65Yll6ejpZWVmVtmOz2RgwYMBJaxNpcFvedIQeVxvp8ZN5OvN/AExoN0GhR0SkAdQo+OTk5GC32wkODq7UHhwcTFZWVrXrZGVlVdu/rKyMnJycU/b5/Tb79OnD4sWL+fTTT3nhhRfIysqiX79+HD58uGIbv653urUVFxeTl5dX6SVSb/IPwif3A1B68T08kPsixaaYPj59GBY4zMnFiYi0DLW6udlisVR6b4yp0vZn/f/Y/mfbjI+PZ+jQofTo0YPLL7+cjz76CIBXXnml1rXNnDkTPz+/ileHDh1O+hlE/rKV90DRMQg5m+ejPPih8Af8XP14OOJhjc4sItJAavTbNiAgAFdX1ypnULKzs6ucaflVSEhItf3d3Nzw9/c/ZZ+TbRPA29ubHj16sHPnzoptADXazuTJk8nNza147d2796T7E/lLtr0P21eAixub4ifycvarAEwJn0Kge6CTixMRaTlqFHzc3d2JjY0lKSmpUntSUhL9+vWrdp2+fftW6b9q1Sp69eqF1Wo9ZZ+TbRMcl6m2b99OaGgoAFFRUYSEhFTaTklJCSkpKSfdjs1mw9fXt9JLpM4VHIGP7gHg+IXj+Hf+K5RTzt/b/p3L21zu5OJERFqWGk9ZkZiYyIgRI+jVqxd9+/bl+eefJyMjgzFjHI/hTp48mczMTBYvXgw4nuCaM2cOiYmJjB49mtTUVBYuXFjxtBbAxIkTueiii5g9ezZXX30177//Pp999hlffvllRZ977rmHK6+8kvDwcLKzs5k+fTp5eXmMHDkScFziSkhIYMaMGXTq1IlOnToxY8YMvLy8uOkmzW4tTvTpFDiRDYFd+E9UOfuP7ifMPYx7O9zr7MpERFoeUwtz5841ERERxt3d3cTExJiUlJSKZSNHjjQDBgyo1D85Odn07NnTuLu7m8jISDN//vwq23z77bfNmWeeaaxWq+nSpYtZtmxZpeXDhg0zoaGhxmq1mrCwMHPttdeatLS0Sn3Ky8vNQw89ZEJCQozNZjMXXXSR2bp162l/rtzcXAOY3Nzc015H5JR2fGrMQ77GPORnknbONzEbY0yvjb3Mt/nfOrsyEZFmoybf3zUex6c50zg+UqeK8mDe+ZCXSXa/27kheDO59lxuC76Nse3GOrs6EZFmo97G8RGRGvjsIcjLpLxNJNMiCsi159LVqyv/L/T/ObsyEZEWS8FHpD6kr4ENLwGwNO461p1Yj81i49HIR7G6WJ1cnIhIy6XgI1LXSgpgxXgAdvW5nmeKHGNOJbRPIMojypmViYi0eAo+InXt88fgaDolfu14IDyXElPCBb4XcF3Adc6uTESkxVPwEalL+zbA1/MAWHD5QH4s3kVrt9b8O+LfpxzdXEREGoaCj0hdKSuG98eCKWfDeYNYXLoGgAfCHyDAGuDk4kREBBR8ROrOF0/AoR/I9wvk3+2PYjBc7X81l7S+xNmViYjILxR8ROpC1lb48ikAZl9yAQfLDtHe1p572t/j5MJEROT3FHxE/ip7Gbw/DsrL+LT3RXxc/j2uuDI9cjperl7Ork5ERH5HwUfkr0p9Fg5sJsuvLTPb5QFwW8ht9PDu4eTCRETkjxR8RP6KnJ3w+UzKgYcGnEN++XG6eXXj9tDbnV2ZiIhUQ8FHpLbKyx2XuOzFvHZebzawFw8XD6ZHTsdq0ejMIiKNkYKPSG2tfwH2fs3Otq2ZG5IPwL/a/4twj3AnFyYiIiej4CNSG0f3wGcPU+xiYeoFnSmljIv8LuIa/2ucXZmIiJyCgo9ITRkDH0yA0hPM7dODXZYjtHVry4PhD2p0ZhGRRk7BR6SmNi2Bn5P5JtiP1wILAHgw4kHaWts6uTAREfkzCj4iNZF3AD6dSq7VhYfOiwBgaMBQLvK7yMmFiYjI6VDwETldxsBHiZjiXGb26Uy2pYBwWzh3t7vb2ZWJiMhpUvAROV3fL4MdK/m4Q2uS2pZVjM7s6erp7MpEROQ0KfiInI4TOfDxfRzwdGPWuSEAjA4dTTfvbk4uTEREakLBR+R0fDwJe8Fh/t0nihOWMs72PptbQ251dlUiIlJDCj4if2bHx/D9O7zasQ3f+hq8XLx4JPIR3Cxuzq5MRERqSMFH5FQKj8GHd/ODrzvzuwYAcE/7e+hg6+DcukREpFYUfEROZdUDFJ3I4oHzOlBmMVzidwlX+V/l7KpERKSWFHxETmbX57DpVZ45y590Lwv+bv5MjZiq0ZlFRJowBR+R6hQfhw8msDbQi6XRrQGYFjGNNm5tnFuXiIj8JQo+ItVZ/ShHC/YxLSYUgGGBw+jn18/JRYmIyF+l4CPyRxlfY9Y9x4yzgzhssxBpi2R8u/HOrkpEROqAgo/I75UWwfvj+KBDK1aHtXKMzhw1HU8Xjc4sItIcKPiI/F7KbPYVpvOfHkEAjAkbQ1evrk4uSkRE6oqCj8iv9m+mbO3/+HdMMAVuFs71PpeRwSOdXZWIiNQhBR8RAHspvD+OV87wZUtbT7xdvHk08lFcLa7OrkxEROqQgo8IwJdPk1b8I8+d2RaA+zrcR5gtzMlFiYhIXVPwEcn+gcKv/sODMcHYXSxc3vpyBrcd7OyqRESkHij4SMtWbof3x/L0mb7saeVOoDWQKeFTNDqziEgzpeAjLdu6Bawp3cY7UX6AY3RmPzc/JxclIiL1RcFHWq7Duziy5jEe6el4dP2moJs43/d8JxclIiL1ScFHWqbycswHE5jezYcjNjfO8IhmXNg4Z1clIiL1TMFHWqZvX+a98u9ICW2FFTemRz6GzcXm7KpERKSeKfhIy5O7j4wvH+bJ7gEA3NVuLJ29Oju5KBERaQi1Cj7z5s0jKioKDw8PYmNjWbNmzSn7p6SkEBsbi4eHB9HR0SxYsKBKn2XLlnHWWWdhs9k466yzeO+99yotnzlzJr1798bHx4egoCCGDBnCjh07KvUZNWoUFoul0uv883XPhvyOMZR+mMADPVpR5OZCbKtYhgcNd3ZVIiLSQGocfJYuXUpCQgJTp05l06ZN9O/fn/j4eDIyMqrtn56ezqBBg+jfvz+bNm1iypQpTJgwgWXLllX0SU1NZdiwYYwYMYItW7YwYsQIrr/+etatW1fRJyUlhbFjx/L111+TlJREWVkZcXFxnDhxotL+rrjiCg4cOFDxWrlyZU0/ojRn373FSy4bSWvjQSuLFw9HPoyLRSc+RURaCosxxtRkhT59+hATE8P8+fMr2rp27cqQIUOYOXNmlf6TJk1ixYoVbN++vaJtzJgxbNmyhdTUVACGDRtGXl4eH3/8cUWfK664gjZt2vDGG29UW8ehQ4cICgoiJSWFiy66CHCc8Tl27BjLly+vyUeqkJeXh5+fH7m5ufj6+tZqG9KIHc9m65J+3N7LF7uLhemR04lvG+/sqkRE5C+qyfd3jf5Xt6SkhI0bNxIXF1epPS4ujrVr11a7TmpqapX+AwcOZMOGDZSWlp6yz8m2CZCbmwtA27ZtK7UnJycTFBRE586dGT16NNnZ2SfdRnFxMXl5eZVe0nwVfHw3D3TzxO5iYWDrOIUeEZEWqEbBJycnB7vdTnBwcKX24OBgsrKyql0nKyur2v5lZWXk5OScss/JtmmMITExkQsvvJDu3btXtMfHx/Paa6+xevVqnnzySdavX8+ll15KcXFxtduZOXMmfn5+Fa8OHTqc+gBI07VtBU+5rmOftzvBLm25P3yysysSEREncKvNSn8czt8Yc8oh/qvr/8f2mmxz3LhxfPfdd3z55ZeV2ocNG1bxc/fu3enVqxcRERF89NFHXHvttVW2M3nyZBITEyve5+XlKfw0RwVHSF43iffO8cNi4OEzZuDrpkuZIiItUY2CT0BAAK6urlXOxGRnZ1c5Y/OrkJCQavu7ubnh7+9/yj7VbXP8+PGsWLGCL774gvbt25+y3tDQUCIiIti5c2e1y202Gzabxm5p7nKS7uXRLo7/zsMDb6S3T28nVyQiIs5So0td7u7uxMbGkpSUVKk9KSmJfv36VbtO3759q/RftWoVvXr1wmq1nrLP77dpjGHcuHG8++67rF69mqioqD+t9/Dhw+zdu5fQ0NDT+nzS/Jgfk3jE/SuO2Vzp5NqOu9pPcHZJIiLiTKaG3nzzTWO1Ws3ChQvNtm3bTEJCgvH29ja7d+82xhhz//33mxEjRlT0//nnn42Xl5e5++67zbZt28zChQuN1Wo177zzTkWfr776yri6uppZs2aZ7du3m1mzZhk3Nzfz9ddfV/S58847jZ+fn0lOTjYHDhyoeBUUFBhjjMnPzzf/+te/zNq1a016err5/PPPTd++fU27du1MXl7eaX223NxcA5jc3NyaHhZpjApzzVtv9DAxG2PM+Rt6mR8LfnR2RSIiUg9q8v1d4+BjjDFz5841ERERxt3d3cTExJiUlJSKZSNHjjQDBgyo1D85Odn07NnTuLu7m8jISDN//vwq23z77bfNmWeeaaxWq+nSpYtZtmxZ5UKh2teiRYuMMcYUFBSYuLg4ExgYaKxWqwkPDzcjR440GRkZp/25FHyal/SVd5i+6841MRtjzGuZLzm7HBERqSc1+f6u8Tg+zZnG8Wk+StNTuDX9Lra39uA8t07M7fG6BioUEWmm6m0cH5EmoaSAF7bczfbWHviWuzGty/8UekREBFDwkWZo85f3suiXUQkmd5hCsHv1TxyKiEjLo+AjzcrxvV/yb+sayi0WBrudS1zw1c4uSUREGhEFH2k+yop5Ii2BTG8roaXu3NvtaWdXJCIijYyCjzQb/5eayAdBBosxPBo9Ex9XH2eXJCIijYyCjzQLhzLX8Jj1KwBGul5Az8CLnVuQiIg0Sgo+0uSVl5Uw7cd/kevuyplFNsb0eNLZJYmISCOl4CNN3lsbJvC1nx2b3TC989NYXd2dXZKIiDRStZqdXaRBGQPF+XDiEJzIgROHOHoig7SinXxfnsFiz3TAwkTXS4kOOM/Z1YqISCOm4CPOUVZcEWJ++/NQNe9zKCzM4QcfSGvtQVobG2mtPcj0toLXrxuz0Dffg+v7P+7MTyQiIk2Ago/UjfJyKDx6igBzqHLQKc6tdjNlFkj3cef71h6kBdpI6+zBLp922F0sVfpGlHrSvdyf7tZorurzbywuunIrIiKnpuAj1TMGSk6c1hkZThyCghww5TXbhYsbB/yD+D7Qz3E2p5Vhu62QIou9Sl9/t7Z09+5Bd+/udPPqxlleZ+HjpsfVRUSkZhR8BMrtkJ4CW5dB9rbfwkxZYc235dkGvAN/eQVU+vmYlzfb3E/wvUsOafYM0op+5GjZUaD0l5eDt4s3Xb260s27G929utPNuxtB1iAslqpnfURERGpCwaelMgYObIbv3oLvl8Hxg9X3c/N0BJdW1YWZP7z38gdXKwBF5UXsKNhBWkEa35/4nrSCVPYV74Piypt3xZXOXp3p5tWtIuhEeETganGt388vIiItkoJPS3MkHba+7Qg8h3f+1u7ZBrpdA2dcBq2Cfws07t7wJ2da7MZOelE6aUfXVQSdnwp/wk7VS1bhtvDfQo53dzp7dsbmYqvrTykiIlItBZ+W4EQOpL0H3y2Ffet/a3fzgDPj4exhjsDj9ufj3xhjyCrNIu1EmuNVkMa2gm0Ulle9LObv5k937+6c5XUW3bwd9+X4ufnV5ScTERGpEQWf5qrkBPywEra+BT/9H5hfzr5YXCBqAJx9PXT5O3j4nnIzuWW5bCvYVhFy0k6kcbjscJV+Xi5edPXqWnHzcTfvbgRbg3VfjoiINCoKPs2JvQx+TnaEne0fQumJ35aFnusIO92Hgk9ItasXlRexs3Cn456cX4JORnFGlX6uuNLJs1PF5apuXt2I9IjUfTkiItLoKfg0dcZA5reOsPP9MsfTWL9qHeEIOz2uh8DOFc1Hy46yu2g36UXp7C7aXfHaX7Ifg6myiw62DpVuPu7s1RkPF4+G+HQiIiJ1SsGnqTq8y3GD8ta34ciu39o920L3aynvcR0Hgtqzu3gP6UXfsHvPW46gU7ybY2XHTrrZNm5tKh4h//W+nNZurev944iIiDQEBZ+moLTI8QRW9g9waLvjclbmRgCKXSxktPEhvfN57O7Qkd1eLqQX72HP4USKc4pPuslQ91AiPSKJ8ogi0vbLnx6RtHFro/tyRESk2VLwaUzKiuHwT5C9HQ79gD17G3lHd3C4cC9H3F046u7KEZsrWX5u7G4fSnobP/a7l/1ycWoPlOyBkt82Z7VYCbeF/xZwfvkzwhaBp6unkz6kiIiI8yj4NLBCeyGHi7I4euR7jhxN40jeLo4W7OVIySGOmOMccXfhiM2Vo56uHOvkSrnFAnQ4ydbKAPBx9SHKI6oi3ER6RBJliyLMFqYbjkVERH5HwacB5B/fx00/3MgRCily+cPNw178bpbxVlXWtWDBz8WHtu4BtHFrQ1u3tgRYAyqdxWnr1laXp0RERE6Dgk8D8HLzIcty4pezN2ArK8e/xNDGuNPWxYe21kDaeLXD3+cM2rSKoq21LW3d2tLG2obWbq1xs+g/k4iISF3QN2oDcPXw4+Vj/fDzDKVtwDl4BZ0Dfu3/dCoIERERqVsKPg2k22VznF2CiIhIi+fi7AJEREREGoqCj4iIiLQYCj4iIiLSYij4iIiISIuh4CMiIiIthoKPiIiItBgKPiIiItJiKPiIiIhIi6HgIyIiIi2Ggo+IiIi0GAo+IiIi0mIo+IiIiEiLoeAjIiIiLYZmZ/8dYwwAeXl5Tq5ERERETtev39u/fo+fioLP7+Tn5wPQoUMHJ1ciIiIiNZWfn4+fn98p+1jM6cSjFqK8vJz9+/fj4+ODxWKp9Xby8vLo0KEDe/fuxdfXtw4rlOroeDccHeuGo2PdcHSsG059HWtjDPn5+YSFheHicuq7eHTG53dcXFxo3759nW3P19dX/4gakI53w9Gxbjg61g1Hx7rh1Mex/rMzPb/Szc0iIiLSYij4iIiISIuh4FMPbDYbDz30EDabzdmltAg63g1Hx7rh6Fg3HB3rhtMYjrVubhYREZEWQ2d8REREpMVQ8BEREZEWQ8FHREREWgwFHxEREWkxFHzqwbx584iKisLDw4PY2FjWrFnj7JKavJkzZ9K7d298fHwICgpiyJAh7Nixo1IfYwzTpk0jLCwMT09PLr74YtLS0pxUcfMwc+ZMLBYLCQkJFW06znUrMzOT4cOH4+/vj5eXF+eeey4bN26sWK7jXTfKysp44IEHiIqKwtPTk+joaB555BHKy8sr+uhY184XX3zBlVdeSVhYGBaLheXLl1dafjrHtbi4mPHjxxMQEIC3tzdXXXUV+/btq5+CjdSpN99801itVvPCCy+Ybdu2mYkTJxpvb2+zZ88eZ5fWpA0cONAsWrTIfP/992bz5s1m8ODBJjw83Bw/fryiz6xZs4yPj49ZtmyZ2bp1qxk2bJgJDQ01eXl5Tqy86frmm29MZGSkOfvss83EiRMr2nWc686RI0dMRESEGTVqlFm3bp1JT083n332mfnpp58q+uh4143p06cbf39/8+GHH5r09HTz9ttvm1atWpmnn366oo+Ode2sXLnSTJ061SxbtswA5r333qu0/HSO65gxY0y7du1MUlKS+fbbb80ll1xizjnnHFNWVlbn9Sr41LHzzjvPjBkzplJbly5dzP333++kipqn7OxsA5iUlBRjjDHl5eUmJCTEzJo1q6JPUVGR8fPzMwsWLHBWmU1Wfn6+6dSpk0lKSjIDBgyoCD46znVr0qRJ5sILLzzpch3vujN48GBz2223VWq79tprzfDhw40xOtZ15Y/B53SO67Fjx4zVajVvvvlmRZ/MzEzj4uJiPvnkkzqvUZe66lBJSQkbN24kLi6uUntcXBxr1651UlXNU25uLgBt27YFID09naysrErH3mazMWDAAB37Whg7diyDBw/m8ssvr9Su41y3VqxYQa9evbjuuusICgqiZ8+evPDCCxXLdbzrzoUXXsj//d//8eOPPwKwZcsWvvzySwYNGgToWNeX0zmuGzdupLS0tFKfsLAwunfvXi/HXpOU1qGcnBzsdjvBwcGV2oODg8nKynJSVc2PMYbExEQuvPBCunfvDlBxfKs79nv27GnwGpuyN998k2+//Zb169dXWabjXLd+/vln5s+fT2JiIlOmTOGbb75hwoQJ2Gw2brnlFh3vOjRp0iRyc3Pp0qULrq6u2O12HnvsMW688UZAf7fry+kc16ysLNzd3WnTpk2VPvXx3angUw8sFkul98aYKm1Se+PGjeO7777jyy+/rLJMx/6v2bt3LxMnTmTVqlV4eHictJ+Oc90oLy+nV69ezJgxA4CePXuSlpbG/PnzueWWWyr66Xj/dUuXLmXJkiW8/vrrdOvWjc2bN5OQkEBYWBgjR46s6KdjXT9qc1zr69jrUlcdCggIwNXVtUpCzc7OrpJ2pXbGjx/PihUr+Pzzz2nfvn1Fe0hICICO/V+0ceNGsrOziY2Nxc3NDTc3N1JSUnjmmWdwc3OrOJY6znUjNDSUs846q1Jb165dycjIAPT3ui7de++93H///dxwww306NGDESNGcPfddzNz5kxAx7q+nM5xDQkJoaSkhKNHj560T11S8KlD7u7uxMbGkpSUVKk9KSmJfv36Oamq5sEYw7hx43j33XdZvXo1UVFRlZZHRUUREhJS6diXlJSQkpKiY18Dl112GVu3bmXz5s0Vr169enHzzTezefNmoqOjdZzr0AUXXFBlWIYff/yRiIgIQH+v61JBQQEuLpW/8lxdXSseZ9exrh+nc1xjY2OxWq2V+hw4cIDvv/++fo59nd8u3cL9+jj7woULzbZt20xCQoLx9vY2u3fvdnZpTdqdd95p/Pz8THJysjlw4EDFq6CgoKLPrFmzjJ+fn3n33XfN1q1bzY033qhHUevA75/qMkbHuS598803xs3NzTz22GNm586d5rXXXjNeXl5myZIlFX10vOvGyJEjTbt27SoeZ3/33XdNQECAue+++yr66FjXTn5+vtm0aZPZtGmTAcxTTz1lNm3aVDGMy+kc1zFjxpj27dubzz77zHz77bfm0ksv1ePsTcncuXNNRESEcXd3NzExMRWPXEvtAdW+Fi1aVNGnvLzcPPTQQyYkJMTYbDZz0UUXma1btzqv6Gbij8FHx7luffDBB6Z79+7GZrOZLl26mOeff77Sch3vupGXl2cmTpxowsPDjYeHh4mOjjZTp041xcXFFX10rGvn888/r/b388iRI40xp3dcCwsLzbhx40zbtm2Np6en+fvf/24yMjLqpV6LMcbU/XkkERERkcZH9/iIiIhIi6HgIyIiIi2Ggo+IiIi0GAo+IiIi0mIo+IiIiEiLoeAjIiIiLYaCj4iIiLQYCj4iIiLSYij4iIiISIuh4CMiIiIthoKPiIiItBgKPiIiItJi/H9Wd9osbAGjygAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "tps_execution = [results[i][\"exec_time\"] for i in states]\n", - "tps_execution_op = [results_op[i][\"exec_time\"] for i in states]\n", - "plt.plot(states, tps_execution, color=\"C1\")\n", - "plt.plot(states, tps_execution_op, color=\"limegreen\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "287cb6ef", - "metadata": {}, - "source": [ - "### Handmade Numpy Backpropagation " - ] - }, - { - "cell_type": "code", - "execution_count": 81, - "id": "e1469fb0", - "metadata": {}, - "outputs": [], - "source": [ - "def compute_grad(observations, a0, P0, a_pred_seq, P_pred_seq, Z, H, T):\n", - " # Constant\n", - " SHAPE_a0 = a0.shape[0]\n", - " NB_obs = len(observations)\n", - "\n", - " # Initialisation for the backprop\n", - " PZT = P_pred_seq[-2].dot(Z.T)\n", - " F = Z.dot(PZT) + H\n", - " F_inv = np.linalg.solve(F, np.eye(F.shape[0]))\n", - " \n", - " grad = [0 for _ in range(NB_obs)]\n", - " grad[-1] = - 2 * Z.T @ F_inv @ (observations[-1] - Z @ a_pred_seq[-2])\n", - "\n", - " # Backprop\n", - " for i in range(3, NB_obs+1):\n", - "\n", - " PZT = P_pred_seq[-i].dot(Z.T)\n", - " F = Z.dot(PZT) + H\n", - " F_inv = np.linalg.solve(F, np.eye(F.shape[0]))\n", - "\n", - " K = PZT.dot(F_inv)\n", - " I_KZ = np.eye(SHAPE_a0) - K.dot(Z)\n", - "\n", - " grad[1-i] = I_KZ.T @ T.T @ grad[2-i] - (2 * Z.T @ F_inv @ (observations[1-i] - Z @ a_pred_seq[-i])).T \n", - "\n", - " # Last iter with a0/P0\n", - " PZT = P0.dot(Z.T)\n", - " F = Z.dot(PZT) + H\n", - " F_inv = np.linalg.solve(F, np.eye(F.shape[0]))\n", - "\n", - " K = PZT.dot(F_inv)\n", - " I_KZ = np.eye(SHAPE_a0) - K.dot(Z)\n", - "\n", - " grad[0] = I_KZ.T @ T.T @ grad[1] - (2 * Z.T @ F_inv @ (observations[0] - Z @ a0)).T\n", - "\n", - " return grad" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "id": "479b8832", - "metadata": {}, - "outputs": [], - "source": [ - "def benchmark_kalman_gradients_np(loss, a_pred_seq, P_pred_seq, state_dims, N=3):\n", - " results = defaultdict(dict)\n", - " kalman_fn = pytensor.function(inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", - " outputs=(a_pred_seq, P_pred_seq))\n", - " \n", - " for _ in range(10):\n", - " for n in state_dims:\n", - " data = generate_kalman_dataset(n, N=N, seed=42 + n)\n", - "\n", - " # --- forward pass ---\n", - " t0 = perf_counter()\n", - " a_pred, P_pred = kalman_fn(data[\"y\"],\n", - " data[\"A0\"],\n", - " data[\"P0\"],\n", - " data[\"T\"],\n", - " data[\"Z\"],\n", - " data[\"H\"],\n", - " data[\"Q\"],)\n", - " t1 = perf_counter()\n", - " forward_pass = t1 - t0\n", - " results[n][\"Forward pass\"] = forward_pass/10\n", - "\n", - " # --- Backprop ---\n", - " t0 = perf_counter()\n", - " grad = compute_grad(data[\"y\"],\n", - " data[\"A0\"],\n", - " data[\"P0\"],\n", - " a_pred,\n", - " P_pred,\n", - " data[\"Z\"],\n", - " data[\"H\"],\n", - " data[\"T\"],)\n", - " t1 = perf_counter()\n", - " compile_time = t1 - t0\n", - " results[n][\"Backprop\"] = compile_time/10\n", - "\n", - " return results" - ] - }, - { - "cell_type": "code", - "execution_count": 121, - "id": "1e633e75", - "metadata": {}, - "outputs": [], - "source": [ - "results_np = benchmark_kalman_gradients_np(loss, a_pred_seq, P_pred_seq, states, N=30)" - ] - }, - { - "cell_type": "code", - "execution_count": 122, - "id": "7109d7bf", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 122, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAGdCAYAAADqsoKGAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAUUxJREFUeJzt3Xl8VPW9//FXkkkmIZAAiSRQtgRRQFRIYjEg4NYgixXLVdoqYnsvv6ZVIaRVNq11waBWy7UIFEu9WqpQRSsoKHEhgsSFEBYBQQUMAjGEZYYlZP3+/jgwMGQhExJOMnk/+5gH53zne8585jyQefd7lm+AMcYgIiIi0sQF2l2AiIiISH1QqBERERG/oFAjIiIifkGhRkRERPyCQo2IiIj4BYUaERER8QsKNSIiIuIXFGpERETELzjsLuBCqqioYO/evbRq1YqAgAC7yxEREZFaMMZw5MgROnToQGBg9eMxzSrU7N27l06dOtldhoiIiNTB7t276dixY7XvN6tQ06pVK8A6KBERETZXIyIiIrXhdrvp1KmT53e8Os0q1Jw65RQREaFQIyIi0sSc69IRXSgsIiIifkGhRkRERPyCQo2IiIj4hTqFmtmzZxMXF0doaCiJiYmsWrWqxv5ZWVkkJiYSGhpKfHw8c+fO9Xp/8+bNjBo1iq5duxIQEMDMmTOr3M+ePXu48847iYqKokWLFvTp04ecnJy6fAURERHxMz6HmkWLFpGWlsa0adPIzc1l4MCBDB06lLy8vCr779y5k2HDhjFw4EByc3OZOnUq48ePZ/HixZ4+x48fJz4+nhkzZhAbG1vlfg4dOsSAAQMIDg5m+fLlbNmyhWeeeYbWrVv7+hVERETEDwUYY4wvG/Tr14+EhATmzJnjaevZsycjR44kIyOjUv9JkyaxZMkStm7d6mlLTU1lw4YNZGdnV+rftWtX0tLSSEtL82qfPHkyn3zyyTlHhWridruJjIzE5XLp7icREZEmora/3z6N1JSUlJCTk0NKSopXe0pKCmvWrKlym+zs7Er9hwwZwtq1ayktLa31Zy9ZsoSkpCRuu+022rVrR9++fXnhhRd8KV9ERET8mE+hprCwkPLycmJiYrzaY2JiyM/Pr3Kb/Pz8KvuXlZVRWFhY68/esWMHc+bMoXv37rz33nukpqYyfvx4Xn755Wq3KS4uxu12e71ERETEP9Xp4XtnP/zGGFPjA3Gq6l9Ve00qKipISkriiSeeAKBv375s3ryZOXPmcNddd1W5TUZGBo888kitP0NERESaLp9GaqKjowkKCqo0KlNQUFBpNOaU2NjYKvs7HA6ioqJq/dnt27enV69eXm09e/as9gJlgClTpuByuTyv3bt31/rzREREpGnxKdSEhISQmJhIZmamV3tmZib9+/evcpvk5ORK/VesWEFSUhLBwcG1/uwBAwawbds2r7bt27fTpUuXardxOp2eKRE0NYKIiIh/8/mW7vT0dP7+97/zj3/8g61btzJx4kTy8vJITU0FrNGRM08Hpaam8t1335Gens7WrVv5xz/+wfz58/nDH/7g6VNSUsL69etZv349JSUl7Nmzh/Xr1/PNN994+kycOJFPP/2UJ554gm+++YZXXnmFefPmcc8995zP9xcRERF/Yerg+eefN126dDEhISEmISHBZGVled4bO3asGTx4sFf/lStXmr59+5qQkBDTtWtXM2fOHK/3d+7caYBKr7P3s3TpUtO7d2/jdDpNjx49zLx583yq2+VyGcC4XC6fthMREZEalJcZ8/nfjfnPPQ2y+9r+fvv8nJqmTM+pERERqWd7c+HtdNi7zlofuxTiBtXrR9T297tOdz+JiIhIM1d0GD6aDl/8HUwFOCPg+oegywDbSlKoERERkdozBja9Bu9Ng2MFVtvlt0HK49Cq6qmOLhSFGhEREamd/dvhnXTYdXLKoqjuMPzPEH+trWWdolAjIiIiNSs5Dqv+DJ88BxWl4AiFQX+A/uPB4bS7Og+FGhEREanetndh+f1w+OTDbrsPgWFPQZuutpZVFYUaERERqexwHiyfDNvesdYjOsLQJ6HHcPBhmqMLSaFGRERETisrgU+fh6ynoPQ4BDog+R4YPAlCwu2urkYKNSIiImLZtRre+T3s/8pa7zIAhj8D7XraW1ctKdSIiIg0d0cLYMVDsHGhtd4i2rpF+8qfN9pTTVVRqBEREWmuKsoh50X44FE44QICIOlXcMMfIayN3dX5TKFGRESkOTp7eoP2V8Lwv0DHRHvrOg8KNSIiIs1J0WH48HFregPM6ekNrvpvCAyyu7rzolAjIiLSHFQ7vcF0aBVjb231RKFGRETE3+3fZt3V5DW9wTMQP9jeuuqZQo2IiIi/KjkOHz8Na/56xvQG90P/+xrV9Ab1RaFGRETEH21bDsseANfJ6Q0uucl6InAjnN6gvijUiIiI+JPDebB8EmxbZq1HdrLCzKXDmtQzZ+pCoUZERMQflJVA9ixreoOyopPTG9wLgx9o9NMb1BeFGhERkaZu5yrrQuDCbdZ6l2tOTm/Qw966LjCFGhERkabqaAGseBA2LrLWwy+ypje4YrTfn2qqikKNiIhIU1NRDmv/AR88BsUnpze46r/h+geb5PQG9UWhRkREpCnZsw7eSbemOQBo3wdGPAs/arrTG9QXhRoREZGmoOgwfPgYfDEfa3qDSLjhIUj6dZOf3qC+KNSIiIg0ZsbAxn/DimlwbL/VdsVo+MljfjO9QX1RqBEREWmszp7eIPoS666muEH21tVIKdSIiIg0NiXH4eOnYM2sk9MbhMHg+yH5PnCE2F1do6VQIyIi0liUnoBN/4asp8+Y3mDoyekNuthbWxOgUCMiImK3o/th7Xz4/AU4Xmi1RXaCoU9Bj2H21taEKNSIiIjYpeAr+HQ2bFgI5cVWW0RHuDrVuqupmUxvUF8UakRERC4kY2DHSsh+Hr7JPN3eIQH63ws9b4Eg/TzXhY6aiIjIhVBWDJtet8JMweaTjQHQYzj0vw869WuWUxvUJ4UaERGRhnT84OnrZY7+YLUFh0PfO63TTG3j7a3PjyjUiIiINITCr63rZda/CmVFVlurDtDvN5A4tlnP0dRQAuuy0ezZs4mLiyM0NJTExERWrVpVY/+srCwSExMJDQ0lPj6euXPner2/efNmRo0aRdeuXQkICGDmzJk17i8jI4OAgADS0tLqUr6IiEjDMAZ2roJXRsOsJGvSybIiiL0CfvYCpG2Ea9IUaBqIz6Fm0aJFpKWlMW3aNHJzcxk4cCBDhw4lLy+vyv47d+5k2LBhDBw4kNzcXKZOncr48eNZvHixp8/x48eJj49nxowZxMbG1vj5X3zxBfPmzeOKK67wtXQREZGGUVYCGxbB3wbBSyNg+7tW+yVD4e534DcfwxW3Q1CwvXX6uQBjjPFlg379+pGQkMCcOXM8bT179mTkyJFkZGRU6j9p0iSWLFnC1q1bPW2pqals2LCB7OzsSv27du1KWlpalaMwR48eJSEhgdmzZ/P444/Tp0+fc47qnMntdhMZGYnL5SIiIqLW24mIiFSp6BDk/B98Ng+O7LXaHGHQ55dw9e8g+mJby/MXtf399mmkpqSkhJycHFJSUrzaU1JSWLNmTZXbZGdnV+o/ZMgQ1q5dS2lpqS8fzz333MPw4cO58cYba9W/uLgYt9vt9RIRETlvB76FZffDs73g/T9ZgaZlDFz/EKRvgRHPKtDYwKcLhQsLCykvLycmxntW0JiYGPLz86vcJj8/v8r+ZWVlFBYW0r59+1p99sKFC1m3bh1ffPFFrevNyMjgkUceqXV/ERGRahkDeZ9C9iz46h3g5ImOmN6QfA/0HgUOp60lNnd1uvsp4Kz76I0xldrO1b+q9urs3r2bCRMmsGLFCkJDQ2td55QpU0hPT/esu91uOnXqVOvtRUREKC+FLW9Zz5fZu+50e/cUK8zEDdbzZRoJn0JNdHQ0QUFBlUZlCgoKKo3GnBIbG1tlf4fDQVRUVK0+Nycnh4KCAhITEz1t5eXlfPzxx8yaNYvi4mKCgoIqbed0OnE6lZpFRKQOTrgg5yX47G/g/t5qC3LClT+3wsxFl9pbn1TiU6gJCQkhMTGRzMxMbr31Vk97ZmYmt9xyS5XbJCcns3TpUq+2FStWkJSURHBw7a4Cv+GGG9i0aZNX269+9St69OjBpEmTqgw0IiIidXJolxVk1r0MJUettvCL4Kpx1nxMLS+ytTypns+nn9LT0xkzZgxJSUkkJyczb9488vLySE1NBaxTPnv27OHll18GrDudZs2aRXp6OuPGjSM7O5v58+fz6quvevZZUlLCli1bPMt79uxh/fr1tGzZkosvvphWrVrRu3dvrzrCw8OJioqq1C4iIlInu7+A7L/C1qVgKqy2i3pYozKX3w7Btb/8Qezhc6gZPXo0Bw4c4NFHH2Xfvn307t2bZcuW0aVLFwD27dvn9cyauLg4li1bxsSJE3n++efp0KEDzz33HKNGjfL02bt3L3379vWs//nPf+bPf/4zgwcPZuXKlefx9URERGpQXgZfvW1dL/P956fbu11vhZluN+h6mSbE5+fUNGV6To2IiABQfATW/RM+mwOHT/4f8aAQ6wF5V/8OYi6ztz7xUtvfb839JCIizcfh3fD536wLgItPPrssrC1c9T/Wq1XVN71I06BQIyIi/m9PjnWKafN/wJRbbVHdIfl3cMXPIaSFreVJ/VCoERER/1RRDtuWWw/LyztjWp64QZB8L1z8Ewis07zO0kgp1IiIiH8pOQa5/4JPZ8OhnVZbYDBc/l/W9TLtNSGyv1KoERER/+DeC5/Pg7UvwonDVltoa+vZMj8eBxEd7KxOLgCFGhERadr2bbCul/lyMVSUWW1t461RmT6/hJBwe+uTC0ahRkREmp6KCvh6hXW9zK5Vp9u7DLCeL3PJTRCop803Nwo1IiLSdJQchw2vWtfLHPjGagsIgt4/s0ZmfpRgb31iK4UaERFp/I78cPJ6mflQdMhqc0ZC4ljo9xuI7GhvfdIoKNSIiEjjlf+lNSqz6TUoL7HaWnexRmX63gHOVvbWJ42KQo2IiDQuxsA3H1iTS+5Yebq9Uz/repkeI3S9jFRJoUZERBqH0hOwcZE1MrP/K6stIBB63QJX3wOdrrK3Pmn0FGpERMRexwrh8xfgi7/D8UKrLaQVJNxlXS/Tpou99UmToVAjIiL2KDlujcqsngklR6y2yE7QLxUSxkBopK3lSdOjUCMiIhdWRTlsWAgfPg5H9lptsVfANWnQ8xYI0k+T1I3+5oiIyIXz7Yew4o/wwyZrPbIT3PAw9B6lySXlvCnUiIhIw8v/EjL/CN9+YK07I2HQ7+HHv4HgUHtrE7+hUCMiIg3HvRc+nA7r/wUYa7bsH4+DQfdDi7Z2Vyd+RqFGRETqX/ER+OR/Yc0sKCuy2nqNhBsftiabFGkACjUiIlJ/ystg3f/ByhlwbL/V1qkfpDwOnX5sa2ni/xRqRETk/BkD25bD+w9D4XarrW083PgI9LwZAgLsrU+aBYUaERE5P3vWwYqH4LvV1npYW7h2MiT+Chwh9tYmzYpCjYiI1M2h7+CDR+HL1611Ryhc/Vu4ZqIenCe2UKgRERHfFB2CVc/AZ387OXN2AFwxGq5/EFp3srs6acYUakREpHbKiq35mbKeghOHrba4wZDyGLS/0tbSREChRkREzsUY2PwmvP8nOPyd1XZRTyvMXHyjLgKWRkOhRkREqvddNqx4EPastdZbxsJ1U6HPHZqjSRod/Y0UEZHKCr+xbs/+6m1rPTgcBkyA/vdCSLi9tYlUQ6FGREROO1ZoPTgv50WoKIOAQEi4C66dCq1i7K5OpEYKNSIiAiXH4dPZsHomlByx2i65yXp4XrsetpYmUlsKNSIizVlFBWxcCB8+Du49Vlv7K61pDeIG2VubiI8UakREmqtvP4LMhyB/k7Ue2Qlu+CP0/i8IDLS3NpE6qNPf2tmzZxMXF0doaCiJiYmsWrWqxv5ZWVkkJiYSGhpKfHw8c+fO9Xp/8+bNjBo1iq5duxIQEMDMmTMr7SMjI4OrrrqKVq1a0a5dO0aOHMm2bdvqUr6ISPP2w2ZYMAr+OdIKNM5I6zTTvWvhitsVaKTJ8vlv7qJFi0hLS2PatGnk5uYycOBAhg4dSl5eXpX9d+7cybBhwxg4cCC5ublMnTqV8ePHs3jxYk+f48ePEx8fz4wZM4iNja1yP1lZWdxzzz18+umnZGZmUlZWRkpKCseOHfP1K4iINE/uffDWvTD3GvjmfQh0QL/fwvhcuCYNgkPtrlDkvAQYY4wvG/Tr14+EhATmzJnjaevZsycjR44kIyOjUv9JkyaxZMkStm7d6mlLTU1lw4YNZGdnV+rftWtX0tLSSEtLq7GO/fv3065dO7Kyshg0qHbnfd1uN5GRkbhcLiIiImq1jYhIk1d8BD55DrJnQelxq63XLXDDwxDVzd7aRGqhtr/fPl1TU1JSQk5ODpMnT/ZqT0lJYc2aNVVuk52dTUpKilfbkCFDmD9/PqWlpQQHB/tSgofL5QKgbdu21fYpLi6muLjYs+52u+v0WSIiTVJ5GeS+DB9lwLECq61TP+si4E4/trc2kQbgU6gpLCykvLycmBjvZxXExMSQn59f5Tb5+flV9i8rK6OwsJD27dv7WDIYY0hPT+eaa66hd+/e1fbLyMjgkUce8Xn/IiJNmjGw/V3IfBgKT1572DYebvwT9PyppjUQv1Wnu58CzvoPwhhTqe1c/atqr617772XjRs3snr16hr7TZkyhfT0dM+62+2mUyfNICsifmzPOljxEHx38t/HsLZw7WRI/BU4QuytTaSB+RRqoqOjCQoKqjQqU1BQUGk05pTY2Ngq+zscDqKionwsF+677z6WLFnCxx9/TMeOHWvs63Q6cTqdPn+GiEiTc+g7+PAx2PSatR7khKt/CwPTITTS3tpELhCf7n4KCQkhMTGRzMxMr/bMzEz69+9f5TbJycmV+q9YsYKkpCSfrqcxxnDvvffyxhtv8OGHHxIXF+dL6SIi/qnosDUyMyvpdKC54udwXw785BEFGmlWfD79lJ6ezpgxY0hKSiI5OZl58+aRl5dHamoqYJ3y2bNnDy+//DJg3ek0a9Ys0tPTGTduHNnZ2cyfP59XX33Vs8+SkhK2bNniWd6zZw/r16+nZcuWXHzxxQDcc889vPLKK7z11lu0atXKM/oTGRlJWFjY+R0FEZGmpqwEvvg7fPwUFB2y2uIGwU8egw59bC1NxC4+39IN1sP3nnrqKfbt20fv3r35y1/+4rmt+u6772bXrl2sXLnS0z8rK4uJEyeyefNmOnTowKRJkzwhCGDXrl1VjrwMHjzYs5/qrr958cUXufvuu2tVt27pFpEmzxjY8h94/09waJfVdlEPK8x0/4kuAha/VNvf7zqFmqZKoUZEmrS8T2HFg/D9F9Z6yxi4bir0uROCNOuN+K8GeU6NiIjYoPAbeP9h+Optaz24BQyYAMn3grOlvbWJNCIKNSIijdWxQsh6Etb+AyrKICAQ+o6xRmdaVT2ljEhzplAjItLYlBbBp7Nh9UwoPvkk9O5DrLuZ2vW0tTSRxkyhRkSksaiogI2L4MPHwf291RZ7hTWtQfxge2sTaQIUakREGoNvP4LMhyB/k7Ue2Qmufwguvw0CfXqkmEizpVAjImKnH7ZA5h/hm5MPKXVGWE8B7pcKwXoGl4gvFGpEROxwJB8+mg65C8BUQKADrvofGPQAhPs+hYyIKNSIiFxYxUdhzXOw5q9Qetxq6/lTawbtqG62libS1CnUiIhcCOVlkPtP+OgJOFZgtXX8sXURcOd+9tYm4icUakREGpIxsP0967qZwm1WW5s4a2Sm1y2a1kCkHinUiIg0lL251gzau1ZZ62FtYfAkSPo1OELsrU3EDynUiIjUt8N58MFjsOnf1nqQE65OhWvSIay1raWJ+DOFGhGR+lJ0GFY9A5/9DcqLrbYrRsP1D0LrzraWJtIcKNSIiJyvshJYO9+ap6nokNXWdSCkPAYd+tpbm0gzolAjIlJXxsCW/8D7j8ChnVbbRT3gJ49C9xRdBCxygSnUiIjURd5nsOJB+P5zaz28HVw/DfrcCUH6p1XEDvovT0TEFwe+hff/BFuXWOvBLaD/eOh/Hzhb2lqaSHOnUCMiUhvHDljXzKydDxVlEBAIfe+Ea6dCRHu7qxMRFGpERGpWWgSfzYVVz0Kx22rrngI3PgIxveytTUS8KNSIiFSlosJ6zswHj4H7e6st9nJrWoP4a20tTUSqplAjInK2HSutJwHnb7TWIzrCDQ/B5bdDYKCtpYlI9RRqREROKdhqzdH09Qpr3RkB10yEq38LwWH21iYi56RQIyJyJB8+mg65C8BUQKADkv4bBj8A4dF2VycitaRQIyLNV/FRWPNXWPMclB632nrebF0EHNXN3tpExGcKNSLS/JSXwfoF8NETcPQHq63jVdZFwJ2vtrc2EakzhRoRaT6Msa6Xyfwj7P/KamvT1RqZ6XWLpjUQaeIUakSkedi73prWYNcqaz2sDQyeZF074wixtTQRqR8KNSLi3w7vhg8fg42LrPUgJ/T7DQz8PYS1trU0EalfCjUi4p+KDsPqZ+HTuVBebLVdfrv1vJnWnW0tTUQahkKNiPiXshJY+w9rnqaig1Zb14Hwk0fhRwn21iYiDUqhRkT8gzGw5S344BE4uMNqi77UCjOXDNFFwCLNgEKNiDR9eZ9ZFwF//7m1Ht4OrpsCfe+CIP0zJ9Jc1GkSk9mzZxMXF0doaCiJiYmsWrWqxv5ZWVkkJiYSGhpKfHw8c+fO9Xp/8+bNjBo1iq5duxIQEMDMmTPr5XNFxM8d3g3/vgv+kWIFmuAW1h1N49dB0q8VaESaGZ9DzaJFi0hLS2PatGnk5uYycOBAhg4dSl5eXpX9d+7cybBhwxg4cCC5ublMnTqV8ePHs3jxYk+f48ePEx8fz4wZM4iNja2XzxURP3dwJ/xjiHXKKSAQEu6C+9bBdVPB2cru6kTEBgHGGOPLBv369SMhIYE5c+Z42nr27MnIkSPJyMio1H/SpEksWbKErVu3etpSU1PZsGED2dnZlfp37dqVtLQ00tLSzutzq+J2u4mMjMTlchEREVGrbUSkETqcBy8OB1eedd3Mbf8HMb3srkpEGkhtf799GqkpKSkhJyeHlJQUr/aUlBTWrFlT5TbZ2dmV+g8ZMoS1a9dSWlraYJ8LUFxcjNvt9nqJSBPn3gsv3WwFmqiLYewSBRoRAXwMNYWFhZSXlxMTE+PVHhMTQ35+fpXb5OfnV9m/rKyMwsLCBvtcgIyMDCIjIz2vTp061erzRKSROpJvBZpDu6zpDcYuhVZVn7IWkeanThcKB5x1a6QxplLbufpX1V7fnztlyhRcLpfntXv3bp8+T0QakaP74aWfwoFvILKzFWgiOthdlYg0Ij7dGhAdHU1QUFCl0ZGCgoJKoyinxMbGVtnf4XAQFRXVYJ8L4HQ6cTqdtfoMEWnEjh+El2+Bwm0Q8SPrlJOeCiwiZ/FppCYkJITExEQyMzO92jMzM+nfv3+V2yQnJ1fqv2LFCpKSkggODm6wzxURP1F0yAo0BZuhZaw1QtM2zu6qRKQR8vkhDunp6YwZM4akpCSSk5OZN28eeXl5pKamAtYpnz179vDyyy8D1p1Os2bNIj09nXHjxpGdnc38+fN59dVXPfssKSlhy5YtnuU9e/awfv16WrZsycUXX1yrzxURP3TCBf/8GeRvhPCLrEAT1c3uqkSksTJ18Pzzz5suXbqYkJAQk5CQYLKysjzvjR071gwePNir/8qVK03fvn1NSEiI6dq1q5kzZ47X+zt37jRApdfZ+6npc2vD5XIZwLhcLp+2ExEbnHAb8/efGPNwhDEzuhqTv9nuikTEJrX9/fb5OTVNmZ5TI9JElByDf90G330Coa2tEZr2V9hdlYjYpEGeUyMi0uBKi+DVn1uBxhkBY95UoBGRWlGoEZHGo/QELLwDdn4MIS3hzjfgRwl2VyUiTYRCjYg0DmUl8NpY+PYDa2LKO16HTlfZXZWINCEKNSJiv/JSeP1XsP1dcITCLxdBl2S7qxKRJkahRkTsVV4Gb4yDr96GICf8/BWIG2R3VSLSBCnUiIh9Ksrhrd/B5jchMBhGL4CLb7C7KhFpohRqRMQeFRWwZDxsXASBDrj9Jbgkxe6qRKQJU6gRkQvPGHhnIqxfAAFBMGo+9Bhud1Ui0sQp1IjIhWUMLH8Acv4PAgLh1r/BZSPtrkpE/IBCjYhcOMbAigfh83lAANzyPFxxm91ViYifUKgRkQvDGPjgEcieZa3fPBP6/NLWkkTEvyjUiMiFsXIGrP6LtTzsz5B4t63liIj/UagRkYb38Z8ha4a1PCQDfjzO3npExC8p1IhIw/rkOfjwMWv5xkcg+Xf21iMifkuhRkQazqdzIfMha/m6B+GaNFvLERH/plAjIg3ji/nw7iRredADMPh+e+sREb+nUCMi9W/dP+GddGt5wAS4bqq99YhIs6BQIyL1a8NCWHKftXz176zraAIC7K1JRJoFhRoRqT9fLob//BYwcNX/wJAnFGhE5IJRqBGR+rFlCSweB6YCEu6CoU8r0IjIBaVQIyLnb9tyeP1XYMrhyl/CiP+FQP3zIiIXlv7VEZHz8/X78O+7oKIMLr8NbpmlQCMittC/PCJSd99+BAt/CeUl0OsWGDkXAoPsrkpEmimFGhGpm12r4dVfQHkxXDocRs2HIIfdVYlIM6ZQIyK+y/sU/nU7lBVB9xS47UUICra7KhFp5hRqRMQ33+fAgv+C0mMQfx3c/k9wOO2uSkREoUZEfLB3PfzzVig5Al0Hws9fgeBQu6sSEQEUakSktvI3wT9HQrELOifDLxZCSAu7qxIR8VCoEZFzK9gKL98CRYeg41Vwx2vgbGl3VSIiXhRqRKRmhV/DSz+F4wegfR+443VwtrK7KhGRShRqRKR6B76Fl26GYwUQczmMeRPCWttdlYhIlRRqRKRqh76zRmiO7IN2veCut6BFW7urEhGpVp1CzezZs4mLiyM0NJTExERWrVpVY/+srCwSExMJDQ0lPj6euXPnVuqzePFievXqhdPppFevXrz55pte75eVlfHggw8SFxdHWFgY8fHxPProo1RUVNTlK4hITVzfw0sjwP09RF9iBZrwKLurEhGpkc+hZtGiRaSlpTFt2jRyc3MZOHAgQ4cOJS8vr8r+O3fuZNiwYQwcOJDc3FymTp3K+PHjWbx4sadPdnY2o0ePZsyYMWzYsIExY8Zw++2389lnn3n6PPnkk8ydO5dZs2axdetWnnrqKZ5++mn++te/1uFri0i13Hvh/0bA4TxoGw93LYGW7eyuSkTknAKMMcaXDfr160dCQgJz5szxtPXs2ZORI0eSkZFRqf+kSZNYsmQJW7du9bSlpqayYcMGsrOzARg9ejRut5vly5d7+tx00020adOGV199FYARI0YQExPD/PnzPX1GjRpFixYt+Oc//1mr2t1uN5GRkbhcLiIiInz52iLNw5Ef4P+Gw4GvoXUX+NUyiOxod1Ui0szV9vfbp5GakpIScnJySElJ8WpPSUlhzZo1VW6TnZ1dqf+QIUNYu3YtpaWlNfY5c5/XXHMNH3zwAdu3bwdgw4YNrF69mmHDhlVbb3FxMW632+slItU4Vggv/9QKNJGdYOxSBRoRaVJ8mn2usLCQ8vJyYmJivNpjYmLIz8+vcpv8/Pwq+5eVlVFYWEj79u2r7XPmPidNmoTL5aJHjx4EBQVRXl7O9OnT+cUvflFtvRkZGTzyyCO+fEWR5un4Qes5NPu/glbtYewSaNPF7qpERHxSpwuFAwICvNaNMZXaztX/7PZz7XPRokUsWLCAV155hXXr1vHSSy/x5z//mZdeeqnaz50yZQoul8vz2r1797m/nEhzU3TYelLwD19CyxhrhKZtvN1ViYj4zKeRmujoaIKCgiqNyhQUFFQaaTklNja2yv4Oh4OoqKga+5y5z/vvv5/Jkyfz85//HIDLL7+c7777joyMDMaOHVvlZzudTpxOTbQnUq0TbljwM9i3AVpEWxcFR3e3uyoRkTrxaaQmJCSExMREMjMzvdozMzPp379/ldskJydX6r9ixQqSkpIIDg6usc+Z+zx+/DiBgd7lBgUF6ZZukboqPgr/ug325EBYG+u27XY97K5KRKTOfBqpAUhPT2fMmDEkJSWRnJzMvHnzyMvLIzU1FbBO+ezZs4eXX34ZsO50mjVrFunp6YwbN47s7Gzmz5/vuasJYMKECQwaNIgnn3ySW265hbfeeov333+f1atXe/rcfPPNTJ8+nc6dO3PZZZeRm5vLs88+y69//evzPQYizU/JcXhlNOz+FEIjYcx/ILa33VWJiJwfUwfPP/+86dKliwkJCTEJCQkmKyvL897YsWPN4MGDvfqvXLnS9O3b14SEhJiuXbuaOXPmVNrna6+9Zi699FITHBxsevToYRYvXuz1vtvtNhMmTDCdO3c2oaGhJj4+3kybNs0UFxfXum6Xy2UA43K5fPvCIv6kpMiYl35qzMMRxkz/kTG719pdkYhIjWr7++3zc2qaMj2nRpq9smJYeAd8kwnB4dZcTp372V2ViEiNGuQ5NSLShJWVwL/HWoHGEQZ3vKZAIyJ+RaFGpDkoL4XFv4bty8ERCr9cCF0H2F2ViEi9UqgR8XcV5fDmb2DrUggKgdH/gvhr7a5KRKTeKdSI+LOKcvjP7+DLxRAYDLf/E7rfaHdVIiINQqFGxF9VVMDSCbBxIQQEwW0vwqU32V2ViEiDUagR8UfGwLI/QO4/ISAQRv0det5sd1UiIg1KoUbE3xgD706BtfOBABg5F3r/zO6qREQanEKNiD8xBjIfgs/mWOu3zIIrR9tbk4jIBaJQI+IvjIEPH4M1f7XWR/wF+t5pb00iIheQQo2Iv8h6ClY9Yy0PfQqSNC+aiDQvCjUi/mDVs7DyCWs5ZTr0+4299YiI2EChRqSpWzMLPnjEWr7hj9D/XnvrERGxiUKNSFP22TxYMc1avnYKDPy9vfWIiNhIoUakqVr7Iiy/31oe+HsYPMneekREbKZQI9IU5S6At9Os5eR74fqHICDA1pJEROymUCPS1Gz8N7x18rqZH/8GUh5XoBERQaFGpGnZ/KY14zbGumV76JMKNCIiJynUiDQVhV/DG/8PTIX1UL1hzyjQiIicQaFGpCkwBt5Jh/IS6HY93PwcBOo/XxGRM+lfRZGmYNPrsPNjcITC8GchMMjuikREGh2FGpHGrugwvDfVWh74B2gbZ2s5IiKNlUKNSGP34WNwrACiusOA8XZXIyLSaCnUiDRme3Lgi/nW8vBnwOG0tx4RkUZMoUaksaooh7fTAQOX3w7xg+2uSESkUVOoEWmsvpgP+9aDMxKGTLe7GhGRRk+hRqQxOpJvXUsDcOMfoWU7e+sREWkCFGpEGqP3pkKxGzokQOKv7K5GRKRJUKgRaWy+/RC+XAwBgTDiL3omjYhILSnUiDQmpSfgnd9byz/+f9Chj63liIg0JQo1Io3JJzPh4A5oGQvXTbO7GhGRJkWhRqSxOPAtrHrWWr7pCQiNsLceEZEmRqFGpDEwBpb9AcqLIf46uOxndlckItLk1CnUzJ49m7i4OEJDQ0lMTGTVqlU19s/KyiIxMZHQ0FDi4+OZO3dupT6LFy+mV69eOJ1OevXqxZtvvlmpz549e7jzzjuJioqiRYsW9OnTh5ycnLp8BZHGZfMb1gXCQU7rycEBAXZXJCLS5PgcahYtWkRaWhrTpk0jNzeXgQMHMnToUPLy8qrsv3PnToYNG8bAgQPJzc1l6tSpjB8/nsWLF3v6ZGdnM3r0aMaMGcOGDRsYM2YMt99+O5999pmnz6FDhxgwYADBwcEsX76cLVu28Mwzz9C6dWvfv7VIY3LCDe+emrAyHaK62VuPiEgTFWCMMb5s0K9fPxISEpgzZ46nrWfPnowcOZKMjIxK/SdNmsSSJUvYunWrpy01NZUNGzaQnZ0NwOjRo3G73SxfvtzT56abbqJNmza8+uqrAEyePJlPPvnknKNCNXG73URGRuJyuYiI0PUK0kgsnwSfzYW28fDbbAgOtbsiEZFGpba/3z6N1JSUlJCTk0NKSopXe0pKCmvWrKlym+zs7Er9hwwZwtq1ayktLa2xz5n7XLJkCUlJSdx22220a9eOvn378sILL9RYb3FxMW632+sl0qjsXQ+fz7OWhz+jQCMich58CjWFhYWUl5cTExPj1R4TE0N+fn6V2+Tn51fZv6ysjMLCwhr7nLnPHTt2MGfOHLp37857771Hamoq48eP5+WXX6623oyMDCIjIz2vTp06+fJ1RRpWRTm8PRFMBfQeBd2ut7siEZEmrU4XCgecdRGjMaZS27n6n91+rn1WVFSQkJDAE088Qd++ffnNb37DuHHjvE6DnW3KlCm4XC7Pa/fu3ef+ciIXSs6LsHcdOCNgyBN2VyMi0uT5FGqio6MJCgqqNCpTUFBQaaTllNjY2Cr7OxwOoqKiauxz5j7bt29Pr169vPr07Nmz2guUAZxOJxEREV4vkUbhaAG8/6i1fP1D0CrW3npERPyAT6EmJCSExMREMjMzvdozMzPp379/ldskJydX6r9ixQqSkpIIDg6usc+Z+xwwYADbtm3z6rN9+3a6dOniy1cQaRzemwbFLmjfB676b7urERHxD8ZHCxcuNMHBwWb+/Plmy5YtJi0tzYSHh5tdu3YZY4yZPHmyGTNmjKf/jh07TIsWLczEiRPNli1bzPz5801wcLB5/fXXPX0++eQTExQUZGbMmGG2bt1qZsyYYRwOh/n00089fT7//HPjcDjM9OnTzddff23+9a9/mRYtWpgFCxbUunaXy2UA43K5fP3aIvXn25XGPBxhzMORxnyfY3c1IiKNXm1/v30ONcYY8/zzz5suXbqYkJAQk5CQYLKysjzvjR071gwePNir/8qVK03fvn1NSEiI6dq1q5kzZ06lfb722mvm0ksvNcHBwaZHjx5m8eLFlfosXbrU9O7d2zidTtOjRw8zb948n+pWqBHblZ4w5rlEK9S8nW53NSIiTUJtf799fk5NU6bn1Ijtsp6Gjx6H8HZw7xcQ1truikREGr0GeU6NiJyHgzth1Z+t5SFPKNCIiNQzhRqRC8EYWHY/lJ2AuMFw+X/ZXZGIiN9RqBG5ELYugW8yIShEE1aKiDQQhRqRhlZ8BJZPtpYHpEF0d1vLERHxVwo1Ig3toww4shfadLVm4RYRkQahUCPSkPZttGbgBhj2DASH2VuPiIgfU6gRaSgVFfBOOphy6DUSut9od0UiIn5NoUakoax7Cb7/AkJawk0ZdlcjIuL3FGpEGsLR/fD+n6zl66ZBRAdbyxERaQ4UakQaQuYf4cRhiL0cfvz/7K5GRKRZUKgRqW+7VsOGV4AAGDETghx2VyQi0iwo1IjUp7ISePvkbduJd0PHJFvLERFpThRqROpT9iwo3AYtouHGh+2uRkSkWVGoEakvh3ZB1lPW8pDpENbG1nJERJobhRqR+mAMLJ8EZUXQdSBcMdruikREmh2FGpH68NU7sP1dCAzWhJUiIjZRqBE5X8VHrVEagAHj4aJL7a1HRKSZUqgROV9ZM8D9PbTuDAP/YHc1IiLNlkKNyPn4YTNkz7aWh/0ZQlrYW4+ISDOmUCNSVxUV1jNpTDn0vBkuGWJ3RSIizZpCjUhdrV8Auz+F4HC4aYbd1YiINHsKNSJ1ceyANb8TwHVTILKjvfWIiIhCjUidvP9HKDoE7S6Dfql2VyMiIijUiPjuu2zIXWAtj/gLBAXbW4+IiAAKNSK+KS+Fd05OWJlwF3TuZ289IiLioVAj4otPZ0PBFmgRBTc+Ync1IiJyBoUakdo6vBtWnrzL6SePQYu29tYjIiJeFGpEamv5JCg9Dp37Q59f2l2NiIicRaFGpDa2LYdt70CgA0Y8qwkrRUQaIYUakXMpOQbLHrCWk++Fdj3trUdERKqkUCNyLh8/Da48iOwMgx+wuxoREamGQo1ITQq2wpq/WstDn4SQcHvrERGRatUp1MyePZu4uDhCQ0NJTExk1apVNfbPysoiMTGR0NBQ4uPjmTt3bqU+ixcvplevXjidTnr16sWbb75Z7f4yMjIICAggLS2tLuWL1I4x8M7voaIMLh0GPYbZXZGIiNTA51CzaNEi0tLSmDZtGrm5uQwcOJChQ4eSl5dXZf+dO3cybNgwBg4cSG5uLlOnTmX8+PEsXrzY0yc7O5vRo0czZswYNmzYwJgxY7j99tv57LPPKu3viy++YN68eVxxxRW+li7imw2vwnefQHALa5RGREQatQBjjPFlg379+pGQkMCcOXM8bT179mTkyJFkZGRU6j9p0iSWLFnC1q1bPW2pqals2LCB7OxsAEaPHo3b7Wb58uWePjfddBNt2rTh1Vdf9bQdPXqUhIQEZs+ezeOPP06fPn2YOXNmrWt3u91ERkbicrmIiIjw5WtLc3P8IMxKguMHrIfsXZNmd0UiIs1WbX+/fRqpKSkpIScnh5SUFK/2lJQU1qxZU+U22dnZlfoPGTKEtWvXUlpaWmOfs/d5zz33MHz4cG688cZa1VtcXIzb7fZ6idTK+3+yAs1FPSH5HrurERGRWvAp1BQWFlJeXk5MTIxXe0xMDPn5+VVuk5+fX2X/srIyCgsLa+xz5j4XLlzIunXrqhwNqk5GRgaRkZGeV6dOnWq9rTRjuz+HdS9ZyyOe1YSVIiJNRJ0uFA4468FjxphKbefqf3Z7TfvcvXs3EyZMYMGCBYSGhta6zilTpuByuTyv3bt313pbaabKy+DtidZynzuhS3976xERkVpz+NI5OjqaoKCgSqMyBQUFlUZaTomNja2yv8PhICoqqsY+p/aZk5NDQUEBiYmJnvfLy8v5+OOPmTVrFsXFxQQFBVX6bKfTidPp9OUrSnP32Vz44UsIawM/edTuakRExAc+jdSEhISQmJhIZmamV3tmZib9+1f9/2iTk5Mr9V+xYgVJSUkEBwfX2OfUPm+44QY2bdrE+vXrPa+kpCTuuOMO1q9fX2WgEfGZaw+sPHl688ZHIDzK3npERMQnPo3UAKSnpzNmzBiSkpJITk5m3rx55OXlkZqaClinfPbs2cPLL78MWHc6zZo1i/T0dMaNG0d2djbz58/3uqtpwoQJDBo0iCeffJJbbrmFt956i/fff5/Vq1cD0KpVK3r37u1VR3h4OFFRUZXaRers3clQchQ69YO+Y+yuRkREfORzqBk9ejQHDhzg0UcfZd++ffTu3Ztly5bRpUsXAPbt2+f1zJq4uDiWLVvGxIkTef755+nQoQPPPfcco0aN8vTp378/Cxcu5MEHH+Shhx6iW7duLFq0iH79+tXDVxSphe0rYOsSCAiC4c9CoB62LSLS1Pj8nJqmTM+pkSqVHIfZV8Ph76wJK4dMt7siERE5Q4M8p0bEL616xgo0ET+Ca6fYXY2IiNSRQo00b/u3wyf/ay0PfRKcLe2tR0RE6kyhRpovY+CddKgohe5DoMcIuysSEZHzoFAjzdfGf8OuVeAIg2FPQQ0PkBQRkcZPoUaap6JDsGKatTz4fmjT1dZyRETk/CnUSPP0waNwbD9EXwrJ99ldjYiI1AOFGml+vs+BtS9ay8OfAUeIvfWIiEi9UKiR5qW8DN5OAwxc+QuIG2h3RSIiUk8UaqR5+eLvkL8RQlvDTx6zuxoREb/hLnPz5bEvba3B52kSRJos9z748HFr+caHoeVF9tYjItIEHSk7wo4TO/j2xLfsKNrhWS4sLQRg1ZWraBHUwpbaFGqk+XhvCpQcgY5XQcLddlcjItKoHSk/ws6inVZ4ObGDb4u+ZeeJnRSUFlS7TUxwDPtL99MlqMsFrPQ0hRppHr55Hza/CQGBmrBSROQMR8uPsvPETnYU7fAEmB1FO/ih9Idqt4kJjiEuNI5uYd2ID42nW1g34kLjaBlk71PZFWrE/5UWwTt/sJb7pUL7K+ytR0TEBsfLj1uB5eSoy6k/awovFwVf5Akt8aHxdAvtRlxYHK2CWl3AymtPoUb83+q/wKGd0Ko9XDfV7mpERBrU8fLj7Dyx0+ualx0ndrCvZF+120QHR9Mt1Aou8WHx1p+h8UQ4qp8RuzFSqBH/VviNFWoAbpoBzsb5/y5ERHxVVF5knTY6a/Rlb8neareJckR5Qku30G6e5UhH5AWsvOEo1Ij/MgaW/R7KS+DiG6HXLXZXJCLis6KKInad2HX6mpeToy97S/ZiMFVu09bR1vu00clrXlo7Wl/Y4i8whRrxX18uhh0rwREKw57WhJUi0iiVVJRwoPQAhWWF7C/ZT2FZIfkl+Z5RmD3Fe6oNL20cbbzCy6nTR20cbS7wt2gcFGrEP51wwXsnr58Z+AdoG29vPSLS7JyoOEFhaSGFpYXsL91f5XJhaSGuctc59xUZFOk16nLq9FGb4OYZXqqjUCP+6cPH4egPENUdBoy3uxoR8SPHy49XGU72l1qjLKeWj5YfrfU+gwOCiQ6Otl6OaC4KuYiuzq7Eh50ML442BGi0+ZwUasT/7FkHn79gLQ9/BhxOe+sRkUbPGMPRiqPVjqac2X684nit9+sMcJ4OK8HRXBR8UZXrkUGRCi31QKFG/EtFObw9ETBw+e0QP9juikTERsYY3OXu6kdWSgs917IUm+Ja7zcsMMwroHiWHdFEh0RzkcNabxnUUmHlAlKoEf+y9h+wbz04I2HIdLurEZEGUmEqcJW5Kp32qSq8lJiSWu+3ZVBLK5jUMKoSHRxNeFB4A347qSuFGvEfR36ADx61lm94CFq2s7ceEfFZuSnnUNmhc15cW1haSDnltd5vZFCkV0Cp7nRQWGBYA347aWgKNeI/3psKxW7o0BeSfm13NSIClFaUcrj8MIfLDuMqc3n9ebj89PLBsoMUlhZysPSgT2GljaONd0CpYpQlKjgKZ6CurWsOFGrEP3z7EXz5ujVh5Yi/QGCQ3RWJ+J2SihIrhJwMKV5BpdxVqe1w2WGOVRzz+XMCCfSElZpOAUU5oggODG6AbypNlUKNNH2lJ+Cd31vLV42zRmpEpEaegFJ2mEPlhyqPolQRVOoSUMAKKZGOSCKDImntaE1rR2siHaeXT71OjbS0CW6DI0A/T+I7/a2Rpu+T/4WD30LLGLh+mt3ViFxwxRXFXqd0Ko2inDG6cqrNl9uSzxREEBGOiNPhpKqgEtTaq61VUCsCAwLr+VuLVKZQI03bgW9h1TPW8k0ZEOofk7JJ81VcUVxpxORcp3uKKorq9FlBBHmCSKQj0hNGzhxFiXRE0sbRhtZB1nLLoJYKKNJoKdRI02UMLLsfyosh/jq47Gd2VyTi5UTFCe/TOWddj3J2u6vMdV4B5ewRk2pP95wML+FB4Qoo4lcUaqTp2vIf+PYDCHJaTw7WA66kARVVFJ3zuhOvoFLu4kTFiTp91pkBxSuUnDWScmZQaRmoh7yJKNRI03TCDcsnW8vXTISobvbWI01KUUVR5XBSxW3GZ758edrsmRwBDq8RE6+gUsVISqQjUgFFpI4UaqRp+ugJOJpvzb59zUS7qxEbnQooVd1OXN1txucTUE6dvqnq7p0z20/9GR4YroAicoHUKdTMnj2bp59+mn379nHZZZcxc+ZMBg4cWG3/rKws0tPT2bx5Mx06dOCBBx4gNTXVq8/ixYt56KGH+Pbbb+nWrRvTp0/n1ltv9byfkZHBG2+8wVdffUVYWBj9+/fnySef5NJLL63LV5CmbN8G+Pxv1vLwZyA41N565IJzlbl479B7vH3gbTYf31ynfQQHBFcaManqupMz21sEtlBAEWnEfA41ixYtIi0tjdmzZzNgwAD+9re/MXToULZs2ULnzp0r9d+5cyfDhg1j3LhxLFiwgE8++YTf/e53XHTRRYwaNQqA7OxsRo8ezWOPPcatt97Km2++ye23387q1avp168fYAWje+65h6uuuoqysjKmTZtGSkoKW7ZsITxcc3A0G6cmrDQV1oXB3a63uyK5QMpMGdnubJYeWMrHro8pNaWe90ICQirdUlzVdSdnBpWwwDAFFBE/E2CMMb5s0K9fPxISEpgzZ46nrWfPnowcOZKMjIxK/SdNmsSSJUvYunWrpy01NZUNGzaQnZ0NwOjRo3G73SxfvtzT56abbqJNmza8+uqrVdaxf/9+2rVrR1ZWFoMGDapV7W63m8jISFwuFxEREbXaRhqZL+bDO+ngjIB7PoeI9nZXJA3s26JvWXpgKcsOLuNA2QFP+6VhlzIiagQpbVKIckQpoIj4sdr+fvs0UlNSUkJOTg6TJ0/2ak9JSWHNmjVVbpOdnU1KSopX25AhQ5g/fz6lpaUEBweTnZ3NxIkTK/WZOXNmtbW4XC4A2rZt68tXkKbsaAF88Ii1fP2DCjR+7NTppaUHlrLl+BZPextHG4a2HcqItiO4tIVOPYuIN59CTWFhIeXl5cTExHi1x8TEkJ+fX+U2+fn5VfYvKyujsLCQ9u3bV9unun0aY0hPT+eaa66hd+/e1dZbXFxMcfHpCwLdbneN308auRUPwQkXtL8Srvofu6uRelbd6aUgghgUOYibo26mf2R/ggM014+IVK1OFwqfPcxrjKlx6Leq/me3+7LPe++9l40bN7J69eoa68zIyOCRRx6psY80ETs/ho0LgQBNWOlnvin6hrcPvF3l6aWbo27mpjY30Sa4jY0VikhT4VOoiY6OJigoqNIISkFBQaWRllNiY2Or7O9wOIiKiqqxT1X7vO+++1iyZAkff/wxHTt2rLHeKVOmkJ6e7ll3u9106tSpxm2kESorOWPCyv+GHyXaW4+ct8Nlh3nv4HssPbiUrcdPX2/XxtGGYW2HMaLtCC5pcYmNFYpIU+RTqAkJCSExMZHMzEyv260zMzO55ZZbqtwmOTmZpUuXerWtWLGCpKQkgoODPX0yMzO9rqtZsWIF/fv396wbY7jvvvt48803WblyJXFxcees1+l04nQ6ffmK0hiteQ4Kt0N4O7j+IburkToqM2Wsca/h7QNvk+XKosyUATq9JCL1x+fTT+np6YwZM4akpCSSk5OZN28eeXl5nufOTJkyhT179vDyyy8D1p1Os2bNIj09nXHjxpGdnc38+fO97mqaMGECgwYN4sknn+SWW27hrbfe4v333/c6vXTPPffwyiuv8NZbb9GqVSvPyE5kZCRhYWHndRCkETu4Ez5+2loeMh3CWttajvju66KvefvA2yw/uNzr9FKPsB6MiBqh00siUm98DjWjR4/mwIEDPProo+zbt4/evXuzbNkyunTpAsC+ffvIy8vz9I+Li2PZsmVMnDiR559/ng4dOvDcc895nlED0L9/fxYuXMiDDz7IQw89RLdu3Vi0aJHnGTWA5xbya6+91queF198kbvvvtvXryGNXUUFFB2yJqwsOwFxg+Dy2+yuSmqputNLbR1tGdp2KDe3vZnuLbrbWKGI+COfn1PTlOk5NTYzxgoqRwvgWIH1Z3XLx/ZDhXV6gqAQ+O0aiNaPYGNWakrJdmWz9KB199Kp00uOAId1eqntzSRHJuv0koj4rEGeUyNSiTFw4nDNAeXomUGl9Jy79NIiGm58WIGmEfu66GuWHljK8oPLOVh20NPeI6wHN0fdzJC2Q2jj0OklEWl4CjVSmTHW82CO7YejP5wOJKeWPYFlv/VneYlv+w+NhJYx1oW/LU++wi+y2jzrJ9scIQ3zHeW8HCo7ZJ1eOrCUr4q+8rTr9JKI2EmhprkwBordp4PI0R+s5aM/nA4oR384GV4KoNzHWYydkVUElIvOCC+nli8Ch+5Ia4pqOr00OHIwI6JGkByh00siYh+FGn/n3gsbXoX1r8CBb3zb1hnhHVDC21UdVsLbaaZsP/b18a9ZerDy6aWeLXpyc1vr9FJrR2v7ChQROUmhxh+VnoBt70Duv2DHR9aM1qeEtDojoJx6nRxBOXs5WLfKN1eHyg7x7sF3WXpgKduKtnna2zraWg/HixpB9zCdXhKRxkWhxl8YA/vWW0Fm02vWxbundO4Pfe+Anjdb17OIVKHUlLLGtYalB5ayyr1Kp5dEpMlRqGnqju6HTf+2wkzB5tPtET+CK38BfX4JUd3sq08avVOnl5YdXMahskOedp1eEpGmRqGmKSovha9XWEHm6/fOeJ6LE3qOgD53QPy1mvRRqlXd6aUoR5Tn9NLFYRfbWKGIiO8UapqSH7bA+n/BxkXWXUqndEiwTi/1HgVheh6IVK2600vBAcGeuZeSI5JxBOifBRFpmvSvV2NXdAg2vW6Fmb25p9vDL4IrRlujMjG97KtPGr3tx7d77l468/RSrxa9uDnqZlLapOj0koj4BYWaxqii3LprKfdf8NU7p58ZE+iAS26ygkz3n0CQLtiUqh0qPcTyQ8t5+8DbOr0kIs2GQk1jceyAdaHvjpWwYSG495x+r91l1umly2+3bscWqUKpKeUT1yfW6SXXKsopB3R6SUSaD/3rdqGVFUPhdvhhs/fraL53v9DW1qzUfe+A9n0gIMCOaqUJ2HZ8G0sPLOXdQ+96nV66rMVljIgawZA2Q4h06FZ+EfF/CjUNxRhrtOXs8HLg69N3K52tTVeIvRwu+xlcOkxP6ZVqHSw9yLuHrLuXthdt97RHOaIY3nY4I6JG0C1Mt/KLSPOiUFMfio9CwVb44UsruBRssZZPuKruHxppnVKKOePVric4W13YuqVJ2V+yn3VH1/HeofdY7VrtdXppcORgbo66masjrtbpJRFptvSv3/kqL4On4queADLQAVHdvcNLzGXWg/F0OklqYIxhV/Eu1h9dT+7RXNYfXc+ekj1efXR6SUTEm0LN+QpyQPQl1nNjzg4v0ZdoRmqplTJTxrbj206HmGPrva6PAQgkkO5h3bk64mqGtx2u00siImdRqKkP/70CQlrYXYU0IUUVRWw6ton1R9ez/uh6Nh7bSFFFkVefkIAQeof3pm/LvvRp2Ycrwq+gZVBLmyoWEWn8FGrqgwKNnMPhssOeAJN7NJetx7d6rok5pVVQK64Mv5K+LfvSt2VferboSUhgiE0Vi4g0PQo1Ig1gX/E+z2mk3KO57Dixo1KfdsHtPKMwfVv2pVtoNwIDAm2oVkTEPyjUiJynClPBjhM7yD2a67mo94fSHyr1iwuNs0JMuBVi2oe0J0AXjIuI1BuFGhEflVaUsvX4Vk+I2XBsA+5yt1efIILo0aKHZySmT8s+tHFoslERkYakUCNyDsfKj7Hx2EbPKMyXx76k2Hjfwh8aGMoV4Vd4RmIuD7+csKAwmyoWEWmeFGpEznKg9IDngt7co7lsL9pOBRVefVo7WntOI/Vt2ZdLWlxCcIAmGBURsZNCjTRrxhi+L/6e3GO5niCTV5xXqV+HkA5eF/V2dXbV9TAiIo2MQo00K+WmnK+LvvZ6yF1haaFXnwAC6BbazTMK06dlH2JCYmyqWEREakuhRvxacUUxm49t9txaveHoBo5VHPPqExwQTK8WvTyjMFeGX0mEI8KmikVEpK4UasSvHCk7wvpjpx9yt+X4FkpNqVef8MBwrmx5peeamF7hvQgN1IzoIiJNnUKNNGkFJQVeD7n7pugbDMarT5QjyutUUvew7gQFBNlUsYiINBSFGmkyTs1cferW6qpmrgbo7Ox8+qLe8L50dHbURb0iIs2AQo3YqtyUU2JKKKsoo8SUUGpKKa0opdSUUmJKKKooYvOxzZ7RmMNlh722DySQS8Iu8YzEXNnySqKDo+35MiIiYiuFGj9XYSooM2WUmTJPUPAKECdDRFWBoqb3ykwZJRVn7MOUVrleZqoOK6eWz37+y7k4A5z0Du/tuaj38vDLNXO1iIgACjX14om8JzhafhSwbgcODAgkgIA6rQOeEHIqiJSZMk8IOLPN6/2z/6yw/jx7JujGLjgg+PQrMBhngJP40HjPSEyPFj00c7WIiFSpTqFm9uzZPP300+zbt4/LLruMmTNnMnDgwGr7Z2VlkZ6ezubNm+nQoQMPPPAAqampXn0WL17MQw89xLfffku3bt2YPn06t95663l97oWy8vBKDpQdsLuMWgkggJCAEIIDreAQEhCCI8BBSGCIV6Coat0R4PDa9tT2Xutn9jsZTDz9zrHuCHDo2hcREakzn0PNokWLSEtLY/bs2QwYMIC//e1vDB06lC1bttC5c+dK/Xfu3MmwYcMYN24cCxYs4JNPPuF3v/sdF110EaNGjQIgOzub0aNH89hjj3Hrrbfy5ptvcvvtt7N69Wr69etXp8+9kH7b4bcUVRRhjMHrf2etg3U66Mz1U33OPA3jCHDgCHAQHBBc6c9Ty47Ayn2q6n+qr2c5QINzIiLinwKMMebc3U7r168fCQkJzJkzx9PWs2dPRo4cSUZGRqX+kyZNYsmSJWzdutXTlpqayoYNG8jOzgZg9OjRuN1uli9f7ulz00030aZNG1599dU6fW5V3G43kZGRuFwuIiL0cDUREZGmoLa/34G+7LSkpIScnBxSUlK82lNSUlizZk2V22RnZ1fqP2TIENauXUtpaWmNfU7tsy6fC1BcXIzb7fZ6iYiIiH/yKdQUFhZSXl5OTIz3PDgxMTHk5+dXuU1+fn6V/cvKyigsLKyxz6l91uVzATIyMoiMjPS8OnXqVLsvKiIiIk2OT6HmlLMv5jTG1HiBZ1X9z26vzT59/dwpU6bgcrk8r927d1fbV0RERJo2n64ajY6OJigoqNLoSEFBQaVRlFNiY2Or7O9wOIiKiqqxz6l91uVzAZxOJ06ns3ZfTkRERJo0n0ZqQkJCSExMJDMz06s9MzOT/v37V7lNcnJypf4rVqwgKSmJ4ODgGvuc2mddPldERESaGeOjhQsXmuDgYDN//nyzZcsWk5aWZsLDw82uXbuMMcZMnjzZjBkzxtN/x44dpkWLFmbixIlmy5YtZv78+SY4ONi8/vrrnj6ffPKJCQoKMjNmzDBbt241M2bMMA6Hw3z66ae1/tzacLlcBjAul8vXry0iIiI2qe3vt8+hxhhjnn/+edOlSxcTEhJiEhISTFZWlue9sWPHmsGDB3v1X7lypenbt68JCQkxXbt2NXPmzKm0z9dee81ceumlJjg42PTo0cMsXrzYp8+tDYUaERGRpqe2v98+P6emKdNzakRERJqeBnlOjYiIiEhjpVAjIiIifkGhRkRERPyCQo2IiIj4BYUaERER8Qs+PVG4qTt1o5cmthQREWk6Tv1un+uG7WYVao4cOQKgiS1FRESaoCNHjhAZGVnt+83qOTUVFRXs3buXVq1a1TgR5rm43W46derE7t279bybBqZjfeHoWF84OtYXjo71hdOQx9oYw5EjR+jQoQOBgdVfOdOsRmoCAwPp2LFjve0vIiJC/5FcIDrWF46O9YWjY33h6FhfOA11rGsaoTlFFwqLiIiIX1CoEREREb+gUFMHTqeThx9+GKfTaXcpfk/H+sLRsb5wdKwvHB3rC6cxHOtmdaGwiIiI+C+N1IiIiIhfUKgRERERv6BQIyIiIn5BoUZERET8gkKNj2bPnk1cXByhoaEkJiayatUqu0tq8jIyMrjqqqto1aoV7dq1Y+TIkWzbts2rjzGGP/3pT3To0IGwsDCuvfZaNm/ebFPF/iMjI4OAgADS0tI8bTrW9WfPnj3ceeedREVF0aJFC/r06UNOTo7nfR3r+lFWVsaDDz5IXFwcYWFhxMfH8+ijj1JRUeHpo2NdNx9//DE333wzHTp0ICAggP/85z9e79fmuBYXF3PfffcRHR1NeHg4P/3pT/n+++8bpmAjtbZw4UITHBxsXnjhBbNlyxYzYcIEEx4ebr777ju7S2vShgwZYl588UXz5ZdfmvXr15vhw4ebzp07m6NHj3r6zJgxw7Rq1cosXrzYbNq0yYwePdq0b9/euN1uGytv2j7//HPTtWtXc8UVV5gJEyZ42nWs68fBgwdNly5dzN13320+++wzs3PnTvP++++bb775xtNHx7p+PP744yYqKsq8/fbbZufOnea1114zLVu2NDNnzvT00bGum2XLlplp06aZxYsXG8C8+eabXu/X5rimpqaaH/3oRyYzM9OsW7fOXHfddebKK680ZWVl9V6vQo0PfvzjH5vU1FSvth49epjJkyfbVJF/KigoMIDJysoyxhhTUVFhYmNjzYwZMzx9Tpw4YSIjI83cuXPtKrNJO3LkiOnevbvJzMw0gwcP9oQaHev6M2nSJHPNNddU+76Odf0ZPny4+fWvf+3V9rOf/czceeedxhgd6/pydqipzXE9fPiwCQ4ONgsXLvT02bNnjwkMDDTvvvtuvdeo00+1VFJSQk5ODikpKV7tKSkprFmzxqaq/JPL5QKgbdu2AOzcuZP8/HyvY+90Ohk8eLCOfR3dc889DB8+nBtvvNGrXce6/ixZsoSkpCRuu+022rVrR9++fXnhhRc87+tY159rrrmGDz74gO3btwOwYcMGVq9ezbBhwwAd64ZSm+Oak5NDaWmpV58OHTrQu3fvBjn2zWpCy/NRWFhIeXk5MTExXu0xMTHk5+fbVJX/McaQnp7ONddcQ+/evQE8x7eqY//dd99d8BqbuoULF7Ju3Tq++OKLSu/pWNefHTt2MGfOHNLT05k6dSqff/4548ePx+l0ctddd+lY16NJkybhcrno0aMHQUFBlJeXM336dH7xi18A+nvdUGpzXPPz8wkJCaFNmzaV+jTEb6dCjY8CAgK81o0xldqk7u699142btzI6tWrK72nY3/+du/ezYQJE1ixYgWhoaHV9tOxPn8VFRUkJSXxxBNPANC3b182b97MnDlzuOuuuzz9dKzP36JFi1iwYAGvvPIKl112GevXryctLY0OHTowduxYTz8d64ZRl+PaUMdep59qKTo6mqCgoErJsqCgoFJKlbq57777WLJkCR999BEdO3b0tMfGxgLo2NeDnJwcCgoKSExMxOFw4HA4yMrK4rnnnsPhcHiOp471+Wvfvj29evXyauvZsyd5eXmA/l7Xp/vvv5/Jkyfz85//nMsvv5wxY8YwceJEMjIyAB3rhlKb4xobG0tJSQmHDh2qtk99UqippZCQEBITE8nMzPRqz8zMpH///jZV5R+MMdx777288cYbfPjhh8TFxXm9HxcXR2xsrNexLykpISsrS8feRzfccAObNm1i/fr1nldSUhJ33HEH69evJz4+Xse6ngwYMKDSowm2b99Oly5dAP29rk/Hjx8nMND75ywoKMhzS7eOdcOozXFNTEwkODjYq8++ffv48ssvG+bY1/ulx37s1C3d8+fPN1u2bDFpaWkmPDzc7Nq1y+7SmrTf/va3JjIy0qxcudLs27fP8zp+/Linz4wZM0xkZKR54403zKZNm8wvfvEL3Y5ZT868+8kYHev68vnnnxuHw2GmT59uvv76a/Ovf/3LtGjRwixYsMDTR8e6fowdO9b86Ec/8tzS/cYbb5jo6GjzwAMPeProWNfNkSNHTG5ursnNzTWAefbZZ01ubq7nUSa1Oa6pqammY8eO5v333zfr1q0z119/vW7pbiyef/5506VLFxMSEmISEhI8tx1L3QFVvl588UVPn4qKCvPwww+b2NhY43Q6zaBBg8ymTZvsK9qPnB1qdKzrz9KlS03v3r2N0+k0PXr0MPPmzfN6X8e6frjdbjNhwgTTuXNnExoaauLj4820adNMcXGxp4+Odd189NFHVf77PHbsWGNM7Y5rUVGRuffee03btm1NWFiYGTFihMnLy2uQegOMMab+x39ERERELixdUyMiIiJ+QaFGRERE/IJCjYiIiPgFhRoRERHxCwo1IiIi4hcUakRERMQvKNSIiIiIX1CoEREREb+gUCMiIiJ+QaFGRERE/IJCjYiIiPgFhRoRERHxC/8fLd2ekRekri0AAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "tps_execution = [results[i][\"exec_time\"] for i in states]\n", - "tps_execution_np = [results_np[i][\"Backprop\"] + results_np[i][\"Forward pass\"] for i in states]\n", - "plt.plot(states, tps_execution, color=\"C1\")\n", - "plt.plot(states, tps_execution_np, color=\"limegreen\")" - ] - }, - { - "cell_type": "code", - "execution_count": 124, - "id": "f2dbe75f", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.0017101200064644217\n" - ] - } - ], - "source": [ - "print(results[20][\"exec_time\"])" - ] - }, - { - "cell_type": "code", - "execution_count": 126, - "id": "614d5f94", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.00022822999162599444\n" - ] - } - ], - "source": [ - "print(results_np[20][\"Backprop\"] + results_np[20][\"Forward pass\"])" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "CausalPy", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From fc48f87287413c717465ee1e625c4a0e52c09ab3 Mon Sep 17 00:00:00 2001 From: Jean Van Dyk Date: Thu, 7 Aug 2025 17:54:07 +0200 Subject: [PATCH 3/4] Adding Gradient with respect to T --- notebooks/Kalman_Filter_Gradient.ipynb | 144 +++++++++++++++++-------- 1 file changed, 100 insertions(+), 44 deletions(-) diff --git a/notebooks/Kalman_Filter_Gradient.ipynb b/notebooks/Kalman_Filter_Gradient.ipynb index 7bea8141..ff31dbbb 100644 --- a/notebooks/Kalman_Filter_Gradient.ipynb +++ b/notebooks/Kalman_Filter_Gradient.ipynb @@ -10,7 +10,7 @@ }, { "cell_type": "code", - "execution_count": 214, + "execution_count": 1, "id": "90979a41", "metadata": {}, "outputs": [], @@ -40,18 +40,10 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "fdb156d6", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - ".f at 0x000001820DC942C0>\n" - ] - } - ], + "outputs": [], "source": [ "mod = (\n", " sts.LevelTrendComponent(order=2, innovations_order=[0, 1], name='level') +\n", @@ -87,7 +79,7 @@ }, { "cell_type": "code", - "execution_count": 218, + "execution_count": 3, "id": "3661408d", "metadata": {}, "outputs": [], @@ -140,7 +132,7 @@ }, { "cell_type": "code", - "execution_count": 219, + "execution_count": 4, "id": "35351096", "metadata": {}, "outputs": [], @@ -268,7 +260,7 @@ }, { "cell_type": "code", - "execution_count": 132, + "execution_count": 5, "id": "ee21ef4e", "metadata": {}, "outputs": [], @@ -330,7 +322,7 @@ }, { "cell_type": "code", - "execution_count": 133, + "execution_count": 6, "id": "8c89b018", "metadata": {}, "outputs": [], @@ -387,7 +379,7 @@ }, { "cell_type": "code", - "execution_count": 134, + "execution_count": 7, "id": "bba53a26", "metadata": {}, "outputs": [], @@ -426,7 +418,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 8, "id": "c17949b7", "metadata": {}, "outputs": [], @@ -441,7 +433,7 @@ "id": "f0bc0287", "metadata": {}, "source": [ - "## Gradient with respect to **H**\n", + "### Gradient with respect to **H**\n", "\n", "From the article we have :\n", "\n", @@ -472,7 +464,7 @@ }, { "cell_type": "code", - "execution_count": 135, + "execution_count": 9, "id": "84cb6867", "metadata": {}, "outputs": [], @@ -499,6 +491,68 @@ " return K.T @ P_filtered_grad @ K - 0.5 * K.T @ a_filtered_grad @ v.T @ F_inv - 0.5 * F_inv @ v @ a_filtered_grad.T @ K + F_inv - F_inv @ v @ v.T @ F_inv" ] }, + { + "cell_type": "markdown", + "id": "4fa2ffc0", + "metadata": {}, + "source": [ + "### Gradient with respect to **T**\n", + "\n", + "This gradient was not given in the article. Here are the steps that got me to this expression :\n", + "\n", + "1 - Only $x_{n|n-1}$ and $P_{n|n-1}$ depends on $T_n$. Hence :\n", + "$$\n", + "\\frac{\\partial L}{\\partial T} = \\frac{\\partial L}{\\partial x_{n|n-1}} \\frac{\\partial x_{n|n-1}}{\\partial T} + \\frac{\\partial L}{\\partial P_{n|n-1}} \\frac{\\partial T}{\\partial P_{n|n-1}}\n", + "$$\n", + "2 - Using the equation (11) and (12) of the article, on the (1), we directly got that :\n", + "$$\n", + "\\frac{\\partial L}{\\partial x_{n|n-1}} \\frac{\\partial x_{n|n-1}}{\\partial T} = \\frac{\\partial L}{\\partial x_{n|n-1}} x_{n-1|n-1}^T\n", + "$$\n", + "3 - Recognizing the first quadratic form in the equation (2), and using equation (11) we got :\n", + "$$\n", + "\\frac{\\partial L}{\\partial P_{n|n-1}} \\frac{\\partial P_{n|n-1}}{\\partial T^T} = P_{n|n-1}T_n^T \\frac{\\partial L}{\\partial P_{n|n-1}}^T + P_{n|n-1}^T T_n^T \\frac{\\partial L}{\\partial P_{n|n-1}}\n", + "$$\n", + "4 - Now transposing to get the dependencies on T :\n", + "$$\n", + "\\frac{\\partial L}{\\partial P_{n|n-1}} \\frac{\\partial P_{n|n-1}}{\\partial T} = \\frac{\\partial L}{\\partial P_{n|n-1}} T_n P_{n|n-1}^T +\\frac{\\partial L}{\\partial P_{n|n-1}}^T T_n P_{n|n-1}\n", + "$$\n", + "5 - Finally, we have :\n", + "$$\n", + "\\frac{\\partial L}{\\partial T} = \\frac{\\partial L}{\\partial x_{n|n-1}} x_{n-1|n-1}^T + \\frac{\\partial L}{\\partial P_{n|n-1}} T_n P_{n|n-1}^T +\\frac{\\partial L}{\\partial P_{n|n-1}}^T T_n P_{n|n-1}\n", + "$$" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "9a560ed9", + "metadata": {}, + "outputs": [], + "source": [ + "def grad_T(inp, out, out_grad):\n", + " y, a, P, T, Z, H, Q = inp\n", + " a_hat_grad, P_h_grad, y_grad = out_grad\n", + "\n", + " y_hat = Z.dot(a)\n", + " v = y - y_hat\n", + "\n", + " PZT = P.dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + " F_inv = pt.linalg.inv(F)\n", + "\n", + " K = PZT.dot(F_inv)\n", + " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", + "\n", + " v = v.dimshuffle(0, 'x')\n", + " a = a.dimshuffle(0, 'x')\n", + " a_hat_grad = a_hat_grad.dimshuffle(0, 'x')\n", + "\n", + " a_filtered = a + K.dot(v)\n", + " P_filtered = I_KZ @ P\n", + "\n", + " return a_hat_grad @ a_filtered.T + P_h_grad @ T @ P_filtered.T + P_h_grad.T @ T @ P_filtered" + ] + }, { "cell_type": "markdown", "id": "bd458dee", @@ -509,7 +563,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "afb362e5", "metadata": {}, "outputs": [], @@ -566,7 +620,7 @@ }, { "cell_type": "code", - "execution_count": 259, + "execution_count": 12, "id": "7cead2c1", "metadata": {}, "outputs": [], @@ -576,7 +630,7 @@ "kalman_step_op = OpFromGraph(\n", " inputs=[y_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", " outputs=kalman_step(y_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym),\n", - " lop_overrides=[grad_y, grad_a_hat, grad_P_hat, None, None, grad_H, grad_Q],\n", + " lop_overrides=[grad_y, grad_a_hat, grad_P_hat, grad_T, None, grad_H, grad_Q],\n", " inline=True\n", ")\n", "\n", @@ -604,7 +658,7 @@ }, { "cell_type": "code", - "execution_count": 264, + "execution_count": 13, "id": "b6eb5d48", "metadata": {}, "outputs": [], @@ -665,7 +719,7 @@ }, { "cell_type": "code", - "execution_count": 253, + "execution_count": 14, "id": "908946b0", "metadata": {}, "outputs": [], @@ -712,7 +766,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "a85fe92e", "metadata": {}, "outputs": [], @@ -769,7 +823,7 @@ }, { "cell_type": "code", - "execution_count": 254, + "execution_count": 16, "id": "27a60fb3", "metadata": {}, "outputs": [], @@ -779,7 +833,7 @@ }, { "cell_type": "code", - "execution_count": 257, + "execution_count": 17, "id": "a413c8e9", "metadata": {}, "outputs": [ @@ -787,7 +841,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "defaultdict(, {'exec_time': 0.016296579997288063})\n" + "defaultdict(, {'exec_time': 0.017576184973586352})\n" ] } ], @@ -797,7 +851,7 @@ }, { "cell_type": "code", - "execution_count": 260, + "execution_count": 18, "id": "d35b98d6", "metadata": {}, "outputs": [], @@ -807,7 +861,7 @@ }, { "cell_type": "code", - "execution_count": 261, + "execution_count": 19, "id": "539c18c2", "metadata": {}, "outputs": [ @@ -815,7 +869,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "defaultdict(, {'exec_time': 0.015451419999590144})\n" + "defaultdict(, {'exec_time': 0.021262520016171044})\n" ] } ], @@ -825,7 +879,7 @@ }, { "cell_type": "code", - "execution_count": 267, + "execution_count": 20, "id": "1e633e75", "metadata": {}, "outputs": [], @@ -835,7 +889,7 @@ }, { "cell_type": "code", - "execution_count": 268, + "execution_count": 21, "id": "7118dfec", "metadata": {}, "outputs": [ @@ -843,7 +897,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "defaultdict(, {'Forward pass': 0.00269013000652194, 'Backprop': 0.002321220003068447})\n" + "defaultdict(, {'Forward pass': 0.002400995010975749, 'Backprop': 0.0018996200058609247})\n" ] } ], @@ -869,7 +923,7 @@ }, { "cell_type": "code", - "execution_count": 274, + "execution_count": 22, "id": "fbae0189", "metadata": {}, "outputs": [], @@ -905,7 +959,7 @@ }, { "cell_type": "code", - "execution_count": 278, + "execution_count": 23, "id": "c3a114b2", "metadata": {}, "outputs": [ @@ -914,25 +968,25 @@ "output_type": "stream", "text": [ "Comparison between classic a0 gradient and our custom OpFromGraph : True\n", - "Comparison between classic a0 gradient and our handmaid NumPy backprop : True\n" + "Comparison between classic a0 gradient and our handmade NumPy backprop : True\n" ] } ], "source": [ "print(\"Comparison between classic a0 gradient and our custom OpFromGraph :\", np.allclose(grad_a0, grad_a0_op))\n", - "print(\"Comparison between classic a0 gradient and our handmaid NumPy backprop :\", np.allclose(grad_a0, grad_a0_np))" + "print(\"Comparison between classic a0 gradient and our handmade NumPy backprop :\", np.allclose(grad_a0, grad_a0_np))" ] }, { "cell_type": "code", - "execution_count": 279, + "execution_count": 24, "id": "867d5e2f", "metadata": {}, "outputs": [], "source": [ "# First the classic way with autodiff\n", "\n", - "grad_list = pt.grad(loss, [data_sym, a0_sym, P0_sym, H_sym, Q_sym])\n", + "grad_list = pt.grad(loss, [data_sym, a0_sym, P0_sym, T_sym, H_sym, Q_sym])\n", "f_grad = pytensor.function(\n", " inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", " outputs=grad_list,\n", @@ -942,7 +996,7 @@ "\n", "# Now using our OpFromGraph custom gradient\n", "\n", - "grad_list_op = pt.grad(loss_op, [data_sym, a0_sym, P0_sym, H_sym, Q_sym])\n", + "grad_list_op = pt.grad(loss_op, [data_sym, a0_sym, P0_sym, T_sym, H_sym, Q_sym])\n", "f_grad = pytensor.function(\n", " inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", " outputs=grad_list_op,\n", @@ -953,7 +1007,7 @@ }, { "cell_type": "code", - "execution_count": 289, + "execution_count": 25, "id": "25f0a57b", "metadata": {}, "outputs": [ @@ -964,6 +1018,7 @@ "Comparison between classic y gradient and our custom OpFromGraph : True\n", "Comparison between classic a0 gradient and our custom OpFromGraph : True\n", "Comparison between classic P0 gradient and our custom OpFromGraph : True\n", + "Comparison between classic T gradient and our custom OpFromGraph : True\n", "Comparison between classic H gradient and our custom OpFromGraph : True\n", "Comparison between classic Q gradient and our custom OpFromGraph : True\n" ] @@ -973,8 +1028,9 @@ "print(\"Comparison between classic y gradient and our custom OpFromGraph :\", np.allclose(grad_a0[0], grad_a0_op[0]))\n", "print(\"Comparison between classic a0 gradient and our custom OpFromGraph :\", np.allclose(grad_a0[1], grad_a0_op[1]))\n", "print(\"Comparison between classic P0 gradient and our custom OpFromGraph :\", np.allclose((grad_a0[2] + grad_a0[2].T)/2, grad_a0_op[2]))\n", - "print(\"Comparison between classic H gradient and our custom OpFromGraph :\", np.allclose(grad_a0[3], grad_a0_op[3]))\n", - "print(\"Comparison between classic Q gradient and our custom OpFromGraph :\", np.allclose((grad_a0[4] + grad_a0[4].T)/2, grad_a0_op[4]))" + "print(\"Comparison between classic T gradient and our custom OpFromGraph :\", np.allclose(grad_a0[3], grad_a0_op[3]))\n", + "print(\"Comparison between classic H gradient and our custom OpFromGraph :\", np.allclose(grad_a0[4], grad_a0_op[4]))\n", + "print(\"Comparison between classic Q gradient and our custom OpFromGraph :\", np.allclose((grad_a0[5] + grad_a0[5].T)/2, grad_a0_op[5]))" ] } ], From 38385822541d2954427cdda3e3bd10a74efcd0e6 Mon Sep 17 00:00:00 2001 From: Jean Van Dyk Date: Thu, 14 Aug 2025 14:33:41 +0200 Subject: [PATCH 4/4] Adding the gradient with respect to Z --- notebooks/Kalman_Filter_Gradient.ipynb | 300 +++++++++++++++++++++---- 1 file changed, 261 insertions(+), 39 deletions(-) diff --git a/notebooks/Kalman_Filter_Gradient.ipynb b/notebooks/Kalman_Filter_Gradient.ipynb index ff31dbbb..ea7ed556 100644 --- a/notebooks/Kalman_Filter_Gradient.ipynb +++ b/notebooks/Kalman_Filter_Gradient.ipynb @@ -10,10 +10,19 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "90979a41", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING (pytensor.configdefaults): g++ not available, if using conda: `conda install gxx`\n", + "WARNING (pytensor.configdefaults): g++ not detected! PyTensor will be unable to compile C-implementations and will default to Python. Performance may be severely degraded. To remove this warning, set PyTensor flags cxx to an empty string.\n" + ] + } + ], "source": [ "import numpy as np\n", "import pandas as pd\n", @@ -40,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "fdb156d6", "metadata": {}, "outputs": [], @@ -79,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "3661408d", "metadata": {}, "outputs": [], @@ -132,7 +141,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "35351096", "metadata": {}, "outputs": [], @@ -260,7 +269,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "ee21ef4e", "metadata": {}, "outputs": [], @@ -322,7 +331,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "8c89b018", "metadata": {}, "outputs": [], @@ -379,7 +388,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "bba53a26", "metadata": {}, "outputs": [], @@ -418,7 +427,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "c17949b7", "metadata": {}, "outputs": [], @@ -464,7 +473,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "id": "84cb6867", "metadata": {}, "outputs": [], @@ -524,7 +533,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "id": "9a560ed9", "metadata": {}, "outputs": [], @@ -553,6 +562,177 @@ " return a_hat_grad @ a_filtered.T + P_h_grad @ T @ P_filtered.T + P_h_grad.T @ T @ P_filtered" ] }, + { + "cell_type": "markdown", + "id": "32c8125d", + "metadata": {}, + "source": [ + "### Gradient with respect to Z\n", + "\n", + "To obtain this gradient, I used the matrix differential + trace trick. Meaning that is we consider that $Z_n$ influences the loss only through $P_{n|n}$ and $a_{n|n}$, then $\\frac{dL}{dZ_n}$ is the matrix that verify :\n", + "\n", + "$$\n", + "dL = tr((\\frac{dL}{dP_{n|n}})^TdP_{n|n}) + tr((\\frac{dL}{da_{n|n}})^Tda_{n|n}) = tr((\\frac{dL}{dZ_n})^TdZ_n)\n", + "$$\n", + "\n", + "We now list the intermediate quantities that depend on $Z_n$ :\n", + "\n", + "$$\n", + "\\begin{align}\n", + "dv_n &= - dZ_n a_{n|n-1} \\\\\n", + "dS_n &= dZ_n P_{n|n-1} Z_n^T + Z_n P_{n|n-1} (dZ_n)^T \\\\\n", + "d(S_n^{-1}) &= -S_n^{-1} dS_n S_n^{-1} \\\\\n", + "dK_n &= P_{n|n-1}((dZ_n)^T S_n^{-1} + Z_n^T d(S_n^{-1})) \\\\\n", + "da_{n|n} &= dK_n v_n + K_n dv_n \\\\\n", + "dP_{n|n} &= - (dK_n Z_n + K_n dZ_n) P_{n|n-1} \\\\\n", + "\\end{align}\n", + "$$\n", + "\n", + "Okay, now let's put all that together to get an expression of $da_{n|n}$ and $dP_{n|n}$ that only relies on $dZ_n$ :\n", + "\n", + "$$\n", + "\\begin{align}\n", + "da_{n|n} &= dK_n v_n + K_n dv_n \\\\\n", + "&= P_{n|n-1}((dZ_n)^T S_n^{-1} + Z_n^T d(S_n^{-1})) v_n - K_n dZ_n a_{n|n-1} \\\\\n", + "&= P_{n|n-1}((dZ_n)^T S_n^{-1} - Z_n^T S_n^{-1} dS_n S_n^{-1}) v_n - K_n dZ_n a_{n|n-1} \\\\\n", + "&= P_{n|n-1}((dZ_n)^T S_n^{-1} - Z_n^T S_n^{-1} (dZ_n P_{n|n-1} Z_n^T + Z_n P_{n|n-1} (dZ_n)^T) S_n^{-1}) v_n - K_n dZ_n a_{n|n-1} \\\\\n", + "&= P_{n|n-1} (dZ_n)^T S_n^{-1} v_n \\\\\n", + "&- P_{n|n-1} Z_n^T S_n^{-1} dZ_n P_{n|n-1} Z_n^T S_n^{-1} v_n \\\\\n", + "&- P_{n|n-1} Z_n^T S_n^{-1} Z_n P_{n|n-1} (dZ_n)^T S_n^{-1} v_n \\\\\n", + "&- K_n dZ_n a_{n|n-1} \\\\\n", + "\\\\\n", + "dP_{n|n} &= - (dK_n Z_n + K_n dZ_n) P_{n|n-1} \\\\\n", + "&= - (P_{n|n-1}((dZ_n)^T S_n^{-1} + Z_n^T d(S_n^{-1})) Z_n + K_n dZ_n) P_{n|n-1} \\\\\n", + "&= - (P_{n|n-1}((dZ_n)^T S_n^{-1} - Z_n^T S_n^{-1} dS_n S_n^{-1}) Z_n + K_n dZ_n) P_{n|n-1} \\\\\n", + "&= - (P_{n|n-1}((dZ_n)^T S_n^{-1} - Z_n^T S_n^{-1} (dZ_n P_{n|n-1} Z_n^T + Z_n P_{n|n-1} (dZ_n)^T) S_n^{-1}) Z_n + K_n dZ_n) P_{n|n-1} \\\\\n", + "&= - P_{n|n-1}(dZ_n)^T S_n^{-1} Z_n P_{n|n-1} \\\\\n", + "&+ P_{n|n-1} Z_n^T S_n^{-1} dZ_n P_{n|n-1} Z_n^T S_n^{-1} Z_n P_{n|n-1} \\\\\n", + "&+ P_{n|n-1} Z_n^T S_n^{-1} Z_n P_{n|n-1} (dZ_n)^T S_n^{-1} Z_n P_{n|n-1} \\\\\n", + "&- K_n dZ_n P_{n|n-1} \\\\\n", + "\\end{align}\n", + "$$\n", + "\n", + "Now, going back to the trace. We'll use the following facts : \n", + "- **ciclicity of the trace** : $tr(A) = tr(A^T)$ ; \n", + "- **transpose invariance** : $tr(ABC) = tr(CAB) = tr(BCA)$. \n", + "\n", + "Also to make this easier to follow, I'll go term by term, using the additivity of the trace. Let's start with the terms of $da_{n|n}$:\n", + "\n", + "$$\n", + "\\begin{align}\n", + "&tr((\\frac{dL}{da_{n|n}})^T P_{n|n-1} (dZ_n)^T S_n^{-1} v_n) = tr(v_n^T S_n^{-T} dZ_n P_{n|n-1}^T \\frac{dL}{da_{n|n}}) = tr(P_{n|n-1}^T \\frac{dL}{da_{n|n}} v_n^T S_n^{-T} dZ_n) \\\\\n", + "&tr((\\frac{dL}{da_{n|n}})^T P_{n|n-1} Z_n^T S_n^{-1} dZ_n P_{n|n-1} Z_n^T S_n^{-1} v_n) = tr(P_{n|n-1} Z_n^T S_n^{-1} v_n (\\frac{dL}{da_{n|n}})^T P_{n|n-1} Z_n^T S_n^{-1} dZ_n) \\\\\n", + "&tr((\\frac{dL}{da_{n|n}})^T P_{n|n-1} Z_n^T S_n^{-1} Z_n P_{n|n-1} (dZ_n)^T S_n^{-1} v_n) = tr(v_n^T S_n^{-T} dZ_n P_n{n|n-1}^T Z_n^T S_n^{-T} Z_n P_{n|n-1}^T \\frac{dL}{da_{n|n}}) \\\\\n", + "&= tr(P_n{n|n-1}^T Z_n^T S_n^{-T} Z_n P_{n|n-1}^T \\frac{dL}{a_{n|n}} v_n^T S_n^{-T} dZ_n) \\\\\n", + "&tr((\\frac{dL}{da_{n|n}})^T K_n dZ_n a_{n|n-1}) = tr(a_{n|n-1} (\\frac{dL}{dda_{n|n}})^T K_n dZ_n) \\\\\n", + "\\\\\n", + "\\end{align}\n", + "$$\n", + "\n", + "So the contibution of $Z_n$ through $a_{n|n}$ is :\n", + "$$\n", + "\\begin{align}\n", + "&tr((\\frac{dL}{da_{n|n}})^T da_{n|n}) = tr((P_{n|n-1}^T \\frac{dL}{da_{n|n}} v_n^T S_n^{-T} - P_{n|n-1} Z_n^T S_n^{-1} v_n (\\frac{dL}{da_{n|n}})^T P_{n|n-1} Z_n^T S_n^{-1} \\\\\n", + "&- P_n{n|n-1}^T Z_n^T S_n^{-T} Z_n P_{n|n-1}^T \\frac{dL}{da_{n|n}} v_n^T S_n^{-T} - a_{n|n-1} (\\frac{dL}{da_{n|n}})^T K_n) dZ_n) \\\\\n", + "&= tr((P_{n|n-1}^T \\frac{dL}{da_{n|n}} v_n^T S_n^{-T} - K_n v_n (\\frac{dL}{da_{n|n}})^T K_n - P_n{n|n-1}^T Z_n^T K_n^T \\frac{dL}{da_{n|n}} v_n^T S_n^{-T} - a_{n|n-1} (\\frac{dL}{da_{n|n}})^T K_n) dZ_n)\n", + "\\end{align}\n", + "$$\n", + "\n", + "Let's do the same with the terms of $dP_{n|n}$:\n", + "\n", + "$$\n", + "\\begin{align}\n", + "&tr((\\frac{dL}{dP_{n|n}})^T P_{n|n-1}(dZ_n)^T S_n^{-1} Z_n P_{n|n-1}) = tr(P_{n|n-1}^T Z_n^T S_n^{-T} dZ_n P_{n|n-1}^T \\frac{dL}{dP_{n|n}}) = tr(P_{n|n-1}^T \\frac{dL}{dP_{n|n}} P_{n|n-1}^T Z_n^T S_n^{-T} dZ_n) \\\\\n", + "&tr((\\frac{dL}{dP_{n|n}})^T P_{n|n-1} Z_n^T S_n^{-1} dZ_n P_{n|n-1} Z_n^T S_n^{-1} Z_n P_{n|n-1}) = tr(P_{n|n-1} Z_n^T S_n^{-1} Z_n P_{n|n-1} (\\frac{dL}{dP_{n|n}})^T P_{n|n-1} Z_n^T S_n^{-1} dZ_n) \\\\\n", + "&tr((\\frac{dL}{dP_{n|n}})^T P_{n|n-1} Z_n^T S_n^{-1} Z_n P_{n|n-1} (dZ_n)^T S_n^{-1} Z_n P_{n|n-1}) = tr(P_{n|n-1}^T Z_n^T S_n^{-T} dZ_n P_n{n|n-1}^T Z_n^T S_n^{-T} Z_n P_{n|n-1}^T \\frac{dL}{dP_{n|n}}) \\\\\n", + "&= tr(P_{n|n-1}^T Z_n^T S_n^{-T} Z_n P_{n|n-1}^T \\frac{dL}{dP_{n|n}} P_{n|n-1}^T Z_n^T S_n^{-T} dZ_n) \\\\\n", + "&tr((\\frac{dL}{dP_{n|n}})^T K_n dZ_n P_{n|n-1}) = tr(P_{n|n-1} (\\frac{dL}{dP_{n|n}})^T K_n dZ_n) \\\\\n", + "\\\\\n", + "\\end{align}\n", + "$$\n", + "\n", + "So the contibution of $Z_n$ through $P_{n|n}$ is :\n", + "\n", + "$$\n", + "\\begin{align}\n", + "&tr((\\frac{dL}{dP_{n|n}})^T dP_{n|n}) = tr((- P_{n|n-1}^T \\frac{dL}{dP_{n|n}} P_{n|n-1}^T Z_n^T S_n^{-T} + P_{n|n-1} Z_n^T S_n^{-1} Z_n P_{n|n-1} (\\frac{dL}{dP_{n|n}})^T P_{n|n-1} Z_n^T S_n^{-1} \\\\\n", + "&+ P_{n|n-1}^T Z_n^T S_n^{-T} Z_n P_{n|n-1}^T \\frac{dL}{dP_{n|n}} P_{n|n-1}^T Z_n^T S_n^{-T} - P_{n|n-1} (\\frac{dL}{dP_{n|n}})^T K_n) dZ_n) \\\\\n", + "&= tr((- P_{n|n-1}^T \\frac{dL}{dP_{n|n}} P_{n|n-1}^T Z_n^T S_n^{-T} + K_n Z_n P_{n|n-1} (\\frac{dL}{dP_{n|n}})^T K_n + P_{n|n-1}^T Z_n^T K_n^T \\frac{dL}{dP_{n|n}} P_{n|n-1}^T Z_n^T S_n^{-T} - P_{n|n-1} (\\frac{dL}{dP_{n|n}})^T K_n) dZ_n)\n", + "\\end{align}\n", + "$$\n", + "\n", + "We finally get the gradient of the loss according to $Z_n$, considering his dependencies on $a_{n|n}$ and $P_{n|n}$ :\n", + "\n", + "$$\n", + "\\begin{align}\n", + "(\\frac{dL}{dZ_n})^T &= (P_{n|n-1}^T \\frac{dL}{da_{n|n}} v_n^T S_n^{-T} - K_n v_n (\\frac{dL}{da_{n|n}})^T K_n - P_{n|n-1}^T Z_n^T K_n^T \\frac{dL}{da_{n|n}} v_n^T S_n^{-T} - a_{n|n-1} (\\frac{dL}{da_{n|n}})^T K_n) \\\\\n", + "&+ (- P_{n|n-1}^T \\frac{dL}{dP_{n|n}} P_{n|n-1}^T Z_n^T S_n^{-T} + K_n Z_n P_{n|n-1} (\\frac{dL}{dP_{n|n}})^T K_n + P_{n|n-1}^T Z_n^T K_n^T \\frac{dL}{dP_{n|n}} P_{n|n-1}^T Z_n^T S_n^{-T} - P_{n|n-1} (\\frac{dL}{dP_{n|n}})^T K_n) \\\\\n", + "\\end{align}\n", + "$$\n", + "\n", + "And after transposition :\n", + "\n", + "$$\n", + "\\begin{align}\n", + "\\frac{dL}{dZ_n} &= S_n^{-1} v_n (\\frac{dL}{da_{n|n}})^T P_{n|n-1} - K_n^T \\frac{dL}{da_{n|n}} v_n^T K_n^T - S_n^{-1} v_n (\\frac{dL}{da_{n|n}})^T K_n Z_n P_{n|n-1} - K_n^T \\frac{dL}{da_{n|n}} a_{n|n-1}^T \\\\\n", + "&- S_n^{-1} Z_n P_{n|n-1} (\\frac{dL}{dP_{n|n}})^T P_{n|n-1} + K_n^T \\frac{dL}{dP_{n|n}} P_{n|n-1}^T Z_n^T K_n^T + S_n^{-1} Z_n P_{n|n-1} (\\frac{dL}{dP_{n|n}})^T K_n Z_n P_{n|n-1} - K_n^T \\frac{dL}{dP_{n|n}} P_{n|n-1}^T \\\\\n", + "\\end{align}\n", + "$$" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "df925ee8", + "metadata": {}, + "outputs": [], + "source": [ + "def grad_Z(inp, out, out_grad):\n", + "\n", + " y, a, P, T, Z, H, Q = inp\n", + " a_h_grad, P_h_grad, y_grad = out_grad\n", + "\n", + " y_hat = Z.dot(a)\n", + " v = y - y_hat\n", + "\n", + " PZT = P.dot(Z.T)\n", + " F = Z.dot(PZT) + H\n", + " F_inv = pt.linalg.inv(F)\n", + "\n", + " K = PZT.dot(F_inv)\n", + " I_KZ = pt.eye(a.shape[0]) - K.dot(Z)\n", + "\n", + " v = v.dimshuffle(0, 'x')\n", + " a = a.dimshuffle(0, 'x')\n", + " a_h_grad = a_h_grad.dimshuffle(0, 'x')\n", + "\n", + " a_filtered = a + K.dot(v)\n", + " P_filtered = I_KZ @ P\n", + "\n", + " a_filtered_grad = T.T @ a_h_grad\n", + " P_filtered_grad = T.T @ P_h_grad @ T\n", + "\n", + " # Contribution via Pnn\n", + "\n", + " term_P_1 = - F_inv @ Z @ P @ P_filtered_grad.T @ P\n", + " term_P_2 = K.T @ P_filtered_grad @ P.T @ Z.T @ K.T\n", + " term_P_3 = F_inv @ Z @ P @ P_filtered_grad.T @ K @ Z @ P\n", + " term_P_4 = - K.T @ P_filtered_grad @ P.T\n", + "\n", + " contrib_P = term_P_1 + term_P_2 + term_P_3 + term_P_4\n", + "\n", + " # Contibution via xnn\n", + "\n", + " term_x_1 = F_inv @ v @ a_filtered_grad.T @ P\n", + " term_x_2 = - K.T @ a_filtered_grad @ v.T @ K.T\n", + " term_x_3 = - F_inv @ v @ a_filtered_grad.T @ K @ Z @ P\n", + " term_x_4 = - K.T @ a_filtered_grad @ a.T\n", + "\n", + " contrib_x = term_x_1 + term_x_2 + term_x_3 + term_x_4\n", + "\n", + " return contrib_x + contrib_P" + ] + }, { "cell_type": "markdown", "id": "bd458dee", @@ -563,7 +743,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "id": "afb362e5", "metadata": {}, "outputs": [], @@ -620,7 +800,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 33, "id": "7cead2c1", "metadata": {}, "outputs": [], @@ -630,7 +810,7 @@ "kalman_step_op = OpFromGraph(\n", " inputs=[y_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", " outputs=kalman_step(y_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym),\n", - " lop_overrides=[grad_y, grad_a_hat, grad_P_hat, grad_T, None, grad_H, grad_Q],\n", + " lop_overrides=[grad_y, grad_a_hat, grad_P_hat, grad_T, grad_Z, grad_H, grad_Q],\n", " inline=True\n", ")\n", "\n", @@ -658,7 +838,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 15, "id": "b6eb5d48", "metadata": {}, "outputs": [], @@ -719,7 +899,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 35, "id": "908946b0", "metadata": {}, "outputs": [], @@ -728,7 +908,7 @@ " results = defaultdict(dict)\n", " exec_time = 0\n", "\n", - " grad_list = pt.grad(loss, [a0_sym])\n", + " grad_list = pt.grad(loss, [data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym])\n", " f_grad = pytensor.function(\n", " inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", " outputs=grad_list,\n", @@ -766,7 +946,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 17, "id": "a85fe92e", "metadata": {}, "outputs": [], @@ -823,17 +1003,26 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 36, "id": "27a60fb3", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\pytensor\\tensor\\rewriting\\elemwise.py:954: UserWarning: Loop fusion failed because the resulting node would exceed the kernel argument limit.\n", + " warn(\n" + ] + } + ], "source": [ "results = benchmark_kalman_gradients(loss, obs_data, a0, P0, T, Z, R, H, Q)" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 37, "id": "a413c8e9", "metadata": {}, "outputs": [ @@ -841,7 +1030,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "defaultdict(, {'exec_time': 0.017576184973586352})\n" + "defaultdict(, {'exec_time': 0.07424874001881107})\n" ] } ], @@ -851,17 +1040,26 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 38, "id": "d35b98d6", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\pytensor\\tensor\\rewriting\\elemwise.py:954: UserWarning: Loop fusion failed because the resulting node would exceed the kernel argument limit.\n", + " warn(\n" + ] + } + ], "source": [ "results_op = benchmark_kalman_gradients(loss_op, obs_data, a0, P0, T, Z, R, H, Q)" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 39, "id": "539c18c2", "metadata": {}, "outputs": [ @@ -869,7 +1067,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "defaultdict(, {'exec_time': 0.021262520016171044})\n" + "defaultdict(, {'exec_time': 0.11171145999105649})\n" ] } ], @@ -879,7 +1077,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 22, "id": "1e633e75", "metadata": {}, "outputs": [], @@ -889,7 +1087,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 23, "id": "7118dfec", "metadata": {}, "outputs": [ @@ -897,7 +1095,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "defaultdict(, {'Forward pass': 0.002400995010975749, 'Backprop': 0.0018996200058609247})\n" + "defaultdict(, {'Forward pass': 0.1035015100147575, 'Backprop': 0.024320614989846952})\n" ] } ], @@ -923,10 +1121,21 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 40, "id": "fbae0189", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\pytensor\\tensor\\rewriting\\elemwise.py:954: UserWarning: Loop fusion failed because the resulting node would exceed the kernel argument limit.\n", + " warn(\n", + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\pytensor\\tensor\\rewriting\\elemwise.py:954: UserWarning: Loop fusion failed because the resulting node would exceed the kernel argument limit.\n", + " warn(\n" + ] + } + ], "source": [ "# First the classic way with autodiff\n", "\n", @@ -959,7 +1168,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 41, "id": "c3a114b2", "metadata": {}, "outputs": [ @@ -979,14 +1188,25 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 42, "id": "867d5e2f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\pytensor\\tensor\\rewriting\\elemwise.py:954: UserWarning: Loop fusion failed because the resulting node would exceed the kernel argument limit.\n", + " warn(\n", + "c:\\Users\\jeanv\\miniconda3\\envs\\CausalPy\\Lib\\site-packages\\pytensor\\tensor\\rewriting\\elemwise.py:954: UserWarning: Loop fusion failed because the resulting node would exceed the kernel argument limit.\n", + " warn(\n" + ] + } + ], "source": [ "# First the classic way with autodiff\n", "\n", - "grad_list = pt.grad(loss, [data_sym, a0_sym, P0_sym, T_sym, H_sym, Q_sym])\n", + "grad_list = pt.grad(loss, [data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym])\n", "f_grad = pytensor.function(\n", " inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", " outputs=grad_list,\n", @@ -996,7 +1216,7 @@ "\n", "# Now using our OpFromGraph custom gradient\n", "\n", - "grad_list_op = pt.grad(loss_op, [data_sym, a0_sym, P0_sym, T_sym, H_sym, Q_sym])\n", + "grad_list_op = pt.grad(loss_op, [data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym])\n", "f_grad = pytensor.function(\n", " inputs=[data_sym, a0_sym, P0_sym, T_sym, Z_sym, H_sym, Q_sym],\n", " outputs=grad_list_op,\n", @@ -1007,7 +1227,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 43, "id": "25f0a57b", "metadata": {}, "outputs": [ @@ -1019,6 +1239,7 @@ "Comparison between classic a0 gradient and our custom OpFromGraph : True\n", "Comparison between classic P0 gradient and our custom OpFromGraph : True\n", "Comparison between classic T gradient and our custom OpFromGraph : True\n", + "Comparison between classic Z gradient and our custom OpFromGraph : False\n", "Comparison between classic H gradient and our custom OpFromGraph : True\n", "Comparison between classic Q gradient and our custom OpFromGraph : True\n" ] @@ -1029,8 +1250,9 @@ "print(\"Comparison between classic a0 gradient and our custom OpFromGraph :\", np.allclose(grad_a0[1], grad_a0_op[1]))\n", "print(\"Comparison between classic P0 gradient and our custom OpFromGraph :\", np.allclose((grad_a0[2] + grad_a0[2].T)/2, grad_a0_op[2]))\n", "print(\"Comparison between classic T gradient and our custom OpFromGraph :\", np.allclose(grad_a0[3], grad_a0_op[3]))\n", - "print(\"Comparison between classic H gradient and our custom OpFromGraph :\", np.allclose(grad_a0[4], grad_a0_op[4]))\n", - "print(\"Comparison between classic Q gradient and our custom OpFromGraph :\", np.allclose((grad_a0[5] + grad_a0[5].T)/2, grad_a0_op[5]))" + "print(\"Comparison between classic Z gradient and our custom OpFromGraph :\", np.allclose(grad_a0[4], grad_a0_op[4]))\n", + "print(\"Comparison between classic H gradient and our custom OpFromGraph :\", np.allclose(grad_a0[5], grad_a0_op[5]))\n", + "print(\"Comparison between classic Q gradient and our custom OpFromGraph :\", np.allclose((grad_a0[6] + grad_a0[6].T)/2, grad_a0_op[6]))" ] } ],