From b11743ca4b954f20333d152b4f83b1ae64ef46e9 Mon Sep 17 00:00:00 2001 From: Dikra Date: Fri, 9 Aug 2024 15:10:50 +0200 Subject: [PATCH 01/24] Remove TensorFlow references --- dlclive/dlclive.py | 287 +++++++++++++++++++++++---------------------- test.py | 8 ++ 2 files changed, 152 insertions(+), 143 deletions(-) create mode 100644 test.py diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 210671e..66cac67 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -10,27 +10,27 @@ import glob import warnings import numpy as np -import tensorflow as tf +# import tensorflow as tf import typing from pathlib import Path from typing import Optional, Tuple, List -try: - TFVER = [int(v) for v in tf.__version__.split(".")] - if TFVER[1] < 14: - from tensorflow.contrib.tensorrt import trt_convert as trt - else: - from tensorflow.python.compiler.tensorrt import trt_convert as trt -except Exception: - pass - -from dlclive.graph import ( - read_graph, - finalize_graph, - get_output_nodes, - get_output_tensors, - extract_graph, -) +# try: +# TFVER = [int(v) for v in tf.__version__.split(".")] +# if TFVER[1] < 14: +# from tensorflow.contrib.tensorrt import trt_convert as trt +# else: +# from tensorflow.python.compiler.tensorrt import trt_convert as trt +# except Exception: +# pass + +# from dlclive.graph import ( +# read_graph, +# finalize_graph, +# get_output_nodes, +# get_output_tensors, +# extract_graph, +# ) from dlclive.pose import extract_cnn_output, argmax_pose_predict, multi_pose_predict from dlclive.display import Display from dlclive import utils @@ -145,12 +145,12 @@ def __init__( # checks - if self.model_type == "tflite" and self.dynamic[0]: - self.dynamic = (False, *self.dynamic[1:]) - warnings.warn( - "Dynamic cropping is not supported for tensorflow lite inference. Dynamic cropping will not be used...", - DLCLiveWarning, - ) + # if self.model_type == "tflite" and self.dynamic[0]: + # self.dynamic = (False, *self.dynamic[1:]) + # warnings.warn( + # "Dynamic cropping is not supported for tensorflow lite inference. Dynamic cropping will not be used...", + # DLCLiveWarning, + # ) self.read_config() @@ -261,11 +261,12 @@ def init_inference(self, frame=None, **kwargs): ) # process frame - - if frame is None and (self.model_type == "tflite"): - raise DLCLiveError( - "No image was passed to initialize inference. An image must be passed to the init_inference method" - ) + + # ! TODO replace this if statement + # if frame is None and (self.model_type == "tflite"): + # raise DLCLiveError( + # "No image was passed to initialize inference. An image must be passed to the init_inference method" + # ) if frame is not None: if frame.ndim == 2: @@ -274,96 +275,96 @@ def init_inference(self, frame=None, **kwargs): # load model - if self.model_type == "base": + # if self.model_type == "base": - graph_def = read_graph(model_file) - graph = finalize_graph(graph_def) - self.sess, self.inputs, self.outputs = extract_graph( - graph, tf_config=self.tf_config - ) + # graph_def = read_graph(model_file) + # graph = finalize_graph(graph_def) + # self.sess, self.inputs, self.outputs = extract_graph( + # graph, tf_config=self.tf_config + # ) - elif self.model_type == "tflite": + # elif self.model_type == "tflite": - ### - # the frame size needed to initialize the tflite model as - # tflite does not support saving a model with dynamic input size - ### + # ### + # # the frame size needed to initialize the tflite model as + # # tflite does not support saving a model with dynamic input size + # ### - # get input and output tensor names from graph_def - graph_def = read_graph(model_file) - graph = finalize_graph(graph_def) - output_nodes = get_output_nodes(graph) - output_nodes = [on.replace("DLC/", "") for on in output_nodes] + # # get input and output tensor names from graph_def + # graph_def = read_graph(model_file) + # graph = finalize_graph(graph_def) + # output_nodes = get_output_nodes(graph) + # output_nodes = [on.replace("DLC/", "") for on in output_nodes] - tf_version_2 = tf.__version__[0] == '2' - - if tf_version_2: - converter = tf.compat.v1.lite.TFLiteConverter.from_frozen_graph( - model_file, - ["Placeholder"], - output_nodes, - input_shapes={"Placeholder": [1, processed_frame.shape[0], processed_frame.shape[1], 3]}, - ) - else: - converter = tf.lite.TFLiteConverter.from_frozen_graph( - model_file, - ["Placeholder"], - output_nodes, - input_shapes={"Placeholder": [1, processed_frame.shape[0], processed_frame.shape[1], 3]}, - ) + # tf_version_2 = tf.__version__[0] == '2' + + # if tf_version_2: + # converter = tf.compat.v1.lite.TFLiteConverter.from_frozen_graph( + # model_file, + # ["Placeholder"], + # output_nodes, + # input_shapes={"Placeholder": [1, processed_frame.shape[0], processed_frame.shape[1], 3]}, + # ) + # else: + # converter = tf.lite.TFLiteConverter.from_frozen_graph( + # model_file, + # ["Placeholder"], + # output_nodes, + # input_shapes={"Placeholder": [1, processed_frame.shape[0], processed_frame.shape[1], 3]}, + # ) - try: - tflite_model = converter.convert() - except Exception: - raise DLCLiveError( - ( - "This model cannot be converted to tensorflow lite format. " - "To use tensorflow lite for live inference, " - "make sure to set TFGPUinference=False " - "when exporting the model from DeepLabCut" - ) - ) - - self.tflite_interpreter = tf.lite.Interpreter(model_content=tflite_model) - self.tflite_interpreter.allocate_tensors() - self.inputs = self.tflite_interpreter.get_input_details() - self.outputs = self.tflite_interpreter.get_output_details() - - elif self.model_type == "tensorrt": - - graph_def = read_graph(model_file) - graph = finalize_graph(graph_def) - output_tensors = get_output_tensors(graph) - output_tensors = [ot.replace("DLC/", "") for ot in output_tensors] - - if (TFVER[0] > 1) | (TFVER[0] == 1 & TFVER[1] >= 14): - converter = trt.TrtGraphConverter( - input_graph_def=graph_def, - nodes_blacklist=output_tensors, - is_dynamic_op=True, - ) - graph_def = converter.convert() - else: - graph_def = trt.create_inference_graph( - input_graph_def=graph_def, - outputs=output_tensors, - max_batch_size=1, - precision_mode=self.precision, - is_dynamic_op=True, - ) - - graph = finalize_graph(graph_def) - self.sess, self.inputs, self.outputs = extract_graph( - graph, tf_config=self.tf_config - ) - - else: - - raise DLCLiveError( - "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( - self.model_type - ) - ) + # try: + # tflite_model = converter.convert() + # except Exception: + # raise DLCLiveError( + # ( + # "This model cannot be converted to tensorflow lite format. " + # "To use tensorflow lite for live inference, " + # "make sure to set TFGPUinference=False " + # "when exporting the model from DeepLabCut" + # ) + # ) + + # self.tflite_interpreter = tf.lite.Interpreter(model_content=tflite_model) + # self.tflite_interpreter.allocate_tensors() + # self.inputs = self.tflite_interpreter.get_input_details() + # self.outputs = self.tflite_interpreter.get_output_details() + + # elif self.model_type == "tensorrt": + + # graph_def = read_graph(model_file) + # graph = finalize_graph(graph_def) + # output_tensors = get_output_tensors(graph) + # output_tensors = [ot.replace("DLC/", "") for ot in output_tensors] + + # if (TFVER[0] > 1) | (TFVER[0] == 1 & TFVER[1] >= 14): + # converter = trt.TrtGraphConverter( + # input_graph_def=graph_def, + # nodes_blacklist=output_tensors, + # is_dynamic_op=True, + # ) + # graph_def = converter.convert() + # else: + # graph_def = trt.create_inference_graph( + # input_graph_def=graph_def, + # outputs=output_tensors, + # max_batch_size=1, + # precision_mode=self.precision, + # is_dynamic_op=True, + # ) + + # graph = finalize_graph(graph_def) + # self.sess, self.inputs, self.outputs = extract_graph( + # graph, tf_config=self.tf_config + # ) + + # else: + + # raise DLCLiveError( + # "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( + # self.model_type + # ) + # ) # get pose of first frame (first inference is often very slow) @@ -396,37 +397,37 @@ def get_pose(self, frame=None, **kwargs): frame = self.process_frame(frame) - if self.model_type in ["base", "tensorrt"]: - - pose_output = self.sess.run( - self.outputs, feed_dict={self.inputs: np.expand_dims(frame, axis=0)} - ) - - elif self.model_type == "tflite": - - self.tflite_interpreter.set_tensor( - self.inputs[0]["index"], - np.expand_dims(frame, axis=0).astype(np.float32), - ) - self.tflite_interpreter.invoke() - - if len(self.outputs) > 1: - pose_output = [ - self.tflite_interpreter.get_tensor(self.outputs[0]["index"]), - self.tflite_interpreter.get_tensor(self.outputs[1]["index"]), - ] - else: - pose_output = self.tflite_interpreter.get_tensor( - self.outputs[0]["index"] - ) - - else: - - raise DLCLiveError( - "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( - self.model_type - ) - ) + # if self.model_type in ["base", "tensorrt"]: + + # pose_output = self.sess.run( + # self.outputs, feed_dict={self.inputs: np.expand_dims(frame, axis=0)} + # ) + + # elif self.model_type == "tflite": + + # self.tflite_interpreter.set_tensor( + # self.inputs[0]["index"], + # np.expand_dims(frame, axis=0).astype(np.float32), + # ) + # self.tflite_interpreter.invoke() + + # if len(self.outputs) > 1: + # pose_output = [ + # self.tflite_interpreter.get_tensor(self.outputs[0]["index"]), + # self.tflite_interpreter.get_tensor(self.outputs[1]["index"]), + # ] + # else: + # pose_output = self.tflite_interpreter.get_tensor( + # self.outputs[0]["index"] + # ) + + # else: + + # raise DLCLiveError( + # "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( + # self.model_type + # ) + # ) # check if using TFGPUinference flag # if not, get pose from network output diff --git a/test.py b/test.py new file mode 100644 index 0000000..bfabb22 --- /dev/null +++ b/test.py @@ -0,0 +1,8 @@ +from dlclive import DLCLive, Processor +import dlclive +import cv2 + +dlc_proc = Processor() +dlc_live = DLCLive("/media1/data/dikra/dlc-live-tmp", processor=dlc_proc) +img = cv2.imread("/media1/data/dikra/fly-kevin/img001.png") +dlc_live.init_inference(frame=img) \ No newline at end of file From c8f58394b2b362e46c4e614cb7686f3c55a17613 Mon Sep 17 00:00:00 2001 From: AnnaStuckert <47814177+annastuckert@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:31:11 +0100 Subject: [PATCH 02/24] Add comments on TF questions for changing dlc-live pipeline to Pytorch --- dlclive/dlclive.py | 425 +++++++++++++++++++++++----------------- docs/install_desktop.md | 9 +- run_dlc-live.py | 18 ++ 3 files changed, 267 insertions(+), 185 deletions(-) create mode 100644 run_dlc-live.py diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 66cac67..477a4bc 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -5,39 +5,65 @@ Licensed under GNU Lesser General Public License v3.0 """ -import os -import ruamel.yaml import glob -import warnings -import numpy as np -# import tensorflow as tf +import os import typing +import warnings from pathlib import Path -from typing import Optional, Tuple, List - -# try: -# TFVER = [int(v) for v in tf.__version__.split(".")] -# if TFVER[1] < 14: -# from tensorflow.contrib.tensorrt import trt_convert as trt -# else: -# from tensorflow.python.compiler.tensorrt import trt_convert as trt -# except Exception: -# pass - -# from dlclive.graph import ( -# read_graph, -# finalize_graph, -# get_output_nodes, -# get_output_tensors, -# extract_graph, -# ) -from dlclive.pose import extract_cnn_output, argmax_pose_predict, multi_pose_predict -from dlclive.display import Display +from typing import List, Optional, Tuple + +import numpy as np +import ruamel.yaml +import tensorflow as tf + +try: + TFVER = [int(v) for v in tf.__version__.split(".")] + if TFVER[1] < 14: + from tensorflow.contrib.tensorrt import trt_convert as trt + else: + from tensorflow.python.compiler.tensorrt import trt_convert as trt +except Exception: + pass + from dlclive import utils +from dlclive.display import Display from dlclive.exceptions import DLCLiveError, DLCLiveWarning +from dlclive.graph import ( + extract_graph, + finalize_graph, + get_output_nodes, + get_output_tensors, + read_graph, +) +from dlclive.pose import argmax_pose_predict, extract_cnn_output, multi_pose_predict + if typing.TYPE_CHECKING: from dlclive.processor import Processor + +# TODO: +# graph.py the main element to import TF model - convert to pytorch implementation +# add pcutoffn to docstring + +# Q: What is the best way to test the code as we go? +# Q: if self.pose is not None: - ask Niels to go through this! + +# Q: what exactly does model_type reference? +# Q: is precision a type of qunatization? +# Q: for dynamic: First key points are predicted, then dynamic cropping is performed to 'single out' the animal, and then pose is estimated, we think. What is the difference from key point prediction to pose prediction? +# Q: what is the processor? see processor code F12 from init file - what is the 'user defined process' - could it be that if mouse = standing, perform some action? or is the process the prediction of a certain pose/set of keypoints +# Q: why have the convert2rgb function, is the stream coming from the camera different from the input needed to DLC live? +# Q: what is the parameter 'cfg'? + +# What do these do? +# self.inputs = None +# self.outputs = None +# self.tflite_interpreter = None +# self.pose = None +# self.is_initialized = False +# self.sess = None + + class DLCLive(object): """ Object that loads a DLC network and performs inference on single images (e.g. images captured from a camera feed) @@ -55,9 +81,9 @@ class DLCLive(object): precision of model weights, only for model_type='tensorrt'. Can be 'FP16' (default), 'FP32', or 'INT8' cropping : list of int - cropping parameters in pixel number: [x1, x2, y1, y2] + cropping parameters in pixel number: [x1, x2, y1, y2] #A: Maybe this is the dynamic cropping of each frame to speed of processing, so instead of analyzing the whole frame, it analyses only the part of the frame where the animal is - dynamic: triple containing (state, detectiontreshold, margin) + dynamic: triple containing (state, detectiontreshold, margin) #A: margin adds some space so the 'bbox' isn't too narrow around the animal'. First key points are predicted, then dynamic cropping is performed to 'single out' the animal, and then pose is estimated, we think. If the state is true, then dynamic cropping will be performed. That means that if an object is detected (i.e. any body part > detectiontreshold), then object boundaries are computed according to the smallest/largest x position and smallest/largest y position of all body parts. This window is expanded by the margin and from then on only the posture within this crop is analyzed (until the object is lost, i.e. dict: + def parameterization( + self, + ) -> ( + dict + ): # A: constructs a dictionary based on the object attributes based on the list of parameters """ Return Returns @@ -181,7 +215,7 @@ def parameterization(self) -> dict: """ return {param: getattr(self, param) for param in self.PARAMETERS} - def process_frame(self, frame): + def process_frame(self, frame): #'self' holds all the arguments """ Crops an image according to the object's cropping and dynamic properties. @@ -200,13 +234,15 @@ def process_frame(self, frame): frame = utils.convert_to_ubyte(frame) - if self.cropping: + if self.cropping: # if cropping is specified, it will be applied - frame = frame[ + frame = frame[ # A: this produces a cropped image based on incoming coordinates x1,x2,y1,y2 self.cropping[2] : self.cropping[3], self.cropping[0] : self.cropping[1] ] - if self.dynamic[0]: + if self.dynamic[ + 0 + ]: # to go through this if statement, the boolean would have to be = True. for it to react to false you'd have to write if not self.dynamic[0] if self.pose is not None: @@ -217,7 +253,9 @@ def process_frame(self, frame): x = self.pose[detected, 0] y = self.pose[detected, 1] - x1 = int(max([0, int(np.amin(x)) - self.dynamic[2]])) + x1 = int( + max([0, int(np.amin(x)) - self.dynamic[2]]) + ) # We think it is dtected if keypoint likelihood exceeds the dynamic threshold for dynamic cropping x2 = int(min([frame.shape[1], int(np.amax(x)) + self.dynamic[2]])) y1 = int(max([0, int(np.amin(y)) - self.dynamic[2]])) y2 = int(min([frame.shape[0], int(np.amax(y)) + self.dynamic[2]])) @@ -254,19 +292,20 @@ def init_inference(self, frame=None, **kwargs): # get model file - model_file = glob.glob(os.path.normpath(self.path + "/*.pb"))[0] + model_file = glob.glob(os.path.normpath(self.path + "/*.pb"))[ + 0 + ] # TODO TF_ref - maybe .pb format will be changed when using pytorch if not os.path.isfile(model_file): raise FileNotFoundError( "The model file {} does not exist.".format(model_file) ) # process frame - - # ! TODO replace this if statement - # if frame is None and (self.model_type == "tflite"): - # raise DLCLiveError( - # "No image was passed to initialize inference. An image must be passed to the init_inference method" - # ) + + if frame is None and (self.model_type == "tflite"): + raise DLCLiveError( + "No image was passed to initialize inference. An image must be passed to the init_inference method" + ) if frame is not None: if frame.ndim == 2: @@ -275,96 +314,112 @@ def init_inference(self, frame=None, **kwargs): # load model - # if self.model_type == "base": - - # graph_def = read_graph(model_file) - # graph = finalize_graph(graph_def) - # self.sess, self.inputs, self.outputs = extract_graph( - # graph, tf_config=self.tf_config - # ) - - # elif self.model_type == "tflite": - - # ### - # # the frame size needed to initialize the tflite model as - # # tflite does not support saving a model with dynamic input size - # ### - - # # get input and output tensor names from graph_def - # graph_def = read_graph(model_file) - # graph = finalize_graph(graph_def) - # output_nodes = get_output_nodes(graph) - # output_nodes = [on.replace("DLC/", "") for on in output_nodes] - - # tf_version_2 = tf.__version__[0] == '2' - - # if tf_version_2: - # converter = tf.compat.v1.lite.TFLiteConverter.from_frozen_graph( - # model_file, - # ["Placeholder"], - # output_nodes, - # input_shapes={"Placeholder": [1, processed_frame.shape[0], processed_frame.shape[1], 3]}, - # ) - # else: - # converter = tf.lite.TFLiteConverter.from_frozen_graph( - # model_file, - # ["Placeholder"], - # output_nodes, - # input_shapes={"Placeholder": [1, processed_frame.shape[0], processed_frame.shape[1], 3]}, - # ) - - # try: - # tflite_model = converter.convert() - # except Exception: - # raise DLCLiveError( - # ( - # "This model cannot be converted to tensorflow lite format. " - # "To use tensorflow lite for live inference, " - # "make sure to set TFGPUinference=False " - # "when exporting the model from DeepLabCut" - # ) - # ) - - # self.tflite_interpreter = tf.lite.Interpreter(model_content=tflite_model) - # self.tflite_interpreter.allocate_tensors() - # self.inputs = self.tflite_interpreter.get_input_details() - # self.outputs = self.tflite_interpreter.get_output_details() - - # elif self.model_type == "tensorrt": - - # graph_def = read_graph(model_file) - # graph = finalize_graph(graph_def) - # output_tensors = get_output_tensors(graph) - # output_tensors = [ot.replace("DLC/", "") for ot in output_tensors] - - # if (TFVER[0] > 1) | (TFVER[0] == 1 & TFVER[1] >= 14): - # converter = trt.TrtGraphConverter( - # input_graph_def=graph_def, - # nodes_blacklist=output_tensors, - # is_dynamic_op=True, - # ) - # graph_def = converter.convert() - # else: - # graph_def = trt.create_inference_graph( - # input_graph_def=graph_def, - # outputs=output_tensors, - # max_batch_size=1, - # precision_mode=self.precision, - # is_dynamic_op=True, - # ) - - # graph = finalize_graph(graph_def) - # self.sess, self.inputs, self.outputs = extract_graph( - # graph, tf_config=self.tf_config - # ) - - # else: - - # raise DLCLiveError( - # "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( - # self.model_type - # ) - # ) + if self.model_type == "base": + + graph_def = read_graph( + model_file + ) # TODO TF_ref - this load model section seems to be highly dependent on TF - likely have to be changed + graph = finalize_graph(graph_def) + self.sess, self.inputs, self.outputs = extract_graph( + graph, tf_config=self.tf_config + ) + + elif self.model_type == "tflite": + + ### + # the frame size needed to initialize the tflite model as + # tflite does not support saving a model with dynamic input size + ### + + # get input and output tensor names from graph_def + graph_def = read_graph(model_file) + graph = finalize_graph(graph_def) + output_nodes = get_output_nodes(graph) + output_nodes = [on.replace("DLC/", "") for on in output_nodes] + + tf_version_2 = tf.__version__[0] == "2" + + if tf_version_2: + converter = tf.compat.v1.lite.TFLiteConverter.from_frozen_graph( + model_file, + ["Placeholder"], + output_nodes, + input_shapes={ + "Placeholder": [ + 1, + processed_frame.shape[0], + processed_frame.shape[1], + 3, + ] + }, + ) + else: + converter = tf.lite.TFLiteConverter.from_frozen_graph( + model_file, + ["Placeholder"], + output_nodes, + input_shapes={ + "Placeholder": [ + 1, + processed_frame.shape[0], + processed_frame.shape[1], + 3, + ] + }, + ) + + try: + tflite_model = converter.convert() + except Exception: + raise DLCLiveError( + ( + "This model cannot be converted to tensorflow lite format. " + "To use tensorflow lite for live inference, " + "make sure to set TFGPUinference=False " + "when exporting the model from DeepLabCut" + ) + ) + + self.tflite_interpreter = tf.lite.Interpreter(model_content=tflite_model) + self.tflite_interpreter.allocate_tensors() + self.inputs = self.tflite_interpreter.get_input_details() + self.outputs = self.tflite_interpreter.get_output_details() + + elif self.model_type == "tensorrt": + + graph_def = read_graph(model_file) + graph = finalize_graph(graph_def) + output_tensors = get_output_tensors(graph) + output_tensors = [ot.replace("DLC/", "") for ot in output_tensors] + + if (TFVER[0] > 1) | (TFVER[0] == 1 & TFVER[1] >= 14): + converter = trt.TrtGraphConverter( + input_graph_def=graph_def, + nodes_blacklist=output_tensors, + is_dynamic_op=True, + ) + graph_def = converter.convert() + else: + graph_def = trt.create_inference_graph( + input_graph_def=graph_def, + outputs=output_tensors, + max_batch_size=1, + precision_mode=self.precision, + is_dynamic_op=True, + ) + + graph = finalize_graph(graph_def) + self.sess, self.inputs, self.outputs = extract_graph( + graph, tf_config=self.tf_config + ) + + else: + + raise DLCLiveError( + "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( + self.model_type + ) + ) # get pose of first frame (first inference is often very slow) @@ -397,45 +452,50 @@ def get_pose(self, frame=None, **kwargs): frame = self.process_frame(frame) - # if self.model_type in ["base", "tensorrt"]: + if self.model_type in [ + "base", + "tensorrt", + ]: # TODO: from here to 480 (raise DLCLiveError()) we think it is doing inference - # pose_output = self.sess.run( - # self.outputs, feed_dict={self.inputs: np.expand_dims(frame, axis=0)} - # ) + pose_output = self.sess.run( + self.outputs, feed_dict={self.inputs: np.expand_dims(frame, axis=0)} + ) - # elif self.model_type == "tflite": + elif self.model_type == "tflite": - # self.tflite_interpreter.set_tensor( - # self.inputs[0]["index"], - # np.expand_dims(frame, axis=0).astype(np.float32), - # ) - # self.tflite_interpreter.invoke() + self.tflite_interpreter.set_tensor( + self.inputs[0]["index"], + np.expand_dims(frame, axis=0).astype(np.float32), + ) + self.tflite_interpreter.invoke() - # if len(self.outputs) > 1: - # pose_output = [ - # self.tflite_interpreter.get_tensor(self.outputs[0]["index"]), - # self.tflite_interpreter.get_tensor(self.outputs[1]["index"]), - # ] - # else: - # pose_output = self.tflite_interpreter.get_tensor( - # self.outputs[0]["index"] - # ) + if len(self.outputs) > 1: + pose_output = [ + self.tflite_interpreter.get_tensor(self.outputs[0]["index"]), + self.tflite_interpreter.get_tensor(self.outputs[1]["index"]), + ] + else: + pose_output = self.tflite_interpreter.get_tensor( + self.outputs[0]["index"] + ) - # else: + else: - # raise DLCLiveError( - # "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( - # self.model_type - # ) - # ) + raise DLCLiveError( + "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( + self.model_type + ) + ) # check if using TFGPUinference flag # if not, get pose from network output if len(pose_output) > 1: - scmap, locref = extract_cnn_output(pose_output, self.cfg) + scmap, locref = extract_cnn_output( + pose_output, self.cfg + ) # scmap = the heatmaps of likelihood of different key points being placed at different positions in the frame. locref = think it is something with refining how the 'peaks' in the heatmap is created. num_outputs = self.cfg.get("num_outputs", 1) - if num_outputs > 1: + if num_outputs > 1: # seems to be if detecting more than 1 key point self.pose = multi_pose_predict( scmap, locref, self.cfg["stride"], num_outputs ) @@ -470,9 +530,8 @@ def get_pose(self, frame=None, **kwargs): return self.pose - def close(self): - """ Close tensorflow session - """ + def close(self): # seems to be deletable + """Close tensorflow session""" self.sess.close() self.sess = None diff --git a/docs/install_desktop.md b/docs/install_desktop.md index 5eb710c..923568d 100755 --- a/docs/install_desktop.md +++ b/docs/install_desktop.md @@ -7,9 +7,14 @@ We recommend that you install DeepLabCut-live in a conda environment (It is a st Create a conda environment with python 3.7 and tensorflow: +New version: ``` -conda create -n dlc-live python=3.7 tensorflow-gpu==1.13.1 # if using GPU -conda create -n dlc-live python=3.7 tensorflow==1.13.1 # if not using GPU +conda create -n dlc-live python=3.8 +conda activate dlc-live +conda install -c conda-forge pytables==3.8.0 +pip install "tensorflow-macos<2.13.0" "tensorflow-metal" "tensorpack>=0.11" "tf_slim>=1.1.0" +pip install deeplabcut-live +dlc-live-test ``` Activate the conda environment, install the DeepLabCut-live package, then test the installation: diff --git a/run_dlc-live.py b/run_dlc-live.py new file mode 100644 index 0000000..73d9ed5 --- /dev/null +++ b/run_dlc-live.py @@ -0,0 +1,18 @@ +import numpy as np +from PIL import Image + +from dlclive import DLCLive, Processor + +image = Image.open( + "/Users/annastuckert/Downloads/exported DLC model for dlc-live/img049.png" +) +img = np.asarray(image) + +dlc_proc = Processor() +dlc_live = DLCLive( + "/Users/annastuckert/Downloads/exported DLC model for dlc-live/DLC_dev-single-animal_resnet_50_iteration-1_shuffle-1", + processor=dlc_proc, +) +dlc_live.init_inference(img) +pose = dlc_live.get_pose(img) +print(pose) From 030c42ca82406161ddfbb702581c26f7254ea7dd Mon Sep 17 00:00:00 2001 From: Dikra Date: Mon, 24 Feb 2025 14:32:27 +0100 Subject: [PATCH 03/24] Vanilla pytorch inference done; Commenting out TensorFlow references in code --- dlclive/__init__.py | 3 +- dlclive/display.py | 2 +- dlclive/dlclive.py | 381 ++++++++++++++++++++++++-------------------- 3 files changed, 209 insertions(+), 177 deletions(-) diff --git a/dlclive/__init__.py b/dlclive/__init__.py index 2eff208..e4ce7e9 100644 --- a/dlclive/__init__.py +++ b/dlclive/__init__.py @@ -7,5 +7,6 @@ from dlclive.version import __version__, VERSION from dlclive.dlclive import DLCLive +from dlclive.display import Display from dlclive.processor import Processor -from dlclive.benchmark import benchmark, benchmark_videos, download_benchmarking_data +# from dlclive.benchmark import benchmark, benchmark_videos, download_benchmarking_data diff --git a/dlclive/display.py b/dlclive/display.py index cc324d8..e0acfec 100644 --- a/dlclive/display.py +++ b/dlclive/display.py @@ -64,7 +64,7 @@ def display_frame(self, frame, pose=None): pose :class:`numpy.ndarray` the pose estimated by DeepLabCut for the image """ - + im_size = (frame.shape[1], frame.shape[0]) if pose is not None: diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 477a4bc..39a5a4d 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -6,6 +6,7 @@ """ import glob +import ruamel.yaml import os import typing import warnings @@ -13,29 +14,35 @@ from typing import List, Optional, Tuple import numpy as np -import ruamel.yaml -import tensorflow as tf - -try: - TFVER = [int(v) for v in tf.__version__.split(".")] - if TFVER[1] < 14: - from tensorflow.contrib.tensorrt import trt_convert as trt - else: - from tensorflow.python.compiler.tensorrt import trt_convert as trt -except Exception: - pass - -from dlclive import utils +# import tensorflow as tf +import typing +from pathlib import Path +from typing import Optional, Tuple, List +import torch + +# try: +# TFVER = [int(v) for v in tf.__version__.split(".")] +# if TFVER[1] < 14: +# from tensorflow.contrib.tensorrt import trt_convert as trt +# else: +# from tensorflow.python.compiler.tensorrt import trt_convert as trt +# except Exception: +# pass + +# from dlclive.graph import ( +# read_graph, +# finalize_graph, +# get_output_nodes, +# get_output_tensors, +# extract_graph, +# ) + +import deeplabcut as dlc +from deeplabcut.pose_estimation_pytorch.models import PoseModel +from dlclive.pose import extract_cnn_output, argmax_pose_predict, multi_pose_predict from dlclive.display import Display +from dlclive import utils from dlclive.exceptions import DLCLiveError, DLCLiveWarning -from dlclive.graph import ( - extract_graph, - finalize_graph, - get_output_nodes, - get_output_tensors, - read_graph, -) -from dlclive.pose import argmax_pose_predict, extract_cnn_output, multi_pose_predict if typing.TYPE_CHECKING: from dlclive.processor import Processor @@ -129,10 +136,12 @@ class DLCLive(object): def __init__( self, - model_path: str, + model_path: str = None, model_type: str = "base", precision: str = "FP32", tf_config=None, + pytorch_cfg=str, + snapshot=str, cropping: Optional[List[int]] = None, dynamic: Tuple[bool, float, float] = (False, 0.5, 10), resize: Optional[float] = None, @@ -148,6 +157,8 @@ def __init__( self.cfg = None # type: typing.Optional[dict] self.model_type = model_type self.tf_config = tf_config + self.pytorch_cfg = pytorch_cfg + self.snapshot = snapshot self.precision = precision self.cropping = cropping self.dynamic = dynamic @@ -173,12 +184,12 @@ def __init__( # checks - if self.model_type == "tflite" and self.dynamic[0]: # TODO TF_ref - self.dynamic = (False, *self.dynamic[1:]) - warnings.warn( - "Dynamic cropping is not supported for tensorflow lite inference. Dynamic cropping will not be used...", - DLCLiveWarning, - ) + # if self.model_type == "tflite" and self.dynamic[0]: + # self.dynamic = (False, *self.dynamic[1:]) + # warnings.warn( + # "Dynamic cropping is not supported for tensorflow lite inference. Dynamic cropping will not be used...", + # DLCLiveWarning, + # ) self.read_config() @@ -192,7 +203,7 @@ def read_config(self): """ cfg_path = ( - Path(self.path).resolve() / "pose_cfg.yaml" + Path(self.pytorch_cfg).resolve() / "pytorch_config.yaml" ) # TODO TF_ref - replace by pytorch config - consider importing read_config function from DLC 3 - and the new config may have both detector and 'normal' config - e.g batch size could refer both to detector and key points. should be handled in the read_config from DLC3 if not cfg_path.exists(): raise FileNotFoundError( @@ -202,6 +213,7 @@ def read_config(self): ruamel_file = ruamel.yaml.YAML() self.cfg = ruamel_file.load(open(str(cfg_path), "r")) + @property def parameterization( self, @@ -275,6 +287,19 @@ def process_frame(self, frame): #'self' holds all the arguments return frame + + def load_model(self): + self.read_config() + weights = torch.load(self.snapshot) + print("Loaded weights") + print(self.cfg) + pose_model = PoseModel.build(self.cfg['model']) + print("Built pose model") + pose_model.load_state_dict(weights["model"]) + print('Loaded pretrained weights') + return pose_model + + def init_inference(self, frame=None, **kwargs): """ Load model and perform inference on first frame -- the first inference is usually very slow. @@ -292,7 +317,7 @@ def init_inference(self, frame=None, **kwargs): # get model file - model_file = glob.glob(os.path.normpath(self.path + "/*.pb"))[ + model_file = glob.glob(os.path.normpath(self.pytorch_cfg + "/*.pt"))[ 0 ] # TODO TF_ref - maybe .pb format will be changed when using pytorch if not os.path.isfile(model_file): @@ -302,10 +327,11 @@ def init_inference(self, frame=None, **kwargs): # process frame - if frame is None and (self.model_type == "tflite"): - raise DLCLiveError( - "No image was passed to initialize inference. An image must be passed to the init_inference method" - ) + # ! TODO replace this if statement + # if frame is None and (self.model_type == "tflite"): + # raise DLCLiveError( + # "No image was passed to initialize inference. An image must be passed to the init_inference method" + # ) if frame is not None: if frame.ndim == 2: @@ -314,112 +340,97 @@ def init_inference(self, frame=None, **kwargs): # load model - if self.model_type == "base": - - graph_def = read_graph( - model_file - ) # TODO TF_ref - this load model section seems to be highly dependent on TF - likely have to be changed - graph = finalize_graph(graph_def) - self.sess, self.inputs, self.outputs = extract_graph( - graph, tf_config=self.tf_config - ) + # if self.model_type == "base": + + # graph_def = read_graph(model_file) + # graph = finalize_graph(graph_def) + # self.sess, self.inputs, self.outputs = extract_graph( + # graph, tf_config=self.tf_config + # ) + + # elif self.model_type == "tflite": + + # ### + # # the frame size needed to initialize the tflite model as + # # tflite does not support saving a model with dynamic input size + # ### + + # # get input and output tensor names from graph_def + # graph_def = read_graph(model_file) + # graph = finalize_graph(graph_def) + # output_nodes = get_output_nodes(graph) + # output_nodes = [on.replace("DLC/", "") for on in output_nodes] + + # tf_version_2 = tf.__version__[0] == '2' + + # if tf_version_2: + # converter = tf.compat.v1.lite.TFLiteConverter.from_frozen_graph( + # model_file, + # ["Placeholder"], + # output_nodes, + # input_shapes={"Placeholder": [1, processed_frame.shape[0], processed_frame.shape[1], 3]}, + # ) + # else: + # converter = tf.lite.TFLiteConverter.from_frozen_graph( + # model_file, + # ["Placeholder"], + # output_nodes, + # input_shapes={"Placeholder": [1, processed_frame.shape[0], processed_frame.shape[1], 3]}, + # ) + + # try: + # tflite_model = converter.convert() + # except Exception: + # raise DLCLiveError( + # ( + # "This model cannot be converted to tensorflow lite format. " + # "To use tensorflow lite for live inference, " + # "make sure to set TFGPUinference=False " + # "when exporting the model from DeepLabCut" + # ) + # ) + + # self.tflite_interpreter = tf.lite.Interpreter(model_content=tflite_model) + # self.tflite_interpreter.allocate_tensors() + # self.inputs = self.tflite_interpreter.get_input_details() + # self.outputs = self.tflite_interpreter.get_output_details() + + # elif self.model_type == "tensorrt": + + # graph_def = read_graph(model_file) + # graph = finalize_graph(graph_def) + # output_tensors = get_output_tensors(graph) + # output_tensors = [ot.replace("DLC/", "") for ot in output_tensors] + + # if (TFVER[0] > 1) | (TFVER[0] == 1 & TFVER[1] >= 14): + # converter = trt.TrtGraphConverter( + # input_graph_def=graph_def, + # nodes_blacklist=output_tensors, + # is_dynamic_op=True, + # ) + # graph_def = converter.convert() + # else: + # graph_def = trt.create_inference_graph( + # input_graph_def=graph_def, + # outputs=output_tensors, + # max_batch_size=1, + # precision_mode=self.precision, + # is_dynamic_op=True, + # ) + + # graph = finalize_graph(graph_def) + # self.sess, self.inputs, self.outputs = extract_graph( + # graph, tf_config=self.tf_config + # ) + + # else: + + # raise DLCLiveError( + # "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( + # self.model_type + # ) + # ) - elif self.model_type == "tflite": - - ### - # the frame size needed to initialize the tflite model as - # tflite does not support saving a model with dynamic input size - ### - - # get input and output tensor names from graph_def - graph_def = read_graph(model_file) - graph = finalize_graph(graph_def) - output_nodes = get_output_nodes(graph) - output_nodes = [on.replace("DLC/", "") for on in output_nodes] - - tf_version_2 = tf.__version__[0] == "2" - - if tf_version_2: - converter = tf.compat.v1.lite.TFLiteConverter.from_frozen_graph( - model_file, - ["Placeholder"], - output_nodes, - input_shapes={ - "Placeholder": [ - 1, - processed_frame.shape[0], - processed_frame.shape[1], - 3, - ] - }, - ) - else: - converter = tf.lite.TFLiteConverter.from_frozen_graph( - model_file, - ["Placeholder"], - output_nodes, - input_shapes={ - "Placeholder": [ - 1, - processed_frame.shape[0], - processed_frame.shape[1], - 3, - ] - }, - ) - - try: - tflite_model = converter.convert() - except Exception: - raise DLCLiveError( - ( - "This model cannot be converted to tensorflow lite format. " - "To use tensorflow lite for live inference, " - "make sure to set TFGPUinference=False " - "when exporting the model from DeepLabCut" - ) - ) - - self.tflite_interpreter = tf.lite.Interpreter(model_content=tflite_model) - self.tflite_interpreter.allocate_tensors() - self.inputs = self.tflite_interpreter.get_input_details() - self.outputs = self.tflite_interpreter.get_output_details() - - elif self.model_type == "tensorrt": - - graph_def = read_graph(model_file) - graph = finalize_graph(graph_def) - output_tensors = get_output_tensors(graph) - output_tensors = [ot.replace("DLC/", "") for ot in output_tensors] - - if (TFVER[0] > 1) | (TFVER[0] == 1 & TFVER[1] >= 14): - converter = trt.TrtGraphConverter( - input_graph_def=graph_def, - nodes_blacklist=output_tensors, - is_dynamic_op=True, - ) - graph_def = converter.convert() - else: - graph_def = trt.create_inference_graph( - input_graph_def=graph_def, - outputs=output_tensors, - max_batch_size=1, - precision_mode=self.precision, - is_dynamic_op=True, - ) - - graph = finalize_graph(graph_def) - self.sess, self.inputs, self.outputs = extract_graph( - graph, tf_config=self.tf_config - ) - - else: - - raise DLCLiveError( - "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( - self.model_type - ) - ) # get pose of first frame (first inference is often very slow) @@ -452,44 +463,43 @@ def get_pose(self, frame=None, **kwargs): frame = self.process_frame(frame) - if self.model_type in [ - "base", - "tensorrt", - ]: # TODO: from here to 480 (raise DLCLiveError()) we think it is doing inference + # if self.model_type in ["base", "tensorrt"]: - pose_output = self.sess.run( - self.outputs, feed_dict={self.inputs: np.expand_dims(frame, axis=0)} - ) + # pose_output = self.sess.run( + # self.outputs, feed_dict={self.inputs: np.expand_dims(frame, axis=0)} + # ) - elif self.model_type == "tflite": + # elif self.model_type == "tflite": - self.tflite_interpreter.set_tensor( - self.inputs[0]["index"], - np.expand_dims(frame, axis=0).astype(np.float32), - ) - self.tflite_interpreter.invoke() + # self.tflite_interpreter.set_tensor( + # self.inputs[0]["index"], + # np.expand_dims(frame, axis=0).astype(np.float32), + # ) + # self.tflite_interpreter.invoke() - if len(self.outputs) > 1: - pose_output = [ - self.tflite_interpreter.get_tensor(self.outputs[0]["index"]), - self.tflite_interpreter.get_tensor(self.outputs[1]["index"]), - ] - else: - pose_output = self.tflite_interpreter.get_tensor( - self.outputs[0]["index"] - ) + # if len(self.outputs) > 1: + # pose_output = [ + # self.tflite_interpreter.get_tensor(self.outputs[0]["index"]), + # self.tflite_interpreter.get_tensor(self.outputs[1]["index"]), + # ] + # else: + # pose_output = self.tflite_interpreter.get_tensor( + # self.outputs[0]["index"] + # ) - else: + # else: - raise DLCLiveError( - "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( - self.model_type - ) - ) + # raise DLCLiveError( + # "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( + # self.model_type + # ) + # ) # check if using TFGPUinference flag # if not, get pose from network output + # ! to be replaced + ''' if len(pose_output) > 1: scmap, locref = extract_cnn_output( pose_output, self.cfg @@ -526,15 +536,36 @@ def get_pose(self, frame=None, **kwargs): # process the pose if self.processor: - self.pose = self.processor.process(self.pose, **kwargs) + self.pose = self.processor.process(self.pose, **kwargs)''' + + # Mock pose + num_individuals = 1 + num_kpts = 3 + + # ! Multi animal OR single animal: display only supports single for now + # self.pose = np.ones((num_individuals, num_kpts, 3)) # Multi animal + # self.pose = np.ones((num_kpts, 3)) # Single animal + + # mock_frame = np.ones((1, 3, 128, 128)) + frame = torch.Tensor(frame).permute(2, 0, 1) + + # Pytorch pose prediction + pose_model = self.load_model() + outputs = pose_model(frame) + self.pose = pose_model.get_predictions(outputs) + + # debug + print(pose_model) + print(self.pose, self.pose['bodypart']['poses'].shape()) return self.pose - def close(self): # seems to be deletable - """Close tensorflow session""" + # def close(self): + # """ Close tensorflow session + # """ - self.sess.close() - self.sess = None - self.is_initialized = False - if self.display is not None: - self.display.destroy() + # self.sess.close() + # self.sess = None + # self.is_initialized = False + # if self.display is not None: + # self.display.destroy() From d75339addb75fb59db2934827e9e6f94d6bff221 Mon Sep 17 00:00:00 2001 From: AnnaStuckert <47814177+annastuckert@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:36:27 +0100 Subject: [PATCH 04/24] change testing directory for Anna and allow to run on CPU --- dlclive/dlclive.py | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 39a5a4d..5fd8516 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -6,19 +6,24 @@ """ import glob -import ruamel.yaml import os + +# import tensorflow as tf import typing import warnings from pathlib import Path from typing import List, Optional, Tuple +import deeplabcut as dlc import numpy as np -# import tensorflow as tf -import typing -from pathlib import Path -from typing import Optional, Tuple, List +import ruamel.yaml import torch +from deeplabcut.pose_estimation_pytorch.models import PoseModel + +from dlclive import utils +from dlclive.display import Display +from dlclive.exceptions import DLCLiveError, DLCLiveWarning +from dlclive.pose import argmax_pose_predict, extract_cnn_output, multi_pose_predict # try: # TFVER = [int(v) for v in tf.__version__.split(".")] @@ -37,12 +42,6 @@ # extract_graph, # ) -import deeplabcut as dlc -from deeplabcut.pose_estimation_pytorch.models import PoseModel -from dlclive.pose import extract_cnn_output, argmax_pose_predict, multi_pose_predict -from dlclive.display import Display -from dlclive import utils -from dlclive.exceptions import DLCLiveError, DLCLiveWarning if typing.TYPE_CHECKING: from dlclive.processor import Processor @@ -213,7 +212,6 @@ def read_config(self): ruamel_file = ruamel.yaml.YAML() self.cfg = ruamel_file.load(open(str(cfg_path), "r")) - @property def parameterization( self, @@ -287,19 +285,19 @@ def process_frame(self, frame): #'self' holds all the arguments return frame - def load_model(self): self.read_config() - weights = torch.load(self.snapshot) + weights = torch.load( + self.snapshot, map_location=torch.device("cpu") + ) # added this to run on CPU print("Loaded weights") print(self.cfg) - pose_model = PoseModel.build(self.cfg['model']) + pose_model = PoseModel.build(self.cfg["model"]) print("Built pose model") pose_model.load_state_dict(weights["model"]) - print('Loaded pretrained weights') + print("Loaded pretrained weights") return pose_model - def init_inference(self, frame=None, **kwargs): """ Load model and perform inference on first frame -- the first inference is usually very slow. @@ -431,7 +429,6 @@ def init_inference(self, frame=None, **kwargs): # ) # ) - # get pose of first frame (first inference is often very slow) if frame is not None: @@ -499,7 +496,7 @@ def get_pose(self, frame=None, **kwargs): # if not, get pose from network output # ! to be replaced - ''' + """ if len(pose_output) > 1: scmap, locref = extract_cnn_output( pose_output, self.cfg @@ -536,7 +533,7 @@ def get_pose(self, frame=None, **kwargs): # process the pose if self.processor: - self.pose = self.processor.process(self.pose, **kwargs)''' + self.pose = self.processor.process(self.pose, **kwargs)""" # Mock pose num_individuals = 1 @@ -557,7 +554,7 @@ def get_pose(self, frame=None, **kwargs): # debug print(pose_model) - print(self.pose, self.pose['bodypart']['poses'].shape()) + # print(self.pose, self.pose["bodypart"]["poses"].shape()) return self.pose # def close(self): From 4de5829625cc4ca8977d6f9f00180c4823d37f13 Mon Sep 17 00:00:00 2001 From: Dikra Date: Mon, 24 Feb 2025 14:38:01 +0100 Subject: [PATCH 05/24] Fix display, clean code, adapt frame processing, onnx inference --- dlclive/__init__.py | 1 + dlclive/display.py | 10 +- dlclive/dlclive.py | 338 +++++++------------------- dlclive/pose.py | 29 ++- dlclive/predictor/__init__.py | 1 + dlclive/predictor/base.py | 66 +++++ dlclive/predictor/single_predictor.py | 179 ++++++++++++++ 7 files changed, 360 insertions(+), 264 deletions(-) create mode 100644 dlclive/predictor/__init__.py create mode 100644 dlclive/predictor/base.py create mode 100644 dlclive/predictor/single_predictor.py diff --git a/dlclive/__init__.py b/dlclive/__init__.py index e4ce7e9..b21df4a 100644 --- a/dlclive/__init__.py +++ b/dlclive/__init__.py @@ -9,4 +9,5 @@ from dlclive.dlclive import DLCLive from dlclive.display import Display from dlclive.processor import Processor +from dlclive.predictor import HeatmapPredictor # from dlclive.benchmark import benchmark, benchmark_videos, download_benchmarking_data diff --git a/dlclive/display.py b/dlclive/display.py index e0acfec..573af3e 100644 --- a/dlclive/display.py +++ b/dlclive/display.py @@ -9,7 +9,8 @@ from tkinter import Tk, Label import colorcet as cc from PIL import Image, ImageTk, ImageDraw - +import numpy as np +from dlclive import utils class Display(object): """ @@ -35,7 +36,7 @@ def __init__(self, cmap="bmy", radius=3, pcutoff=0.5): def set_display(self, im_size, bodyparts): """ Create tkinter window to display image - + Parameters ---------- im_size : tuple @@ -64,7 +65,10 @@ def display_frame(self, frame, pose=None): pose :class:`numpy.ndarray` the pose estimated by DeepLabCut for the image """ - + frame = np.squeeze(frame) + frame = frame.astype(np.uint8) + # frame = np.transpose(frame, (1, 2, 0)) + pose = pose["poses"].squeeze() im_size = (frame.shape[1], frame.shape[0]) if pose is not None: diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 5fd8516..d345293 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -7,7 +7,6 @@ import glob import os - # import tensorflow as tf import typing import warnings @@ -16,32 +15,17 @@ import deeplabcut as dlc import numpy as np +import onnx +import onnxruntime as ort import ruamel.yaml import torch from deeplabcut.pose_estimation_pytorch.models import PoseModel - from dlclive import utils from dlclive.display import Display from dlclive.exceptions import DLCLiveError, DLCLiveWarning -from dlclive.pose import argmax_pose_predict, extract_cnn_output, multi_pose_predict - -# try: -# TFVER = [int(v) for v in tf.__version__.split(".")] -# if TFVER[1] < 14: -# from tensorflow.contrib.tensorrt import trt_convert as trt -# else: -# from tensorflow.python.compiler.tensorrt import trt_convert as trt -# except Exception: -# pass - -# from dlclive.graph import ( -# read_graph, -# finalize_graph, -# get_output_nodes, -# get_output_tensors, -# extract_graph, -# ) - +from dlclive.pose import (argmax_pose_predict, extract_cnn_output, + multi_pose_predict) +from dlclive.predictor import HeatmapPredictor if typing.TYPE_CHECKING: from dlclive.processor import Processor @@ -135,11 +119,10 @@ class DLCLive(object): def __init__( self, - model_path: str = None, - model_type: str = "base", + path: str, + model_type: str = "onnx", precision: str = "FP32", - tf_config=None, - pytorch_cfg=str, + device: str = "cpu", snapshot=str, cropping: Optional[List[int]] = None, dynamic: Tuple[bool, float, float] = (False, 0.5, 10), @@ -152,11 +135,9 @@ def __init__( display_cmap: str = "bmy", ): - self.path = model_path - self.cfg = None # type: typing.Optional[dict] + self.path = path self.model_type = model_type - self.tf_config = tf_config - self.pytorch_cfg = pytorch_cfg + self.device = device self.snapshot = snapshot self.precision = precision self.cropping = cropping @@ -174,22 +155,11 @@ def __init__( else: self.display = None + self.cfg = None + self.cfg_path = None self.sess = None - self.inputs = None - self.outputs = None - self.tflite_interpreter = None + self.pose_model = None self.pose = None - self.is_initialized = False - - # checks - - # if self.model_type == "tflite" and self.dynamic[0]: - # self.dynamic = (False, *self.dynamic[1:]) - # warnings.warn( - # "Dynamic cropping is not supported for tensorflow lite inference. Dynamic cropping will not be used...", - # DLCLiveWarning, - # ) - self.read_config() def read_config(self): @@ -201,9 +171,7 @@ def read_config(self): error thrown if pose configuration file does nott exist """ - cfg_path = ( - Path(self.pytorch_cfg).resolve() / "pytorch_config.yaml" - ) # TODO TF_ref - replace by pytorch config - consider importing read_config function from DLC 3 - and the new config may have both detector and 'normal' config - e.g batch size could refer both to detector and key points. should be handled in the read_config from DLC3 + cfg_path = Path(self.path).resolve() / "pytorch_config.yaml" if not cfg_path.exists(): raise FileNotFoundError( f"The pose configuration file for the exported model at {str(cfg_path)} was not found. Please check the path to the exported model directory" @@ -225,7 +193,7 @@ def parameterization( """ return {param: getattr(self, param) for param in self.PARAMETERS} - def process_frame(self, frame): #'self' holds all the arguments + def process_frame(self, frame): """ Crops an image according to the object's cropping and dynamic properties. @@ -240,19 +208,15 @@ def process_frame(self, frame): #'self' holds all the arguments processed frame: convert type, crop, convert color """ - if frame.dtype != np.uint8: + # ! NORMALISE FRAMES - frame = utils.convert_to_ubyte(frame) - - if self.cropping: # if cropping is specified, it will be applied - - frame = frame[ # A: this produces a cropped image based on incoming coordinates x1,x2,y1,y2 + if self.cropping: + frame = frame[ self.cropping[2] : self.cropping[3], self.cropping[0] : self.cropping[1] ] - if self.dynamic[ 0 - ]: # to go through this if statement, the boolean would have to be = True. for it to react to false you'd have to write if not self.dynamic[0] + ]: if self.pose is not None: @@ -265,7 +229,7 @@ def process_frame(self, frame): #'self' holds all the arguments x1 = int( max([0, int(np.amin(x)) - self.dynamic[2]]) - ) # We think it is dtected if keypoint likelihood exceeds the dynamic threshold for dynamic cropping + ) x2 = int(min([frame.shape[1], int(np.amax(x)) + self.dynamic[2]])) y1 = int(max([0, int(np.amin(y)) - self.dynamic[2]])) y2 = int(min([frame.shape[0], int(np.amax(y)) + self.dynamic[2]])) @@ -286,17 +250,32 @@ def process_frame(self, frame): #'self' holds all the arguments return frame def load_model(self): - self.read_config() - weights = torch.load( - self.snapshot, map_location=torch.device("cpu") - ) # added this to run on CPU - print("Loaded weights") - print(self.cfg) - pose_model = PoseModel.build(self.cfg["model"]) - print("Built pose model") - pose_model.load_state_dict(weights["model"]) - print("Loaded pretrained weights") - return pose_model + if self.model_type == "pytorch": + # Requires DLC 3.0 to be imported + model_path = os.path.join(self.path, self.snapshot) + if not os.path.isfile(model_path): + raise FileNotFoundError( + "The model file {} does not exist.".format(model_path) + ) + weights = torch.load(model_path, map_location=torch.device(self.device)) + self.pose_model = PoseModel.build(self.cfg["model"]) + self.pose_model.load_state_dict(weights["model"]) + + elif self.model_type == "onnx": + model_path = glob.glob(os.path.normpath(self.path + "/*.onnx"))[0] + self.sess = ort.InferenceSession(model_path) + + if not os.path.isfile(model_path): + raise FileNotFoundError( + "The model file {} does not exist.".format(model_path) + ) + + else: + raise DLCLiveError( + "model_type = {} is not supported. model_type must be 'pytorch' or 'onnx'".format( + self.model_type + ) + ) def init_inference(self, frame=None, **kwargs): """ @@ -313,121 +292,13 @@ def init_inference(self, frame=None, **kwargs): the pose estimated by DeepLabCut for the input image """ - # get model file - - model_file = glob.glob(os.path.normpath(self.pytorch_cfg + "/*.pt"))[ - 0 - ] # TODO TF_ref - maybe .pb format will be changed when using pytorch - if not os.path.isfile(model_file): - raise FileNotFoundError( - "The model file {} does not exist.".format(model_file) - ) - - # process frame - - # ! TODO replace this if statement - # if frame is None and (self.model_type == "tflite"): - # raise DLCLiveError( - # "No image was passed to initialize inference. An image must be passed to the init_inference method" - # ) - - if frame is not None: - if frame.ndim == 2: - self.convert2rgb = True - processed_frame = self.process_frame(frame) + # if frame is not None: + # if frame.ndim == 2: + # self.convert2rgb = True + # processed_frame = self.process_frame(frame) # load model - - # if self.model_type == "base": - - # graph_def = read_graph(model_file) - # graph = finalize_graph(graph_def) - # self.sess, self.inputs, self.outputs = extract_graph( - # graph, tf_config=self.tf_config - # ) - - # elif self.model_type == "tflite": - - # ### - # # the frame size needed to initialize the tflite model as - # # tflite does not support saving a model with dynamic input size - # ### - - # # get input and output tensor names from graph_def - # graph_def = read_graph(model_file) - # graph = finalize_graph(graph_def) - # output_nodes = get_output_nodes(graph) - # output_nodes = [on.replace("DLC/", "") for on in output_nodes] - - # tf_version_2 = tf.__version__[0] == '2' - - # if tf_version_2: - # converter = tf.compat.v1.lite.TFLiteConverter.from_frozen_graph( - # model_file, - # ["Placeholder"], - # output_nodes, - # input_shapes={"Placeholder": [1, processed_frame.shape[0], processed_frame.shape[1], 3]}, - # ) - # else: - # converter = tf.lite.TFLiteConverter.from_frozen_graph( - # model_file, - # ["Placeholder"], - # output_nodes, - # input_shapes={"Placeholder": [1, processed_frame.shape[0], processed_frame.shape[1], 3]}, - # ) - - # try: - # tflite_model = converter.convert() - # except Exception: - # raise DLCLiveError( - # ( - # "This model cannot be converted to tensorflow lite format. " - # "To use tensorflow lite for live inference, " - # "make sure to set TFGPUinference=False " - # "when exporting the model from DeepLabCut" - # ) - # ) - - # self.tflite_interpreter = tf.lite.Interpreter(model_content=tflite_model) - # self.tflite_interpreter.allocate_tensors() - # self.inputs = self.tflite_interpreter.get_input_details() - # self.outputs = self.tflite_interpreter.get_output_details() - - # elif self.model_type == "tensorrt": - - # graph_def = read_graph(model_file) - # graph = finalize_graph(graph_def) - # output_tensors = get_output_tensors(graph) - # output_tensors = [ot.replace("DLC/", "") for ot in output_tensors] - - # if (TFVER[0] > 1) | (TFVER[0] == 1 & TFVER[1] >= 14): - # converter = trt.TrtGraphConverter( - # input_graph_def=graph_def, - # nodes_blacklist=output_tensors, - # is_dynamic_op=True, - # ) - # graph_def = converter.convert() - # else: - # graph_def = trt.create_inference_graph( - # input_graph_def=graph_def, - # outputs=output_tensors, - # max_batch_size=1, - # precision_mode=self.precision, - # is_dynamic_op=True, - # ) - - # graph = finalize_graph(graph_def) - # self.sess, self.inputs, self.outputs = extract_graph( - # graph, tf_config=self.tf_config - # ) - - # else: - - # raise DLCLiveError( - # "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( - # self.model_type - # ) - # ) + self.load_model() # get pose of first frame (first inference is often very slow) @@ -436,7 +307,7 @@ def init_inference(self, frame=None, **kwargs): else: pose = None - self.is_initialized = True + # self.is_initialized = True return pose @@ -458,64 +329,45 @@ def get_pose(self, frame=None, **kwargs): if frame is None: raise DLCLiveError("No frame provided for live pose estimation") - frame = self.process_frame(frame) - - # if self.model_type in ["base", "tensorrt"]: - - # pose_output = self.sess.run( - # self.outputs, feed_dict={self.inputs: np.expand_dims(frame, axis=0)} - # ) - - # elif self.model_type == "tflite": - - # self.tflite_interpreter.set_tensor( - # self.inputs[0]["index"], - # np.expand_dims(frame, axis=0).astype(np.float32), - # ) - # self.tflite_interpreter.invoke() - - # if len(self.outputs) > 1: - # pose_output = [ - # self.tflite_interpreter.get_tensor(self.outputs[0]["index"]), - # self.tflite_interpreter.get_tensor(self.outputs[1]["index"]), - # ] - # else: - # pose_output = self.tflite_interpreter.get_tensor( - # self.outputs[0]["index"] - # ) + if frame is not None: + if frame.ndim >= 2: + self.convert2rgb = True - # else: + processed_frame = self.process_frame(frame) - # raise DLCLiveError( - # "model_type = {} is not supported. model_type must be 'base', 'tflite', or 'tensorrt'".format( - # self.model_type - # ) - # ) + if self.model_type == "pytorch": + frame = torch.Tensor(processed_frame) + frame = frame.permute(2, 0, 1).unsqueeze(0) + self.pose_model.eval() + outputs = self.pose_model(frame) + self.pose = self.pose_model.get_predictions(outputs) + self.pose = self.pose["bodypart"] + + elif self.model_type == "onnx": + frame = processed_frame.astype(np.float32) + frame = np.transpose(frame, (2, 0, 1)) + frame = np.expand_dims(frame, axis=0) + ort_inputs = {self.sess.get_inputs()[0].name: frame} + outputs = self.sess.run(None, ort_inputs) + outputs = {# ! optimise: make it one var 'outputs' + "heatmap": torch.Tensor(outputs[0]), + "locref": torch.Tensor(outputs[1]), + } + predictor = HeatmapPredictor.build(self.cfg) + self.pose = predictor(outputs=outputs) - # check if using TFGPUinference flag - # if not, get pose from network output + else: - # ! to be replaced - """ - if len(pose_output) > 1: - scmap, locref = extract_cnn_output( - pose_output, self.cfg - ) # scmap = the heatmaps of likelihood of different key points being placed at different positions in the frame. locref = think it is something with refining how the 'peaks' in the heatmap is created. - num_outputs = self.cfg.get("num_outputs", 1) - if num_outputs > 1: # seems to be if detecting more than 1 key point - self.pose = multi_pose_predict( - scmap, locref, self.cfg["stride"], num_outputs + raise DLCLiveError( + "model_type = {} is not supported. model_type must be 'pytorch' or 'onnx'".format( + self.model_type ) - else: - self.pose = argmax_pose_predict(scmap, locref, self.cfg["stride"]) - else: - pose = np.array(pose_output[0]) - self.pose = pose[:, [1, 0, 2]] + ) # display image if display=True before correcting pose for cropping/resizing if self.display is not None: - self.display.display_frame(frame, self.pose) + self.display.display_frame(processed_frame, self.pose) # if frame is cropped, convert pose coordinates to original frame coordinates @@ -523,8 +375,8 @@ def get_pose(self, frame=None, **kwargs): self.pose[:, :2] *= 1 / self.resize if self.cropping is not None: - self.pose[:, 0] += self.cropping[0] - self.pose[:, 1] += self.cropping[2] + self.pose["poses"][:, :, :, 0][0] += self.cropping[0] + self.pose["poses"][:, :, :, 1][0] += self.cropping[2] if self.dynamic_cropping is not None: self.pose[:, 0] += self.dynamic_cropping[0] @@ -533,28 +385,8 @@ def get_pose(self, frame=None, **kwargs): # process the pose if self.processor: - self.pose = self.processor.process(self.pose, **kwargs)""" - - # Mock pose - num_individuals = 1 - num_kpts = 3 - - # ! Multi animal OR single animal: display only supports single for now - - # self.pose = np.ones((num_individuals, num_kpts, 3)) # Multi animal - # self.pose = np.ones((num_kpts, 3)) # Single animal - - # mock_frame = np.ones((1, 3, 128, 128)) - frame = torch.Tensor(frame).permute(2, 0, 1) - - # Pytorch pose prediction - pose_model = self.load_model() - outputs = pose_model(frame) - self.pose = pose_model.get_predictions(outputs) + self.pose = self.processor.process(self.pose, **kwargs) - # debug - print(pose_model) - # print(self.pose, self.pose["bodypart"]["poses"].shape()) return self.pose # def close(self): diff --git a/dlclive/pose.py b/dlclive/pose.py index 3e69bb9..df39f0b 100644 --- a/dlclive/pose.py +++ b/dlclive/pose.py @@ -36,11 +36,12 @@ def extract_cnn_output(outputs, cfg): scmap = outputs[0] scmap = np.squeeze(scmap) locref = None - if cfg["location_refinement"]: + if cfg["model"]["heads"]["bodypart"]["predictor"]["location_refinement"]: locref = np.squeeze(outputs[1]) shape = locref.shape - locref = np.reshape(locref, (shape[0], shape[1], -1, 2)) - locref *= cfg["locref_stdev"] + print(shape, scmap.shape) + locref = np.reshape(locref, (shape[1], shape[2], -1, 2)) + locref *= cfg["model"]["heads"]["bodypart"]["predictor"]["locref_std"] if len(scmap.shape) == 2: # for single body part! scmap = np.expand_dims(scmap, axis=2) return scmap, locref @@ -67,15 +68,27 @@ def argmax_pose_predict(scmap, offmat, stride): pose as a numpy array """ - num_joints = scmap.shape[2] + num_joints = scmap.shape[0] + # debug + print('joints', num_joints) pose = [] for joint_idx in range(num_joints): maxloc = np.unravel_index( - np.argmax(scmap[:, :, joint_idx]), scmap[:, :, joint_idx].shape + np.argmax(scmap[joint_idx, :, :]), scmap[joint_idx, :, :].shape ) - offset = np.array(offmat[maxloc][joint_idx])[::-1] - pos_f8 = np.array(maxloc).astype("float") * stride + 0.5 * stride + offset - pose.append(np.hstack((pos_f8[::-1], [scmap[maxloc][joint_idx]]))) + # debug + print("maxloc", maxloc) + # print(offmat.shape) + # offset = np.array(offmat[maxloc][joint_idx])[::-1] + # print(offmat[maxloc][joint_idx]) + offset = np.array(offmat[maxloc])[::-1] + print(offset[:,0].shape) + # print(np.array(offmat[maxloc]).shape) + print('offset', offset.shape) + print('offmat*stride+offset', (offmat * stride + offset).shape, (offmat * stride + offset)) + pos_f8 = np.array(offmat).astype("float") * stride + 0.5 * stride + offset + print("pos_f8", pos_f8[::-1].shape) + pose.append(np.hstack((pos_f8[::-1], [scmap[joint_idx][maxloc]]))) return np.array(pose) diff --git a/dlclive/predictor/__init__.py b/dlclive/predictor/__init__.py new file mode 100644 index 0000000..47c531c --- /dev/null +++ b/dlclive/predictor/__init__.py @@ -0,0 +1 @@ +from dlclive.predictor.single_predictor import HeatmapPredictor \ No newline at end of file diff --git a/dlclive/predictor/base.py b/dlclive/predictor/base.py new file mode 100644 index 0000000..dc9b38a --- /dev/null +++ b/dlclive/predictor/base.py @@ -0,0 +1,66 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +from abc import ABC, abstractmethod + +import torch +from torch import nn + +from deeplabcut.pose_estimation_pytorch.registry import build_from_cfg, Registry + +PREDICTORS = Registry("predictors", build_func=build_from_cfg) + + +class BasePredictor(ABC, nn.Module): + """The base Predictor class. + + This class is an abstract base class (ABC) for defining predictors used in the DeepLabCut Toolbox. + All predictor classes should inherit from this base class and implement the forward method. + Regresses keypoint coordinates from a models output maps + + Attributes: + num_animals: Number of animals in the project. Should be set in subclasses. + + Example: + # Create a subclass that inherits from BasePredictor and implements the forward method. + class MyPredictor(BasePredictor): + def __init__(self, num_animals): + super().__init__() + self.num_animals = num_animals + + def forward(self, outputs): + # Implement the forward pass of your custom predictor here. + pass + """ + + def __init__(self): + super().__init__() + self.num_animals = None + + @abstractmethod + def forward( + self, stride: float, outputs: dict[str, torch.Tensor] + ) -> dict[str, torch.Tensor]: + """Abstract method for the forward pass of the Predictor. + + Args: + stride: the stride of the model + outputs: outputs of the model heads + + Returns: + A dictionary containing a "poses" key with the output tensor as value, and + optionally a "unique_bodyparts" with the unique bodyparts tensor as value. + + Raises: + NotImplementedError: This method must be implemented in subclasses. + """ + pass diff --git a/dlclive/predictor/single_predictor.py b/dlclive/predictor/single_predictor.py new file mode 100644 index 0000000..528039b --- /dev/null +++ b/dlclive/predictor/single_predictor.py @@ -0,0 +1,179 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +from typing import Tuple + +import torch + +from deeplabcut.pose_estimation_pytorch.models.predictors.base import ( + BasePredictor, + PREDICTORS, +) + + +# @PREDICTORS.register_module +class HeatmapPredictor(BasePredictor): + """Predictor class for pose estimation from heatmaps (and optionally locrefs). + + Args: + location_refinement: Enable location refinement. + locref_std: Standard deviation for location refinement. + apply_sigmoid: Apply sigmoid to heatmaps. Defaults to True. + + Returns: + Regressed keypoints from heatmaps and locref_maps of baseline DLC model (ResNet + Deconv). + """ + + def __init__( + self, + apply_sigmoid: bool = True, + clip_scores: bool = False, + location_refinement: bool = True, + locref_std: float = 7.2801, + stride: float = 8. + ): + """ + Args: + apply_sigmoid: Apply sigmoid to heatmaps. Defaults to True. + clip_scores: If a sigmoid is not applied, this can be used to clip scores + for predicted keypoints to values in [0, 1]. + location_refinement : Enable location refinement. + locref_std: Standard deviation for location refinement. + """ + super().__init__() + self.apply_sigmoid = apply_sigmoid + self.clip_scores = clip_scores + self.sigmoid = torch.nn.Sigmoid() + self.location_refinement = location_refinement + self.locref_std = locref_std + self.stride = stride + + def forward( + self, outputs: dict[str, torch.Tensor] + ) -> dict[str, torch.Tensor]: + """Forward pass of SinglePredictor. Gets predictions from model output. + + Args: + stride: the stride of the model + outputs: output of the model heads (heatmap, locref) + + Returns: + A dictionary containing a "poses" key with the output tensor as value. + + Example: + >>> predictor = HeatmapPredictor(location_refinement=True, locref_std=7.2801) + >>> stride = 8 + >>> output = {"heatmap": torch.rand(32, 17, 64, 64), "locref": torch.rand(32, 17, 64, 64)} + >>> poses = predictor.forward(stride, output) + """ + heatmaps = outputs["heatmap"] + scale_factors = self.stride, self.stride + + if self.apply_sigmoid: + heatmaps = self.sigmoid(heatmaps) + + heatmaps = heatmaps.permute(0, 2, 3, 1) + batch_size, height, width, num_joints = heatmaps.shape + + locrefs = None + if self.location_refinement: + locrefs = outputs["locref"] + locrefs = locrefs.permute(0, 2, 3, 1).reshape( + batch_size, height, width, num_joints, 2 + ) + locrefs = locrefs * self.locref_std + + poses = self.get_pose_prediction(heatmaps, locrefs, scale_factors) + + if self.clip_scores: + poses[..., 2] = torch.clip(poses[..., 2], min=0, max=1) + + return {"poses": poses} + + def get_top_values( + self, heatmap: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Get the top values from the heatmap. + + Args: + heatmap: Heatmap tensor. + + Returns: + Y and X indices of the top values. + + Example: + >>> predictor = HeatmapPredictor(location_refinement=True, locref_std=7.2801) + >>> heatmap = torch.rand(32, 17, 64, 64) + >>> Y, X = predictor.get_top_values(heatmap) + """ + batchsize, ny, nx, num_joints = heatmap.shape + heatmap_flat = heatmap.reshape(batchsize, nx * ny, num_joints) + heatmap_top = torch.argmax(heatmap_flat, dim=1) + y, x = heatmap_top // nx, heatmap_top % nx + return y, x + + def get_pose_prediction( + self, heatmap: torch.Tensor, locref: torch.Tensor | None, scale_factors + ) -> torch.Tensor: + """Gets the pose prediction given the heatmaps and locref. + + Args: + heatmap: Heatmap tensor with the following format (batch_size, height, width, num_joints) + locref: Locref tensor with the following format (batch_size, height, width, num_joints, 2) + scale_factors: Scale factors for the poses. + + Returns: + Pose predictions of the format: (batch_size, num_people = 1, num_joints, 3) + + Example: + >>> predictor = HeatmapPredictor(location_refinement=True, locref_std=7.2801) + >>> heatmap = torch.rand(32, 17, 64, 64) + >>> locref = torch.rand(32, 17, 64, 64, 2) + >>> scale_factors = (0.5, 0.5) + >>> poses = predictor.get_pose_prediction(heatmap, locref, scale_factors) + """ + y, x = self.get_top_values(heatmap) + + batch_size, num_joints = x.shape + + dz = torch.zeros((batch_size, 1, num_joints, 3)).to(x.device) + for b in range(batch_size): + for j in range(num_joints): + dz[b, 0, j, 2] = heatmap[b, y[b, j], x[b, j], j] + if locref is not None: + dz[b, 0, j, :2] = locref[b, y[b, j], x[b, j], j, :] + + x, y = torch.unsqueeze(x, 1), torch.unsqueeze(y, 1) + x = x * scale_factors[1] + 0.5 * scale_factors[1] + dz[:, :, :, 0] + y = y * scale_factors[0] + 0.5 * scale_factors[0] + dz[:, :, :, 1] + pose = torch.empty((batch_size, 1, num_joints, 3)) + pose[:, :, :, 0] = x + pose[:, :, :, 1] = y + pose[:, :, :, 2] = dz[:, :, :, 2] + return pose + + @staticmethod + def build(cfg: dict) -> HeatmapPredictor: + apply_sigmoid = cfg["model"]["heads"]["bodypart"]["predictor"]["apply_sigmoid"] + clip_scores = cfg["model"]["heads"]["bodypart"]["predictor"]["clip_scores"] + loc_ref = cfg["model"]["heads"]["bodypart"]["predictor"]["location_refinement"] + loc_ref_std = cfg["model"]["heads"]["bodypart"]["predictor"]["locref_std"] + stride = float(cfg["model"]["backbone"]["output_stride"]) / float( + cfg["model"]["heads"]["bodypart"]["heatmap_config"]["strides"][0] + ) + + predictor = HeatmapPredictor( + apply_sigmoid=apply_sigmoid, stride=stride, clip_scores=clip_scores, + location_refinement=loc_ref, locref_std=loc_ref_std + ) + + return predictor \ No newline at end of file From 40a9dd40a68e5cd64443ca942dab98995013c622 Mon Sep 17 00:00:00 2001 From: Niels Poulsen Date: Mon, 24 Feb 2025 14:43:53 +0100 Subject: [PATCH 06/24] add screenshots --- dlclive/dlclive.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index d345293..cc4d68e 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -263,7 +263,7 @@ def load_model(self): elif self.model_type == "onnx": model_path = glob.glob(os.path.normpath(self.path + "/*.onnx"))[0] - self.sess = ort.InferenceSession(model_path) + self.sess = ort.InferenceSession(model_path) # ! give GPU or CPU provider depending on self.device!! if not os.path.isfile(model_path): raise FileNotFoundError( @@ -292,23 +292,15 @@ def init_inference(self, frame=None, **kwargs): the pose estimated by DeepLabCut for the input image """ - # if frame is not None: - # if frame.ndim == 2: - # self.convert2rgb = True - # processed_frame = self.process_frame(frame) - # load model self.load_model() # get pose of first frame (first inference is often very slow) - if frame is not None: pose = self.get_pose(frame, **kwargs) else: pose = None - # self.is_initialized = True - return pose def get_pose(self, frame=None, **kwargs): @@ -349,7 +341,7 @@ def get_pose(self, frame=None, **kwargs): frame = np.expand_dims(frame, axis=0) ort_inputs = {self.sess.get_inputs()[0].name: frame} outputs = self.sess.run(None, ort_inputs) - outputs = {# ! optimise: make it one var 'outputs' + outputs = { "heatmap": torch.Tensor(outputs[0]), "locref": torch.Tensor(outputs[1]), } From 2fc48f152bee093b72d671e16cb5c8e43c7ad5ca Mon Sep 17 00:00:00 2001 From: Dikra Date: Mon, 24 Feb 2025 14:44:58 +0100 Subject: [PATCH 07/24] Fix CPU inference crash + GPU (cuda) & CPU support for Pytorch and ONNX inference --- dlclive/display.py | 11 +++++----- dlclive/dlclive.py | 50 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/dlclive/display.py b/dlclive/display.py index 573af3e..7260997 100644 --- a/dlclive/display.py +++ b/dlclive/display.py @@ -5,12 +5,13 @@ Licensed under GNU Lesser General Public License v3.0 """ +from tkinter import Label, Tk -from tkinter import Tk, Label import colorcet as cc -from PIL import Image, ImageTk, ImageDraw import numpy as np from dlclive import utils +from PIL import Image, ImageDraw, ImageTk + class Display(object): """ @@ -25,8 +26,7 @@ class Display(object): """ def __init__(self, cmap="bmy", radius=3, pcutoff=0.5): - """ Constructor method - """ + """Constructor method""" self.cmap = cmap self.colors = None @@ -35,7 +35,7 @@ def __init__(self, cmap="bmy", radius=3, pcutoff=0.5): self.window = None def set_display(self, im_size, bodyparts): - """ Create tkinter window to display image + """Create tkinter window to display image Parameters ---------- @@ -67,7 +67,6 @@ def display_frame(self, frame, pose=None): """ frame = np.squeeze(frame) frame = frame.astype(np.uint8) - # frame = np.transpose(frame, (1, 2, 0)) pose = pose["poses"].squeeze() im_size = (frame.shape[1], frame.shape[0]) diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index cc4d68e..a61ef12 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -7,7 +7,7 @@ import glob import os -# import tensorflow as tf +import time import typing import warnings from pathlib import Path @@ -159,6 +159,7 @@ def __init__( self.cfg_path = None self.sess = None self.pose_model = None + self.predictor = None self.pose = None self.read_config() @@ -214,12 +215,10 @@ def process_frame(self, frame): frame = frame[ self.cropping[2] : self.cropping[3], self.cropping[0] : self.cropping[1] ] - if self.dynamic[ - 0 - ]: + if self.dynamic[0]: if self.pose is not None: - + print(self.pose["bodypart"]) detected = self.pose[:, 2] > self.dynamic[1] if np.any(detected): @@ -227,9 +226,7 @@ def process_frame(self, frame): x = self.pose[detected, 0] y = self.pose[detected, 1] - x1 = int( - max([0, int(np.amin(x)) - self.dynamic[2]]) - ) + x1 = int(max([0, int(np.amin(x)) - self.dynamic[2]])) x2 = int(min([frame.shape[1], int(np.amax(x)) + self.dynamic[2]])) y1 = int(max([0, int(np.amin(y)) - self.dynamic[2]])) y2 = int(min([frame.shape[0], int(np.amax(y)) + self.dynamic[2]])) @@ -260,10 +257,22 @@ def load_model(self): weights = torch.load(model_path, map_location=torch.device(self.device)) self.pose_model = PoseModel.build(self.cfg["model"]) self.pose_model.load_state_dict(weights["model"]) + self.pose_model = self.pose_model.to(self.device) + self.pose_model.eval() elif self.model_type == "onnx": model_path = glob.glob(os.path.normpath(self.path + "/*.onnx"))[0] - self.sess = ort.InferenceSession(model_path) # ! give GPU or CPU provider depending on self.device!! + opts = ort.SessionOptions() + opts.enable_profiling = False + if self.device == "cuda": + self.sess = ort.InferenceSession( + model_path, opts, providers=["CUDAExecutionProvider"] + ) + elif self.device == "cpu": + self.sess = ort.InferenceSession( + model_path, opts, providers=["CPUExecutionProvider"] + ) + self.predictor = HeatmapPredictor.build(self.cfg) if not os.path.isfile(model_path): raise FileNotFoundError( @@ -292,8 +301,11 @@ def init_inference(self, frame=None, **kwargs): the pose estimated by DeepLabCut for the input image """ + start = time.time() # load model self.load_model() + end = time.time() + print(f"Loading the model took {end - start} sec") # get pose of first frame (first inference is often very slow) if frame is not None: @@ -330,8 +342,14 @@ def get_pose(self, frame=None, **kwargs): if self.model_type == "pytorch": frame = torch.Tensor(processed_frame) frame = frame.permute(2, 0, 1).unsqueeze(0) - self.pose_model.eval() - outputs = self.pose_model(frame) + frame = frame.to(self.device) + + with torch.no_grad(): + start = time.time() + outputs = self.pose_model(frame) + end = time.time() + print(f"PyTorch inference took {end - start} sec") + self.pose = self.pose_model.get_predictions(outputs) self.pose = self.pose["bodypart"] @@ -339,14 +357,20 @@ def get_pose(self, frame=None, **kwargs): frame = processed_frame.astype(np.float32) frame = np.transpose(frame, (2, 0, 1)) frame = np.expand_dims(frame, axis=0) + ort_inputs = {self.sess.get_inputs()[0].name: frame} + + start = time.time() outputs = self.sess.run(None, ort_inputs) + end = time.time() + print(f"ONNX inference took {end - start} sec") + outputs = { "heatmap": torch.Tensor(outputs[0]), "locref": torch.Tensor(outputs[1]), } - predictor = HeatmapPredictor.build(self.cfg) - self.pose = predictor(outputs=outputs) + + self.pose = self.predictor(outputs=outputs) else: From 125a07216792fdce8a892d5027e5ec409b2fdc83 Mon Sep 17 00:00:00 2001 From: AnnaStuckert <47814177+annastuckert@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:46:03 +0100 Subject: [PATCH 08/24] Video analysis feature --- analyze_video.py | 65 +++++++++++ dlclive/benchmark_pytorch.py | 207 +++++++++++++++++++++++++++++++++++ 2 files changed, 272 insertions(+) create mode 100644 analyze_video.py create mode 100644 dlclive/benchmark_pytorch.py diff --git a/analyze_video.py b/analyze_video.py new file mode 100644 index 0000000..bb77aa8 --- /dev/null +++ b/analyze_video.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +import cv2 +import numpy as np +import torch + +from dlclive import DLCLive, Processor +from dlclive.display import Display + + +def analyze_video2(video_path: str, dlc_live): + # Load video + cap = cv2.VideoCapture(video_path) + poses = [] + frame_index = 0 + + while True: + ret, frame = cap.read() + if not ret: + break # End of video + + # Prepare the frame for the model + frame = np.array(frame, dtype=np.float32) + frame = np.transpose(frame, (2, 0, 1)) + frame = frame.reshape(1, frame.shape[0], frame.shape[1], frame.shape[2]) + frame = frame / 255.0 + + # Analyze the frame using the get_pose function + pose = dlc_live.get_pose(frame) + + # Store the pose for this frame + poses.append(pose) + + frame_index += 1 + print(frame_index) + + # Release the video capture object + cap.release() + + return poses + + +def main(): + # Paths provided by you + video_path = "/Users/annastuckert/Documents/DLC_AI_Residency/DLC_AI2024/DeepLabCut-live/Ventral_gait_model/1_20cms_0degUP_first_1s.avi" + model_dir = "/Users/annastuckert/Documents/DLC_AI_Residency/DLC_AI2024/DeepLabCut-live/Ventral_gait_model/train" + snapshot = "/Users/annastuckert/Documents/DLC_AI_Residency/DLC_AI2024/DeepLabCut-live/Ventral_gait_model/train/snapshot-263.pt" + model_type = "pytorch" + + # Initialize the DLCLive model + dlc_proc = Processor() + dlc_live = DLCLive( + pytorch_cfg=model_dir, + processor=dlc_proc, + snapshot=snapshot, + model_type=model_type, + ) + + # Analyze the video + poses = analyze_video2(video_path, dlc_live) + print("Pose analysis complete.") + + +if __name__ == "__main__": + main() diff --git a/dlclive/benchmark_pytorch.py b/dlclive/benchmark_pytorch.py new file mode 100644 index 0000000..94e9d1d --- /dev/null +++ b/dlclive/benchmark_pytorch.py @@ -0,0 +1,207 @@ +import csv +import os + +import colorcet as cc +import cv2 +import h5py +import numpy as np +from PIL import ImageColor + + +def analyze_video( + video_path: str, + dlc_live, + pcutoff=0.5, + display_radius=5, + resize=None, + save_poses=False, + save_dir="model_predictions", + draw_keypoint_names=False, + cmap="bmy", +): + """ + Analyze a video to track keypoints using an imported DeepLabCut model, visualize keypoints on the video, and optionally save the keypoint data and the labelled video. + + Parameters: + ----------- + video_path : str + The path to the video file to be analyzed. + dlc_live : DLCLive + An instance of the DLCLive class. + pcutoff : float, optional, default=0.5 + The probability cutoff value below which keypoints are not visualized. + display_radius : int, optional, default=5 + The radius of the circles drawn to represent keypoints on the video frames. + resize : tuple of int (width, height) or None, optional, default=None + The size to which the frames should be resized. If None, the frames are not resized. + save_poses : bool, optional, default=False + Whether to save the detected poses to CSV and HDF5 files. + save_dir : str, optional, default="model_predictions" + The directory where the output video and pose data will be saved. + draw_keypoint_names : bool, optional, default=False + Whether to draw the names of the keypoints on the video frames. + cmap : str, optional, default="bmy" + The colormap from the colorcet library to use for keypoint visualization. + + Returns: + -------- + poses : list of dict + A list of dictionaries where each dictionary contains the frame number and the corresponding pose data. + """ + # Ensure save directory exists + os.makedirs(name=save_dir, exist_ok=True) + + # Load video + cap = cv2.VideoCapture(video_path) + if not cap.isOpened(): + print(f"Error: Could not open video file {video_path}") + return + # Start empty dict to save poses to for each frame + poses = [] + # Create variable indicate current frame. Later in the code +1 is added to frame_index + frame_index = 0 + + # Load the DLC model + try: + pose_model = dlc_live.load_model() + except Exception as e: + print(f"Error: Could not load DLC model. Details: {e}") + return + + # Retrieve bodypart names and number of keypoints + bodyparts = dlc_live.cfg["metadata"]["bodyparts"] + num_keypoints = len(bodyparts) + + # Set colors and convert to RGB + cmap_colors = getattr(cc, cmap) + colors = [ + ImageColor.getrgb(color) + for color in cmap_colors[:: int(len(cmap_colors) / num_keypoints)] + ] + + # Define output video path + video_name = os.path.splitext(os.path.basename(video_path))[0] + output_video_path = os.path.join(save_dir, f"{video_name}_DLCLIVE_LABELLED.mp4") + + # Get video writer setup + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + fps = cap.get(cv2.CAP_PROP_FPS) + frame_width, frame_height = ( + int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), + int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)), + ) + + if resize: + frame_width, frame_height = resize + vwriter = cv2.VideoWriter( + filename=output_video_path, + fourcc=fourcc, + fps=fps, + frameSize=(frame_width, frame_height), + ) + + while True: + ret, frame = cap.read() + if not ret: + break + + try: + pose = dlc_live.get_pose(frame, pose_model=pose_model) + except Exception as e: + print(f"Error analyzing frame {frame_index}: {e}") + continue + + poses.append({"frame": frame_index, "pose": pose}) + + # Visualize keypoints + this_pose = pose["poses"][0][0] + for j in range(this_pose.shape[0]): + if this_pose[j, 2] > pcutoff: + x, y = map(int, this_pose[j, :2]) + cv2.circle( + frame, + center=(x, y), + radius=display_radius, + color=colors[j], + thickness=-1, + ) + + if draw_keypoint_names: + cv2.putText( + frame, + text=bodyparts[j], + org=(x + 10, y), + fontFace=cv2.FONT_HERSHEY_SIMPLEX, + fontScale=0.5, + color=colors[j], + thickness=1, + lineType=cv2.LINE_AA, + ) + + if resize: + frame = cv2.resize(src=frame, dsize=resize) + + vwriter.write(image=frame) + frame_index += 1 + + cap.release() + vwriter.release() + + if save_poses: + save_poses_to_files(video_path, save_dir, bodyparts, poses) + + return poses + + +def save_poses_to_files(video_path, save_dir, bodyparts, poses): + """ + Save the keypoint poses detected in the video to CSV and HDF5 files. + + Parameters: + ----------- + video_path : str + The path to the video file that was analyzed. + save_dir : str + The directory where the pose data files will be saved. + bodyparts : list of str + A list of body part names corresponding to the keypoints. + poses : list of dict + A list of dictionaries where each dictionary contains the frame number and the corresponding pose data. + + Returns: + -------- + None + """ + base_filename = os.path.splitext(os.path.basename(video_path))[0] + csv_save_path = os.path.join(save_dir, f"{base_filename}_poses.csv") + h5_save_path = os.path.join(save_dir, f"{base_filename}_poses.h5") + + # Save to CSV + with open(csv_save_path, mode="w", newline="") as file: + writer = csv.writer(file) + header = ["frame"] + [ + f"{bp}_{axis}" for bp in bodyparts for axis in ["x", "y", "confidence"] + ] + writer.writerow(header) + for entry in poses: + frame_num = entry["frame"] + pose = entry["pose"]["poses"][0][0] + row = [frame_num] + [item for kp in pose for item in kp] + writer.writerow(row) + + # Save to HDF5 + with h5py.File(h5_save_path, "w") as hf: + hf.create_dataset(name="frames", data=[entry["frame"] for entry in poses]) + for i, bp in enumerate(bodyparts): + hf.create_dataset( + name=f"{bp}_x", + data=[entry["pose"]["poses"][0][0][i, 0].item() for entry in poses], + ) + hf.create_dataset( + name=f"{bp}_y", + data=[entry["pose"]["poses"][0][0][i, 1].item() for entry in poses], + ) + hf.create_dataset( + name=f"{bp}_confidence", + data=[entry["pose"]["poses"][0][0][i, 2].item() for entry in poses], + ) From 9d2350e0c6b4df0861b07db74aa754f6ec392d32 Mon Sep 17 00:00:00 2001 From: AnnaStuckert <47814177+annastuckert@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:46:48 +0100 Subject: [PATCH 09/24] Improvements on benchmark_pytorch.py --- dlclive/benchmark_pytorch.py | 171 +++++++++++++++++++++++++++++++---- dlclive/dlclive.py | 41 +++++---- 2 files changed, 174 insertions(+), 38 deletions(-) diff --git a/dlclive/benchmark_pytorch.py b/dlclive/benchmark_pytorch.py index 94e9d1d..c742903 100644 --- a/dlclive/benchmark_pytorch.py +++ b/dlclive/benchmark_pytorch.py @@ -1,23 +1,139 @@ import csv import os +import platform +import subprocess +import sys +import time import colorcet as cc import cv2 import h5py import numpy as np +import torch from PIL import ImageColor +from pip._internal.operations import freeze + +from dlclive import VERSION, DLCLive + +# def download_benchmarking_data( +# target_dir=".", +# url="http://deeplabcut.rowland.harvard.edu/datasets/dlclivebenchmark.tar.gz", +# ): +# """ +# Downloads a DeepLabCut-Live benchmarking Data (videos & DLC models). +# """ +# import tarfile +# import urllib.request + +# from tqdm import tqdm + +# def show_progress(count, block_size, total_size): +# pbar.update(block_size) + +# def tarfilenamecutting(tarf): +# """' auxfun to extract folder path +# ie. /xyz-trainsetxyshufflez/ +# """ +# for memberid, member in enumerate(tarf.getmembers()): +# if memberid == 0: +# parent = str(member.path) +# l = len(parent) + 1 +# if member.path.startswith(parent): +# member.path = member.path[l:] +# yield member + +# response = urllib.request.urlopen(url) +# print( +# "Downloading the benchmarking data from the DeepLabCut server @Harvard -> Go Crimson!!! {}....".format( +# url +# ) +# ) +# total_size = int(response.getheader("Content-Length")) +# pbar = tqdm(unit="B", total=total_size, position=0) +# filename, _ = urllib.request.urlretrieve(url, reporthook=show_progress) +# with tarfile.open(filename, mode="r:gz") as tar: +# tar.extractall(target_dir, members=tarfilenamecutting(tar)) + + +def get_system_info() -> dict: + """Return summary info for system running benchmark. + + Returns + ------- + dict + Dictionary containing the following system information: + * ``host_name`` (str): name of machine + * ``op_sys`` (str): operating system + * ``python`` (str): path to python (which conda/virtual environment) + * ``device`` (tuple): (device type (``'GPU'`` or ``'CPU'```), device information) + * ``freeze`` (list): list of installed packages and versions + * ``python_version`` (str): python version + * ``git_hash`` (str, None): If installed from git repository, hash of HEAD commit + * ``dlclive_version`` (str): dlclive version from :data:`dlclive.VERSION` + """ + + # Get OS and host name + op_sys = platform.platform() + host_name = platform.node().replace(" ", "") + + # Get Python executable path + if platform.system() == "Windows": + host_python = sys.executable.split(os.path.sep)[-2] + else: + host_python = sys.executable.split(os.path.sep)[-3] + + # Try to get git hash if possible + git_hash = None + dlc_basedir = os.path.dirname(os.path.dirname(__file__)) + try: + git_hash = ( + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=dlc_basedir) + .decode("utf-8") + .strip() + ) + except subprocess.CalledProcessError: + # Not installed from git repo, e.g., pypi + pass + + # Get device info (GPU or CPU) + if torch.cuda.is_available(): + dev_type = "GPU" + dev = [torch.cuda.get_device_name(torch.cuda.current_device())] + else: + from cpuinfo import get_cpu_info + + dev_type = "CPU" + dev = [get_cpu_info()["brand_raw"]] + + return { + "host_name": host_name, + "op_sys": op_sys, + "python": host_python, + "device_type": dev_type, + "device": dev, + "freeze": list(freeze.freeze()), + "python_version": sys.version, + "git_hash": git_hash, + "dlclive_version": VERSION, + } def analyze_video( video_path: str, - dlc_live, + model_path: str, + model_type=str, + device=str, + display=True, pcutoff=0.5, display_radius=5, resize=None, + cropping=None, # Adding cropping to the function parameters + dynamic=(False, 0.5, 10), save_poses=False, save_dir="model_predictions", draw_keypoint_names=False, cmap="bmy", + get_sys_info=True, ): """ Analyze a video to track keypoints using an imported DeepLabCut model, visualize keypoints on the video, and optionally save the keypoint data and the labelled video. @@ -34,6 +150,8 @@ def analyze_video( The radius of the circles drawn to represent keypoints on the video frames. resize : tuple of int (width, height) or None, optional, default=None The size to which the frames should be resized. If None, the frames are not resized. + cropping : list of int, optional, default=None + Cropping parameters in pixel number: [x1, x2, y1, y2] save_poses : bool, optional, default=False Whether to save the detected poses to CSV and HDF5 files. save_dir : str, optional, default="model_predictions" @@ -48,6 +166,17 @@ def analyze_video( poses : list of dict A list of dictionaries where each dictionary contains the frame number and the corresponding pose data. """ + # Create the DLCLive object with cropping + dlc_live = DLCLive( + path=model_path, + model_type=model_type, + device=device, + display=display, + resize=resize, + cropping=cropping, # Pass the cropping parameter + dynamic=dynamic, + ) + # Ensure save directory exists os.makedirs(name=save_dir, exist_ok=True) @@ -56,18 +185,11 @@ def analyze_video( if not cap.isOpened(): print(f"Error: Could not open video file {video_path}") return + # Start empty dict to save poses to for each frame poses = [] - # Create variable indicate current frame. Later in the code +1 is added to frame_index frame_index = 0 - # Load the DLC model - try: - pose_model = dlc_live.load_model() - except Exception as e: - print(f"Error: Could not load DLC model. Details: {e}") - return - # Retrieve bodypart names and number of keypoints bodyparts = dlc_live.cfg["metadata"]["bodyparts"] num_keypoints = len(bodyparts) @@ -86,13 +208,9 @@ def analyze_video( # Get video writer setup fourcc = cv2.VideoWriter_fourcc(*"mp4v") fps = cap.get(cv2.CAP_PROP_FPS) - frame_width, frame_height = ( - int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), - int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)), - ) + frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - if resize: - frame_width, frame_height = resize vwriter = cv2.VideoWriter( filename=output_video_path, fourcc=fourcc, @@ -101,16 +219,29 @@ def analyze_video( ) while True: + start_time = time.time() + ret, frame = cap.read() if not ret: break - + # if frame_index == 0: + # pose = dlc_live.init_inference(frame) # load DLC model try: - pose = dlc_live.get_pose(frame, pose_model=pose_model) + # pose = dlc_live.get_pose(frame) + if frame_index == 0: + # dlc_live.dynamic = (False, dynamic[1], dynamic[2]) # TODO trying to fix issues with dynamic cropping jumping back and forth between dyanmic cropped and original image + pose = dlc_live.init_inference(frame) # load DLC model + else: + # dlc_live.dynamic = dynamic + pose = dlc_live.get_pose(frame) except Exception as e: print(f"Error analyzing frame {frame_index}: {e}") continue + end_time = time.time() + processing_time = end_time - start_time + print(f"Frame {frame_index} processing time: {processing_time:.4f} seconds") + poses.append({"frame": frame_index, "pose": pose}) # Visualize keypoints @@ -138,15 +269,15 @@ def analyze_video( lineType=cv2.LINE_AA, ) - if resize: - frame = cv2.resize(src=frame, dsize=resize) - vwriter.write(image=frame) frame_index += 1 cap.release() vwriter.release() + if get_sys_info: + print(get_system_info()) + if save_poses: save_poses_to_files(video_path, save_dir, bodyparts, poses) diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index a61ef12..bb0e95c 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -20,11 +20,11 @@ import ruamel.yaml import torch from deeplabcut.pose_estimation_pytorch.models import PoseModel + from dlclive import utils from dlclive.display import Display from dlclive.exceptions import DLCLiveError, DLCLiveWarning -from dlclive.pose import (argmax_pose_predict, extract_cnn_output, - multi_pose_predict) +from dlclive.pose import argmax_pose_predict, extract_cnn_output, multi_pose_predict from dlclive.predictor import HeatmapPredictor if typing.TYPE_CHECKING: @@ -210,7 +210,7 @@ def process_frame(self, frame): """ # ! NORMALISE FRAMES - + print(self.dynamic) if self.cropping: frame = frame[ self.cropping[2] : self.cropping[3], self.cropping[0] : self.cropping[1] @@ -218,18 +218,23 @@ def process_frame(self, frame): if self.dynamic[0]: if self.pose is not None: - print(self.pose["bodypart"]) - detected = self.pose[:, 2] > self.dynamic[1] - - if np.any(detected): - - x = self.pose[detected, 0] - y = self.pose[detected, 1] - - x1 = int(max([0, int(np.amin(x)) - self.dynamic[2]])) - x2 = int(min([frame.shape[1], int(np.amax(x)) + self.dynamic[2]])) - y1 = int(max([0, int(np.amin(y)) - self.dynamic[2]])) - y2 = int(min([frame.shape[0], int(np.amax(y)) + self.dynamic[2]])) + detected = self.pose["poses"][0][0][:, 2] > self.dynamic[1] + + # if np.any(detected.numpy()): + if torch.any(detected): + # if detected.any(): # Use PyTorch's any() method + + x = self.pose["poses"][0][0][detected, 0] + y = self.pose["poses"][0][0][detected, 1] + + x1 = int(max([0, int(torch.amin(x)) - self.dynamic[2]])) + x2 = int( + min([frame.shape[1], int(torch.amax(x)) + self.dynamic[2]]) + ) + y1 = int(max([0, int(torch.amin(y)) - self.dynamic[2]])) + y2 = int( + min([frame.shape[0], int(torch.amax(y)) + self.dynamic[2]]) + ) self.dynamic_cropping = [x1, x2, y1, y2] frame = frame[y1:y2, x1:x2] @@ -388,15 +393,15 @@ def get_pose(self, frame=None, **kwargs): # if frame is cropped, convert pose coordinates to original frame coordinates if self.resize is not None: - self.pose[:, :2] *= 1 / self.resize + self.pose["poses"][0][0][:, :2] *= 1 / self.resize if self.cropping is not None: self.pose["poses"][:, :, :, 0][0] += self.cropping[0] self.pose["poses"][:, :, :, 1][0] += self.cropping[2] if self.dynamic_cropping is not None: - self.pose[:, 0] += self.dynamic_cropping[0] - self.pose[:, 1] += self.dynamic_cropping[2] + self.pose["poses"][0][0][:, 0] += self.dynamic_cropping[0] + self.pose["poses"][0][0][:, 1] += self.dynamic_cropping[2] # process the pose From 13b8faec488ff0a2c220fbb2d16832d8aa632419 Mon Sep 17 00:00:00 2001 From: Dikra Date: Mon, 24 Feb 2025 14:47:44 +0100 Subject: [PATCH 10/24] Implement TensorRT optimisation on ONNX models and FP16 precision inference --- dlclive/benchmark_pytorch.py | 13 +++++---- dlclive/dlclive.py | 54 +++++++++++++++++++++++++++--------- 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/dlclive/benchmark_pytorch.py b/dlclive/benchmark_pytorch.py index c742903..b710d68 100644 --- a/dlclive/benchmark_pytorch.py +++ b/dlclive/benchmark_pytorch.py @@ -121,8 +121,9 @@ def get_system_info() -> dict: def analyze_video( video_path: str, model_path: str, - model_type=str, - device=str, + model_type: str, + device: str, + precision:str, display=True, pcutoff=0.5, display_radius=5, @@ -175,6 +176,7 @@ def analyze_video( resize=resize, cropping=cropping, # Pass the cropping parameter dynamic=dynamic, + precision=precision ) # Ensure save directory exists @@ -187,7 +189,8 @@ def analyze_video( return # Start empty dict to save poses to for each frame - poses = [] + poses, times = [], [] + # Create variable indicate current frame. Later in the code +1 is added to frame_index frame_index = 0 # Retrieve bodypart names and number of keypoints @@ -245,7 +248,7 @@ def analyze_video( poses.append({"frame": frame_index, "pose": pose}) # Visualize keypoints - this_pose = pose["poses"][0][0] + this_pose = pose[0]["poses"][0][0] for j in range(this_pose.shape[0]): if this_pose[j, 2] > pcutoff: x, y = map(int, this_pose[j, :2]) @@ -281,7 +284,7 @@ def analyze_video( if save_poses: save_poses_to_files(video_path, save_dir, bodyparts, poses) - return poses + return poses, times def save_poses_to_files(video_path, save_dir, bodyparts, poses): diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index bb0e95c..feb5809 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -120,10 +120,10 @@ class DLCLive(object): def __init__( self, path: str, + snapshot: str = None, model_type: str = "onnx", precision: str = "FP32", device: str = "cpu", - snapshot=str, cropping: Optional[List[int]] = None, dynamic: Tuple[bool, float, float] = (False, 0.5, 10), resize: Optional[float] = None, @@ -209,8 +209,8 @@ def process_frame(self, frame): processed frame: convert type, crop, convert color """ - # ! NORMALISE FRAMES - print(self.dynamic) + # ! NORMALISATION ?? + if self.cropping: frame = frame[ self.cropping[2] : self.cropping[3], self.cropping[0] : self.cropping[1] @@ -266,17 +266,32 @@ def load_model(self): self.pose_model.eval() elif self.model_type == "onnx": - model_path = glob.glob(os.path.normpath(self.path + "/*.onnx"))[0] + model_paths = glob.glob(os.path.normpath(self.path + "/*.onnx")) + if self.precision == "FP16": + model_path = [model_paths[i] for i in range(len(model_paths)) if "fp16" in model_paths[i]][0] + print(model_path) + else: + model_path = model_paths[0] opts = ort.SessionOptions() opts.enable_profiling = False if self.device == "cuda": self.sess = ort.InferenceSession( model_path, opts, providers=["CUDAExecutionProvider"] ) + print(self.sess) elif self.device == "cpu": self.sess = ort.InferenceSession( model_path, opts, providers=["CPUExecutionProvider"] ) + # ! TODO implement if statements for choice of tensorrt engine options (precision, and caching) + elif self.device == "tensorrt": + provider = [("TensorrtExecutionProvider", { + "trt_engine_cache_enable": True, + "trt_engine_cache_path": "./trt_engines" + })] + self.sess = ort.InferenceSession( + model_path, opts, providers=provider + ) self.predictor = HeatmapPredictor.build(self.cfg) if not os.path.isfile(model_path): @@ -314,11 +329,11 @@ def init_inference(self, frame=None, **kwargs): # get pose of first frame (first inference is often very slow) if frame is not None: - pose = self.get_pose(frame, **kwargs) + pose, inf_time = self.get_pose(frame, **kwargs) else: pose = None - return pose + return pose, inf_time def get_pose(self, frame=None, **kwargs): """ @@ -352,14 +367,21 @@ def get_pose(self, frame=None, **kwargs): with torch.no_grad(): start = time.time() outputs = self.pose_model(frame) + torch.cuda.synchronize() end = time.time() - print(f"PyTorch inference took {end - start} sec") + inf_time = end - start + print(f"PyTorch inference took {inf_time} sec") + start = time.time() self.pose = self.pose_model.get_predictions(outputs) + end = time.time() + print(f"PyTorch postprocessing took {end - start} sec") self.pose = self.pose["bodypart"] elif self.model_type == "onnx": - frame = processed_frame.astype(np.float32) + if self.precision == "FP32": frame = processed_frame.astype(np.float32) + elif self.precision == "FP16": frame = processed_frame.astype(np.float16) + frame = np.transpose(frame, (2, 0, 1)) frame = np.expand_dims(frame, axis=0) @@ -368,17 +390,23 @@ def get_pose(self, frame=None, **kwargs): start = time.time() outputs = self.sess.run(None, ort_inputs) end = time.time() - print(f"ONNX inference took {end - start} sec") + inf_time = end - start + print(f"ONNX inference took {inf_time} sec") outputs = { "heatmap": torch.Tensor(outputs[0]), "locref": torch.Tensor(outputs[1]), } + start = time.time() self.pose = self.predictor(outputs=outputs) + end = time.time() + print(f"ONNX postprocessing took {end - start} sec") - else: + # elif self.model_type == "torch_tensorrt": + + else: raise DLCLiveError( "model_type = {} is not supported. model_type must be 'pytorch' or 'onnx'".format( self.model_type @@ -396,8 +424,8 @@ def get_pose(self, frame=None, **kwargs): self.pose["poses"][0][0][:, :2] *= 1 / self.resize if self.cropping is not None: - self.pose["poses"][:, :, :, 0][0] += self.cropping[0] - self.pose["poses"][:, :, :, 1][0] += self.cropping[2] + self.pose["poses"][0][0][0] += self.cropping[0] + self.pose["poses"][0][0][0] += self.cropping[2] if self.dynamic_cropping is not None: self.pose["poses"][0][0][:, 0] += self.dynamic_cropping[0] @@ -408,7 +436,7 @@ def get_pose(self, frame=None, **kwargs): if self.processor: self.pose = self.processor.process(self.pose, **kwargs) - return self.pose + return self.pose, inf_time # def close(self): # """ Close tensorflow session From 8b41df62019597e28d4e27a31906f3998dd48e80 Mon Sep 17 00:00:00 2001 From: AnnaStuckert <47814177+annastuckert@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:49:34 +0100 Subject: [PATCH 11/24] Avs live feed --- dlclive/LiveVideoInference.py | 317 ++++++++++++++++++++++++++++++++++ dlclive/benchmark_pytorch.py | 6 +- dlclive/dlclive.py | 2 +- 3 files changed, 322 insertions(+), 3 deletions(-) create mode 100644 dlclive/LiveVideoInference.py diff --git a/dlclive/LiveVideoInference.py b/dlclive/LiveVideoInference.py new file mode 100644 index 0000000..6c9d128 --- /dev/null +++ b/dlclive/LiveVideoInference.py @@ -0,0 +1,317 @@ +import csv +import os +import platform +import subprocess +import sys +import time + +import colorcet as cc +import cv2 +import h5py +import numpy as np +import torch +from PIL import ImageColor +from pip._internal.operations import freeze + +from dlclive import VERSION, DLCLive + + +def get_system_info() -> dict: + """Return summary info for system running benchmark. + + Returns + ------- + dict + Dictionary containing the following system information: + * ``host_name`` (str): name of machine + * ``op_sys`` (str): operating system + * ``python`` (str): path to python (which conda/virtual environment) + * ``device`` (tuple): (device type (``'GPU'`` or ``'CPU'```), device information) + * ``freeze`` (list): list of installed packages and versions + * ``python_version`` (str): python version + * ``git_hash`` (str, None): If installed from git repository, hash of HEAD commit + * ``dlclive_version`` (str): dlclive version from :data:`dlclive.VERSION` + """ + + # Get OS and host name + op_sys = platform.platform() + host_name = platform.node().replace(" ", "") + + # Get Python executable path + if platform.system() == "Windows": + host_python = sys.executable.split(os.path.sep)[-2] + else: + host_python = sys.executable.split(os.path.sep)[-3] + + # Try to get git hash if possible + git_hash = None + dlc_basedir = os.path.dirname(os.path.dirname(__file__)) + try: + git_hash = ( + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=dlc_basedir) + .decode("utf-8") + .strip() + ) + except subprocess.CalledProcessError: + # Not installed from git repo, e.g., pypi + pass + + # Get device info (GPU or CPU) + if torch.cuda.is_available(): + dev_type = "GPU" + dev = [torch.cuda.get_device_name(torch.cuda.current_device())] + else: + from cpuinfo import get_cpu_info + + dev_type = "CPU" + dev = [get_cpu_info()["brand_raw"]] + + return { + "host_name": host_name, + "op_sys": op_sys, + "python": host_python, + "device_type": dev_type, + "device": dev, + "freeze": list(freeze.freeze()), + "python_version": sys.version, + "git_hash": git_hash, + "dlclive_version": VERSION, + } + + +def analyze_live_video( + model_path: str, + model_type: str, + device: str, + camera: float = 0, + experiment_name: str = "Test", + precision: str = "FP32", + snapshot: str = None, + display=True, + pcutoff=0.5, + display_radius=5, + resize=None, + cropping=None, # Adding cropping to the function parameters + dynamic=(False, 0.5, 10), + save_poses=False, + save_dir="model_predictions", + draw_keypoint_names=False, + cmap="bmy", + get_sys_info=True, +): + """ + Analyze a video to track keypoints using an imported DeepLabCut model, visualize keypoints on the video, and optionally save the keypoint data and the labelled video. + + Parameters: + ----------- + camera : float, default=0 (webcam) + The camera to record the live video from + experiment_name : str, default = "Test" + Prefix to label generated pose and video files + pcutoff : float, optional, default=0.5 + The probability cutoff value below which keypoints are not visualized. + display_radius : int, optional, default=5 + The radius of the circles drawn to represent keypoints on the video frames. + resize : tuple of int (width, height) or None, optional, default=None + The size to which the frames should be resized. If None, the frames are not resized. + cropping : list of int, optional, default=None + Cropping parameters in pixel number: [x1, x2, y1, y2] + save_poses : bool, optional, default=False + Whether to save the detected poses to CSV and HDF5 files. + save_dir : str, optional, default="model_predictions" + The directory where the output video and pose data will be saved. + draw_keypoint_names : bool, optional, default=False + Whether to draw the names of the keypoints on the video frames. + cmap : str, optional, default="bmy" + The colormap from the colorcet library to use for keypoint visualization. + + Returns: + -------- + poses : list of dict + A list of dictionaries where each dictionary contains the frame number and the corresponding pose data. + """ + # Create the DLCLive object with cropping + dlc_live = DLCLive( + path=model_path, + model_type=model_type, + device=device, + display=display, + resize=resize, + cropping=cropping, # Pass the cropping parameter + dynamic=dynamic, + precision=precision, + snapshot=snapshot, + ) + + # Ensure save directory exists + os.makedirs(name=save_dir, exist_ok=True) + + # Load video + cap = cv2.VideoCapture(camera) + if not cap.isOpened(): + print(f"Error: Could not open video file {camera}") + return + + # Start empty dict to save poses to for each frame + poses, times = [], [] + frame_index = 0 + + # Retrieve bodypart names and number of keypoints + bodyparts = dlc_live.cfg["metadata"]["bodyparts"] + num_keypoints = len(bodyparts) + + # Set colors and convert to RGB + cmap_colors = getattr(cc, cmap) + colors = [ + ImageColor.getrgb(color) + for color in cmap_colors[:: int(len(cmap_colors) / num_keypoints)] + ] + + # Define output video path + output_video_path = os.path.join( + save_dir, f"{experiment_name}_DLCLIVE_LABELLED.mp4" + ) + + # Get video writer setup + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + fps = cap.get(cv2.CAP_PROP_FPS) + frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + vwriter = cv2.VideoWriter( + filename=output_video_path, + fourcc=fourcc, + fps=fps, + frameSize=(frame_width, frame_height), + ) + + while True: + start_time = time.time() + + ret, frame = cap.read() + if not ret: + break + + try: + if frame_index == 0: + pose = dlc_live.init_inference(frame) # load DLC model + else: + pose = dlc_live.get_pose(frame) + except Exception as e: + print(f"Error analyzing frame {frame_index}: {e}") + continue + + end_time = time.time() + processing_time = end_time - start_time + print(f"Frame {frame_index} processing time: {processing_time:.4f} seconds") + + poses.append({"frame": frame_index, "pose": pose}) + + # Visualize keypoints + this_pose = pose[0]["poses"][0][0] + for j in range(this_pose.shape[0]): + if this_pose[j, 2] > pcutoff: + x, y = map(int, this_pose[j, :2]) + cv2.circle( + frame, + center=(x, y), + radius=display_radius, + color=colors[j], + thickness=-1, + ) + + if draw_keypoint_names: + cv2.putText( + frame, + text=bodyparts[j], + org=(x + 10, y), + fontFace=cv2.FONT_HERSHEY_SIMPLEX, + fontScale=0.5, + color=colors[j], + thickness=1, + lineType=cv2.LINE_AA, + ) + + vwriter.write(image=frame) + frame_index += 1 + + # Display the frame + if display: + cv2.imshow("DLCLive", frame) + + # Add key press check for quitting + if cv2.waitKey(1) & 0xFF == ord("q"): + break + + cap.release() + vwriter.release() + cv2.destroyAllWindows() + + if get_sys_info: + print(get_system_info()) + + if save_poses: + save_poses_to_files(experiment_name, save_dir, bodyparts, poses) + + return poses, times + + +def save_poses_to_files(experiment_name, save_dir, bodyparts, poses): + """ + Save the keypoint poses detected in the video to CSV and HDF5 files. + + Parameters: + ----------- + experiment_name : str + Name of the experiment, used as a prefix for saving files. + save_dir : str + The directory where the pose data files will be saved. + bodyparts : list of str + A list of body part names corresponding to the keypoints. + poses : list of dict + A list of dictionaries where each dictionary contains the frame number and the corresponding pose data. + + Returns: + -------- + None + """ + base_filename = os.path.splitext(os.path.basename(experiment_name))[0] + csv_save_path = os.path.join(save_dir, f"{base_filename}_poses.csv") + h5_save_path = os.path.join(save_dir, f"{base_filename}_poses.h5") + + # Save to CSV + with open(csv_save_path, mode="w", newline="") as file: + writer = csv.writer(file) + header = ["frame"] + [ + f"{bp}_{axis}" for bp in bodyparts for axis in ["x", "y", "confidence"] + ] + writer.writerow(header) + for entry in poses: + frame_num = entry["frame"] + pose_data = entry["pose"][0]["poses"][0][0] + # Convert tensor data to numeric values + row = [frame_num] + [ + item.item() if isinstance(item, torch.Tensor) else item + for kp in pose_data + for item in kp + ] + writer.writerow(row) + + # Save to HDF5 + with h5py.File(h5_save_path, "w") as hf: + hf.create_dataset(name="frames", data=[entry["frame"] for entry in poses]) + for i, bp in enumerate(bodyparts): + hf.create_dataset( + name=f"{bp}_x", + data=[ + ( + entry["pose"][0]["poses"][0][0][i, 0].item() + if isinstance( + entry["pose"][0]["poses"][0][0][i, 0], torch.Tensor + ) + else entry["pose"][0]["poses"][0][0][i, 0] + ) + for entry in poses + ], + ) + hf.create_dataset( diff --git a/dlclive/benchmark_pytorch.py b/dlclive/benchmark_pytorch.py index b710d68..7151664 100644 --- a/dlclive/benchmark_pytorch.py +++ b/dlclive/benchmark_pytorch.py @@ -123,7 +123,8 @@ def analyze_video( model_path: str, model_type: str, device: str, - precision:str, + precision: str = "FP32", + snapshot: str = None, display=True, pcutoff=0.5, display_radius=5, @@ -176,7 +177,8 @@ def analyze_video( resize=resize, cropping=cropping, # Pass the cropping parameter dynamic=dynamic, - precision=precision + precision=precision, + snapshot=snapshot ) # Ensure save directory exists diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index feb5809..e61725b 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -367,7 +367,7 @@ def get_pose(self, frame=None, **kwargs): with torch.no_grad(): start = time.time() outputs = self.pose_model(frame) - torch.cuda.synchronize() + #torch.cuda.synchronize() end = time.time() inf_time = end - start print(f"PyTorch inference took {inf_time} sec") From d1e7df967540115c8210f138011356d90bcf2b1e Mon Sep 17 00:00:00 2001 From: Dikra Date: Mon, 24 Feb 2025 14:50:11 +0100 Subject: [PATCH 12/24] Tutorial notebook in progress --- dlclive/benchmark_pytorch.py | 108 +++++++++++++------------- dlclive/dlclive.py | 27 +++---- dlclive/predictor/single_predictor.py | 23 +++++- 3 files changed, 85 insertions(+), 73 deletions(-) diff --git a/dlclive/benchmark_pytorch.py b/dlclive/benchmark_pytorch.py index 7151664..45393e3 100644 --- a/dlclive/benchmark_pytorch.py +++ b/dlclive/benchmark_pytorch.py @@ -15,6 +15,7 @@ from dlclive import VERSION, DLCLive + # def download_benchmarking_data( # target_dir=".", # url="http://deeplabcut.rowland.harvard.edu/datasets/dlclivebenchmark.tar.gz", @@ -136,6 +137,7 @@ def analyze_video( draw_keypoint_names=False, cmap="bmy", get_sys_info=True, + save_video=False ): """ Analyze a video to track keypoints using an imported DeepLabCut model, visualize keypoints on the video, and optionally save the keypoint data and the labelled video. @@ -199,29 +201,30 @@ def analyze_video( bodyparts = dlc_live.cfg["metadata"]["bodyparts"] num_keypoints = len(bodyparts) - # Set colors and convert to RGB - cmap_colors = getattr(cc, cmap) - colors = [ - ImageColor.getrgb(color) - for color in cmap_colors[:: int(len(cmap_colors) / num_keypoints)] - ] - - # Define output video path - video_name = os.path.splitext(os.path.basename(video_path))[0] - output_video_path = os.path.join(save_dir, f"{video_name}_DLCLIVE_LABELLED.mp4") - - # Get video writer setup - fourcc = cv2.VideoWriter_fourcc(*"mp4v") - fps = cap.get(cv2.CAP_PROP_FPS) - frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) - frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - - vwriter = cv2.VideoWriter( - filename=output_video_path, - fourcc=fourcc, - fps=fps, - frameSize=(frame_width, frame_height), - ) + if save_video: + # Set colors and convert to RGB + cmap_colors = getattr(cc, cmap) + colors = [ + ImageColor.getrgb(color) + for color in cmap_colors[:: int(len(cmap_colors) / num_keypoints)] + ] + + # Define output video path + video_name = os.path.splitext(os.path.basename(video_path))[0] + output_video_path = os.path.join(save_dir, f"{video_name}_DLCLIVE_LABELLED.mp4") + + # Get video writer setup + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + fps = cap.get(cv2.CAP_PROP_FPS) + frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + vwriter = cv2.VideoWriter( + filename=output_video_path, + fourcc=fourcc, + fps=fps, + frameSize=(frame_width, frame_height), + ) while True: start_time = time.time() @@ -235,50 +238,49 @@ def analyze_video( # pose = dlc_live.get_pose(frame) if frame_index == 0: # dlc_live.dynamic = (False, dynamic[1], dynamic[2]) # TODO trying to fix issues with dynamic cropping jumping back and forth between dyanmic cropped and original image - pose = dlc_live.init_inference(frame) # load DLC model + pose, inf_time = dlc_live.init_inference(frame) # load DLC model else: # dlc_live.dynamic = dynamic - pose = dlc_live.get_pose(frame) + pose, inf_time = dlc_live.get_pose(frame) except Exception as e: print(f"Error analyzing frame {frame_index}: {e}") continue - end_time = time.time() - processing_time = end_time - start_time - print(f"Frame {frame_index} processing time: {processing_time:.4f} seconds") - poses.append({"frame": frame_index, "pose": pose}) - - # Visualize keypoints - this_pose = pose[0]["poses"][0][0] - for j in range(this_pose.shape[0]): - if this_pose[j, 2] > pcutoff: - x, y = map(int, this_pose[j, :2]) - cv2.circle( - frame, - center=(x, y), - radius=display_radius, - color=colors[j], - thickness=-1, - ) - - if draw_keypoint_names: - cv2.putText( + times.append(inf_time) + + if save_video: + # Visualize keypoints + this_pose = pose["poses"][0][0] + for j in range(this_pose.shape[0]): + if this_pose[j, 2] > pcutoff: + x, y = map(int, this_pose[j, :2]) + cv2.circle( frame, - text=bodyparts[j], - org=(x + 10, y), - fontFace=cv2.FONT_HERSHEY_SIMPLEX, - fontScale=0.5, + center=(x, y), + radius=display_radius, color=colors[j], - thickness=1, - lineType=cv2.LINE_AA, + thickness=-1, ) - vwriter.write(image=frame) + if draw_keypoint_names: + cv2.putText( + frame, + text=bodyparts[j], + org=(x + 10, y), + fontFace=cv2.FONT_HERSHEY_SIMPLEX, + fontScale=0.5, + color=colors[j], + thickness=1, + lineType=cv2.LINE_AA, + ) + + vwriter.write(image=frame) frame_index += 1 cap.release() - vwriter.release() + if save_video: + vwriter.release() if get_sys_info: print(get_system_info()) diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index e61725b..a5fcff3 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -161,6 +161,15 @@ def __init__( self.pose_model = None self.predictor = None self.pose = None + + if self.model_type == "pytorch" and (self.snapshot) is None: + raise DLCLiveError( + f"The selected model type is '{self.model_type}', but no snapshot was provided" + ) + if self.model_type == "pytorch" and (self.device) == "tensorrt": + raise DLCLiveError( + f"The selected model type is '{self.model_type}' is not enabled by the selected runtime {self.device}" + ) self.read_config() def read_config(self): @@ -321,12 +330,10 @@ def init_inference(self, frame=None, **kwargs): the pose estimated by DeepLabCut for the input image """ - start = time.time() # load model self.load_model() - end = time.time() - print(f"Loading the model took {end - start} sec") + inf_time = 0. # get pose of first frame (first inference is often very slow) if frame is not None: pose, inf_time = self.get_pose(frame, **kwargs) @@ -349,7 +356,7 @@ def get_pose(self, frame=None, **kwargs): pose :class:`numpy.ndarray` the pose estimated by DeepLabCut for the input image """ - + inf_time = 0. if frame is None: raise DLCLiveError("No frame provided for live pose estimation") @@ -367,15 +374,10 @@ def get_pose(self, frame=None, **kwargs): with torch.no_grad(): start = time.time() outputs = self.pose_model(frame) - #torch.cuda.synchronize() end = time.time() inf_time = end - start - print(f"PyTorch inference took {inf_time} sec") - start = time.time() self.pose = self.pose_model.get_predictions(outputs) - end = time.time() - print(f"PyTorch postprocessing took {end - start} sec") self.pose = self.pose["bodypart"] elif self.model_type == "onnx": @@ -391,20 +393,13 @@ def get_pose(self, frame=None, **kwargs): outputs = self.sess.run(None, ort_inputs) end = time.time() inf_time = end - start - print(f"ONNX inference took {inf_time} sec") outputs = { "heatmap": torch.Tensor(outputs[0]), "locref": torch.Tensor(outputs[1]), } - start = time.time() self.pose = self.predictor(outputs=outputs) - end = time.time() - print(f"ONNX postprocessing took {end - start} sec") - - # elif self.model_type == "torch_tensorrt": - else: raise DLCLiveError( diff --git a/dlclive/predictor/single_predictor.py b/dlclive/predictor/single_predictor.py index 528039b..6425d02 100644 --- a/dlclive/predictor/single_predictor.py +++ b/dlclive/predictor/single_predictor.py @@ -163,17 +163,32 @@ def get_pose_prediction( @staticmethod def build(cfg: dict) -> HeatmapPredictor: + # if cfg["method"] == "bu": apply_sigmoid = cfg["model"]["heads"]["bodypart"]["predictor"]["apply_sigmoid"] clip_scores = cfg["model"]["heads"]["bodypart"]["predictor"]["clip_scores"] loc_ref = cfg["model"]["heads"]["bodypart"]["predictor"]["location_refinement"] loc_ref_std = cfg["model"]["heads"]["bodypart"]["predictor"]["locref_std"] - stride = float(cfg["model"]["backbone"]["output_stride"]) / float( - cfg["model"]["heads"]["bodypart"]["heatmap_config"]["strides"][0] - ) - + if len(cfg["model"]["heads"]["bodypart"]["heatmap_config"]["strides"]) > 0: + if cfg["model"]["heads"]["bodypart"]["heatmap_config"]["strides"][0] > 0: + stride = float(cfg["model"]["backbone"]["output_stride"]) / float( + cfg["model"]["heads"]["bodypart"]["heatmap_config"]["strides"][0] + ) + else: + stride = float(cfg["model"]["backbone"]["output_stride"]) * -float( + cfg["model"]["heads"]["bodypart"]["heatmap_config"]["strides"][0] + ) + else: + stride = float(cfg["model"]["backbone"]["output_stride"]) predictor = HeatmapPredictor( apply_sigmoid=apply_sigmoid, stride=stride, clip_scores=clip_scores, location_refinement=loc_ref, locref_std=loc_ref_std ) + # elif cfg["method"] == "td": + # apply_sigmoid = cfg["model"]["heads"]["bodypart"]["predictor"]["apply_sigmoid"] + # clip_scores = cfg["model"]["heads"]["bodypart"]["predictor"]["clip_scores"] + # loc_ref = cfg["model"]["heads"]["bodypart"]["predictor"]["location_refinement"] + # heatmap_stride = cfg[] + # predictor = HeatmapPredictor(apply_sigmoid=apply_sigmoid, clip_scores=clip_scores, location_refinement=loc_ref) + return predictor \ No newline at end of file From 2fb39fcbface642e19a00cb57ff51c2c934cc728 Mon Sep 17 00:00:00 2001 From: AnnaStuckert <47814177+annastuckert@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:50:48 +0100 Subject: [PATCH 13/24] bug fixing h5 saving for live video feed --- dlclive/LiveVideoInference.py | 123 +++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 47 deletions(-) diff --git a/dlclive/LiveVideoInference.py b/dlclive/LiveVideoInference.py index 6c9d128..9e75df6 100644 --- a/dlclive/LiveVideoInference.py +++ b/dlclive/LiveVideoInference.py @@ -98,6 +98,7 @@ def analyze_live_video( draw_keypoint_names=False, cmap="bmy", get_sys_info=True, + save_video=False, ): """ Analyze a video to track keypoints using an imported DeepLabCut model, visualize keypoints on the video, and optionally save the keypoint data and the labelled video. @@ -160,30 +161,31 @@ def analyze_live_video( bodyparts = dlc_live.cfg["metadata"]["bodyparts"] num_keypoints = len(bodyparts) - # Set colors and convert to RGB - cmap_colors = getattr(cc, cmap) - colors = [ - ImageColor.getrgb(color) - for color in cmap_colors[:: int(len(cmap_colors) / num_keypoints)] - ] + if save_video: + # Set colors and convert to RGB + cmap_colors = getattr(cc, cmap) + colors = [ + ImageColor.getrgb(color) + for color in cmap_colors[:: int(len(cmap_colors) / num_keypoints)] + ] - # Define output video path - output_video_path = os.path.join( - save_dir, f"{experiment_name}_DLCLIVE_LABELLED.mp4" - ) + # Define output video path + output_video_path = os.path.join( + save_dir, f"{experiment_name}_DLCLIVE_LABELLED.mp4" + ) - # Get video writer setup - fourcc = cv2.VideoWriter_fourcc(*"mp4v") - fps = cap.get(cv2.CAP_PROP_FPS) - frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) - frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - - vwriter = cv2.VideoWriter( - filename=output_video_path, - fourcc=fourcc, - fps=fps, - frameSize=(frame_width, frame_height), - ) + # Get video writer setup + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + fps = cap.get(cv2.CAP_PROP_FPS) + frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + vwriter = cv2.VideoWriter( + filename=output_video_path, + fourcc=fourcc, + fps=fps, + frameSize=(frame_width, frame_height), + ) while True: start_time = time.time() @@ -206,33 +208,33 @@ def analyze_live_video( print(f"Frame {frame_index} processing time: {processing_time:.4f} seconds") poses.append({"frame": frame_index, "pose": pose}) - - # Visualize keypoints - this_pose = pose[0]["poses"][0][0] - for j in range(this_pose.shape[0]): - if this_pose[j, 2] > pcutoff: - x, y = map(int, this_pose[j, :2]) - cv2.circle( - frame, - center=(x, y), - radius=display_radius, - color=colors[j], - thickness=-1, - ) - - if draw_keypoint_names: - cv2.putText( + if save_video: + # Visualize keypoints + this_pose = pose[0]["poses"][0][0] + for j in range(this_pose.shape[0]): + if this_pose[j, 2] > pcutoff: + x, y = map(int, this_pose[j, :2]) + cv2.circle( frame, - text=bodyparts[j], - org=(x + 10, y), - fontFace=cv2.FONT_HERSHEY_SIMPLEX, - fontScale=0.5, + center=(x, y), + radius=display_radius, color=colors[j], - thickness=1, - lineType=cv2.LINE_AA, + thickness=-1, ) - vwriter.write(image=frame) + if draw_keypoint_names: + cv2.putText( + frame, + text=bodyparts[j], + org=(x + 10, y), + fontFace=cv2.FONT_HERSHEY_SIMPLEX, + fontScale=0.5, + color=colors[j], + thickness=1, + lineType=cv2.LINE_AA, + ) + + vwriter.write(image=frame) frame_index += 1 # Display the frame @@ -244,8 +246,10 @@ def analyze_live_video( break cap.release() - vwriter.release() - cv2.destroyAllWindows() + + if save_video: + vwriter.release() + # cv2.destroyAllWindows() if get_sys_info: print(get_system_info()) @@ -315,3 +319,28 @@ def save_poses_to_files(experiment_name, save_dir, bodyparts, poses): ], ) hf.create_dataset( + name=f"{bp}_y", + data=[ + ( + entry["pose"][0]["poses"][0][0][i, 1].item() + if isinstance( + entry["pose"][0]["poses"][0][0][i, 1], torch.Tensor + ) + else entry["pose"][0]["poses"][0][0][i, 1] + ) + for entry in poses + ], + ) + hf.create_dataset( + name=f"{bp}_confidence", + data=[ + ( + entry["pose"][0]["poses"][0][0][i, 2].item() + if isinstance( + entry["pose"][0]["poses"][0][0][i, 2], torch.Tensor + ) + else entry["pose"][0]["poses"][0][0][i, 2] + ) + for entry in poses + ], + ) From 98b3a13c07adace6f78721fd658ed61653ef8653 Mon Sep 17 00:00:00 2001 From: AnnaStuckert <47814177+annastuckert@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:52:29 +0100 Subject: [PATCH 14/24] add code to save numbers in csv and h5 as numbers, not tensor(number) --- dlclive/benchmark_pytorch.py | 380 ++++++++++++++++++++++++++++++----- 1 file changed, 332 insertions(+), 48 deletions(-) diff --git a/dlclive/benchmark_pytorch.py b/dlclive/benchmark_pytorch.py index 45393e3..773f200 100644 --- a/dlclive/benchmark_pytorch.py +++ b/dlclive/benchmark_pytorch.py @@ -292,54 +292,338 @@ def analyze_video( def save_poses_to_files(video_path, save_dir, bodyparts, poses): - """ - Save the keypoint poses detected in the video to CSV and HDF5 files. + import csv + import os + import platform + import subprocess + import sys + import time + + import colorcet as cc + import cv2 + import h5py + import numpy as np + import torch + from PIL import ImageColor + from pip._internal.operations import freeze + + from dlclive import VERSION, DLCLive + + def get_system_info() -> dict: + """Return summary info for system running benchmark. + + Returns + ------- + dict + Dictionary containing the following system information: + * ``host_name`` (str): name of machine + * ``op_sys`` (str): operating system + * ``python`` (str): path to python (which conda/virtual environment) + * ``device`` (tuple): (device type (``'GPU'`` or ``'CPU'```), device information) + * ``freeze`` (list): list of installed packages and versions + * ``python_version`` (str): python version + * ``git_hash`` (str, None): If installed from git repository, hash of HEAD commit + * ``dlclive_version`` (str): dlclive version from :data:`dlclive.VERSION` + """ + + # Get OS and host name + op_sys = platform.platform() + host_name = platform.node().replace(" ", "") + + # Get Python executable path + if platform.system() == "Windows": + host_python = sys.executable.split(os.path.sep)[-2] + else: + host_python = sys.executable.split(os.path.sep)[-3] + + # Try to get git hash if possible + git_hash = None + dlc_basedir = os.path.dirname(os.path.dirname(__file__)) + try: + git_hash = ( + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=dlc_basedir) + .decode("utf-8") + .strip() + ) + except subprocess.CalledProcessError: + # Not installed from git repo, e.g., pypi + pass + + # Get device info (GPU or CPU) + if torch.cuda.is_available(): + dev_type = "GPU" + dev = [torch.cuda.get_device_name(torch.cuda.current_device())] + else: + from cpuinfo import get_cpu_info + + dev_type = "CPU" + dev = [get_cpu_info()["brand_raw"]] + + return { + "host_name": host_name, + "op_sys": op_sys, + "python": host_python, + "device_type": dev_type, + "device": dev, + "freeze": list(freeze.freeze()), + "python_version": sys.version, + "git_hash": git_hash, + "dlclive_version": VERSION, + } + + def analyze_video( + video_path: str, + model_path: str, + model_type: str, + device: str, + precision: str = "FP32", + snapshot: str = None, + display=True, + pcutoff=0.5, + display_radius=5, + resize=None, + cropping=None, # Adding cropping to the function parameters + dynamic=(False, 0.5, 10), + save_poses=False, + save_dir="model_predictions", + draw_keypoint_names=False, + cmap="bmy", + get_sys_info=True, + save_video=False, + ): + """ + Analyze a video to track keypoints using an imported DeepLabCut model, visualize keypoints on the video, and optionally save the keypoint data and the labelled video. + + Parameters: + ----------- + video_path : str + The path to the video file to be analyzed. + dlc_live : DLCLive + An instance of the DLCLive class. + pcutoff : float, optional, default=0.5 + The probability cutoff value below which keypoints are not visualized. + display_radius : int, optional, default=5 + The radius of the circles drawn to represent keypoints on the video frames. + resize : tuple of int (width, height) or None, optional, default=None + The size to which the frames should be resized. If None, the frames are not resized. + cropping : list of int, optional, default=None + Cropping parameters in pixel number: [x1, x2, y1, y2] + save_poses : bool, optional, default=False + Whether to save the detected poses to CSV and HDF5 files. + save_dir : str, optional, default="model_predictions" + The directory where the output video and pose data will be saved. + draw_keypoint_names : bool, optional, default=False + Whether to draw the names of the keypoints on the video frames. + cmap : str, optional, default="bmy" + The colormap from the colorcet library to use for keypoint visualization. + + Returns: + -------- + poses : list of dict + A list of dictionaries where each dictionary contains the frame number and the corresponding pose data. + """ + # Create the DLCLive object with cropping + dlc_live = DLCLive( + path=model_path, + model_type=model_type, + device=device, + display=display, + resize=resize, + cropping=cropping, # Pass the cropping parameter + dynamic=dynamic, + precision=precision, + snapshot=snapshot, + ) - Parameters: - ----------- - video_path : str - The path to the video file that was analyzed. - save_dir : str - The directory where the pose data files will be saved. - bodyparts : list of str - A list of body part names corresponding to the keypoints. - poses : list of dict - A list of dictionaries where each dictionary contains the frame number and the corresponding pose data. + # Ensure save directory exists + os.makedirs(name=save_dir, exist_ok=True) - Returns: - -------- - None - """ - base_filename = os.path.splitext(os.path.basename(video_path))[0] - csv_save_path = os.path.join(save_dir, f"{base_filename}_poses.csv") - h5_save_path = os.path.join(save_dir, f"{base_filename}_poses.h5") - - # Save to CSV - with open(csv_save_path, mode="w", newline="") as file: - writer = csv.writer(file) - header = ["frame"] + [ - f"{bp}_{axis}" for bp in bodyparts for axis in ["x", "y", "confidence"] - ] - writer.writerow(header) - for entry in poses: - frame_num = entry["frame"] - pose = entry["pose"]["poses"][0][0] - row = [frame_num] + [item for kp in pose for item in kp] - writer.writerow(row) - - # Save to HDF5 - with h5py.File(h5_save_path, "w") as hf: - hf.create_dataset(name="frames", data=[entry["frame"] for entry in poses]) - for i, bp in enumerate(bodyparts): - hf.create_dataset( - name=f"{bp}_x", - data=[entry["pose"]["poses"][0][0][i, 0].item() for entry in poses], - ) - hf.create_dataset( - name=f"{bp}_y", - data=[entry["pose"]["poses"][0][0][i, 1].item() for entry in poses], - ) - hf.create_dataset( - name=f"{bp}_confidence", - data=[entry["pose"]["poses"][0][0][i, 2].item() for entry in poses], + # Load video + cap = cv2.VideoCapture(video_path) + if not cap.isOpened(): + print(f"Error: Could not open video file {video_path}") + return + + # Start empty dict to save poses to for each frame + poses, times = [], [] + # Create variable indicate current frame. Later in the code +1 is added to frame_index + frame_index = 0 + + # Retrieve bodypart names and number of keypoints + bodyparts = dlc_live.cfg["metadata"]["bodyparts"] + num_keypoints = len(bodyparts) + + if save_video: + # Set colors and convert to RGB + cmap_colors = getattr(cc, cmap) + colors = [ + ImageColor.getrgb(color) + for color in cmap_colors[:: int(len(cmap_colors) / num_keypoints)] + ] + + # Define output video path + video_name = os.path.splitext(os.path.basename(video_path))[0] + output_video_path = os.path.join( + save_dir, f"{video_name}_DLCLIVE_LABELLED.mp4" + ) + + # Get video writer setup + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + fps = cap.get(cv2.CAP_PROP_FPS) + frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + vwriter = cv2.VideoWriter( + filename=output_video_path, + fourcc=fourcc, + fps=fps, + frameSize=(frame_width, frame_height), ) + + while True: + start_time = time.time() + + ret, frame = cap.read() + if not ret: + break + # if frame_index == 0: + # pose = dlc_live.init_inference(frame) # load DLC model + try: + # pose = dlc_live.get_pose(frame) + if frame_index == 0: + # dlc_live.dynamic = (False, dynamic[1], dynamic[2]) # TODO trying to fix issues with dynamic cropping jumping back and forth between dyanmic cropped and original image + pose, inf_time = dlc_live.init_inference(frame) # load DLC model + else: + # dlc_live.dynamic = dynamic + pose, inf_time = dlc_live.get_pose(frame) + except Exception as e: + print(f"Error analyzing frame {frame_index}: {e}") + continue + + poses.append({"frame": frame_index, "pose": pose}) + times.append(inf_time) + + if save_video: + # Visualize keypoints + this_pose = pose["poses"][0][0] + for j in range(this_pose.shape[0]): + if this_pose[j, 2] > pcutoff: + x, y = map(int, this_pose[j, :2]) + cv2.circle( + frame, + center=(x, y), + radius=display_radius, + color=colors[j], + thickness=-1, + ) + + if draw_keypoint_names: + cv2.putText( + frame, + text=bodyparts[j], + org=(x + 10, y), + fontFace=cv2.FONT_HERSHEY_SIMPLEX, + fontScale=0.5, + color=colors[j], + thickness=1, + lineType=cv2.LINE_AA, + ) + + vwriter.write(image=frame) + frame_index += 1 + + cap.release() + if save_video: + vwriter.release() + + if get_sys_info: + print(get_system_info()) + + if save_poses: + save_poses_to_files(video_path, save_dir, bodyparts, poses) + + return poses, times + + def save_poses_to_files(video_path, save_dir, bodyparts, poses): + """ + Save the keypoint poses detected in the video to CSV and HDF5 files. + + Parameters: + ----------- + video_path : str + The path to the video file that was analyzed. + save_dir : str + The directory where the pose data files will be saved. + bodyparts : list of str + A list of body part names corresponding to the keypoints. + poses : list of dict + A list of dictionaries where each dictionary contains the frame number and the corresponding pose data. + + Returns: + -------- + None + """ + base_filename = os.path.splitext(os.path.basename(video_path))[0] + csv_save_path = os.path.join(save_dir, f"{base_filename}_poses.csv") + h5_save_path = os.path.join(save_dir, f"{base_filename}_poses.h5") + + # Save to CSV + with open(csv_save_path, mode="w", newline="") as file: + writer = csv.writer(file) + header = ["frame"] + [ + f"{bp}_{axis}" for bp in bodyparts for axis in ["x", "y", "confidence"] + ] + writer.writerow(header) + for entry in poses: + frame_num = entry["frame"] + pose = entry["pose"]["poses"][0][0] + row = [frame_num] + [ + item.item() if isinstance(item, torch.Tensor) else item + for kp in pose + for item in kp + ] + writer.writerow(row) + + # Save to HDF5 + with h5py.File(h5_save_path, "w") as hf: + hf.create_dataset(name="frames", data=[entry["frame"] for entry in poses]) + for i, bp in enumerate(bodyparts): + hf.create_dataset( + name=f"{bp}_x", + data=[ + ( + entry["pose"]["poses"][0][0][i, 0].item() + if isinstance( + entry["pose"]["poses"][0][0][i, 0], torch.Tensor + ) + else entry["pose"]["poses"][0][0][i, 0] + ) + for entry in poses + ], + ) + hf.create_dataset( + name=f"{bp}_y", + data=[ + ( + entry["pose"]["poses"][0][0][i, 1].item() + if isinstance( + entry["pose"]["poses"][0][0][i, 1], torch.Tensor + ) + else entry["pose"]["poses"][0][0][i, 1] + ) + for entry in poses + ], + ) + hf.create_dataset( + name=f"{bp}_confidence", + data=[ + ( + entry["pose"]["poses"][0][0][i, 2].item() + if isinstance( + entry["pose"]["poses"][0][0][i, 2], torch.Tensor + ) + else entry["pose"]["poses"][0][0][i, 2] + ) + for entry in poses + ], + ) From e6a914ab899d32fd63ea9bd7f35a86d579b7990e Mon Sep 17 00:00:00 2001 From: AnnaStuckert <47814177+annastuckert@users.noreply.github.com> Date: Mon, 24 Feb 2025 14:53:24 +0100 Subject: [PATCH 15/24] add timestamp suffix to videos and csv/h5 files --- dlclive/LiveVideoInference.py | 156 ++++---- dlclive/benchmark_pytorch.py | 666 ++++++++++++++-------------------- 2 files changed, 350 insertions(+), 472 deletions(-) diff --git a/dlclive/LiveVideoInference.py b/dlclive/LiveVideoInference.py index 9e75df6..9e0e34b 100644 --- a/dlclive/LiveVideoInference.py +++ b/dlclive/LiveVideoInference.py @@ -17,20 +17,22 @@ def get_system_info() -> dict: - """Return summary info for system running benchmark. + """ + Returns a summary of system information relevant to running benchmarking. Returns ------- dict - Dictionary containing the following system information: - * ``host_name`` (str): name of machine - * ``op_sys`` (str): operating system - * ``python`` (str): path to python (which conda/virtual environment) - * ``device`` (tuple): (device type (``'GPU'`` or ``'CPU'```), device information) - * ``freeze`` (list): list of installed packages and versions - * ``python_version`` (str): python version - * ``git_hash`` (str, None): If installed from git repository, hash of HEAD commit - * ``dlclive_version`` (str): dlclive version from :data:`dlclive.VERSION` + A dictionary containing the following system information: + - host_name (str): Name of the machine. + - op_sys (str): Operating system. + - python (str): Path to the Python executable, indicating the conda/virtual environment in use. + - device_type (str): Type of device used ('GPU' or 'CPU'). + - device (list): List containing the name of the GPU or CPU brand. + - freeze (list): List of installed Python packages with their versions. + - python_version (str): Version of Python in use. + - git_hash (str or None): If installed from git repository, hash of HEAD commit. + - dlclive_version (str): Version of the DLCLive package. """ # Get OS and host name @@ -101,35 +103,55 @@ def analyze_live_video( save_video=False, ): """ - Analyze a video to track keypoints using an imported DeepLabCut model, visualize keypoints on the video, and optionally save the keypoint data and the labelled video. - - Parameters: - ----------- + Analyzes a video to track keypoints using a DeepLabCut model, and optionally saves the keypoint data and the labeled video. + + Parameters + ---------- + model_path : str + Path to the DeepLabCut model. + model_type : str + Type of the model (e.g., 'onnx'). + device : str + Device to run the model on ('cpu' or 'cuda'). camera : float, default=0 (webcam) - The camera to record the live video from + The camera to record the live video from. experiment_name : str, default = "Test" Prefix to label generated pose and video files + precision : str, optional, default='FP32' + Precision type for the model ('FP32' or 'FP16'). + snapshot : str, optional + Snapshot to use for the model, if using pytorch as model type. + display : bool, optional, default=True + Whether to display frame with labelled key points. pcutoff : float, optional, default=0.5 - The probability cutoff value below which keypoints are not visualized. + Probability cutoff below which keypoints are not visualized. display_radius : int, optional, default=5 - The radius of the circles drawn to represent keypoints on the video frames. - resize : tuple of int (width, height) or None, optional, default=None - The size to which the frames should be resized. If None, the frames are not resized. - cropping : list of int, optional, default=None - Cropping parameters in pixel number: [x1, x2, y1, y2] + Radius of circles drawn for keypoints on video frames. + resize : tuple of int (width, height) or None, optional + Resize dimensions for video frames. e.g. if resize = 0.5, the video will be processed in half the original size. If None, no resizing is applied. + cropping : list of int or None, optional + Cropping parameters [x1, x2, y1, y2] in pixels. If None, no cropping is applied. + dynamic : tuple, optional, default=(False, 0.5, 10) (True/false), p cutoff, margin) + Parameters for dynamic cropping. If the state is true, then dynamic cropping will be performed. That means that if an object is detected (i.e. any body part > detectiontreshold), then object boundaries are computed according to the smallest/largest x position and smallest/largest y position of all body parts. This window is expanded by the margin and from then on only the posture within this crop is analyzed (until the object is lost, i.e. pcutoff: x, y = map(int, this_pose[j, :2]) @@ -255,33 +277,35 @@ def analyze_live_video( print(get_system_info()) if save_poses: - save_poses_to_files(experiment_name, save_dir, bodyparts, poses) + save_poses_to_files( + experiment_name, save_dir, bodyparts, poses, timestamp=timestamp + ) return poses, times -def save_poses_to_files(experiment_name, save_dir, bodyparts, poses): +def save_poses_to_files(experiment_name, save_dir, bodyparts, poses, timestamp): """ - Save the keypoint poses detected in the video to CSV and HDF5 files. + Saves the detected keypoint poses from the video to CSV and HDF5 files. - Parameters: - ----------- - experiment_name : str - Name of the experiment, used as a prefix for saving files. + Parameters + ---------- + video_path : str + Path to the analyzed video file. save_dir : str - The directory where the pose data files will be saved. + Directory where the pose data files will be saved. bodyparts : list of str - A list of body part names corresponding to the keypoints. + List of body part names corresponding to the keypoints. poses : list of dict - A list of dictionaries where each dictionary contains the frame number and the corresponding pose data. + List of dictionaries containing frame numbers and corresponding pose data. - Returns: - -------- + Returns + ------- None """ base_filename = os.path.splitext(os.path.basename(experiment_name))[0] - csv_save_path = os.path.join(save_dir, f"{base_filename}_poses.csv") - h5_save_path = os.path.join(save_dir, f"{base_filename}_poses.h5") + csv_save_path = os.path.join(save_dir, f"{base_filename}_poses_{timestamp}.csv") + h5_save_path = os.path.join(save_dir, f"{base_filename}_poses_{timestamp}.h5") # Save to CSV with open(csv_save_path, mode="w", newline="") as file: @@ -292,7 +316,7 @@ def save_poses_to_files(experiment_name, save_dir, bodyparts, poses): writer.writerow(header) for entry in poses: frame_num = entry["frame"] - pose_data = entry["pose"][0]["poses"][0][0] + pose_data = entry["pose"]["poses"][0][0] # Convert tensor data to numeric values row = [frame_num] + [ item.item() if isinstance(item, torch.Tensor) else item @@ -309,11 +333,9 @@ def save_poses_to_files(experiment_name, save_dir, bodyparts, poses): name=f"{bp}_x", data=[ ( - entry["pose"][0]["poses"][0][0][i, 0].item() - if isinstance( - entry["pose"][0]["poses"][0][0][i, 0], torch.Tensor - ) - else entry["pose"][0]["poses"][0][0][i, 0] + entry["pose"]["poses"][0][0][i, 0].item() + if isinstance(entry["pose"]["poses"][0][0][i, 0], torch.Tensor) + else entry["pose"]["poses"][0][0][i, 0] ) for entry in poses ], @@ -322,11 +344,9 @@ def save_poses_to_files(experiment_name, save_dir, bodyparts, poses): name=f"{bp}_y", data=[ ( - entry["pose"][0]["poses"][0][0][i, 1].item() - if isinstance( - entry["pose"][0]["poses"][0][0][i, 1], torch.Tensor - ) - else entry["pose"][0]["poses"][0][0][i, 1] + entry["pose"]["poses"][0][0][i, 1].item() + if isinstance(entry["pose"]["poses"][0][0][i, 1], torch.Tensor) + else entry["pose"]["poses"][0][0][i, 1] ) for entry in poses ], @@ -335,11 +355,9 @@ def save_poses_to_files(experiment_name, save_dir, bodyparts, poses): name=f"{bp}_confidence", data=[ ( - entry["pose"][0]["poses"][0][0][i, 2].item() - if isinstance( - entry["pose"][0]["poses"][0][0][i, 2], torch.Tensor - ) - else entry["pose"][0]["poses"][0][0][i, 2] + entry["pose"]["poses"][0][0][i, 2].item() + if isinstance(entry["pose"]["poses"][0][0][i, 2], torch.Tensor) + else entry["pose"]["poses"][0][0][i, 2] ) for entry in poses ], diff --git a/dlclive/benchmark_pytorch.py b/dlclive/benchmark_pytorch.py index 773f200..fa6be0f 100644 --- a/dlclive/benchmark_pytorch.py +++ b/dlclive/benchmark_pytorch.py @@ -16,61 +16,23 @@ from dlclive import VERSION, DLCLive -# def download_benchmarking_data( -# target_dir=".", -# url="http://deeplabcut.rowland.harvard.edu/datasets/dlclivebenchmark.tar.gz", -# ): -# """ -# Downloads a DeepLabCut-Live benchmarking Data (videos & DLC models). -# """ -# import tarfile -# import urllib.request - -# from tqdm import tqdm - -# def show_progress(count, block_size, total_size): -# pbar.update(block_size) - -# def tarfilenamecutting(tarf): -# """' auxfun to extract folder path -# ie. /xyz-trainsetxyshufflez/ -# """ -# for memberid, member in enumerate(tarf.getmembers()): -# if memberid == 0: -# parent = str(member.path) -# l = len(parent) + 1 -# if member.path.startswith(parent): -# member.path = member.path[l:] -# yield member - -# response = urllib.request.urlopen(url) -# print( -# "Downloading the benchmarking data from the DeepLabCut server @Harvard -> Go Crimson!!! {}....".format( -# url -# ) -# ) -# total_size = int(response.getheader("Content-Length")) -# pbar = tqdm(unit="B", total=total_size, position=0) -# filename, _ = urllib.request.urlretrieve(url, reporthook=show_progress) -# with tarfile.open(filename, mode="r:gz") as tar: -# tar.extractall(target_dir, members=tarfilenamecutting(tar)) - - def get_system_info() -> dict: - """Return summary info for system running benchmark. + """ + Returns a summary of system information relevant to running benchmarking. Returns ------- dict - Dictionary containing the following system information: - * ``host_name`` (str): name of machine - * ``op_sys`` (str): operating system - * ``python`` (str): path to python (which conda/virtual environment) - * ``device`` (tuple): (device type (``'GPU'`` or ``'CPU'```), device information) - * ``freeze`` (list): list of installed packages and versions - * ``python_version`` (str): python version - * ``git_hash`` (str, None): If installed from git repository, hash of HEAD commit - * ``dlclive_version`` (str): dlclive version from :data:`dlclive.VERSION` + A dictionary containing the following system information: + - host_name (str): Name of the machine. + - op_sys (str): Operating system. + - python (str): Path to the Python executable, indicating the conda/virtual environment in use. + - device_type (str): Type of device used ('GPU' or 'CPU'). + - device (list): List containing the name of the GPU or CPU brand. + - freeze (list): List of installed Python packages with their versions. + - python_version (str): Version of Python in use. + - git_hash (str or None): If installed from git repository, hash of HEAD commit. + - dlclive_version (str): Version of the DLCLive package. """ # Get OS and host name @@ -137,39 +99,58 @@ def analyze_video( draw_keypoint_names=False, cmap="bmy", get_sys_info=True, - save_video=False + save_video=False, ): """ - Analyze a video to track keypoints using an imported DeepLabCut model, visualize keypoints on the video, and optionally save the keypoint data and the labelled video. + Analyzes a video to track keypoints using a DeepLabCut model, and optionally saves the keypoint data and the labeled video. - Parameters: - ----------- + Parameters + ---------- video_path : str - The path to the video file to be analyzed. - dlc_live : DLCLive - An instance of the DLCLive class. + Path to the video file to be analyzed. + model_path : str + Path to the DeepLabCut model. + model_type : str + Type of the model (e.g., 'onnx'). + device : str + Device to run the model on ('cpu' or 'cuda'). + precision : str, optional, default='FP32' + Precision type for the model ('FP32' or 'FP16'). + snapshot : str, optional + Snapshot to use for the model, if using pytorch as model type. + display : bool, optional, default=True + Whether to display frame with labelled key points. pcutoff : float, optional, default=0.5 - The probability cutoff value below which keypoints are not visualized. + Probability cutoff below which keypoints are not visualized. display_radius : int, optional, default=5 - The radius of the circles drawn to represent keypoints on the video frames. - resize : tuple of int (width, height) or None, optional, default=None - The size to which the frames should be resized. If None, the frames are not resized. - cropping : list of int, optional, default=None - Cropping parameters in pixel number: [x1, x2, y1, y2] + Radius of circles drawn for keypoints on video frames. + resize : tuple of int (width, height) or None, optional + Resize dimensions for video frames. e.g. if resize = 0.5, the video will be processed in half the original size. If None, no resizing is applied. + cropping : list of int or None, optional + Cropping parameters [x1, x2, y1, y2] in pixels. If None, no cropping is applied. + dynamic : tuple, optional, default=(False, 0.5, 10) (True/false), p cutoff, margin) + Parameters for dynamic cropping. If the state is true, then dynamic cropping will be performed. That means that if an object is detected (i.e. any body part > detectiontreshold), then object boundaries are computed according to the smallest/largest x position and smallest/largest y position of all body parts. This window is expanded by the margin and from then on only the posture within this crop is analyzed (until the object is lost, i.e. dict: - """Return summary info for system running benchmark. - - Returns - ------- - dict - Dictionary containing the following system information: - * ``host_name`` (str): name of machine - * ``op_sys`` (str): operating system - * ``python`` (str): path to python (which conda/virtual environment) - * ``device`` (tuple): (device type (``'GPU'`` or ``'CPU'```), device information) - * ``freeze`` (list): list of installed packages and versions - * ``python_version`` (str): python version - * ``git_hash`` (str, None): If installed from git repository, hash of HEAD commit - * ``dlclive_version`` (str): dlclive version from :data:`dlclive.VERSION` - """ - - # Get OS and host name - op_sys = platform.platform() - host_name = platform.node().replace(" ", "") - - # Get Python executable path - if platform.system() == "Windows": - host_python = sys.executable.split(os.path.sep)[-2] - else: - host_python = sys.executable.split(os.path.sep)[-3] - - # Try to get git hash if possible - git_hash = None - dlc_basedir = os.path.dirname(os.path.dirname(__file__)) - try: - git_hash = ( - subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=dlc_basedir) - .decode("utf-8") - .strip() +def save_poses_to_files(video_path, save_dir, bodyparts, poses, timestamp): + """ + Saves the detected keypoint poses from the video to CSV and HDF5 files. + + Parameters + ---------- + video_path : str + Path to the analyzed video file. + save_dir : str + Directory where the pose data files will be saved. + bodyparts : list of str + List of body part names corresponding to the keypoints. + poses : list of dict + List of dictionaries containing frame numbers and corresponding pose data. + + Returns + ------- + None + """ + + base_filename = os.path.splitext(os.path.basename(video_path))[0] + csv_save_path = os.path.join(save_dir, f"{base_filename}_poses_{timestamp}.csv") + h5_save_path = os.path.join(save_dir, f"{base_filename}_poses_{timestamp}.h5") + + # Save to CSV + with open(csv_save_path, mode="w", newline="") as file: + writer = csv.writer(file) + header = ["frame"] + [ + f"{bp}_{axis}" for bp in bodyparts for axis in ["x", "y", "confidence"] + ] + writer.writerow(header) + for entry in poses: + frame_num = entry["frame"] + pose = entry["pose"]["poses"][0][0] + row = [frame_num] + [ + item.item() if isinstance(item, torch.Tensor) else item + for kp in pose + for item in kp + ] + writer.writerow(row) + + # Save to HDF5 + with h5py.File(h5_save_path, "w") as hf: + hf.create_dataset(name="frames", data=[entry["frame"] for entry in poses]) + for i, bp in enumerate(bodyparts): + hf.create_dataset( + name=f"{bp}_x", + data=[ + ( + entry["pose"]["poses"][0][0][i, 0].item() + if isinstance(entry["pose"]["poses"][0][0][i, 0], torch.Tensor) + else entry["pose"]["poses"][0][0][i, 0] + ) + for entry in poses + ], + ) + hf.create_dataset( + name=f"{bp}_y", + data=[ + ( + entry["pose"]["poses"][0][0][i, 1].item() + if isinstance(entry["pose"]["poses"][0][0][i, 1], torch.Tensor) + else entry["pose"]["poses"][0][0][i, 1] + ) + for entry in poses + ], + ) + hf.create_dataset( + name=f"{bp}_confidence", + data=[ + ( + entry["pose"]["poses"][0][0][i, 2].item() + if isinstance(entry["pose"]["poses"][0][0][i, 2], torch.Tensor) + else entry["pose"]["poses"][0][0][i, 2] + ) + for entry in poses + ], ) - except subprocess.CalledProcessError: - # Not installed from git repo, e.g., pypi - pass - - # Get device info (GPU or CPU) - if torch.cuda.is_available(): - dev_type = "GPU" - dev = [torch.cuda.get_device_name(torch.cuda.current_device())] - else: - from cpuinfo import get_cpu_info - - dev_type = "CPU" - dev = [get_cpu_info()["brand_raw"]] - - return { - "host_name": host_name, - "op_sys": op_sys, - "python": host_python, - "device_type": dev_type, - "device": dev, - "freeze": list(freeze.freeze()), - "python_version": sys.version, - "git_hash": git_hash, - "dlclive_version": VERSION, - } - - def analyze_video( - video_path: str, - model_path: str, - model_type: str, - device: str, - precision: str = "FP32", - snapshot: str = None, - display=True, - pcutoff=0.5, - display_radius=5, - resize=None, - cropping=None, # Adding cropping to the function parameters - dynamic=(False, 0.5, 10), - save_poses=False, - save_dir="model_predictions", - draw_keypoint_names=False, - cmap="bmy", - get_sys_info=True, - save_video=False, - ): - """ - Analyze a video to track keypoints using an imported DeepLabCut model, visualize keypoints on the video, and optionally save the keypoint data and the labelled video. - - Parameters: - ----------- - video_path : str - The path to the video file to be analyzed. - dlc_live : DLCLive - An instance of the DLCLive class. - pcutoff : float, optional, default=0.5 - The probability cutoff value below which keypoints are not visualized. - display_radius : int, optional, default=5 - The radius of the circles drawn to represent keypoints on the video frames. - resize : tuple of int (width, height) or None, optional, default=None - The size to which the frames should be resized. If None, the frames are not resized. - cropping : list of int, optional, default=None - Cropping parameters in pixel number: [x1, x2, y1, y2] - save_poses : bool, optional, default=False - Whether to save the detected poses to CSV and HDF5 files. - save_dir : str, optional, default="model_predictions" - The directory where the output video and pose data will be saved. - draw_keypoint_names : bool, optional, default=False - Whether to draw the names of the keypoints on the video frames. - cmap : str, optional, default="bmy" - The colormap from the colorcet library to use for keypoint visualization. - - Returns: - -------- - poses : list of dict - A list of dictionaries where each dictionary contains the frame number and the corresponding pose data. - """ - # Create the DLCLive object with cropping - dlc_live = DLCLive( - path=model_path, - model_type=model_type, - device=device, - display=display, - resize=resize, - cropping=cropping, # Pass the cropping parameter - dynamic=dynamic, - precision=precision, - snapshot=snapshot, - ) - # Ensure save directory exists - os.makedirs(name=save_dir, exist_ok=True) - # Load video - cap = cv2.VideoCapture(video_path) - if not cap.isOpened(): - print(f"Error: Could not open video file {video_path}") - return +import argparse +import os - # Start empty dict to save poses to for each frame - poses, times = [], [] - # Create variable indicate current frame. Later in the code +1 is added to frame_index - frame_index = 0 - # Retrieve bodypart names and number of keypoints - bodyparts = dlc_live.cfg["metadata"]["bodyparts"] - num_keypoints = len(bodyparts) +def main(): + """Provides a command line interface to analyze_video function.""" - if save_video: - # Set colors and convert to RGB - cmap_colors = getattr(cc, cmap) - colors = [ - ImageColor.getrgb(color) - for color in cmap_colors[:: int(len(cmap_colors) / num_keypoints)] - ] + parser = argparse.ArgumentParser( + description="Analyze a video using a DeepLabCut model and visualize keypoints." + ) + parser.add_argument("model_path", type=str, help="Path to the model.") + parser.add_argument("video_path", type=str, help="Path to the video file.") + parser.add_argument("model_type", type=str, help="Type of the model (e.g., 'DLC').") + parser.add_argument( + "device", type=str, help="Device to run the model on (e.g., 'cuda' or 'cpu')." + ) + parser.add_argument( + "-p", + "--precision", + type=str, + default="FP32", + help="Model precision (e.g., 'FP32', 'FP16').", + ) + parser.add_argument( + "-s", + "--snapshot", + type=str, + default=None, + help="Path to a specific model snapshot.", + ) + parser.add_argument( + "-d", "--display", action="store_true", help="Display keypoints on the video." + ) + parser.add_argument( + "-c", + "--pcutoff", + type=float, + default=0.5, + help="Probability cutoff for keypoints visualization.", + ) + parser.add_argument( + "-dr", + "--display-radius", + type=int, + default=5, + help="Radius of keypoint circles in the display.", + ) + parser.add_argument( + "-r", + "--resize", + type=int, + default=None, + help="Resize video frames to [width, height].", + ) + parser.add_argument( + "-x", + "--cropping", + type=int, + nargs=4, + default=None, + help="Cropping parameters [x1, x2, y1, y2].", + ) + parser.add_argument( + "-y", + "--dynamic", + type=float, + nargs=3, + default=[False, 0.5, 10], + help="Dynamic cropping [flag, pcutoff, margin].", + ) + parser.add_argument( + "--save-poses", action="store_true", help="Save the keypoint poses to files." + ) + parser.add_argument( + "--save-video", + action="store_true", + help="Save the output video with keypoints.", + ) + parser.add_argument( + "--save-dir", + type=str, + default="model_predictions", + help="Directory to save output files.", + ) + parser.add_argument( + "--draw-keypoint-names", + action="store_true", + help="Draw keypoint names on the video.", + ) + parser.add_argument( + "--cmap", type=str, default="bmy", help="Colormap for keypoints visualization." + ) + parser.add_argument( + "--no-sys-info", + action="store_false", + help="Do not print system info.", + dest="get_sys_info", + ) - # Define output video path - video_name = os.path.splitext(os.path.basename(video_path))[0] - output_video_path = os.path.join( - save_dir, f"{video_name}_DLCLIVE_LABELLED.mp4" - ) - - # Get video writer setup - fourcc = cv2.VideoWriter_fourcc(*"mp4v") - fps = cap.get(cv2.CAP_PROP_FPS) - frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) - frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - - vwriter = cv2.VideoWriter( - filename=output_video_path, - fourcc=fourcc, - fps=fps, - frameSize=(frame_width, frame_height), - ) + args = parser.parse_args() + + # Call the analyze_video function with the parsed arguments + analyze_video( + video_path=args.video_path, + model_path=args.model_path, + model_type=args.model_type, + device=args.device, + precision=args.precision, + snapshot=args.snapshot, + display=args.display, + pcutoff=args.pcutoff, + display_radius=args.display_radius, + resize=tuple(args.resize) if args.resize else None, + cropping=args.cropping, + dynamic=tuple(args.dynamic), + save_poses=args.save_poses, + save_dir=args.save_dir, + draw_keypoint_names=args.draw_keypoint_names, + cmap=args.cmap, + get_sys_info=args.get_sys_info, + save_video=args.save_video, + ) - while True: - start_time = time.time() - - ret, frame = cap.read() - if not ret: - break - # if frame_index == 0: - # pose = dlc_live.init_inference(frame) # load DLC model - try: - # pose = dlc_live.get_pose(frame) - if frame_index == 0: - # dlc_live.dynamic = (False, dynamic[1], dynamic[2]) # TODO trying to fix issues with dynamic cropping jumping back and forth between dyanmic cropped and original image - pose, inf_time = dlc_live.init_inference(frame) # load DLC model - else: - # dlc_live.dynamic = dynamic - pose, inf_time = dlc_live.get_pose(frame) - except Exception as e: - print(f"Error analyzing frame {frame_index}: {e}") - continue - - poses.append({"frame": frame_index, "pose": pose}) - times.append(inf_time) - - if save_video: - # Visualize keypoints - this_pose = pose["poses"][0][0] - for j in range(this_pose.shape[0]): - if this_pose[j, 2] > pcutoff: - x, y = map(int, this_pose[j, :2]) - cv2.circle( - frame, - center=(x, y), - radius=display_radius, - color=colors[j], - thickness=-1, - ) - if draw_keypoint_names: - cv2.putText( - frame, - text=bodyparts[j], - org=(x + 10, y), - fontFace=cv2.FONT_HERSHEY_SIMPLEX, - fontScale=0.5, - color=colors[j], - thickness=1, - lineType=cv2.LINE_AA, - ) - - vwriter.write(image=frame) - frame_index += 1 - - cap.release() - if save_video: - vwriter.release() - - if get_sys_info: - print(get_system_info()) - - if save_poses: - save_poses_to_files(video_path, save_dir, bodyparts, poses) - - return poses, times - - def save_poses_to_files(video_path, save_dir, bodyparts, poses): - """ - Save the keypoint poses detected in the video to CSV and HDF5 files. - - Parameters: - ----------- - video_path : str - The path to the video file that was analyzed. - save_dir : str - The directory where the pose data files will be saved. - bodyparts : list of str - A list of body part names corresponding to the keypoints. - poses : list of dict - A list of dictionaries where each dictionary contains the frame number and the corresponding pose data. - - Returns: - -------- - None - """ - base_filename = os.path.splitext(os.path.basename(video_path))[0] - csv_save_path = os.path.join(save_dir, f"{base_filename}_poses.csv") - h5_save_path = os.path.join(save_dir, f"{base_filename}_poses.h5") - - # Save to CSV - with open(csv_save_path, mode="w", newline="") as file: - writer = csv.writer(file) - header = ["frame"] + [ - f"{bp}_{axis}" for bp in bodyparts for axis in ["x", "y", "confidence"] - ] - writer.writerow(header) - for entry in poses: - frame_num = entry["frame"] - pose = entry["pose"]["poses"][0][0] - row = [frame_num] + [ - item.item() if isinstance(item, torch.Tensor) else item - for kp in pose - for item in kp - ] - writer.writerow(row) - - # Save to HDF5 - with h5py.File(h5_save_path, "w") as hf: - hf.create_dataset(name="frames", data=[entry["frame"] for entry in poses]) - for i, bp in enumerate(bodyparts): - hf.create_dataset( - name=f"{bp}_x", - data=[ - ( - entry["pose"]["poses"][0][0][i, 0].item() - if isinstance( - entry["pose"]["poses"][0][0][i, 0], torch.Tensor - ) - else entry["pose"]["poses"][0][0][i, 0] - ) - for entry in poses - ], - ) - hf.create_dataset( - name=f"{bp}_y", - data=[ - ( - entry["pose"]["poses"][0][0][i, 1].item() - if isinstance( - entry["pose"]["poses"][0][0][i, 1], torch.Tensor - ) - else entry["pose"]["poses"][0][0][i, 1] - ) - for entry in poses - ], - ) - hf.create_dataset( - name=f"{bp}_confidence", - data=[ - ( - entry["pose"]["poses"][0][0][i, 2].item() - if isinstance( - entry["pose"]["poses"][0][0][i, 2], torch.Tensor - ) - else entry["pose"]["poses"][0][0][i, 2] - ) - for entry in poses - ], - ) +if __name__ == "__main__": + main() + + +# Example how to run in command line: +# python benchmark_pytorch.py /path/to/model /path/to/video DLC cuda -p FP32 -d -c 0.5 -dr 5 -r 0.5 -x 10 630 10 470 --save-poses --save-video --draw-keypoint-names --cmap bmy --save-dir +# python benchmark_pytorch.py /Users/annastuckert/Documents/DLC_AI_Residency/DLC_AI2024/DeepLabCut-live/dlc-live-dummy/ventral-gait/resnet.onnx /Users/annastuckert/Documents/DLC_AI_Residency/DLC_AI2024/DeepLabCut-live/dlc-live-dummy/ventral-gait/1_20cms_0degUP_first_1s.avi DLC cuda -p FP32 -d -r 0.5 --save-poses --save-video --draw-keypoint-names --cmap bmy --save-dir /Users/annastuckert/Documents/DLC_AI_Residency/DLC_AI2024/DeepLabCut-live/dlc-live-dummy/ventral-gait/out From 29c81226964898062c1aaf10d04cc89fe1549e05 Mon Sep 17 00:00:00 2001 From: Dikra Date: Mon, 24 Feb 2025 14:54:43 +0100 Subject: [PATCH 16/24] fix live inference and display, black and isort --- dlclive/__init__.py | 7 ++- dlclive/benchmark.py | 36 +++++------ dlclive/benchmark_pytorch.py | 8 +-- dlclive/check_install/check_install.py | 50 ++++++++------- dlclive/display.py | 3 +- dlclive/dlclive.py | 36 +++++++---- dlclive/exceptions.py | 4 +- dlclive/graph.py | 1 - ...iveVideoInference.py => live_inference.py} | 63 ++++++++++--------- dlclive/pose.py | 13 ++-- dlclive/predictor/__init__.py | 2 +- dlclive/predictor/base.py | 4 +- dlclive/predictor/single_predictor.py | 29 ++++----- dlclive/processor/__init__.py | 2 +- dlclive/processor/kalmanfilter.py | 5 +- dlclive/utils.py | 15 ++--- dlclive/version.py | 1 - 17 files changed, 147 insertions(+), 132 deletions(-) rename dlclive/{LiveVideoInference.py => live_inference.py} (91%) diff --git a/dlclive/__init__.py b/dlclive/__init__.py index b21df4a..9afe11b 100644 --- a/dlclive/__init__.py +++ b/dlclive/__init__.py @@ -5,9 +5,10 @@ Licensed under GNU Lesser General Public License v3.0 """ -from dlclive.version import __version__, VERSION -from dlclive.dlclive import DLCLive from dlclive.display import Display -from dlclive.processor import Processor +from dlclive.dlclive import DLCLive from dlclive.predictor import HeatmapPredictor +from dlclive.processor import Processor +from dlclive.version import VERSION, __version__ + # from dlclive.benchmark import benchmark, benchmark_videos, download_benchmarking_data diff --git a/dlclive/benchmark.py b/dlclive/benchmark.py index 4cb4fb1..bd347c6 100644 --- a/dlclive/benchmark.py +++ b/dlclive/benchmark.py @@ -5,33 +5,31 @@ Licensed under GNU Lesser General Public License v3.0 """ - -import platform import os -import time -import sys -import warnings +import pickle +import platform import subprocess +import sys +import time import typing -import pickle +import warnings + import colorcet as cc -from PIL import ImageColor import ruamel +from PIL import ImageColor try: from pip._internal.operations import freeze except ImportError: from pip.operations import freeze -from tqdm import tqdm +import cv2 import numpy as np import tensorflow as tf -import cv2 +from tqdm import tqdm -from dlclive import DLCLive -from dlclive import VERSION +from dlclive import VERSION, DLCLive from dlclive import __file__ as dlcfile - from dlclive.utils import decode_fourcc @@ -42,8 +40,9 @@ def download_benchmarking_data( """ Downloads a DeepLabCut-Live benchmarking Data (videos & DLC models). """ - import urllib.request import tarfile + import urllib.request + from tqdm import tqdm def show_progress(count, block_size, total_size): @@ -75,7 +74,7 @@ def tarfilenamecutting(tarf): def get_system_info() -> dict: - """ Return summary info for system running benchmark + """Return summary info for system running benchmark Returns ------- dict @@ -165,7 +164,7 @@ def benchmark( save_video=False, output=None, ) -> typing.Tuple[np.ndarray, tuple, bool, dict]: - """ Analyze DeepLabCut-live exported model on a video: + """Analyze DeepLabCut-live exported model on a video: Calculate inference time, display keypoints, or get poses/create a labeled video @@ -193,7 +192,7 @@ def benchmark( n_frames : int, optional number of frames to run inference on, by default 1000 print_rate : bool, optional - flat to print inference rate frame by frame, by default False + flag to print inference rate frame by frame, by default False display : bool, optional flag to display keypoints on images. Useful for checking the accuracy of exported models. pcutoff : float, optional @@ -440,7 +439,7 @@ def benchmark( def save_inf_times( sys_info, inf_times, im_size, TFGPUinference, model=None, meta=None, output=None ): - """ Save inference time data collected using :function:`benchmark` with system information to a pickle file. + """Save inference time data collected using :function:`benchmark` with system information to a pickle file. This is primarily used through :function:`benchmark_videos` @@ -666,8 +665,7 @@ def benchmark_videos( def main(): - """Provides a command line interface :function:`benchmark_videos` - """ + """Provides a command line interface :function:`benchmark_videos`""" import argparse diff --git a/dlclive/benchmark_pytorch.py b/dlclive/benchmark_pytorch.py index fa6be0f..8ab0cb1 100644 --- a/dlclive/benchmark_pytorch.py +++ b/dlclive/benchmark_pytorch.py @@ -13,7 +13,8 @@ from PIL import ImageColor from pip._internal.operations import freeze -from dlclive import VERSION, DLCLive +from dlclive import DLCLive +from dlclive.version import VERSION def get_system_info() -> dict: @@ -482,8 +483,3 @@ def main(): if __name__ == "__main__": main() - - -# Example how to run in command line: -# python benchmark_pytorch.py /path/to/model /path/to/video DLC cuda -p FP32 -d -c 0.5 -dr 5 -r 0.5 -x 10 630 10 470 --save-poses --save-video --draw-keypoint-names --cmap bmy --save-dir -# python benchmark_pytorch.py /Users/annastuckert/Documents/DLC_AI_Residency/DLC_AI2024/DeepLabCut-live/dlc-live-dummy/ventral-gait/resnet.onnx /Users/annastuckert/Documents/DLC_AI_Residency/DLC_AI2024/DeepLabCut-live/dlc-live-dummy/ventral-gait/1_20cms_0degUP_first_1s.avi DLC cuda -p FP32 -d -r 0.5 --save-poses --save-video --draw-keypoint-names --cmap bmy --save-dir /Users/annastuckert/Documents/DLC_AI_Residency/DLC_AI2024/DeepLabCut-live/dlc-live-dummy/ventral-gait/out diff --git a/dlclive/check_install/check_install.py b/dlclive/check_install/check_install.py index 7601533..6527498 100755 --- a/dlclive/check_install/check_install.py +++ b/dlclive/check_install/check_install.py @@ -5,19 +5,16 @@ Licensed under GNU Lesser General Public License v3.0 """ - -import sys +import argparse import shutil -import warnings - -from dlclive import benchmark_videos +import sys import urllib.request -import argparse +import warnings from pathlib import Path -from dlclibrary.dlcmodelzoo.modelzoo_download import ( - download_huggingface_model, -) +from dlclibrary.dlcmodelzoo.modelzoo_download import download_huggingface_model + +from dlclive import benchmark_videos MODEL_NAME = "superanimal_quadruped" SNAPSHOT_NAME = "snapshot-700000.pb" @@ -27,28 +24,33 @@ def urllib_pbar(count, blockSize, totalSize): percent = int(count * blockSize * 100 / totalSize) outstr = f"{round(percent)}%" sys.stdout.write(outstr) - sys.stdout.write("\b"*len(outstr)) + sys.stdout.write("\b" * len(outstr)) sys.stdout.flush() def main(): parser = argparse.ArgumentParser( - description="Test DLC-Live installation by downloading and evaluating a demo DLC project!") - parser.add_argument('--nodisplay', action='store_false', help="Run the test without displaying tracking") + description="Test DLC-Live installation by downloading and evaluating a demo DLC project!" + ) + parser.add_argument( + "--nodisplay", + action="store_false", + help="Run the test without displaying tracking", + ) args = parser.parse_args() display = args.nodisplay if not display: - print('Running without displaying video') + print("Running without displaying video") # make temporary directory in $HOME # TODO: why create this temp directory in $HOME? print("\nCreating temporary directory...\n") - tmp_dir = Path().home() / 'dlc-live-tmp' - tmp_dir.mkdir(mode=0o775,exist_ok=True) + tmp_dir = Path().home() / "dlc-live-tmp" + tmp_dir.mkdir(mode=0o775, exist_ok=True) - video_file = str(tmp_dir / 'dog_clip.avi') - model_dir = tmp_dir / 'DLC_Dog_resnet_50_iteration-0_shuffle-0' + video_file = str(tmp_dir / "dog_clip.avi") + model_dir = tmp_dir / "DLC_Dog_resnet_50_iteration-0_shuffle-0" # download dog test video from github: # TODO: Should check if the video's already there before downloading it (should have been cloned with the files) @@ -58,25 +60,31 @@ def main(): # download model from the DeepLabCut Model Zoo if Path(model_dir / SNAPSHOT_NAME).exists(): - print('Model already downloaded, using cached version') + print("Model already downloaded, using cached version") else: print("Downloading full_dog model from the DeepLabCut Model Zoo...") download_huggingface_model(MODEL_NAME, model_dir) # assert these things exist so we can give informative error messages assert Path(video_file).exists(), f"Missing video file {video_file}" - assert Path(model_dir / SNAPSHOT_NAME).exists(), f"Missing model file {model_dir / SNAPSHOT_NAME}" + assert Path( + model_dir / SNAPSHOT_NAME + ).exists(), f"Missing model file {model_dir / SNAPSHOT_NAME}" # run benchmark videos print("\n Running inference...\n") - benchmark_videos(str(model_dir), video_file, display=display, resize=0.5, pcutoff=0.25) + benchmark_videos( + str(model_dir), video_file, display=display, resize=0.5, pcutoff=0.25 + ) # deleting temporary files print("\n Deleting temporary files...\n") try: shutil.rmtree(tmp_dir) except PermissionError: - warnings.warn(f'Could not delete temporary directory {str(tmp_dir)} due to a permissions error, but otherwise dlc-live seems to be working fine!') + warnings.warn( + f"Could not delete temporary directory {str(tmp_dir)} due to a permissions error, but otherwise dlc-live seems to be working fine!" + ) print("\nDone!\n") diff --git a/dlclive/display.py b/dlclive/display.py index 7260997..4d73257 100644 --- a/dlclive/display.py +++ b/dlclive/display.py @@ -9,9 +9,10 @@ import colorcet as cc import numpy as np -from dlclive import utils from PIL import Image, ImageDraw, ImageTk +from dlclive import utils + class Display(object): """ diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index a5fcff3..1d57912 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -24,7 +24,8 @@ from dlclive import utils from dlclive.display import Display from dlclive.exceptions import DLCLiveError, DLCLiveWarning -from dlclive.pose import argmax_pose_predict, extract_cnn_output, multi_pose_predict +from dlclive.pose import (argmax_pose_predict, extract_cnn_output, + multi_pose_predict) from dlclive.predictor import HeatmapPredictor if typing.TYPE_CHECKING: @@ -277,7 +278,11 @@ def load_model(self): elif self.model_type == "onnx": model_paths = glob.glob(os.path.normpath(self.path + "/*.onnx")) if self.precision == "FP16": - model_path = [model_paths[i] for i in range(len(model_paths)) if "fp16" in model_paths[i]][0] + model_path = [ + model_paths[i] + for i in range(len(model_paths)) + if "fp16" in model_paths[i] + ][0] print(model_path) else: model_path = model_paths[0] @@ -294,13 +299,16 @@ def load_model(self): ) # ! TODO implement if statements for choice of tensorrt engine options (precision, and caching) elif self.device == "tensorrt": - provider = [("TensorrtExecutionProvider", { - "trt_engine_cache_enable": True, - "trt_engine_cache_path": "./trt_engines" - })] - self.sess = ort.InferenceSession( - model_path, opts, providers=provider - ) + provider = [ + ( + "TensorrtExecutionProvider", + { + "trt_engine_cache_enable": True, + "trt_engine_cache_path": "./trt_engines", + }, + ) + ] + self.sess = ort.InferenceSession(model_path, opts, providers=provider) self.predictor = HeatmapPredictor.build(self.cfg) if not os.path.isfile(model_path): @@ -333,7 +341,7 @@ def init_inference(self, frame=None, **kwargs): # load model self.load_model() - inf_time = 0. + inf_time = 0.0 # get pose of first frame (first inference is often very slow) if frame is not None: pose, inf_time = self.get_pose(frame, **kwargs) @@ -356,7 +364,7 @@ def get_pose(self, frame=None, **kwargs): pose :class:`numpy.ndarray` the pose estimated by DeepLabCut for the input image """ - inf_time = 0. + inf_time = 0.0 if frame is None: raise DLCLiveError("No frame provided for live pose estimation") @@ -381,8 +389,10 @@ def get_pose(self, frame=None, **kwargs): self.pose = self.pose["bodypart"] elif self.model_type == "onnx": - if self.precision == "FP32": frame = processed_frame.astype(np.float32) - elif self.precision == "FP16": frame = processed_frame.astype(np.float16) + if self.precision == "FP32": + frame = processed_frame.astype(np.float32) + elif self.precision == "FP16": + frame = processed_frame.astype(np.float16) frame = np.transpose(frame, (2, 0, 1)) frame = np.expand_dims(frame, axis=0) diff --git a/dlclive/exceptions.py b/dlclive/exceptions.py index 5d7a1aa..13c7c88 100644 --- a/dlclive/exceptions.py +++ b/dlclive/exceptions.py @@ -7,12 +7,12 @@ class DLCLiveError(Exception): - """ Generic error type for incorrect use of the DLCLive class """ + """Generic error type for incorrect use of the DLCLive class""" pass class DLCLiveWarning(Warning): - """ Generic warning for incorrect use of the DLCLive class """ + """Generic warning for incorrect use of the DLCLive class""" pass diff --git a/dlclive/graph.py b/dlclive/graph.py index 0841b46..56ec6c3 100644 --- a/dlclive/graph.py +++ b/dlclive/graph.py @@ -5,7 +5,6 @@ Licensed under GNU Lesser General Public License v3.0 """ - import tensorflow as tf vers = (tf.__version__).split(".") diff --git a/dlclive/LiveVideoInference.py b/dlclive/live_inference.py similarity index 91% rename from dlclive/LiveVideoInference.py rename to dlclive/live_inference.py index 9e0e34b..dea3750 100644 --- a/dlclive/LiveVideoInference.py +++ b/dlclive/live_inference.py @@ -158,7 +158,7 @@ def analyze_live_video( path=model_path, model_type=model_type, device=device, - display=display, + display=False, resize=resize, cropping=cropping, # Pass the cropping parameter dynamic=dynamic, @@ -186,13 +186,14 @@ def analyze_live_video( bodyparts = dlc_live.cfg["metadata"]["bodyparts"] num_keypoints = len(bodyparts) + # Set colors and convert to RGB + cmap_colors = getattr(cc, cmap) + colors = [ + ImageColor.getrgb(color) + for color in cmap_colors[:: int(len(cmap_colors) / num_keypoints)] + ] + if save_video: - # Set colors and convert to RGB - cmap_colors = getattr(cc, cmap) - colors = [ - ImageColor.getrgb(color) - for color in cmap_colors[:: int(len(cmap_colors) / num_keypoints)] - ] # Define output video path output_video_path = os.path.join( @@ -230,32 +231,31 @@ def analyze_live_video( poses.append({"frame": frame_index, "pose": pose}) times.append(inf_time) - if save_video: - # Visualize keypoints - this_pose = pose["poses"][0][0] - for j in range(this_pose.shape[0]): - if this_pose[j, 2] > pcutoff: - x, y = map(int, this_pose[j, :2]) - cv2.circle( + # Visualize keypoints + this_pose = pose["poses"][0][0] + for j in range(this_pose.shape[0]): + if this_pose[j, 2] > pcutoff: + x, y = map(int, this_pose[j, :2]) + cv2.circle( + frame, + center=(x, y), + radius=display_radius, + color=colors[j], + thickness=-1, + ) + + if draw_keypoint_names: + cv2.putText( frame, - center=(x, y), - radius=display_radius, + text=bodyparts[j], + org=(x + 10, y), + fontFace=cv2.FONT_HERSHEY_SIMPLEX, + fontScale=0.5, color=colors[j], - thickness=-1, + thickness=1, + lineType=cv2.LINE_AA, ) - - if draw_keypoint_names: - cv2.putText( - frame, - text=bodyparts[j], - org=(x + 10, y), - fontFace=cv2.FONT_HERSHEY_SIMPLEX, - fontScale=0.5, - color=colors[j], - thickness=1, - lineType=cv2.LINE_AA, - ) - + if save_video: vwriter.write(image=frame) frame_index += 1 @@ -271,7 +271,8 @@ def analyze_live_video( if save_video: vwriter.release() - # cv2.destroyAllWindows() + + cv2.destroyAllWindows() if get_sys_info: print(get_system_info()) diff --git a/dlclive/pose.py b/dlclive/pose.py index df39f0b..30cdb5b 100644 --- a/dlclive/pose.py +++ b/dlclive/pose.py @@ -5,7 +5,6 @@ Licensed under GNU Lesser General Public License v3.0 """ - import numpy as np @@ -70,7 +69,7 @@ def argmax_pose_predict(scmap, offmat, stride): num_joints = scmap.shape[0] # debug - print('joints', num_joints) + print("joints", num_joints) pose = [] for joint_idx in range(num_joints): maxloc = np.unravel_index( @@ -82,10 +81,14 @@ def argmax_pose_predict(scmap, offmat, stride): # offset = np.array(offmat[maxloc][joint_idx])[::-1] # print(offmat[maxloc][joint_idx]) offset = np.array(offmat[maxloc])[::-1] - print(offset[:,0].shape) + print(offset[:, 0].shape) # print(np.array(offmat[maxloc]).shape) - print('offset', offset.shape) - print('offmat*stride+offset', (offmat * stride + offset).shape, (offmat * stride + offset)) + print("offset", offset.shape) + print( + "offmat*stride+offset", + (offmat * stride + offset).shape, + (offmat * stride + offset), + ) pos_f8 = np.array(offmat).astype("float") * stride + 0.5 * stride + offset print("pos_f8", pos_f8[::-1].shape) pose.append(np.hstack((pos_f8[::-1], [scmap[joint_idx][maxloc]]))) diff --git a/dlclive/predictor/__init__.py b/dlclive/predictor/__init__.py index 47c531c..3f6777c 100644 --- a/dlclive/predictor/__init__.py +++ b/dlclive/predictor/__init__.py @@ -1 +1 @@ -from dlclive.predictor.single_predictor import HeatmapPredictor \ No newline at end of file +from dlclive.predictor.single_predictor import HeatmapPredictor diff --git a/dlclive/predictor/base.py b/dlclive/predictor/base.py index dc9b38a..8a15194 100644 --- a/dlclive/predictor/base.py +++ b/dlclive/predictor/base.py @@ -13,10 +13,10 @@ from abc import ABC, abstractmethod import torch +from deeplabcut.pose_estimation_pytorch.registry import (Registry, + build_from_cfg) from torch import nn -from deeplabcut.pose_estimation_pytorch.registry import build_from_cfg, Registry - PREDICTORS = Registry("predictors", build_func=build_from_cfg) diff --git a/dlclive/predictor/single_predictor.py b/dlclive/predictor/single_predictor.py index 6425d02..81b443e 100644 --- a/dlclive/predictor/single_predictor.py +++ b/dlclive/predictor/single_predictor.py @@ -13,14 +13,10 @@ from typing import Tuple import torch +from deeplabcut.pose_estimation_pytorch.models.predictors.base import \ + BasePredictor -from deeplabcut.pose_estimation_pytorch.models.predictors.base import ( - BasePredictor, - PREDICTORS, -) - -# @PREDICTORS.register_module class HeatmapPredictor(BasePredictor): """Predictor class for pose estimation from heatmaps (and optionally locrefs). @@ -39,7 +35,7 @@ def __init__( clip_scores: bool = False, location_refinement: bool = True, locref_std: float = 7.2801, - stride: float = 8. + stride: float = 8.0, ): """ Args: @@ -57,9 +53,7 @@ def __init__( self.locref_std = locref_std self.stride = stride - def forward( - self, outputs: dict[str, torch.Tensor] - ) -> dict[str, torch.Tensor]: + def forward(self, outputs: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: """Forward pass of SinglePredictor. Gets predictions from model output. Args: @@ -172,17 +166,20 @@ def build(cfg: dict) -> HeatmapPredictor: if cfg["model"]["heads"]["bodypart"]["heatmap_config"]["strides"][0] > 0: stride = float(cfg["model"]["backbone"]["output_stride"]) / float( cfg["model"]["heads"]["bodypart"]["heatmap_config"]["strides"][0] - ) + ) else: stride = float(cfg["model"]["backbone"]["output_stride"]) * -float( cfg["model"]["heads"]["bodypart"]["heatmap_config"]["strides"][0] - ) + ) else: stride = float(cfg["model"]["backbone"]["output_stride"]) predictor = HeatmapPredictor( - apply_sigmoid=apply_sigmoid, stride=stride, clip_scores=clip_scores, - location_refinement=loc_ref, locref_std=loc_ref_std - ) + apply_sigmoid=apply_sigmoid, + stride=stride, + clip_scores=clip_scores, + location_refinement=loc_ref, + locref_std=loc_ref_std, + ) # elif cfg["method"] == "td": # apply_sigmoid = cfg["model"]["heads"]["bodypart"]["predictor"]["apply_sigmoid"] @@ -191,4 +188,4 @@ def build(cfg: dict) -> HeatmapPredictor: # heatmap_stride = cfg[] # predictor = HeatmapPredictor(apply_sigmoid=apply_sigmoid, clip_scores=clip_scores, location_refinement=loc_ref) - return predictor \ No newline at end of file + return predictor diff --git a/dlclive/processor/__init__.py b/dlclive/processor/__init__.py index 67e14db..2ec41c2 100644 --- a/dlclive/processor/__init__.py +++ b/dlclive/processor/__init__.py @@ -5,5 +5,5 @@ Licensed under GNU Lesser General Public License v3.0 """ -from dlclive.processor.processor import Processor from dlclive.processor.kalmanfilter import KalmanFilterPredictor +from dlclive.processor.processor import Processor diff --git a/dlclive/processor/kalmanfilter.py b/dlclive/processor/kalmanfilter.py index 447bcae..dbe05b3 100644 --- a/dlclive/processor/kalmanfilter.py +++ b/dlclive/processor/kalmanfilter.py @@ -5,9 +5,10 @@ Licensed under GNU Lesser General Public License v3.0 """ - import time + import numpy as np + from dlclive.processor import Processor @@ -45,7 +46,7 @@ def _get_forward_model(self, dt): F = np.zeros((self.n_states, self.n_states)) for d in range(self.nderiv + 1): for i in range(self.n_states - (d * self.bp * 2)): - F[i, i + (2 * self.bp * d)] = (dt ** d) / max(1, d) + F[i, i + (2 * self.bp * d)] = (dt**d) / max(1, d) return F diff --git a/dlclive/utils.py b/dlclive/utils.py index 4b0deaa..657011b 100644 --- a/dlclive/utils.py +++ b/dlclive/utils.py @@ -5,9 +5,10 @@ Licensed under GNU Lesser General Public License v3.0 """ +import warnings import numpy as np -import warnings + from dlclive.exceptions import DLCLiveWarning try: @@ -32,7 +33,7 @@ def convert_to_ubyte(frame): - """ Converts an image to unsigned 8-bit integer numpy array. + """Converts an image to unsigned 8-bit integer numpy array. If scikit-image is installed, uses skimage.img_as_ubyte, otherwise, uses a similar custom function. Parameters @@ -53,7 +54,7 @@ def convert_to_ubyte(frame): def resize_frame(frame, resize=None): - """ Resizes an image. Uses OpenCV if installed, otherwise, uses pillow + """Resizes an image. Uses OpenCV if installed, otherwise, uses pillow Parameters ---------- @@ -81,7 +82,7 @@ def resize_frame(frame, resize=None): def img_to_rgb(frame): - """ Convert an image to RGB. Uses OpenCV is installed, otherwise uses pillow. + """Convert an image to RGB. Uses OpenCV is installed, otherwise uses pillow. Parameters ---------- @@ -107,7 +108,7 @@ def img_to_rgb(frame): def gray_to_rgb(frame): - """ Convert an image from grayscale to RGB. Uses OpenCV is installed, otherwise uses pillow. + """Convert an image from grayscale to RGB. Uses OpenCV is installed, otherwise uses pillow. Parameters ---------- @@ -127,7 +128,7 @@ def gray_to_rgb(frame): def bgr_to_rgb(frame): - """ Convert an image from BGR to RGB. Uses OpenCV is installed, otherwise uses pillow. + """Convert an image from BGR to RGB. Uses OpenCV is installed, otherwise uses pillow. Parameters ---------- @@ -147,7 +148,7 @@ def bgr_to_rgb(frame): def _img_as_ubyte_np(frame): - """ Converts an image as a numpy array to unsinged 8-bit integer. + """Converts an image as a numpy array to unsinged 8-bit integer. As in scikit-image img_as_ubyte, converts negative pixels to 0 and converts range to [0, 255] Parameters diff --git a/dlclive/version.py b/dlclive/version.py index 7996e03..a35486f 100644 --- a/dlclive/version.py +++ b/dlclive/version.py @@ -6,6 +6,5 @@ Licensed under GNU Lesser General Public License v3.0 """ - __version__ = "1.0.4" VERSION = __version__ From 9508a79e861ddf3aa9f8f958719811964d985f93 Mon Sep 17 00:00:00 2001 From: AnnaStuckert <47814177+annastuckert@users.noreply.github.com> Date: Mon, 24 Feb 2025 15:01:44 +0100 Subject: [PATCH 17/24] cleaning out unused files --- analyze_video.py | 65 ------------------------------------------------ run_dlc-live.py | 18 -------------- 2 files changed, 83 deletions(-) delete mode 100644 analyze_video.py delete mode 100644 run_dlc-live.py diff --git a/analyze_video.py b/analyze_video.py deleted file mode 100644 index bb77aa8..0000000 --- a/analyze_video.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 - -import cv2 -import numpy as np -import torch - -from dlclive import DLCLive, Processor -from dlclive.display import Display - - -def analyze_video2(video_path: str, dlc_live): - # Load video - cap = cv2.VideoCapture(video_path) - poses = [] - frame_index = 0 - - while True: - ret, frame = cap.read() - if not ret: - break # End of video - - # Prepare the frame for the model - frame = np.array(frame, dtype=np.float32) - frame = np.transpose(frame, (2, 0, 1)) - frame = frame.reshape(1, frame.shape[0], frame.shape[1], frame.shape[2]) - frame = frame / 255.0 - - # Analyze the frame using the get_pose function - pose = dlc_live.get_pose(frame) - - # Store the pose for this frame - poses.append(pose) - - frame_index += 1 - print(frame_index) - - # Release the video capture object - cap.release() - - return poses - - -def main(): - # Paths provided by you - video_path = "/Users/annastuckert/Documents/DLC_AI_Residency/DLC_AI2024/DeepLabCut-live/Ventral_gait_model/1_20cms_0degUP_first_1s.avi" - model_dir = "/Users/annastuckert/Documents/DLC_AI_Residency/DLC_AI2024/DeepLabCut-live/Ventral_gait_model/train" - snapshot = "/Users/annastuckert/Documents/DLC_AI_Residency/DLC_AI2024/DeepLabCut-live/Ventral_gait_model/train/snapshot-263.pt" - model_type = "pytorch" - - # Initialize the DLCLive model - dlc_proc = Processor() - dlc_live = DLCLive( - pytorch_cfg=model_dir, - processor=dlc_proc, - snapshot=snapshot, - model_type=model_type, - ) - - # Analyze the video - poses = analyze_video2(video_path, dlc_live) - print("Pose analysis complete.") - - -if __name__ == "__main__": - main() diff --git a/run_dlc-live.py b/run_dlc-live.py deleted file mode 100644 index 73d9ed5..0000000 --- a/run_dlc-live.py +++ /dev/null @@ -1,18 +0,0 @@ -import numpy as np -from PIL import Image - -from dlclive import DLCLive, Processor - -image = Image.open( - "/Users/annastuckert/Downloads/exported DLC model for dlc-live/img049.png" -) -img = np.asarray(image) - -dlc_proc = Processor() -dlc_live = DLCLive( - "/Users/annastuckert/Downloads/exported DLC model for dlc-live/DLC_dev-single-animal_resnet_50_iteration-1_shuffle-1", - processor=dlc_proc, -) -dlc_live.init_inference(img) -pose = dlc_live.get_pose(img) -print(pose) From d61f89288bc6ad1fd03274fb53b25644e434520a Mon Sep 17 00:00:00 2001 From: Dikra Date: Mon, 24 Feb 2025 15:02:08 +0100 Subject: [PATCH 18/24] update docstrings, clean dlclive script --- dlclive/__init__.py | 2 +- dlclive/dlclive.py | 94 ++++++++--------------------------- dlclive/live_inference.py | 2 +- dlclive/processor/__init__.py | 3 -- test.py | 8 --- 5 files changed, 23 insertions(+), 86 deletions(-) delete mode 100644 test.py diff --git a/dlclive/__init__.py b/dlclive/__init__.py index 9afe11b..460490c 100644 --- a/dlclive/__init__.py +++ b/dlclive/__init__.py @@ -8,7 +8,7 @@ from dlclive.display import Display from dlclive.dlclive import DLCLive from dlclive.predictor import HeatmapPredictor -from dlclive.processor import Processor +from dlclive.processor.processor import Processor from dlclive.version import VERSION, __version__ # from dlclive.benchmark import benchmark, benchmark_videos, download_benchmarking_data diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 1d57912..f2eebf8 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -13,9 +13,7 @@ from pathlib import Path from typing import List, Optional, Tuple -import deeplabcut as dlc import numpy as np -import onnx import onnxruntime as ort import ruamel.yaml import torch @@ -24,37 +22,11 @@ from dlclive import utils from dlclive.display import Display from dlclive.exceptions import DLCLiveError, DLCLiveWarning -from dlclive.pose import (argmax_pose_predict, extract_cnn_output, - multi_pose_predict) from dlclive.predictor import HeatmapPredictor if typing.TYPE_CHECKING: from dlclive.processor import Processor - -# TODO: -# graph.py the main element to import TF model - convert to pytorch implementation -# add pcutoffn to docstring - -# Q: What is the best way to test the code as we go? -# Q: if self.pose is not None: - ask Niels to go through this! - -# Q: what exactly does model_type reference? -# Q: is precision a type of qunatization? -# Q: for dynamic: First key points are predicted, then dynamic cropping is performed to 'single out' the animal, and then pose is estimated, we think. What is the difference from key point prediction to pose prediction? -# Q: what is the processor? see processor code F12 from init file - what is the 'user defined process' - could it be that if mouse = standing, perform some action? or is the process the prediction of a certain pose/set of keypoints -# Q: why have the convert2rgb function, is the stream coming from the camera different from the input needed to DLC live? -# Q: what is the parameter 'cfg'? - -# What do these do? -# self.inputs = None -# self.outputs = None -# self.tflite_interpreter = None -# self.pose = None -# self.is_initialized = False -# self.sess = None - - class DLCLive(object): """ Object that loads a DLC network and performs inference on single images (e.g. images captured from a camera feed) @@ -66,10 +38,10 @@ class DLCLive(object): Full path to exported model directory model_type: string, optional - which model to use: 'base', 'tensorrt' for tensorrt optimized graph, 'lite' for tensorflow lite optimized graph + which model to use: 'pytorch' or 'onnx' for exported snapshot precision : string, optional - precision of model weights, only for model_type='tensorrt'. Can be 'FP16' (default), 'FP32', or 'INT8' + precision of model weights, only for model_type='onnx'. Can be 'FP32' (default) or 'FP16' cropping : list of int cropping parameters in pixel number: [x1, x2, y1, y2] #A: Maybe this is the dynamic cropping of each frame to speed of processing, so instead of analyzing the whole frame, it analyses only the part of the frame where the animal is @@ -196,12 +168,7 @@ def parameterization( self, ) -> ( dict - ): # A: constructs a dictionary based on the object attributes based on the list of parameters - """ - Return - Returns - ------- - """ + ): return {param: getattr(self, param) for param in self.PARAMETERS} def process_frame(self, frame): @@ -219,8 +186,6 @@ def process_frame(self, frame): processed frame: convert type, crop, convert color """ - # ! NORMALISATION ?? - if self.cropping: frame = frame[ self.cropping[2] : self.cropping[3], self.cropping[0] : self.cropping[1] @@ -230,9 +195,7 @@ def process_frame(self, frame): if self.pose is not None: detected = self.pose["poses"][0][0][:, 2] > self.dynamic[1] - # if np.any(detected.numpy()): if torch.any(detected): - # if detected.any(): # Use PyTorch's any() method x = self.pose["poses"][0][0][detected, 0] y = self.pose["poses"][0][0][detected, 1] @@ -263,7 +226,7 @@ def process_frame(self, frame): def load_model(self): if self.model_type == "pytorch": - # Requires DLC 3.0 to be imported + # Requires DLC 3.0 to be imported ! model_path = os.path.join(self.path, self.snapshot) if not os.path.isfile(model_path): raise FileNotFoundError( @@ -278,12 +241,7 @@ def load_model(self): elif self.model_type == "onnx": model_paths = glob.glob(os.path.normpath(self.path + "/*.onnx")) if self.precision == "FP16": - model_path = [ - model_paths[i] - for i in range(len(model_paths)) - if "fp16" in model_paths[i] - ][0] - print(model_path) + model_path = [model_paths[i] for i in range(len(model_paths)) if "fp16" in model_paths[i]][0] else: model_path = model_paths[0] opts = ort.SessionOptions() @@ -292,23 +250,19 @@ def load_model(self): self.sess = ort.InferenceSession( model_path, opts, providers=["CUDAExecutionProvider"] ) - print(self.sess) elif self.device == "cpu": self.sess = ort.InferenceSession( model_path, opts, providers=["CPUExecutionProvider"] ) - # ! TODO implement if statements for choice of tensorrt engine options (precision, and caching) + elif self.device == "tensorrt": - provider = [ - ( - "TensorrtExecutionProvider", - { - "trt_engine_cache_enable": True, - "trt_engine_cache_path": "./trt_engines", - }, - ) - ] - self.sess = ort.InferenceSession(model_path, opts, providers=provider) + provider = [("TensorrtExecutionProvider", { + "trt_engine_cache_enable": True, + "trt_engine_cache_path": "./trt_engines" + })] + self.sess = ort.InferenceSession( + model_path, opts, providers=provider + ) self.predictor = HeatmapPredictor.build(self.cfg) if not os.path.isfile(model_path): @@ -336,13 +290,15 @@ def init_inference(self, frame=None, **kwargs): -------- pose :class:`numpy.ndarray` the pose estimated by DeepLabCut for the input image + inf_time:class: `float` + the pose inference time """ # load model self.load_model() - inf_time = 0.0 - # get pose of first frame (first inference is often very slow) + inf_time = 0. + # get pose of first frame (first inference is very slow) if frame is not None: pose, inf_time = self.get_pose(frame, **kwargs) else: @@ -363,8 +319,11 @@ def get_pose(self, frame=None, **kwargs): -------- pose :class:`numpy.ndarray` the pose estimated by DeepLabCut for the input image + inf_time:class: `float` + the pose inference time """ - inf_time = 0.0 + + inf_time = 0. if frame is None: raise DLCLiveError("No frame provided for live pose estimation") @@ -437,18 +396,7 @@ def get_pose(self, frame=None, **kwargs): self.pose["poses"][0][0][:, 1] += self.dynamic_cropping[2] # process the pose - if self.processor: self.pose = self.processor.process(self.pose, **kwargs) return self.pose, inf_time - - # def close(self): - # """ Close tensorflow session - # """ - - # self.sess.close() - # self.sess = None - # self.is_initialized = False - # if self.display is not None: - # self.display.destroy() diff --git a/dlclive/live_inference.py b/dlclive/live_inference.py index dea3750..8a3fddf 100644 --- a/dlclive/live_inference.py +++ b/dlclive/live_inference.py @@ -175,7 +175,7 @@ def analyze_live_video( # Load video cap = cv2.VideoCapture(camera) if not cap.isOpened(): - print(f"Error: Could not open video file {camera}") + print(f"Error: Could not open camera {camera}") return # Start empty dict to save poses to for each frame diff --git a/dlclive/processor/__init__.py b/dlclive/processor/__init__.py index 2ec41c2..a360448 100644 --- a/dlclive/processor/__init__.py +++ b/dlclive/processor/__init__.py @@ -4,6 +4,3 @@ Licensed under GNU Lesser General Public License v3.0 """ - -from dlclive.processor.kalmanfilter import KalmanFilterPredictor -from dlclive.processor.processor import Processor diff --git a/test.py b/test.py deleted file mode 100644 index bfabb22..0000000 --- a/test.py +++ /dev/null @@ -1,8 +0,0 @@ -from dlclive import DLCLive, Processor -import dlclive -import cv2 - -dlc_proc = Processor() -dlc_live = DLCLive("/media1/data/dikra/dlc-live-tmp", processor=dlc_proc) -img = cv2.imread("/media1/data/dikra/fly-kevin/img001.png") -dlc_live.init_inference(frame=img) \ No newline at end of file From f527d38c914ac7c2425a733d062182e51dd4914d Mon Sep 17 00:00:00 2001 From: Niels Poulsen Date: Wed, 18 Sep 2024 17:49:03 +0200 Subject: [PATCH 19/24] Continued DeepLabCut-Live implementation for DeepLabCut 3.0 --- dlclive/__init__.py | 3 - dlclive/benchmark.py | 965 ++++++------ dlclive/benchmark_tf.py | 724 +++++++++ dlclive/check_install/check_install.py | 2 +- dlclive/core/__init__.py | 10 + dlclive/core/config.py | 28 + dlclive/core/inferenceutils.py | 1314 +++++++++++++++++ dlclive/core/runner.py | 96 ++ dlclive/display.py | 76 +- dlclive/dlclive.py | 369 ++--- dlclive/factory.py | 33 + dlclive/graph.py | 3 +- dlclive/live_inference.py | 13 +- dlclive/pose.py | 34 +- dlclive/pose_estimation_pytorch/__init__.py | 15 + .../pose_estimation_pytorch/data/__init__.py | 10 + dlclive/pose_estimation_pytorch/data/image.py | 140 ++ .../dynamic_cropping.py | 543 +++++++ .../models/__init__.py | 9 + .../models/backbones/__init__.py | 14 + .../models/backbones/base.py | 141 ++ .../models/backbones/cspnext.py | 207 +++ .../models/backbones/hrnet.py | 119 ++ .../models/backbones/resnet.py | 148 ++ .../models/detectors/__init__.py | 16 + .../models/detectors/base.py | 56 + .../models/detectors/fasterRCNN.py | 72 + .../models/detectors/ssd.py | 68 + .../models/detectors/torchvision.py | 96 ++ .../models/heads/__init__.py | 16 + .../models/heads/base.py | 57 + .../models/heads/dekr.py | 412 ++++++ .../models/heads/dlcrnet.py | 134 ++ .../models/heads/rtmcc_head.py | 139 ++ .../models/heads/simple_head.py | 224 +++ .../models/heads/transformer.py | 94 ++ .../pose_estimation_pytorch/models/model.py | 127 ++ .../models/modules/__init__.py | 24 + .../models/modules/conv_block.py | 307 ++++ .../models/modules/conv_module.py | 244 +++ .../models/modules/csp.py | 387 +++++ .../models/modules/gated_attention_unit.py | 237 +++ .../models/modules/norm.py | 41 + .../models/necks/__init__.py | 12 + .../models/necks/base.py | 48 + .../models/necks/layers.py | 287 ++++ .../models/necks/transformer.py | 274 ++++ .../models/necks/utils.py | 60 + .../models/predictors/__init__.py | 15 + .../models/predictors/base.py | 64 + .../models/predictors/dekr_predictor.py | 405 +++++ .../models/predictors/identity_predictor.py | 66 + .../models/predictors/paf_predictor.py | 368 +++++ .../models/predictors/sim_cc.py | 163 ++ .../models/predictors/single_predictor.py | 161 ++ .../models/registry.py | 330 +++++ dlclive/pose_estimation_pytorch/runner.py | 326 ++++ .../pose_estimation_tensorflow/__init__.py | 0 dlclive/pose_estimation_tensorflow/graph.py | 138 ++ dlclive/pose_estimation_tensorflow/pose.py | 120 ++ dlclive/pose_estimation_tensorflow/runner.py | 212 +++ dlclive/predictor/base.py | 30 +- dlclive/predictor/single_predictor.py | 58 +- dlclive/processor/__init__.py | 1 + dlclive/processor/processor.py | 2 +- dlclive/utils.py | 28 +- dlclive/version.py | 2 +- docs/DLC Live Benchmark.md | 32 + docs/assets/select_dlc.png | Bin 0 -> 201080 bytes docs/install_desktop.md | 41 +- docs/install_jetson.md | 40 +- pyproject.toml | 36 +- scripts/export.py | 81 + scripts/fix_deeplabcut_imports.py | 82 + 74 files changed, 10375 insertions(+), 874 deletions(-) create mode 100644 dlclive/benchmark_tf.py create mode 100644 dlclive/core/__init__.py create mode 100644 dlclive/core/config.py create mode 100644 dlclive/core/inferenceutils.py create mode 100644 dlclive/core/runner.py create mode 100644 dlclive/factory.py create mode 100644 dlclive/pose_estimation_pytorch/__init__.py create mode 100644 dlclive/pose_estimation_pytorch/data/__init__.py create mode 100644 dlclive/pose_estimation_pytorch/data/image.py create mode 100644 dlclive/pose_estimation_pytorch/dynamic_cropping.py create mode 100644 dlclive/pose_estimation_pytorch/models/__init__.py create mode 100644 dlclive/pose_estimation_pytorch/models/backbones/__init__.py create mode 100644 dlclive/pose_estimation_pytorch/models/backbones/base.py create mode 100644 dlclive/pose_estimation_pytorch/models/backbones/cspnext.py create mode 100644 dlclive/pose_estimation_pytorch/models/backbones/hrnet.py create mode 100644 dlclive/pose_estimation_pytorch/models/backbones/resnet.py create mode 100644 dlclive/pose_estimation_pytorch/models/detectors/__init__.py create mode 100644 dlclive/pose_estimation_pytorch/models/detectors/base.py create mode 100644 dlclive/pose_estimation_pytorch/models/detectors/fasterRCNN.py create mode 100644 dlclive/pose_estimation_pytorch/models/detectors/ssd.py create mode 100644 dlclive/pose_estimation_pytorch/models/detectors/torchvision.py create mode 100644 dlclive/pose_estimation_pytorch/models/heads/__init__.py create mode 100644 dlclive/pose_estimation_pytorch/models/heads/base.py create mode 100644 dlclive/pose_estimation_pytorch/models/heads/dekr.py create mode 100644 dlclive/pose_estimation_pytorch/models/heads/dlcrnet.py create mode 100644 dlclive/pose_estimation_pytorch/models/heads/rtmcc_head.py create mode 100644 dlclive/pose_estimation_pytorch/models/heads/simple_head.py create mode 100644 dlclive/pose_estimation_pytorch/models/heads/transformer.py create mode 100644 dlclive/pose_estimation_pytorch/models/model.py create mode 100644 dlclive/pose_estimation_pytorch/models/modules/__init__.py create mode 100644 dlclive/pose_estimation_pytorch/models/modules/conv_block.py create mode 100644 dlclive/pose_estimation_pytorch/models/modules/conv_module.py create mode 100644 dlclive/pose_estimation_pytorch/models/modules/csp.py create mode 100644 dlclive/pose_estimation_pytorch/models/modules/gated_attention_unit.py create mode 100644 dlclive/pose_estimation_pytorch/models/modules/norm.py create mode 100644 dlclive/pose_estimation_pytorch/models/necks/__init__.py create mode 100644 dlclive/pose_estimation_pytorch/models/necks/base.py create mode 100644 dlclive/pose_estimation_pytorch/models/necks/layers.py create mode 100644 dlclive/pose_estimation_pytorch/models/necks/transformer.py create mode 100644 dlclive/pose_estimation_pytorch/models/necks/utils.py create mode 100644 dlclive/pose_estimation_pytorch/models/predictors/__init__.py create mode 100644 dlclive/pose_estimation_pytorch/models/predictors/base.py create mode 100644 dlclive/pose_estimation_pytorch/models/predictors/dekr_predictor.py create mode 100644 dlclive/pose_estimation_pytorch/models/predictors/identity_predictor.py create mode 100644 dlclive/pose_estimation_pytorch/models/predictors/paf_predictor.py create mode 100644 dlclive/pose_estimation_pytorch/models/predictors/sim_cc.py create mode 100644 dlclive/pose_estimation_pytorch/models/predictors/single_predictor.py create mode 100644 dlclive/pose_estimation_pytorch/models/registry.py create mode 100644 dlclive/pose_estimation_pytorch/runner.py create mode 100644 dlclive/pose_estimation_tensorflow/__init__.py create mode 100644 dlclive/pose_estimation_tensorflow/graph.py create mode 100644 dlclive/pose_estimation_tensorflow/pose.py create mode 100644 dlclive/pose_estimation_tensorflow/runner.py create mode 100755 docs/DLC Live Benchmark.md create mode 100644 docs/assets/select_dlc.png create mode 100644 scripts/export.py create mode 100644 scripts/fix_deeplabcut_imports.py diff --git a/dlclive/__init__.py b/dlclive/__init__.py index 460490c..71a89d9 100644 --- a/dlclive/__init__.py +++ b/dlclive/__init__.py @@ -7,8 +7,5 @@ from dlclive.display import Display from dlclive.dlclive import DLCLive -from dlclive.predictor import HeatmapPredictor from dlclive.processor.processor import Processor from dlclive.version import VERSION, __version__ - -# from dlclive.benchmark import benchmark, benchmark_videos, download_benchmarking_data diff --git a/dlclive/benchmark.py b/dlclive/benchmark.py index bd347c6..d7db674 100644 --- a/dlclive/benchmark.py +++ b/dlclive/benchmark.py @@ -5,132 +5,93 @@ Licensed under GNU Lesser General Public License v3.0 """ -import os -import pickle +import csv import platform import subprocess import sys import time -import typing import warnings +from pathlib import Path import colorcet as cc -import ruamel +import cv2 +import numpy as np +import torch from PIL import ImageColor +from pip._internal.operations import freeze try: - from pip._internal.operations import freeze -except ImportError: - from pip.operations import freeze - -import cv2 -import numpy as np -import tensorflow as tf -from tqdm import tqdm + import pandas as pd -from dlclive import VERSION, DLCLive -from dlclive import __file__ as dlcfile -from dlclive.utils import decode_fourcc + has_pandas = True +except ModuleNotFoundError as err: + has_pandas = False +try: + from tqdm import tqdm -def download_benchmarking_data( - target_dir=".", - url="http://deeplabcut.rowland.harvard.edu/datasets/dlclivebenchmark.tar.gz", -): - """ - Downloads a DeepLabCut-Live benchmarking Data (videos & DLC models). - """ - import tarfile - import urllib.request + has_tqdm = True +except ModuleNotFoundError as err: + has_tqdm = False - from tqdm import tqdm - def show_progress(count, block_size, total_size): - pbar.update(block_size) - - def tarfilenamecutting(tarf): - """' auxfun to extract folder path - ie. /xyz-trainsetxyshufflez/ - """ - for memberid, member in enumerate(tarf.getmembers()): - if memberid == 0: - parent = str(member.path) - l = len(parent) + 1 - if member.path.startswith(parent): - member.path = member.path[l:] - yield member - - response = urllib.request.urlopen(url) - print( - "Downloading the benchmarking data from the DeepLabCut server @Harvard -> Go Crimson!!! {}....".format( - url - ) - ) - total_size = int(response.getheader("Content-Length")) - pbar = tqdm(unit="B", total=total_size, position=0) - filename, _ = urllib.request.urlretrieve(url, reporthook=show_progress) - with tarfile.open(filename, mode="r:gz") as tar: - tar.extractall(target_dir, members=tarfilenamecutting(tar)) +from dlclive import DLCLive +from dlclive.utils import decode_fourcc +from dlclive.version import VERSION def get_system_info() -> dict: - """Return summary info for system running benchmark + """ + Returns a summary of system information relevant to running benchmarking. + Returns ------- dict - Dictionary containing the following system information: - * ``host_name`` (str): name of machine - * ``op_sys`` (str): operating system - * ``python`` (str): path to python (which conda/virtual environment) - * ``device`` (tuple): (device type (``'GPU'`` or ``'CPU'```), device information) - * ``freeze`` (list): list of installed packages and versions - * ``python_version`` (str): python version - * ``git_hash`` (str, None): If installed from git repository, hash of HEAD commit - * ``dlclive_version`` (str): dlclive version from :data:`dlclive.VERSION` + A dictionary containing the following system information: + - host_name (str): Name of the machine. + - op_sys (str): Operating system. + - python (str): Path to the Python executable, indicating the conda/virtual + environment in use. + - device_type (str): Type of device used ('GPU' or 'CPU'). + - device (list): List containing the name of the GPU or CPU brand. + - freeze (list): List of installed Python packages with their versions. + - python_version (str): Version of Python in use. + - git_hash (str or None): If installed from git repository, hash of HEAD commit. + - dlclive_version (str): Version of the DLCLive package. """ - # get os - + # Get OS and host name op_sys = platform.platform() host_name = platform.node().replace(" ", "") - # A string giving the absolute path of the executable binary for the Python interpreter, on systems where this makes sense. + # Get Python executable path if platform.system() == "Windows": host_python = sys.executable.split(os.path.sep)[-2] else: host_python = sys.executable.split(os.path.sep)[-3] - # try to get git hash if possible - dlc_basedir = os.path.dirname(os.path.dirname(dlcfile)) + # Try to get git hash if possible git_hash = None + dlc_basedir = os.path.dirname(os.path.dirname(__file__)) try: - git_hash = subprocess.check_output( - ["git", "rev-parse", "HEAD"], cwd=dlc_basedir + git_hash = ( + subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=dlc_basedir) + .decode("utf-8") + .strip() ) - git_hash = git_hash.decode("utf-8").rstrip("\n") except subprocess.CalledProcessError: - # not installed from git repo, eg. pypi - # fine, pass quietly + # Not installed from git repo, e.g., pypi pass - # get device info (GPU or CPU) - dev = None - if tf.test.is_gpu_available(): - gpu_name = tf.test.gpu_device_name() - from tensorflow.python.client import device_lib - - dev_desc = [ - d.physical_device_desc - for d in device_lib.list_local_devices() - if d.name == gpu_name - ] - dev = [d.split(",")[1].split(":")[1].strip() for d in dev_desc] + # Get device info (GPU or CPU) + if torch.cuda.is_available(): dev_type = "GPU" + dev = [torch.cuda.get_device_name(torch.cuda.current_device())] else: from cpuinfo import get_cpu_info - dev = [get_cpu_info()["brand"]] dev_type = "CPU" + dev = [get_cpu_info()["brand_raw"]] return { "host_name": host_name, @@ -138,7 +99,6 @@ def get_system_info() -> dict: "python": host_python, "device_type": dev_type, "device": dev, - # pip freeze to get versions of all packages "freeze": list(freeze.freeze()), "python_version": sys.version, "git_hash": git_hash, @@ -147,66 +107,78 @@ def get_system_info() -> dict: def benchmark( - model_path, - video_path, - tf_config=None, - resize=None, - pixels=None, - cropping=None, - dynamic=(False, 0.5, 10), - n_frames=1000, - print_rate=False, - display=False, - pcutoff=0.0, - display_radius=3, - cmap="bmy", - save_poses=False, - save_video=False, - output=None, -) -> typing.Tuple[np.ndarray, tuple, bool, dict]: + path: str | Path, + video_path: str | Path, + single_animal: bool = True, + resize: float | None = None, + pixels: int | None = None, + cropping: list[int] = None, + dynamic: tuple[bool, float, int] = (False, 0.5, 10), + n_frames: int = 1000, + print_rate: bool = False, + display: bool = False, + pcutoff: float = 0.0, + max_detections: int = 10, + display_radius: int = 3, + cmap: str = "bmy", + save_poses: bool = False, + save_video: bool = False, + output: str | Path | None = None, +) -> tuple[np.ndarray, tuple, dict]: """Analyze DeepLabCut-live exported model on a video: - Calculate inference time, - display keypoints, or - get poses/create a labeled video + + Calculate inference time, display keypoints, or get poses/create a labeled video. Parameters ---------- - model_path : str + path : str path to exported DeepLabCut model video_path : str path to video file - tf_config : :class:`tensorflow.ConfigProto` - tensorflow session configuration + single_animal: bool + to make code behave like DLCLive for tensorflow models resize : int, optional - resize factor. Can only use one of resize or pixels. If both are provided, will use pixels. by default None + Resize factor. Can only use one of resize or pixels. If both are provided, will + use pixels. by default None pixels : int, optional - downsize image to this number of pixels, maintaining aspect ratio. Can only use one of resize or pixels. If both are provided, will use pixels. by default None + Downsize image to this number of pixels, maintaining aspect ratio. Can only use + one of resize or pixels. If both are provided, will use pixels. by default None cropping : list of int cropping parameters in pixel number: [x1, x2, y1, y2] dynamic: triple containing (state, detectiontreshold, margin) - If the state is true, then dynamic cropping will be performed. That means that if an object is detected (i.e. any body part > detectiontreshold), - then object boundaries are computed according to the smallest/largest x position and smallest/largest y position of all body parts. This window is - expanded by the margin and from then on only the posture within this crop is analyzed (until the object is lost, i.e. detectiontreshold), then object + boundaries are computed according to the smallest/largest x position and + smallest/largest y position of all body parts. This window is expanded by the + margin and from then on only the posture within this crop is analyzed (until the + object is lost, i.e. < detectiontreshold). The current position is utilized for + updating the crop window for the next frame (this is why the margin is important + and should be set large enough given the movement of the animal) n_frames : int, optional number of frames to run inference on, by default 1000 print_rate : bool, optional flag to print inference rate frame by frame, by default False display : bool, optional - flag to display keypoints on images. Useful for checking the accuracy of exported models. + flag to display keypoints on images. Useful for checking the accuracy of + exported models. pcutoff : float, optional likelihood threshold to display keypoints + max_detections: int + for top-down models, the maximum number of individuals to detect in a frame display_radius : int, optional size (radius in pixels) of keypoint to display cmap : str, optional - a string indicating the :package:`colorcet` colormap, `options here `, by default "bmy" + a string indicating the :package:`colorcet` colormap, `options here + `, by default "bmy" save_poses : bool, optional - flag to save poses to an hdf5 file. If True, operates similar to :function:`DeepLabCut.benchmark_videos`, by default False + flag to save poses to an hdf5 file. If True, operates similar to + :function:`DeepLabCut.benchmark_videos`, by default False save_video : bool, optional - flag to save a labeled video. If True, operates similar to :function:`DeepLabCut.create_labeled_video`, by default False + flag to save a labeled video. If True, operates similar to + :function:`DeepLabCut.create_labeled_video`, by default False output : str, optional - path to directory to save pose and/or video file. If not specified, will use the directory of video_path, by default None + path to directory to save pose and/or video file. If not specified, will use + the directory of video_path, by default None Returns ------- @@ -214,8 +186,6 @@ def benchmark( vector of inference times tuple (image width, image height) - bool - tensorflow inference flag dict metadata for video @@ -233,10 +203,19 @@ def benchmark( Analyze a video (save poses to hdf5) and create a labeled video, similar to :function:`DeepLabCut.benchmark_videos` and :function:`create_labeled_video` dlclive.benchmark('/my/exported/model', 'my_video.avi', save_poses=True, save_video=True) """ + path = Path(path) + video_path = Path(video_path) + if not video_path.exists(): + raise ValueError(f"Could not find video: {video_path}: check that it exists!") - ### load video + if output is None: + output = video_path.parent + else: + output = Path(output) + output.mkdir(exist_ok=True, parents=True) - cap = cv2.VideoCapture(video_path) + # load video + cap = cv2.VideoCapture(str(video_path)) ret, frame = cap.read() n_frames = ( n_frames @@ -244,112 +223,107 @@ def benchmark( else (cap.get(cv2.CAP_PROP_FRAME_COUNT) - 1) ) n_frames = int(n_frames) - im_size = (cap.get(cv2.CAP_PROP_FRAME_WIDTH), cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) - - ### get resize factor + im_size = ( + int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), + int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)), + ) + # get resize factor if pixels is not None: resize = np.sqrt(pixels / (im_size[0] * im_size[1])) + if resize is not None: im_size = (int(im_size[0] * resize), int(im_size[1] * resize)) - ### create video writer - + # create video writer if save_video: colors = None - out_dir = ( - output - if output is not None - else os.path.dirname(os.path.realpath(video_path)) - ) - out_vid_base = os.path.basename(video_path) - out_vid_file = os.path.normpath( - f"{out_dir}/{os.path.splitext(out_vid_base)[0]}_DLCLIVE_LABELED.avi" - ) + out_vid_file = output / f"{video_path.stem}_DLCLIVE_LABELED.avi" fourcc = cv2.VideoWriter_fourcc(*"DIVX") fps = cap.get(cv2.CAP_PROP_FPS) - vwriter = cv2.VideoWriter(out_vid_file, fourcc, fps, im_size) - - ### check for pandas installation if using save_poses flag - - if save_poses: - try: - import pandas as pd - - use_pandas = True - except: - use_pandas = False - warnings.warn( - "Could not find installation of pandas; saving poses as a numpy array with the dimensions (n_frames, n_keypoints, [x, y, likelihood])." - ) - - ### initialize DLCLive and perform inference + print(out_vid_file) + print(fourcc) + print(fps) + print(im_size) + vid_writer = cv2.VideoWriter(str(out_vid_file), fourcc, fps, im_size) + # initialize DLCLive and perform inference inf_times = np.zeros(n_frames) poses = [] live = DLCLive( - model_path, - tf_config=tf_config, + model_path=path, + single_animal=single_animal, resize=resize, cropping=cropping, dynamic=dynamic, display=display, + max_detections=max_detections, pcutoff=pcutoff, display_radius=display_radius, display_cmap=cmap, ) poses.append(live.init_inference(frame)) - TFGPUinference = True if len(live.outputs) == 1 else False - iterator = range(n_frames) if (print_rate) or (display) else tqdm(range(n_frames)) - for i in iterator: + iterator = range(n_frames) + if print_rate or display: + iterator = tqdm(iterator) + for i in iterator: ret, frame = cap.read() - if not ret: warnings.warn( - "Did not complete {:d} frames. There probably were not enough frames in the video {}.".format( - n_frames, video_path - ) + f"Did not complete {n_frames:d} frames. There probably were not enough " + f"frames in the video {video_path}." ) break start_pose = time.time() poses.append(live.get_pose(frame)) inf_times[i] = time.time() - start_pose - if save_video: + this_pose = poses[-1] + + if single_animal: + # expand individual dimension + this_pose = this_pose[None] + + num_idv, num_bpt = this_pose.shape[:2] + num_colors = num_bpt if colors is None: all_colors = getattr(cc, cmap) colors = [ ImageColor.getcolor(c, "RGB")[::-1] - for c in all_colors[:: int(len(all_colors) / poses[-1].shape[0])] + for c in all_colors[:: int(len(all_colors) / num_colors)] ] - this_pose = poses[-1] - for j in range(this_pose.shape[0]): - if this_pose[j, 2] > pcutoff: - x = int(this_pose[j, 0]) - y = int(this_pose[j, 1]) - frame = cv2.circle( - frame, (x, y), display_radius, colors[j], thickness=-1 - ) + for j in range(num_idv): + for k in range(num_bpt): + color_idx = k + if this_pose[j, k, 2] > pcutoff: + x = int(this_pose[j, k, 0]) + y = int(this_pose[j, k, 1]) + frame = cv2.circle( + frame, + (x, y), + display_radius, + colors[color_idx], + thickness=-1, + ) if resize is not None: frame = cv2.resize(frame, im_size) - vwriter.write(frame) + vid_writer.write(frame) if print_rate: - print("pose rate = {:d}".format(int(1 / inf_times[i]))) + print(f"pose rate = {int(1 / inf_times[i]):d}") if print_rate: - print("mean pose rate = {:d}".format(int(np.mean(1 / inf_times)))) - - ### gather video and test parameterization + print(f"mean pose rate = {int(np.mean(1 / inf_times)):d}") + # gather video and test parameterization # dont want to fail here so gracefully failing on exception -- # eg. some packages of cv2 don't have CAP_PROP_CODEC_PIXEL_FORMAT try: @@ -359,17 +333,17 @@ def benchmark( try: fps = round(cap.get(cv2.CAP_PROP_FPS)) - except: + except Exception: fps = None try: pix_fmt = decode_fourcc(cap.get(cv2.CAP_PROP_CODEC_PIXEL_FORMAT)) - except: + except Exception: pix_fmt = "" try: frame_count = round(cap.get(cv2.CAP_PROP_FRAME_COUNT)) - except: + except Exception: frame_count = None try: @@ -377,7 +351,7 @@ def benchmark( round(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), round(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)), ) - except: + except Exception: orig_im_size = None meta = { @@ -390,332 +364,411 @@ def benchmark( "dlclive_params": live.parameterization, } - ### close video and tensorflow session - + # close video cap.release() - live.close() - if save_video: - vwriter.release() + vid_writer.release() if save_poses: - - cfg_path = os.path.normpath(f"{model_path}/pose_cfg.yaml") - ruamel_file = ruamel.yaml.YAML() - dlc_cfg = ruamel_file.load(open(cfg_path, "r")) - bodyparts = dlc_cfg["all_joints_names"] - poses = np.array(poses) - - if use_pandas: - - poses = poses.reshape((poses.shape[0], poses.shape[1] * poses.shape[2])) - pdindex = pd.MultiIndex.from_product( - [bodyparts, ["x", "y", "likelihood"]], names=["bodyparts", "coords"] - ) - pose_df = pd.DataFrame(poses, columns=pdindex) - - out_dir = ( - output - if output is not None - else os.path.dirname(os.path.realpath(video_path)) - ) - out_vid_base = os.path.basename(video_path) - out_dlc_file = os.path.normpath( - f"{out_dir}/{os.path.splitext(out_vid_base)[0]}_DLCLIVE_POSES.h5" + bodyparts = live.cfg["metadata"]["bodyparts"] + max_idv = np.max([p.shape[0] for p in poses]) + + poses_array = -np.ones((len(poses), max_idv, len(bodyparts), 3)) + for i, p in enumerate(poses): + num_det = len(p) + poses_array[i, :num_det] = p + poses = poses_array + + num_frames, num_idv, num_bpts = poses.shape[:3] + individuals = [f"individual-{i}" for i in range(num_idv)] + + if has_pandas: + poses = poses.reshape((num_frames, num_idv * num_bpts * 3)) + col_index = pd.MultiIndex.from_product( + [individuals, bodyparts, ["x", "y", "likelihood"]], + names=["individual", "bodyparts", "coords"], ) - pose_df.to_hdf(out_dlc_file, key="df_with_missing", mode="w") + pose_df = pd.DataFrame(poses, columns=col_index) + + out_dlc_file = output / (video_path.stem + "_DLCLIVE_POSES.h5") + try: + pose_df.to_hdf(out_dlc_file, key="df_with_missing", mode="w") + except ImportError as err: + print( + "Cannot export predictions to H5 file. Install ``pytables`` extra " + f"to export to HDF: {err}" + ) + out_csv = Path(out_dlc_file).with_suffix(".csv") + pose_df.to_csv(out_csv) else: - - out_vid_base = os.path.basename(video_path) - out_dlc_file = os.path.normpath( - f"{out_dir}/{os.path.splitext(out_vid_base)[0]}_DLCLIVE_POSES.npy" + warnings.warn( + "Could not find installation of pandas; saving poses as a numpy array " + "with the dimensions (n_frames, n_keypoints, [x, y, likelihood])." ) - np.save(out_dlc_file, poses) + np.save(str(output / (video_path.stem + "_DLCLIVE_POSES.npy")), poses) - return inf_times, im_size, TFGPUinference, meta + return inf_times, im_size, meta -def save_inf_times( - sys_info, inf_times, im_size, TFGPUinference, model=None, meta=None, output=None +def benchmark_videos( + video_path: str, + model_path: str, + model_type: str, + device: str, + precision: str = "FP32", + display=True, + pcutoff=0.5, + display_radius=5, + resize=None, + cropping=None, # Adding cropping to the function parameters + dynamic=(False, 0.5, 10), + save_poses=False, + save_dir="model_predictions", + draw_keypoint_names=False, + cmap="bmy", + get_sys_info=True, + save_video=False, ): - """Save inference time data collected using :function:`benchmark` with system information to a pickle file. - This is primarily used through :function:`benchmark_videos` - + """ + Analyzes a video to track keypoints using a DeepLabCut model, and optionally saves + the keypoint data and the labeled video. Parameters ---------- - sys_info : tuple - system information generated by :func:`get_system_info` - inf_times : :class:`numpy.ndarray` - array of inference times generated by :func:`benchmark` - im_size : tuple or :class:`numpy.ndarray` - image size (width, height) for each benchmark run. If an array, each row corresponds to a row in inf_times - TFGPUinference: bool - flag if using tensorflow inference or numpy inference DLC model - model: str, optional - name of model - meta : dict, optional - metadata returned by :func:`benchmark` - output : str, optional - path to directory to save data. If None, uses pwd, by default None + video_path : str + Path to the video file to be analyzed. + model_path : str + Path to the DeepLabCut model. + model_type : str + Type of the model (e.g., 'onnx'). + device : str + Device to run the model on ('cpu' or 'cuda'). + precision : str, optional, default='FP32' + Precision type for the model ('FP32' or 'FP16'). + display : bool, optional, default=True + Whether to display frame with labelled key points. + pcutoff : float, optional, default=0.5 + Probability cutoff below which keypoints are not visualized. + display_radius : int, optional, default=5 + Radius of circles drawn for keypoints on video frames. + resize : tuple of int (width, height) or None, optional + Resize dimensions for video frames. e.g. if resize = 0.5, the video will be + processed in half the original size. If None, no resizing is applied. + cropping : list of int or None, optional + Cropping parameters [x1, x2, y1, y2] in pixels. If None, no cropping is applied. + dynamic : tuple, optional, default=(False, 0.5, 10) (True/false), p cutoff, margin) + Parameters for dynamic cropping. If the state is true, then dynamic cropping + will be performed. That means that if an object is detected (i.e. any body part + > detectiontreshold), then object boundaries are computed according to the + smallest/largest x position and smallest/largest y position of all body parts. + This window is expanded by the margin and from then on only the posture within + this crop is analyzed (until the object is lost, i.e. detectiontreshold), - then object boundaries are computed according to the smallest/largest x position and smallest/largest y position of all body parts. This window is - expanded by the margin and from then on only the posture within this crop is analyzed (until the object is lost, i.e. `, by default "bmy" - save_poses : bool, optional - flag to save poses to an hdf5 file. If True, operates similar to :function:`DeepLabCut.benchmark_videos`, by default False - save_video : bool, optional - flag to save a labeled video. If True, operates similar to :function:`DeepLabCut.create_labeled_video`, by default False + # Get video writer setup + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + fps = cap.get(cv2.CAP_PROP_FPS) + frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + vwriter = cv2.VideoWriter( + filename=output_video_path, + fourcc=fourcc, + fps=fps, + frameSize=(frame_width, frame_height), + ) - Example - ------- - Return a vector of inference times for 10000 frames on one video or two videos: - dlclive.benchmark_videos('/my/exported/model', 'my_video.avi', n_frames=10000) - dlclive.benchmark_videos('/my/exported/model', ['my_video1.avi', 'my_video2.avi'], n_frames=10000) + while True: - Return a vector of inference times, testing full size and resizing images to half the width and height for inference, for two videos - dlclive.benchmark_videos('/my/exported/model', ['my_video1.avi', 'my_video2.avi'], n_frames=10000, resize=[1.0, 0.5]) + ret, frame = cap.read() + if not ret: + break + # if frame_index == 0: + # pose = dlc_live.init_inference(frame) # load DLC model + try: + # pose = dlc_live.get_pose(frame) + if frame_index == 0: + # TODO trying to fix issues with dynamic cropping jumping back and forth + # between dyanmic cropped and original image + # dlc_live.dynamic = (False, dynamic[1], dynamic[2]) + pose, inf_time = dlc_live.init_inference(frame) # load DLC model + else: + # dlc_live.dynamic = dynamic + pose, inf_time = dlc_live.get_pose(frame) + except Exception as e: + print(f"Error analyzing frame {frame_index}: {e}") + continue + + poses.append({"frame": frame_index, "pose": pose}) + times.append(inf_time) - Display keypoints to check the accuracy of an exported model - dlclive.benchmark_videos('/my/exported/model', 'my_video.avi', display=True) + if save_video: + # Visualize keypoints + this_pose = pose["poses"][0][0] + for j in range(this_pose.shape[0]): + if this_pose[j, 2] > pcutoff: + x, y = map(int, this_pose[j, :2]) + cv2.circle( + frame, + center=(x, y), + radius=display_radius, + color=colors[j], + thickness=-1, + ) - Analyze a video (save poses to hdf5) and create a labeled video, similar to :function:`DeepLabCut.benchmark_videos` and :function:`create_labeled_video` - dlclive.benchmark_videos('/my/exported/model', 'my_video.avi', save_poses=True, save_video=True) + if draw_keypoint_names: + cv2.putText( + frame, + text=bodyparts[j], + org=(x + 10, y), + fontFace=cv2.FONT_HERSHEY_SIMPLEX, + fontScale=0.5, + color=colors[j], + thickness=1, + lineType=cv2.LINE_AA, + ) + + vwriter.write(image=frame) + frame_index += 1 + + cap.release() + if save_video: + vwriter.release() + + if get_sys_info: + print(get_system_info()) + + if save_poses: + save_poses_to_files(video_path, save_dir, bodyparts, poses, timestamp=timestamp) + + return poses, times + + +def save_poses_to_files(video_path, save_dir, bodyparts, poses, timestamp): """ + Saves the detected keypoint poses from the video to CSV and HDF5 files. - # convert video_paths to list + Parameters + ---------- + video_path : str + Path to the analyzed video file. + save_dir : str + Directory where the pose data files will be saved. + bodyparts : list of str + List of body part names corresponding to the keypoints. + poses : list of dict + List of dictionaries containing frame numbers and corresponding pose data. - video_path = video_path if type(video_path) is list else [video_path] + Returns + ------- + None + """ - # fix resize + base_filename = os.path.splitext(os.path.basename(video_path))[0] + csv_save_path = os.path.join(save_dir, f"{base_filename}_poses_{timestamp}.csv") + h5_save_path = os.path.join(save_dir, f"{base_filename}_poses_{timestamp}.h5") - if pixels: - pixels = pixels if type(pixels) is list else [pixels] - resize = [None for p in pixels] - elif resize: - resize = resize if type(resize) is list else [resize] - pixels = [None for r in resize] - else: - resize = [None] - pixels = [None] - - # loop over videos - - for v in video_path: - - # initialize full inference times - - inf_times = [] - im_size_out = [] - - for i in range(len(resize)): - - print(f"\nRun {i+1} / {len(resize)}\n") - - this_inf_times, this_im_size, TFGPUinference, meta = benchmark( - model_path, - v, - tf_config=tf_config, - resize=resize[i], - pixels=pixels[i], - cropping=cropping, - dynamic=dynamic, - n_frames=n_frames, - print_rate=print_rate, - display=display, - pcutoff=pcutoff, - display_radius=display_radius, - cmap=cmap, - save_poses=save_poses, - save_video=save_video, - output=output, - ) + # Save to CSV + with open(csv_save_path, mode="w", newline="") as file: + writer = csv.writer(file) + header = ["frame"] + [ + f"{bp}_{axis}" for bp in bodyparts for axis in ["x", "y", "confidence"] + ] + writer.writerow(header) + for entry in poses: + frame_num = entry["frame"] + pose = entry["pose"]["poses"][0][0] + row = [frame_num] + [ + item.item() if isinstance(item, torch.Tensor) else item + for kp in pose + for item in kp + ] + writer.writerow(row) - inf_times.append(this_inf_times) - im_size_out.append(this_im_size) - inf_times = np.array(inf_times) - im_size_out = np.array(im_size_out) - # save results - if output is not None: - sys_info = get_system_info() - save_inf_times( - sys_info, - inf_times, - im_size_out, - TFGPUinference, - model=os.path.basename(model_path), - meta=meta, - output=output, - ) +import argparse +import os def main(): - """Provides a command line interface :function:`benchmark_videos`""" - - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("model_path", type=str) - parser.add_argument("video_path", type=str, nargs="+") - parser.add_argument("-o", "--output", type=str, default=None) - parser.add_argument("-n", "--n-frames", type=int, default=1000) - parser.add_argument("-r", "--resize", type=float, nargs="+") - parser.add_argument("-p", "--pixels", type=float, nargs="+") - parser.add_argument("-v", "--print-rate", default=False, action="store_true") - parser.add_argument("-d", "--display", default=False, action="store_true") - parser.add_argument("-l", "--pcutoff", default=0.5, type=float) - parser.add_argument("-s", "--display-radius", default=3, type=int) - parser.add_argument("-c", "--cmap", type=str, default="bmy") - parser.add_argument("--cropping", nargs="+", type=int, default=None) - parser.add_argument("--dynamic", nargs="+", type=float, default=[]) - parser.add_argument("--save-poses", action="store_true") - parser.add_argument("--save-video", action="store_true") - args = parser.parse_args() - - if (args.cropping) and (len(args.cropping) < 4): - raise Exception( - "Cropping not properly specified. Must provide 4 values: x1, x2, y1, y2" - ) + """Provides a command line interface to benchmark_videos function.""" + parser = argparse.ArgumentParser( + description="Analyze a video using a DeepLabCut model and visualize keypoints." + ) + parser.add_argument("model_path", type=str, help="Path to the model.") + parser.add_argument("video_path", type=str, help="Path to the video file.") + parser.add_argument("model_type", type=str, help="Type of the model (e.g., 'DLC').") + parser.add_argument( + "device", type=str, help="Device to run the model on (e.g., 'cuda' or 'cpu')." + ) + parser.add_argument( + "-p", + "--precision", + type=str, + default="FP32", + help="Model precision (e.g., 'FP32', 'FP16').", + ) + parser.add_argument( + "-d", "--display", action="store_true", help="Display keypoints on the video." + ) + parser.add_argument( + "-c", + "--pcutoff", + type=float, + default=0.5, + help="Probability cutoff for keypoints visualization.", + ) + parser.add_argument( + "-dr", + "--display-radius", + type=int, + default=5, + help="Radius of keypoint circles in the display.", + ) + parser.add_argument( + "-r", + "--resize", + type=int, + default=None, + help="Resize video frames to [width, height].", + ) + parser.add_argument( + "-x", + "--cropping", + type=int, + nargs=4, + default=None, + help="Cropping parameters [x1, x2, y1, y2].", + ) + parser.add_argument( + "-y", + "--dynamic", + type=float, + nargs=3, + default=[False, 0.5, 10], + help="Dynamic cropping [flag, pcutoff, margin].", + ) + parser.add_argument( + "--save-poses", action="store_true", help="Save the keypoint poses to files." + ) + parser.add_argument( + "--save-video", + action="store_true", + help="Save the output video with keypoints.", + ) + parser.add_argument( + "--save-dir", + type=str, + default="model_predictions", + help="Directory to save output files.", + ) + parser.add_argument( + "--draw-keypoint-names", + action="store_true", + help="Draw keypoint names on the video.", + ) + parser.add_argument( + "--cmap", type=str, default="bmy", help="Colormap for keypoints visualization." + ) + parser.add_argument( + "--no-sys-info", + action="store_false", + help="Do not print system info.", + dest="get_sys_info", + ) - if not args.dynamic: - args.dynamic = (False, 0.5, 10) - elif len(args.dynamic) < 3: - raise Exception( - "Dynamic cropping not properly specified. Must provide three values: 0 or 1 as boolean flag, pcutoff, and margin" - ) - else: - args.dynamic = (bool(args.dynamic[0]), args.dynamic[1], args.dynamic[2]) + args = parser.parse_args() + # Call the benchmark_videos function with the parsed arguments benchmark_videos( - args.model_path, - args.video_path, - output=args.output, - resize=args.resize, - pixels=args.pixels, - cropping=args.cropping, - dynamic=args.dynamic, - n_frames=args.n_frames, - print_rate=args.print_rate, + video_path=args.video_path, + model_path=args.model_path, + model_type=args.model_type, + device=args.device, + precision=args.precision, display=args.display, pcutoff=args.pcutoff, display_radius=args.display_radius, - cmap=args.cmap, + resize=tuple(args.resize) if args.resize else None, + cropping=args.cropping, + dynamic=tuple(args.dynamic), save_poses=args.save_poses, + save_dir=args.save_dir, + draw_keypoint_names=args.draw_keypoint_names, + cmap=args.cmap, + get_sys_info=args.get_sys_info, save_video=args.save_video, ) diff --git a/dlclive/benchmark_tf.py b/dlclive/benchmark_tf.py new file mode 100644 index 0000000..32850b7 --- /dev/null +++ b/dlclive/benchmark_tf.py @@ -0,0 +1,724 @@ +""" +DeepLabCut Toolbox (deeplabcut.org) +© A. & M. Mathis Labs + +Licensed under GNU Lesser General Public License v3.0 +""" + +import os +import pickle +import platform +import subprocess +import sys +import time +import typing +import warnings + +import colorcet as cc +import ruamel +from PIL import ImageColor + +try: + from pip._internal.operations import freeze +except ImportError: + from pip.operations import freeze + +import cv2 +import numpy as np +import tensorflow as tf +from dlclive import VERSION, DLCLive +from dlclive import __file__ as dlcfile +from dlclive.utils import decode_fourcc +from tqdm import tqdm + + +def download_benchmarking_data( + target_dir=".", + url="http://deeplabcut.rowland.harvard.edu/datasets/dlclivebenchmark.tar.gz", +): + """ + Downloads a DeepLabCut-Live benchmarking Data (videos & DLC models). + """ + import tarfile + import urllib.request + + from tqdm import tqdm + + def show_progress(count, block_size, total_size): + pbar.update(block_size) + + def tarfilenamecutting(tarf): + """' auxfun to extract folder path + ie. /xyz-trainsetxyshufflez/ + """ + for memberid, member in enumerate(tarf.getmembers()): + if memberid == 0: + parent = str(member.path) + l = len(parent) + 1 + if member.path.startswith(parent): + member.path = member.path[l:] + yield member + + response = urllib.request.urlopen(url) + print( + "Downloading the benchmarking data from the DeepLabCut server @Harvard -> Go Crimson!!! {}....".format( + url + ) + ) + total_size = int(response.getheader("Content-Length")) + pbar = tqdm(unit="B", total=total_size, position=0) + filename, _ = urllib.request.urlretrieve(url, reporthook=show_progress) + with tarfile.open(filename, mode="r:gz") as tar: + tar.extractall(target_dir, members=tarfilenamecutting(tar)) + + +def get_system_info() -> dict: + """Return summary info for system running benchmark + Returns + ------- + dict + Dictionary containing the following system information: + * ``host_name`` (str): name of machine + * ``op_sys`` (str): operating system + * ``python`` (str): path to python (which conda/virtual environment) + * ``device`` (tuple): (device type (``'GPU'`` or ``'CPU'```), device information) + * ``freeze`` (list): list of installed packages and versions + * ``python_version`` (str): python version + * ``git_hash`` (str, None): If installed from git repository, hash of HEAD commit + * ``dlclive_version`` (str): dlclive version from :data:`dlclive.VERSION` + """ + + # get os + + op_sys = platform.platform() + host_name = platform.node().replace(" ", "") + + # A string giving the absolute path of the executable binary for the Python interpreter, on systems where this makes sense. + if platform.system() == "Windows": + host_python = sys.executable.split(os.path.sep)[-2] + else: + host_python = sys.executable.split(os.path.sep)[-3] + + # try to get git hash if possible + dlc_basedir = os.path.dirname(os.path.dirname(dlcfile)) + git_hash = None + try: + git_hash = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=dlc_basedir + ) + git_hash = git_hash.decode("utf-8").rstrip("\n") + except subprocess.CalledProcessError: + # not installed from git repo, eg. pypi + # fine, pass quietly + pass + + # get device info (GPU or CPU) + dev = None + if tf.test.is_gpu_available(): + gpu_name = tf.test.gpu_device_name() + from tensorflow.python.client import device_lib + + dev_desc = [ + d.physical_device_desc + for d in device_lib.list_local_devices() + if d.name == gpu_name + ] + dev = [d.split(",")[1].split(":")[1].strip() for d in dev_desc] + dev_type = "GPU" + else: + from cpuinfo import get_cpu_info + + dev = [get_cpu_info()["brand"]] + dev_type = "CPU" + + return { + "host_name": host_name, + "op_sys": op_sys, + "python": host_python, + "device_type": dev_type, + "device": dev, + # pip freeze to get versions of all packages + "freeze": list(freeze.freeze()), + "python_version": sys.version, + "git_hash": git_hash, + "dlclive_version": VERSION, + } + + +def benchmark( + model_path, + video_path, + tf_config=None, + resize=None, + pixels=None, + cropping=None, + dynamic=(False, 0.5, 10), + n_frames=1000, + print_rate=False, + display=False, + pcutoff=0.0, + display_radius=3, + cmap="bmy", + save_poses=False, + save_video=False, + output=None, +) -> typing.Tuple[np.ndarray, tuple, bool, dict]: + """Analyze DeepLabCut-live exported model on a video: + Calculate inference time, + display keypoints, or + get poses/create a labeled video + + Parameters + ---------- + model_path : str + path to exported DeepLabCut model + video_path : str + path to video file + tf_config : :class:`tensorflow.ConfigProto` + tensorflow session configuration + resize : int, optional + resize factor. Can only use one of resize or pixels. If both are provided, will use pixels. by default None + pixels : int, optional + downsize image to this number of pixels, maintaining aspect ratio. Can only use one of resize or pixels. If both are provided, will use pixels. by default None + cropping : list of int + cropping parameters in pixel number: [x1, x2, y1, y2] + dynamic: triple containing (state, detectiontreshold, margin) + If the state is true, then dynamic cropping will be performed. That means that if an object is detected (i.e. any body part > detectiontreshold), + then object boundaries are computed according to the smallest/largest x position and smallest/largest y position of all body parts. This window is + expanded by the margin and from then on only the posture within this crop is analyzed (until the object is lost, i.e. `, by default "bmy" + save_poses : bool, optional + flag to save poses to an hdf5 file. If True, operates similar to :function:`DeepLabCut.benchmark_videos`, by default False + save_video : bool, optional + flag to save a labeled video. If True, operates similar to :function:`DeepLabCut.create_labeled_video`, by default False + output : str, optional + path to directory to save pose and/or video file. If not specified, will use the directory of video_path, by default None + + Returns + ------- + :class:`numpy.ndarray` + vector of inference times + tuple + (image width, image height) + bool + tensorflow inference flag + dict + metadata for video + + Example + ------- + Return a vector of inference times for 10000 frames: + dlclive.benchmark('/my/exported/model', 'my_video.avi', n_frames=10000) + + Return a vector of inference times, resizing images to half the width and height for inference + dlclive.benchmark('/my/exported/model', 'my_video.avi', n_frames=10000, resize=0.5) + + Display keypoints to check the accuracy of an exported model + dlclive.benchmark('/my/exported/model', 'my_video.avi', display=True) + + Analyze a video (save poses to hdf5) and create a labeled video, similar to :function:`DeepLabCut.benchmark_videos` and :function:`create_labeled_video` + dlclive.benchmark('/my/exported/model', 'my_video.avi', save_poses=True, save_video=True) + """ + + ### load video + + cap = cv2.VideoCapture(video_path) + ret, frame = cap.read() + n_frames = ( + n_frames + if (n_frames > 0) and (n_frames < cap.get(cv2.CAP_PROP_FRAME_COUNT) - 1) + else (cap.get(cv2.CAP_PROP_FRAME_COUNT) - 1) + ) + n_frames = int(n_frames) + im_size = (cap.get(cv2.CAP_PROP_FRAME_WIDTH), cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + ### get resize factor + + if pixels is not None: + resize = np.sqrt(pixels / (im_size[0] * im_size[1])) + if resize is not None: + im_size = (int(im_size[0] * resize), int(im_size[1] * resize)) + + ### create video writer + + if save_video: + colors = None + out_dir = ( + output + if output is not None + else os.path.dirname(os.path.realpath(video_path)) + ) + out_vid_base = os.path.basename(video_path) + out_vid_file = os.path.normpath( + f"{out_dir}/{os.path.splitext(out_vid_base)[0]}_DLCLIVE_LABELED.avi" + ) + fourcc = cv2.VideoWriter_fourcc(*"DIVX") + fps = cap.get(cv2.CAP_PROP_FPS) + vwriter = cv2.VideoWriter(out_vid_file, fourcc, fps, im_size) + + ### check for pandas installation if using save_poses flag + + if save_poses: + try: + import pandas as pd + + use_pandas = True + except: + use_pandas = False + warnings.warn( + "Could not find installation of pandas; saving poses as a numpy array with the dimensions (n_frames, n_keypoints, [x, y, likelihood])." + ) + + ### initialize DLCLive and perform inference + + inf_times = np.zeros(n_frames) + poses = [] + + live = DLCLive( + model_path, + model_type="base", + tf_config=tf_config, + resize=resize, + cropping=cropping, + dynamic=dynamic, + display=display, + pcutoff=pcutoff, + display_radius=display_radius, + display_cmap=cmap, + ) + + poses.append(live.init_inference(frame)) + TFGPUinference = True if len(live.runner.outputs) == 1 else False + + iterator = range(n_frames) if (print_rate) or (display) else tqdm(range(n_frames)) + for i in iterator: + + ret, frame = cap.read() + + if not ret: + warnings.warn( + "Did not complete {:d} frames. There probably were not enough frames in the video {}.".format( + n_frames, video_path + ) + ) + break + + start_pose = time.time() + poses.append(live.get_pose(frame)) + inf_times[i] = time.time() - start_pose + + if save_video: + + if colors is None: + all_colors = getattr(cc, cmap) + colors = [ + ImageColor.getcolor(c, "RGB")[::-1] + for c in all_colors[:: int(len(all_colors) / poses[-1].shape[0])] + ] + + this_pose = poses[-1] + for j in range(this_pose.shape[0]): + if this_pose[j, 2] > pcutoff: + x = int(this_pose[j, 0]) + y = int(this_pose[j, 1]) + frame = cv2.circle( + frame, (x, y), display_radius, colors[j], thickness=-1 + ) + + if resize is not None: + frame = cv2.resize(frame, im_size) + vwriter.write(frame) + + if print_rate: + print("pose rate = {:d}".format(int(1 / inf_times[i]))) + + if print_rate: + print("mean pose rate = {:d}".format(int(np.mean(1 / inf_times)))) + + ### gather video and test parameterization + + # dont want to fail here so gracefully failing on exception -- + # eg. some packages of cv2 don't have CAP_PROP_CODEC_PIXEL_FORMAT + try: + fourcc = decode_fourcc(cap.get(cv2.CAP_PROP_FOURCC)) + except: + fourcc = "" + + try: + fps = round(cap.get(cv2.CAP_PROP_FPS)) + except: + fps = None + + try: + pix_fmt = decode_fourcc(cap.get(cv2.CAP_PROP_CODEC_PIXEL_FORMAT)) + except: + pix_fmt = "" + + try: + frame_count = round(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + except: + frame_count = None + + try: + orig_im_size = ( + round(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), + round(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)), + ) + except: + orig_im_size = None + + meta = { + "video_path": video_path, + "video_codec": fourcc, + "video_pixel_format": pix_fmt, + "video_fps": fps, + "video_total_frames": frame_count, + "original_frame_size": orig_im_size, + "dlclive_params": live.parameterization, + } + + ### close video and tensorflow session + + cap.release() + live.close() + + if save_video: + vwriter.release() + + if save_poses: + + cfg_path = os.path.normpath(f"{model_path}/pose_cfg.yaml") + ruamel_file = ruamel.yaml.YAML() + dlc_cfg = ruamel_file.load(open(cfg_path, "r")) + bodyparts = dlc_cfg["all_joints_names"] + poses = np.array(poses) + + if use_pandas: + + poses = poses.reshape((poses.shape[0], poses.shape[1] * poses.shape[2])) + pdindex = pd.MultiIndex.from_product( + [bodyparts, ["x", "y", "likelihood"]], names=["bodyparts", "coords"] + ) + pose_df = pd.DataFrame(poses, columns=pdindex) + + out_dir = ( + output + if output is not None + else os.path.dirname(os.path.realpath(video_path)) + ) + out_vid_base = os.path.basename(video_path) + out_dlc_file = os.path.normpath( + f"{out_dir}/{os.path.splitext(out_vid_base)[0]}_DLCLIVE_POSES.h5" + ) + pose_df.to_hdf(out_dlc_file, key="df_with_missing", mode="w") + + else: + + out_vid_base = os.path.basename(video_path) + out_dlc_file = os.path.normpath( + f"{out_dir}/{os.path.splitext(out_vid_base)[0]}_DLCLIVE_POSES.npy" + ) + np.save(out_dlc_file, poses) + + return inf_times, im_size, TFGPUinference, meta + + +def save_inf_times( + sys_info, inf_times, im_size, TFGPUinference, model=None, meta=None, output=None +): + """Save inference time data collected using :function:`benchmark` with system information to a pickle file. + This is primarily used through :function:`benchmark_videos` + + + Parameters + ---------- + sys_info : tuple + system information generated by :func:`get_system_info` + inf_times : :class:`numpy.ndarray` + array of inference times generated by :func:`benchmark` + im_size : tuple or :class:`numpy.ndarray` + image size (width, height) for each benchmark run. If an array, each row corresponds to a row in inf_times + TFGPUinference: bool + flag if using tensorflow inference or numpy inference DLC model + model: str, optional + name of model + meta : dict, optional + metadata returned by :func:`benchmark` + output : str, optional + path to directory to save data. If None, uses pwd, by default None + + Returns + ------- + bool + flag indicating successful save + """ + + output = output if output is not None else os.getcwd() + model_type = None + if model is not None: + if "resnet" in model: + model_type = "resnet" + elif "mobilenet" in model: + model_type = "mobilenet" + else: + model_type = None + + fn_ind = 0 + base_name = ( + f"benchmark_{sys_info['host_name']}_{sys_info['device_type']}_{fn_ind}.pickle" + ) + out_file = os.path.normpath(f"{output}/{base_name}") + while os.path.isfile(out_file): + fn_ind += 1 + base_name = f"benchmark_{sys_info['host_name']}_{sys_info['device_type']}_{fn_ind}.pickle" + out_file = os.path.normpath(f"{output}/{base_name}") + + # summary stats (mean inference time & standard error of mean) + stats = zip( + np.mean(inf_times, 1), + np.std(inf_times, 1) * 1.0 / np.sqrt(np.shape(inf_times)[1]), + ) + + # for stat in stats: + # print("Stats:", stat) + + data = { + "model": model, + "model_type": model_type, + "TFGPUinference": TFGPUinference, + "im_size": im_size, + "inference_times": inf_times, + "stats": stats, + } + + data.update(sys_info) + if meta: + data.update(meta) + + os.makedirs(os.path.normpath(output), exist_ok=True) + pickle.dump(data, open(out_file, "wb")) + + return True + + +def benchmark_videos( + model_path, + video_path, + output=None, + n_frames=1000, + tf_config=None, + resize=None, + pixels=None, + cropping=None, + dynamic=(False, 0.5, 10), + print_rate=False, + display=False, + pcutoff=0.5, + display_radius=3, + cmap="bmy", + save_poses=False, + save_video=False, +): + """Analyze videos using DeepLabCut-live exported models. + Analyze multiple videos and/or multiple options for the size of the video + by specifying a resizing factor or the number of pixels to use in the image (keeping aspect ratio constant). + Options to record inference times (to examine inference speed), + display keypoints to visually check the accuracy, + or save poses to an hdf5 file as in :function:`deeplabcut.benchmark_videos` and + create a labeled video as in :function:`deeplabcut.create_labeled_video`. + + Parameters + ---------- + model_path : str + path to exported DeepLabCut model + video_path : str or list + path to video file or list of paths to video files + output : str + path to directory to save results + tf_config : :class:`tensorflow.ConfigProto` + tensorflow session configuration + resize : int, optional + resize factor. Can only use one of resize or pixels. If both are provided, will use pixels. by default None + pixels : int, optional + downsize image to this number of pixels, maintaining aspect ratio. Can only use one of resize or pixels. If both are provided, will use pixels. by default None + cropping : list of int + cropping parameters in pixel number: [x1, x2, y1, y2] + dynamic: triple containing (state, detectiontreshold, margin) + If the state is true, then dynamic cropping will be performed. That means that if an object is detected (i.e. any body part > detectiontreshold), + then object boundaries are computed according to the smallest/largest x position and smallest/largest y position of all body parts. This window is + expanded by the margin and from then on only the posture within this crop is analyzed (until the object is lost, i.e. `, by default "bmy" + save_poses : bool, optional + flag to save poses to an hdf5 file. If True, operates similar to :function:`DeepLabCut.benchmark_videos`, by default False + save_video : bool, optional + flag to save a labeled video. If True, operates similar to :function:`DeepLabCut.create_labeled_video`, by default False + + Example + ------- + Return a vector of inference times for 10000 frames on one video or two videos: + dlclive.benchmark_videos('/my/exported/model', 'my_video.avi', n_frames=10000) + dlclive.benchmark_videos('/my/exported/model', ['my_video1.avi', 'my_video2.avi'], n_frames=10000) + + Return a vector of inference times, testing full size and resizing images to half the width and height for inference, for two videos + dlclive.benchmark_videos('/my/exported/model', ['my_video1.avi', 'my_video2.avi'], n_frames=10000, resize=[1.0, 0.5]) + + Display keypoints to check the accuracy of an exported model + dlclive.benchmark_videos('/my/exported/model', 'my_video.avi', display=True) + + Analyze a video (save poses to hdf5) and create a labeled video, similar to :function:`DeepLabCut.benchmark_videos` and :function:`create_labeled_video` + dlclive.benchmark_videos('/my/exported/model', 'my_video.avi', save_poses=True, save_video=True) + """ + + # convert video_paths to list + + video_path = video_path if type(video_path) is list else [video_path] + + # fix resize + + if pixels: + pixels = pixels if type(pixels) is list else [pixels] + resize = [None for p in pixels] + elif resize: + resize = resize if type(resize) is list else [resize] + pixels = [None for r in resize] + else: + resize = [None] + pixels = [None] + + # loop over videos + + for v in video_path: + + # initialize full inference times + + inf_times = [] + im_size_out = [] + + for i in range(len(resize)): + + print(f"\nRun {i+1} / {len(resize)}\n") + + this_inf_times, this_im_size, TFGPUinference, meta = benchmark( + model_path, + v, + tf_config=tf_config, + resize=resize[i], + pixels=pixels[i], + cropping=cropping, + dynamic=dynamic, + n_frames=n_frames, + print_rate=print_rate, + display=display, + pcutoff=pcutoff, + display_radius=display_radius, + cmap=cmap, + save_poses=save_poses, + save_video=save_video, + output=output, + ) + + inf_times.append(this_inf_times) + im_size_out.append(this_im_size) + + inf_times = np.array(inf_times) + im_size_out = np.array(im_size_out) + + # save results + + if output is not None: + sys_info = get_system_info() + save_inf_times( + sys_info, + inf_times, + im_size_out, + TFGPUinference, + model=os.path.basename(model_path), + meta=meta, + output=output, + ) + + +def main(): + """Provides a command line interface :function:`benchmark_videos`""" + + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("model_path", type=str) + parser.add_argument("video_path", type=str, nargs="+") + parser.add_argument("-o", "--output", type=str, default=None) + parser.add_argument("-n", "--n-frames", type=int, default=1000) + parser.add_argument("-r", "--resize", type=float, nargs="+") + parser.add_argument("-p", "--pixels", type=float, nargs="+") + parser.add_argument("-v", "--print-rate", default=False, action="store_true") + parser.add_argument("-d", "--display", default=False, action="store_true") + parser.add_argument("-l", "--pcutoff", default=0.5, type=float) + parser.add_argument("-s", "--display-radius", default=3, type=int) + parser.add_argument("-c", "--cmap", type=str, default="bmy") + parser.add_argument("--cropping", nargs="+", type=int, default=None) + parser.add_argument("--dynamic", nargs="+", type=float, default=[]) + parser.add_argument("--save-poses", action="store_true") + parser.add_argument("--save-video", action="store_true") + args = parser.parse_args() + + if (args.cropping) and (len(args.cropping) < 4): + raise Exception( + "Cropping not properly specified. Must provide 4 values: x1, x2, y1, y2" + ) + + if not args.dynamic: + args.dynamic = (False, 0.5, 10) + elif len(args.dynamic) < 3: + raise Exception( + "Dynamic cropping not properly specified. Must provide three values: 0 or 1 as boolean flag, pcutoff, and margin" + ) + else: + args.dynamic = (bool(args.dynamic[0]), args.dynamic[1], args.dynamic[2]) + + benchmark_videos( + args.model_path, + args.video_path, + output=args.output, + resize=args.resize, + pixels=args.pixels, + cropping=args.cropping, + dynamic=args.dynamic, + n_frames=args.n_frames, + print_rate=args.print_rate, + display=args.display, + pcutoff=args.pcutoff, + display_radius=args.display_radius, + cmap=args.cmap, + save_poses=args.save_poses, + save_video=args.save_video, + ) + + +if __name__ == "__main__": + main() diff --git a/dlclive/check_install/check_install.py b/dlclive/check_install/check_install.py index 6527498..30d6e79 100755 --- a/dlclive/check_install/check_install.py +++ b/dlclive/check_install/check_install.py @@ -14,7 +14,7 @@ from dlclibrary.dlcmodelzoo.modelzoo_download import download_huggingface_model -from dlclive import benchmark_videos +from dlclive.benchmark_tf import benchmark_videos MODEL_NAME = "superanimal_quadruped" SNAPSHOT_NAME = "snapshot-700000.pb" diff --git a/dlclive/core/__init__.py b/dlclive/core/__init__.py new file mode 100644 index 0000000..117d127 --- /dev/null +++ b/dlclive/core/__init__.py @@ -0,0 +1,10 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# diff --git a/dlclive/core/config.py b/dlclive/core/config.py new file mode 100644 index 0000000..1305cf9 --- /dev/null +++ b/dlclive/core/config.py @@ -0,0 +1,28 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""Helpers for configuration file IO""" +from pathlib import Path + +import ruamel.yaml + + +def read_yaml(file_path: str | Path) -> dict: + file_path = Path(file_path).resolve() + if not file_path.exists(): + raise FileNotFoundError( + f"The pose configuration file for the exported model at {str(file_path)} " + "was not found. Please check the path to the exported model directory" + ) + + with open(file_path, "r") as f: + cfg = ruamel.yaml.YAML(typ="safe", pure=True).load(f) + + return cfg diff --git a/dlclive/core/inferenceutils.py b/dlclive/core/inferenceutils.py new file mode 100644 index 0000000..64f94f5 --- /dev/null +++ b/dlclive/core/inferenceutils.py @@ -0,0 +1,1314 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +import heapq +import itertools +import multiprocessing +import operator +import pickle +import warnings +from collections import defaultdict +from dataclasses import dataclass +from math import erf, sqrt +from typing import Any, Iterable, Tuple + +import networkx as nx +import numpy as np +import pandas as pd +from scipy.optimize import linear_sum_assignment +from scipy.spatial import cKDTree +from scipy.spatial.distance import cdist, pdist +from scipy.special import softmax +from scipy.stats import chi2, gaussian_kde +from tqdm import tqdm + + +def _conv_square_to_condensed_indices(ind_row, ind_col, n): + if ind_row == ind_col: + raise ValueError("There are no diagonal elements in condensed matrices.") + + if ind_row < ind_col: + ind_row, ind_col = ind_col, ind_row + return n * ind_col - ind_col * (ind_col + 1) // 2 + ind_row - 1 - ind_col + + +Position = Tuple[float, float] + + +@dataclass(frozen=True) +class Joint: + pos: Position + confidence: float = 1.0 + label: int = None + idx: int = None + group: int = -1 + + +class Link: + def __init__(self, j1, j2, affinity=1): + self.j1 = j1 + self.j2 = j2 + self.affinity = affinity + self._length = sqrt((j1.pos[0] - j2.pos[0]) ** 2 + (j1.pos[1] - j2.pos[1]) ** 2) + + def __repr__(self): + return ( + f"Link {self.idx}, affinity={self.affinity:.2f}, length={self.length:.2f}" + ) + + @property + def confidence(self): + return self.j1.confidence * self.j2.confidence + + @property + def idx(self): + return self.j1.idx, self.j2.idx + + @property + def length(self): + return self._length + + @length.setter + def length(self, length): + self._length = length + + def to_vector(self): + return [*self.j1.pos, *self.j2.pos] + + +class Assembly: + def __init__(self, size): + self.data = np.full((size, 4), np.nan) + self.confidence = 0 # 0 by default, overwritten otherwise with `add_joint` + self._affinity = 0 + self._links = [] + self._visible = set() + self._idx = set() + self._dict = dict() + + def __len__(self): + return len(self._visible) + + def __contains__(self, assembly): + return bool(self._visible.intersection(assembly._visible)) + + def __add__(self, other): + if other in self: + raise ValueError("Assemblies contain shared joints.") + + assembly = Assembly(self.data.shape[0]) + for link in self._links + other._links: + assembly.add_link(link) + return assembly + + @classmethod + def from_array(cls, array): + n_bpts, n_cols = array.shape + + # if a single coordinate is NaN for a bodypart, set all to NaN + array[np.isnan(array).any(axis=-1)] = np.nan + + ass = cls(size=n_bpts) + ass.data[:, :n_cols] = array + visible = np.flatnonzero(~np.isnan(array).any(axis=1)) + if n_cols < 3: # Only xy coordinates are being set + ass.data[visible, 2] = 1 # Set detection confidence to 1 + ass._visible.update(visible) + return ass + + @property + def xy(self): + return self.data[:, :2] + + @property + def extent(self): + bbox = np.empty(4) + bbox[:2] = np.nanmin(self.xy, axis=0) + bbox[2:] = np.nanmax(self.xy, axis=0) + return bbox + + @property + def area(self): + x1, y1, x2, y2 = self.extent + return (x2 - x1) * (y2 - y1) + + @property + def confidence(self): + return np.nanmean(self.data[:, 2]) + + @confidence.setter + def confidence(self, confidence): + self.data[:, 2] = confidence + + @property + def soft_identity(self): + data = self.data[~np.isnan(self.data).any(axis=1)] + unq, idx, cnt = np.unique(data[:, 3], return_inverse=True, return_counts=True) + avg = np.bincount(idx, weights=data[:, 2]) / cnt + soft = softmax(avg) + return dict(zip(unq.astype(int), soft)) + + @property + def affinity(self): + n_links = self.n_links + if not n_links: + return 0 + return self._affinity / n_links + + @property + def n_links(self): + return len(self._links) + + def intersection_with(self, other): + x11, y11, x21, y21 = self.extent + x12, y12, x22, y22 = other.extent + x1 = max(x11, x12) + y1 = max(y11, y12) + x2 = min(x21, x22) + y2 = min(y21, y22) + if x2 < x1 or y2 < y1: + return 0 + ll = np.array([x1, y1]) + ur = np.array([x2, y2]) + xy1 = self.xy[~np.isnan(self.xy).any(axis=1)] + xy2 = other.xy[~np.isnan(other.xy).any(axis=1)] + in1 = np.all((xy1 >= ll) & (xy1 <= ur), axis=1).sum() + in2 = np.all((xy2 >= ll) & (xy2 <= ur), axis=1).sum() + return min(in1 / len(self), in2 / len(other)) + + def add_joint(self, joint): + if joint.label in self._visible or joint.label is None: + return False + self.data[joint.label] = *joint.pos, joint.confidence, joint.group + self._visible.add(joint.label) + self._idx.add(joint.idx) + return True + + def remove_joint(self, joint): + if joint.label not in self._visible: + return False + self.data[joint.label] = np.nan + self._visible.remove(joint.label) + self._idx.remove(joint.idx) + return True + + def add_link(self, link, store_dict=False): + if store_dict: + # Selective copy; deepcopy is >5x slower + self._dict = { + "data": self.data.copy(), + "_affinity": self._affinity, + "_links": self._links.copy(), + "_visible": self._visible.copy(), + "_idx": self._idx.copy(), + } + i1, i2 = link.idx + if i1 in self._idx and i2 in self._idx: + self._affinity += link.affinity + self._links.append(link) + return False + if link.j1.label in self._visible and link.j2.label in self._visible: + return False + self.add_joint(link.j1) + self.add_joint(link.j2) + self._affinity += link.affinity + self._links.append(link) + return True + + def calc_pairwise_distances(self): + return pdist(self.xy, metric="sqeuclidean") + + +class Assembler: + def __init__( + self, + data, + *, + max_n_individuals, + n_multibodyparts, + graph=None, + paf_inds=None, + greedy=False, + pcutoff=0.1, + min_affinity=0.05, + min_n_links=2, + max_overlap=0.8, + identity_only=False, + nan_policy="little", + force_fusion=False, + add_discarded=False, + window_size=0, + method="m1", + ): + self.data = data + self.metadata = self.parse_metadata(self.data) + self.max_n_individuals = max_n_individuals + self.n_multibodyparts = n_multibodyparts + self.n_uniquebodyparts = self.n_keypoints - n_multibodyparts + self.greedy = greedy + self.pcutoff = pcutoff + self.min_affinity = min_affinity + self.min_n_links = min_n_links + self.max_overlap = max_overlap + self._has_identity = "identity" in self[0] + if identity_only and not self._has_identity: + warnings.warn( + "The network was not trained with identity; setting `identity_only` to False." + ) + self.identity_only = identity_only & self._has_identity + self.nan_policy = nan_policy + self.force_fusion = force_fusion + self.add_discarded = add_discarded + self.window_size = window_size + self.method = method + self.graph = graph or self.metadata["paf_graph"] + self.paf_inds = paf_inds or self.metadata["paf"] + self._gamma = 0.01 + self._trees = dict() + self.safe_edge = False + self._kde = None + self.assemblies = dict() + self.unique = dict() + + def __getitem__(self, item): + return self.data[self.metadata["imnames"][item]] + + @classmethod + def empty( + cls, + max_n_individuals, + n_multibodyparts, + n_uniquebodyparts, + graph, + paf_inds, + greedy=False, + pcutoff=0.1, + min_affinity=0.05, + min_n_links=2, + max_overlap=0.8, + identity_only=False, + nan_policy="little", + force_fusion=False, + add_discarded=False, + window_size=0, + method="m1", + ): + # Dummy data + n_bodyparts = n_multibodyparts + n_uniquebodyparts + data = { + "metadata": { + "all_joints_names": ["" for _ in range(n_bodyparts)], + "PAFgraph": graph, + "PAFinds": paf_inds, + }, + "0": {}, + } + return cls( + data, + max_n_individuals=max_n_individuals, + n_multibodyparts=n_multibodyparts, + graph=graph, + paf_inds=paf_inds, + greedy=greedy, + pcutoff=pcutoff, + min_affinity=min_affinity, + min_n_links=min_n_links, + max_overlap=max_overlap, + identity_only=identity_only, + nan_policy=nan_policy, + force_fusion=force_fusion, + add_discarded=add_discarded, + window_size=window_size, + method=method, + ) + + @property + def n_keypoints(self): + return self.metadata["num_joints"] + + def calibrate(self, train_data_file): + df = pd.read_hdf(train_data_file) + try: + df.drop("single", level="individuals", axis=1, inplace=True) + except KeyError: + pass + n_bpts = len(df.columns.get_level_values("bodyparts").unique()) + if n_bpts == 1: + warnings.warn("There is only one keypoint; skipping calibration...") + return + + xy = df.to_numpy().reshape((-1, n_bpts, 2)) + frac_valid = np.mean(~np.isnan(xy), axis=(1, 2)) + # Only keeps skeletons that are more than 90% complete + xy = xy[frac_valid >= 0.9] + if not xy.size: + warnings.warn("No complete poses were found. Skipping calibration...") + return + + # TODO Normalize dists by longest length? + # TODO Smarter imputation technique (Bayesian? Grassmann averages?) + dists = np.vstack([pdist(data, "sqeuclidean") for data in xy]) + mu = np.nanmean(dists, axis=0) + missing = np.isnan(dists) + dists = np.where(missing, mu, dists) + try: + kde = gaussian_kde(dists.T) + kde.mean = mu + self._kde = kde + self.safe_edge = True + except np.linalg.LinAlgError: + # Covariance matrix estimation fails due to numerical singularities + warnings.warn( + "The assembler could not be robustly calibrated. Continuing without it..." + ) + + def calc_assembly_mahalanobis_dist( + self, assembly, return_proba=False, nan_policy="little" + ): + if self._kde is None: + raise ValueError("Assembler should be calibrated first with training data.") + + dists = assembly.calc_pairwise_distances() - self._kde.mean + mask = np.isnan(dists) + # Distance is undefined if the assembly is empty + if not len(assembly) or mask.all(): + if return_proba: + return np.inf, 0 + return np.inf + + if nan_policy == "little": + inds = np.flatnonzero(~mask) + dists = dists[inds] + inv_cov = self._kde.inv_cov[np.ix_(inds, inds)] + # Correct distance to account for missing observations + factor = self._kde.d / len(inds) + else: + # Alternatively, reduce contribution of missing values to the Mahalanobis + # distance to zero by substituting the corresponding means. + dists[mask] = 0 + mask.fill(False) + inv_cov = self._kde.inv_cov + factor = 1 + dot = dists @ inv_cov + mahal = factor * sqrt(np.sum((dot * dists), axis=-1)) + if return_proba: + proba = 1 - chi2.cdf(mahal, np.sum(~mask)) + return mahal, proba + return mahal + + def calc_link_probability(self, link): + if self._kde is None: + raise ValueError("Assembler should be calibrated first with training data.") + + i = link.j1.label + j = link.j2.label + ind = _conv_square_to_condensed_indices(i, j, self.n_multibodyparts) + mu = self._kde.mean[ind] + sigma = self._kde.covariance[ind, ind] + z = (link.length**2 - mu) / sigma + return 2 * (1 - 0.5 * (1 + erf(abs(z) / sqrt(2)))) + + @staticmethod + def _flatten_detections(data_dict): + ind = 0 + coordinates = data_dict["coordinates"][0] + confidence = data_dict["confidence"] + ids = data_dict.get("identity", None) + if ids is None: + ids = [np.ones(len(arr), dtype=int) * -1 for arr in confidence] + else: + ids = [arr.argmax(axis=1) for arr in ids] + for i, (coords, conf, id_) in enumerate(zip(coordinates, confidence, ids)): + if not np.any(coords): + continue + for xy, p, g in zip(coords, conf, id_): + joint = Joint(tuple(xy), p.item(), i, ind, g) + ind += 1 + yield joint + + def extract_best_links(self, joints_dict, costs, trees=None): + links = [] + for ind in self.paf_inds: + s, t = self.graph[ind] + dets_s = joints_dict.get(s, None) + dets_t = joints_dict.get(t, None) + if dets_s is None or dets_t is None: + continue + if ind not in costs: + continue + lengths = costs[ind]["distance"] + if np.isinf(lengths).all(): + continue + aff = costs[ind][self.method].copy() + aff[np.isnan(aff)] = 0 + + if trees: + vecs = np.vstack( + [[*det_s.pos, *det_t.pos] for det_s in dets_s for det_t in dets_t] + ) + dists = [] + for n, tree in enumerate(trees, start=1): + d, _ = tree.query(vecs) + dists.append(np.exp(-self._gamma * n * d)) + w = np.mean(dists, axis=0) + aff *= w.reshape(aff.shape) + + if self.greedy: + conf = np.asarray( + [ + [det_s.confidence * det_t.confidence for det_t in dets_t] + for det_s in dets_s + ] + ) + rows, cols = np.where( + (conf >= self.pcutoff * self.pcutoff) & (aff >= self.min_affinity) + ) + candidates = sorted( + zip(rows, cols, aff[rows, cols], lengths[rows, cols]), + key=lambda x: x[2], + reverse=True, + ) + i_seen = set() + j_seen = set() + for i, j, w, l in candidates: + if i not in i_seen and j not in j_seen: + i_seen.add(i) + j_seen.add(j) + links.append(Link(dets_s[i], dets_t[j], w)) + if len(i_seen) == self.max_n_individuals: + break + else: # Optimal keypoint pairing + inds_s = sorted( + range(len(dets_s)), key=lambda x: dets_s[x].confidence, reverse=True + )[: self.max_n_individuals] + inds_t = sorted( + range(len(dets_t)), key=lambda x: dets_t[x].confidence, reverse=True + )[: self.max_n_individuals] + keep_s = [ + ind for ind in inds_s if dets_s[ind].confidence >= self.pcutoff + ] + keep_t = [ + ind for ind in inds_t if dets_t[ind].confidence >= self.pcutoff + ] + aff = aff[np.ix_(keep_s, keep_t)] + rows, cols = linear_sum_assignment(aff, maximize=True) + for row, col in zip(rows, cols): + w = aff[row, col] + if w >= self.min_affinity: + links.append(Link(dets_s[keep_s[row]], dets_t[keep_t[col]], w)) + return links + + def _fill_assembly(self, assembly, lookup, assembled, safe_edge, nan_policy): + stack = [] + visited = set() + tabu = [] + counter = itertools.count() + + def push_to_stack(i): + for j, link in lookup[i].items(): + if j in assembly._idx: + continue + if link.idx in visited: + continue + heapq.heappush(stack, (-link.affinity, next(counter), link)) + visited.add(link.idx) + + for idx in assembly._idx: + push_to_stack(idx) + + while stack and len(assembly) < self.n_multibodyparts: + _, _, best = heapq.heappop(stack) + i, j = best.idx + if i in assembly._idx: + new_ind = j + elif j in assembly._idx: + new_ind = i + else: + continue + if new_ind in assembled: + continue + if safe_edge: + d_old = self.calc_assembly_mahalanobis_dist( + assembly, nan_policy=nan_policy + ) + success = assembly.add_link(best, store_dict=True) + if not success: + assembly._dict = dict() + continue + d = self.calc_assembly_mahalanobis_dist(assembly, nan_policy=nan_policy) + if d < d_old: + push_to_stack(new_ind) + try: + _, _, link = heapq.heappop(tabu) + heapq.heappush(stack, (-link.affinity, next(counter), link)) + except IndexError: + pass + else: + heapq.heappush(tabu, (d - d_old, next(counter), best)) + assembly.__dict__.update(assembly._dict) + assembly._dict = dict() + else: + assembly.add_link(best) + push_to_stack(new_ind) + + def build_assemblies(self, links): + lookup = defaultdict(dict) + for link in links: + i, j = link.idx + lookup[i][j] = link + lookup[j][i] = link + + assemblies = [] + assembled = set() + + # Fill the subsets with unambiguous, complete individuals + G = nx.Graph([link.idx for link in links]) + for chain in nx.connected_components(G): + if len(chain) == self.n_multibodyparts: + edges = [tuple(sorted(edge)) for edge in G.edges(chain)] + assembly = Assembly(self.n_multibodyparts) + for link in links: + i, j = link.idx + if (i, j) in edges: + success = assembly.add_link(link) + if success: + lookup[i].pop(j) + lookup[j].pop(i) + assembled.update(assembly._idx) + assemblies.append(assembly) + + if len(assemblies) == self.max_n_individuals: + return assemblies, assembled + + for link in sorted(links, key=lambda x: x.affinity, reverse=True): + if any(i in assembled for i in link.idx): + continue + assembly = Assembly(self.n_multibodyparts) + assembly.add_link(link) + self._fill_assembly( + assembly, lookup, assembled, self.safe_edge, self.nan_policy + ) + for link in assembly._links: + i, j = link.idx + lookup[i].pop(j) + lookup[j].pop(i) + assembled.update(assembly._idx) + assemblies.append(assembly) + + # Fuse superfluous assemblies + n_extra = len(assemblies) - self.max_n_individuals + if n_extra > 0: + if self.safe_edge: + ds_old = [ + self.calc_assembly_mahalanobis_dist(assembly) + for assembly in assemblies + ] + while len(assemblies) > self.max_n_individuals: + ds = [] + for i, j in itertools.combinations(range(len(assemblies)), 2): + if assemblies[j] not in assemblies[i]: + temp = assemblies[i] + assemblies[j] + d = self.calc_assembly_mahalanobis_dist(temp) + delta = d - max(ds_old[i], ds_old[j]) + ds.append((i, j, delta, d, temp)) + if not ds: + break + min_ = sorted(ds, key=lambda x: x[2]) + i, j, delta, d, new = min_[0] + if delta < 0 or len(min_) == 1: + assemblies[i] = new + assemblies.pop(j) + ds_old[i] = d + ds_old.pop(j) + else: + break + elif self.force_fusion: + assemblies = sorted(assemblies, key=len) + for nrow in range(n_extra): + assembly = assemblies[nrow] + candidates = [a for a in assemblies[nrow:] if assembly not in a] + if not candidates: + continue + if len(candidates) == 1: + candidate = candidates[0] + else: + dists = [] + for cand in candidates: + d = cdist(assembly.xy, cand.xy) + dists.append(np.nanmin(d)) + candidate = candidates[np.argmin(dists)] + ind = assemblies.index(candidate) + assemblies[ind] += assembly + else: + store = dict() + for assembly in assemblies: + if len(assembly) != self.n_multibodyparts: + for i in assembly._idx: + store[i] = assembly + used = [link for assembly in assemblies for link in assembly._links] + unconnected = [link for link in links if link not in used] + for link in unconnected: + i, j = link.idx + try: + if store[j] not in store[i]: + temp = store[i] + store[j] + store[i].__dict__.update(temp.__dict__) + assemblies.remove(store[j]) + for idx in store[j]._idx: + store[idx] = store[i] + except KeyError: + pass + + # Second pass without edge safety + for assembly in assemblies: + if len(assembly) != self.n_multibodyparts: + self._fill_assembly(assembly, lookup, assembled, False, "") + assembled.update(assembly._idx) + + return assemblies, assembled + + def _assemble(self, data_dict, ind_frame): + joints = list(self._flatten_detections(data_dict)) + if not joints: + return None, None + + bag = defaultdict(list) + for joint in joints: + bag[joint.label].append(joint) + + assembled = set() + + if self.n_uniquebodyparts: + unique = np.full((self.n_uniquebodyparts, 3), np.nan) + for n, ind in enumerate(range(self.n_multibodyparts, self.n_keypoints)): + dets = bag[ind] + if not dets: + continue + if len(dets) > 1: + det = max(dets, key=lambda x: x.confidence) + else: + det = dets[0] + # Mark the unique body parts as assembled anyway so + # they are not used later on to fill assemblies. + assembled.update(d.idx for d in dets) + if det.confidence <= self.pcutoff and not self.add_discarded: + continue + unique[n] = *det.pos, det.confidence + if np.isnan(unique).all(): + unique = None + else: + unique = None + + if not any(i in bag for i in range(self.n_multibodyparts)): + return None, unique + + if self.n_multibodyparts == 1: + assemblies = [] + for joint in bag[0]: + if joint.confidence >= self.pcutoff: + ass = Assembly(self.n_multibodyparts) + ass.add_joint(joint) + assemblies.append(ass) + return assemblies, unique + + if self.max_n_individuals == 1: + get_attr = operator.attrgetter("confidence") + ass = Assembly(self.n_multibodyparts) + for ind in range(self.n_multibodyparts): + joints = bag[ind] + if not joints: + continue + ass.add_joint(max(joints, key=get_attr)) + return [ass], unique + + if self.identity_only: + assemblies = [] + get_attr = operator.attrgetter("group") + temp = sorted( + (joint for joint in joints if np.isfinite(joint.confidence)), + key=get_attr, + ) + groups = itertools.groupby(temp, get_attr) + for _, group in groups: + ass = Assembly(self.n_multibodyparts) + for joint in sorted(group, key=lambda x: x.confidence, reverse=True): + if ( + joint.confidence >= self.pcutoff + and joint.label < self.n_multibodyparts + ): + ass.add_joint(joint) + if len(ass): + assemblies.append(ass) + assembled.update(ass._idx) + else: + trees = [] + for j in range(1, self.window_size + 1): + tree = self._trees.get(ind_frame - j, None) + if tree is not None: + trees.append(tree) + + links = self.extract_best_links(bag, data_dict["costs"], trees) + if self._kde: + for link in links[::-1]: + p = max(self.calc_link_probability(link), 0.001) + link.affinity *= p + if link.affinity < self.min_affinity: + links.remove(link) + + if self.window_size >= 1 and links: + # Store selected edges for subsequent frames + vecs = np.vstack([link.to_vector() for link in links]) + self._trees[ind_frame] = cKDTree(vecs) + + assemblies, assembled_ = self.build_assemblies(links) + assembled.update(assembled_) + + # Remove invalid assemblies + discarded = set( + joint + for joint in joints + if joint.idx not in assembled and np.isfinite(joint.confidence) + ) + for assembly in assemblies[::-1]: + if 0 < assembly.n_links < self.min_n_links or not len(assembly): + for link in assembly._links: + discarded.update((link.j1, link.j2)) + assemblies.remove(assembly) + if 0 < self.max_overlap < 1: # Non-maximum pose suppression + if self._kde is not None: + scores = [ + -self.calc_assembly_mahalanobis_dist(ass) for ass in assemblies + ] + else: + scores = [ass._affinity for ass in assemblies] + lst = list(zip(scores, assemblies)) + assemblies = [] + while lst: + temp = max(lst, key=lambda x: x[0]) + lst.remove(temp) + assemblies.append(temp[1]) + for pair in lst[::-1]: + if temp[1].intersection_with(pair[1]) >= self.max_overlap: + lst.remove(pair) + if len(assemblies) > self.max_n_individuals: + assemblies = sorted(assemblies, key=len, reverse=True) + for assembly in assemblies[self.max_n_individuals :]: + for link in assembly._links: + discarded.update((link.j1, link.j2)) + assemblies = assemblies[: self.max_n_individuals] + + if self.add_discarded and discarded: + # Fill assemblies with unconnected body parts + for joint in sorted(discarded, key=lambda x: x.confidence, reverse=True): + if self.safe_edge: + for assembly in assemblies: + if joint.label in assembly._visible: + continue + d_old = self.calc_assembly_mahalanobis_dist(assembly) + assembly.add_joint(joint) + d = self.calc_assembly_mahalanobis_dist(assembly) + if d < d_old: + break + assembly.remove_joint(joint) + else: + dists = [] + for i, assembly in enumerate(assemblies): + if joint.label in assembly._visible: + continue + d = cdist(assembly.xy, np.atleast_2d(joint.pos)) + dists.append((i, np.nanmin(d))) + if not dists: + continue + min_ = sorted(dists, key=lambda x: x[1]) + ind, _ = min_[0] + assemblies[ind].add_joint(joint) + + return assemblies, unique + + def assemble(self, chunk_size=1, n_processes=None): + self.assemblies = dict() + self.unique = dict() + # Spawning (rather than forking) multiple processes does not + # work nicely with the GUI or interactive sessions. + # In that case, we fall back to the serial assembly. + if chunk_size == 0 or multiprocessing.get_start_method() == "spawn": + + for i, data_dict in enumerate(tqdm(self)): + assemblies, unique = self._assemble(data_dict, i) + if assemblies: + self.assemblies[i] = assemblies + if unique is not None: + self.unique[i] = unique + else: + global wrapped # Hack to make the function pickable + + def wrapped(i): + return i, self._assemble(self[i], i) + + n_frames = len(self.metadata["imnames"]) + with multiprocessing.Pool(n_processes) as p: + with tqdm(total=n_frames) as pbar: + for i, (assemblies, unique) in p.imap_unordered( + wrapped, range(n_frames), chunksize=chunk_size + ): + if assemblies: + self.assemblies[i] = assemblies + if unique is not None: + self.unique[i] = unique + pbar.update() + + def from_pickle(self, pickle_path): + with open(pickle_path, "rb") as file: + data = pickle.load(file) + self.unique = data.pop("single", {}) + self.assemblies = data + + @staticmethod + def parse_metadata(data): + params = dict() + params["joint_names"] = data["metadata"]["all_joints_names"] + params["num_joints"] = len(params["joint_names"]) + params["paf_graph"] = data["metadata"]["PAFgraph"] + params["paf"] = data["metadata"].get( + "PAFinds", np.arange(len(params["joint_names"])) + ) + params["bpts"] = params["ibpts"] = range(params["num_joints"]) + params["imnames"] = [fn for fn in list(data) if fn != "metadata"] + return params + + def to_h5(self, output_name): + data = np.full( + ( + len(self.metadata["imnames"]), + self.max_n_individuals, + self.n_multibodyparts, + 4, + ), + fill_value=np.nan, + ) + for ind, assemblies in self.assemblies.items(): + for n, assembly in enumerate(assemblies): + data[ind, n] = assembly.data + index = pd.MultiIndex.from_product( + [ + ["scorer"], + map(str, range(self.max_n_individuals)), + map(str, range(self.n_multibodyparts)), + ["x", "y", "likelihood"], + ], + names=["scorer", "individuals", "bodyparts", "coords"], + ) + temp = data[..., :3].reshape((data.shape[0], -1)) + df = pd.DataFrame(temp, columns=index) + df.to_hdf(output_name, key="ass") + + def to_pickle(self, output_name): + data = dict() + for ind, assemblies in self.assemblies.items(): + data[ind] = [ass.data for ass in assemblies] + if self.unique: + data["single"] = self.unique + with open(output_name, "wb") as file: + pickle.dump(data, file, pickle.HIGHEST_PROTOCOL) + + +@dataclass +class MatchedPrediction: + """A match between a prediction and a ground truth assembly + + The ground truth assembly should be None f the prediction was not matched to any GT, + and the OKS should be 0. + + Attributes: + prediction: A prediction made by a pose model. + score: The confidence score for the prediction. + ground_truth: If None, then this prediction is not matched to any ground truth + (this can happen when there are more predicted individuals than GT). + Otherwise, the ground truth assembly to which this prediction is matched. + oks: The OKS score between the prediction and the ground truth pose. + """ + + prediction: Assembly + score: float + ground_truth: Assembly | None + oks: float + + +def calc_object_keypoint_similarity( + xy_pred, + xy_true, + sigma, + margin=0, + symmetric_kpts=None, +): + visible_gt = ~np.isnan(xy_true).all(axis=1) + if visible_gt.sum() < 2: # At least 2 points needed to calculate scale + return np.nan + + true = xy_true[visible_gt] + scale_squared = np.product(np.ptp(true, axis=0) + np.spacing(1) + margin * 2) + if np.isclose(scale_squared, 0): + return np.nan + + k_squared = (2 * sigma) ** 2 + denom = 2 * scale_squared * k_squared + if symmetric_kpts is None: + pred = xy_pred[visible_gt] + pred[np.isnan(pred)] = np.inf + dist_squared = np.sum((pred - true) ** 2, axis=1) + oks = np.exp(-dist_squared / denom) + return np.mean(oks) + else: + oks = [] + xy_preds = [xy_pred] + combos = ( + pair + for l in range(len(symmetric_kpts)) + for pair in itertools.combinations(symmetric_kpts, l + 1) + ) + for pairs in combos: + # Swap corresponding keypoints + tmp = xy_pred.copy() + for pair in pairs: + tmp[pair, :] = tmp[pair[::-1], :] + xy_preds.append(tmp) + for xy_pred in xy_preds: + pred = xy_pred[visible_gt] + pred[np.isnan(pred)] = np.inf + dist_squared = np.sum((pred - true) ** 2, axis=1) + oks.append(np.mean(np.exp(-dist_squared / denom))) + return max(oks) + + +def match_assemblies( + predictions: list[Assembly], + ground_truth: list[Assembly], + sigma: float, + margin: int = 0, + symmetric_kpts: list[tuple[int, int]] | None = None, + greedy_matching: bool = False, + greedy_oks_threshold: float = 0.0, +) -> tuple[int, list[MatchedPrediction]]: + """Matches assemblies to ground truth predictions + + Returns: + int: the total number of valid ground truth assemblies + list[MatchedPrediction]: a list containing all valid predictions, potentially + matched to ground truth assemblies. + """ + # Only consider assemblies of at least two keypoints + predictions = [a for a in predictions if len(a) > 1] + ground_truth = [a for a in ground_truth if len(a) > 1] + num_ground_truth = len(ground_truth) + + # Sort predictions by score + inds_pred = np.argsort( + [ins.affinity if ins.n_links else ins.confidence for ins in predictions] + )[::-1] + predictions = np.asarray(predictions)[inds_pred] + + # indices of unmatched ground truth assemblies + matched = [ + MatchedPrediction( + prediction=p, + score=(p.affinity if p.n_links else p.confidence), + ground_truth=None, + oks=0.0, + ) + for p in predictions + ] + + # Greedy assembly matching like in pycocotools + if greedy_matching: + matched_gt_indices = set() + for idx, pred in enumerate(predictions): + oks = [ + calc_object_keypoint_similarity( + pred.xy, + gt.xy, + sigma, + margin, + symmetric_kpts, + ) + for gt in ground_truth + ] + if np.all(np.isnan(oks)): + continue + + ind_best = np.nanargmax(oks) + + # if this gt already matched, and not a crowd, continue + if ind_best in matched_gt_indices: + continue + + # Only match the pred to the GT if the OKS value is above a given threshold + if oks[ind_best] < greedy_oks_threshold: + continue + + matched_gt_indices.add(ind_best) + matched[idx].ground_truth = ground_truth[ind_best] + matched[idx].oks = oks[ind_best] + + # Global rather than greedy assembly matching + else: + inds_true = list(range(len(ground_truth))) + mat = np.zeros((len(predictions), len(ground_truth))) + for i, a_pred in enumerate(predictions): + for j, a_true in enumerate(ground_truth): + oks = calc_object_keypoint_similarity( + a_pred.xy, + a_true.xy, + sigma, + margin, + symmetric_kpts, + ) + if ~np.isnan(oks): + mat[i, j] = oks + rows, cols = linear_sum_assignment(mat, maximize=True) + for row, col in zip(rows, cols): + matched[row].ground_truth = ground_truth[col] + matched[row].oks = mat[row, col] + _ = inds_true.remove(col) + + return num_ground_truth, matched + + +def parse_ground_truth_data_file(h5_file): + df = pd.read_hdf(h5_file) + try: + df.drop("single", axis=1, level="individuals", inplace=True) + except KeyError: + pass + # Cast columns of dtype 'object' to float to avoid TypeError + # further down in _parse_ground_truth_data. + cols = df.select_dtypes(include="object").columns + if cols.to_list(): + df[cols] = df[cols].astype("float") + n_individuals = len(df.columns.get_level_values("individuals").unique()) + n_bodyparts = len(df.columns.get_level_values("bodyparts").unique()) + data = df.to_numpy().reshape((df.shape[0], n_individuals, n_bodyparts, -1)) + return _parse_ground_truth_data(data) + + +def _parse_ground_truth_data(data): + gt = dict() + for i, arr in enumerate(data): + temp = [] + for row in arr: + if np.isnan(row[:, :2]).all(): + continue + ass = Assembly.from_array(row) + temp.append(ass) + if not temp: + continue + gt[i] = temp + return gt + + +def find_outlier_assemblies(dict_of_assemblies, criterion="area", qs=(5, 95)): + if not hasattr(Assembly, criterion): + raise ValueError(f"Invalid criterion {criterion}.") + + if len(qs) != 2: + raise ValueError( + "Two percentiles (for lower and upper bounds) should be given." + ) + + tuples = [] + for frame_ind, assemblies in dict_of_assemblies.items(): + for assembly in assemblies: + tuples.append((frame_ind, getattr(assembly, criterion))) + frame_inds, vals = zip(*tuples) + vals = np.asarray(vals) + lo, up = np.percentile(vals, qs, interpolation="nearest") + inds = np.flatnonzero((vals < lo) | (vals > up)).tolist() + return list(set(frame_inds[i] for i in inds)) + + +def _compute_precision_and_recall( + num_gt_assemblies: int, + oks_values: np.ndarray, + oks_threshold: float, + recall_thresholds: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: + """Computes the precision and recall scores at a given OKS threshold + + Args: + num_gt_assemblies: the number of ground truth assemblies (used to compute false + negatives + true positives). + oks_values: the OKS value to the matched GT assembly for each prediction + oks_threshold: the OKS threshold at which recall and precision are being + computed + recall_thresholds: the recall thresholds to use to compute scores + + Returns: + The precision and recall arrays at each recall threshold + """ + tp = np.cumsum(oks_values >= oks_threshold) + fp = np.cumsum(oks_values < oks_threshold) + rc = tp / num_gt_assemblies + pr = tp / (fp + tp + np.spacing(1)) + recall = rc[-1] + + # Guarantee precision decreases monotonically, see + # https://jonathan-hui.medium.com/map-mean-average-precision-for-object-detection-45c121a31173 + for i in range(len(pr) - 1, 0, -1): + if pr[i] > pr[i - 1]: + pr[i - 1] = pr[i] + + inds_rc = np.searchsorted(rc, recall_thresholds, side="left") + precision = np.zeros(inds_rc.shape) + valid = inds_rc < len(pr) + precision[valid] = pr[inds_rc[valid]] + return precision, recall + + +def evaluate_assembly_greedy( + assemblies_gt: dict[Any, list[Assembly]], + assemblies_pred: dict[Any, list[Assembly]], + oks_sigma: float, + oks_thresholds: Iterable[float], + margin: int | float = 0, + symmetric_kpts: list[tuple[int, int]] | None = None, +) -> dict: + """Runs greedy mAP evaluation, as done by pycocotools + + Args: + assemblies_gt: A dictionary mapping image ID (e.g. filepath) to ground truth + assemblies. Should contain all the same keys as ``assemblies_pred``. + assemblies_pred: A dictionary mapping image ID (e.g. filepath) to predicted + assemblies. Should contain all the same keys as ``assemblies_gt``. + oks_sigma: The sigma to use to compute OKS values for keypoints . + oks_thresholds: The OKS thresholds at which to compute precision & recall. + margin: The margin to use to compute bounding boxes from keypoints. + symmetric_kpts: The symmetric keypoints in the dataset. + """ + recall_thresholds = np.linspace( # np.linspace(0, 1, 101) + start=0.0, stop=1.00, num=int(np.round((1.00 - 0.0) / 0.01)) + 1, endpoint=True + ) + precisions = [] + recalls = [] + for oks_t in oks_thresholds: + all_matched = [] + total_gt_assemblies = 0 + for ind, gt_assembly in assemblies_gt.items(): + pred_assemblies = assemblies_pred.get(ind, []) + num_gt_assemblies, matched = match_assemblies( + pred_assemblies, + gt_assembly, + oks_sigma, + margin, + symmetric_kpts, + greedy_matching=True, + greedy_oks_threshold=oks_t, + ) + all_matched.extend(matched) + total_gt_assemblies += num_gt_assemblies + + if len(all_matched) == 0: + precisions.append(0.0) + recalls.append(0.0) + continue + + # Global sort of assemblies (across all images) by score + scores = np.asarray([-m.score for m in all_matched]) + sorted_pred_indices = np.argsort(scores, kind="mergesort") + oks = np.asarray([match.oks for match in all_matched])[sorted_pred_indices] + + # Compute prediction and recall + p, r = _compute_precision_and_recall( + total_gt_assemblies, oks, oks_t, recall_thresholds + ) + precisions.append(p) + recalls.append(r) + + precisions = np.asarray(precisions) + recalls = np.asarray(recalls) + return { + "precisions": precisions, + "recalls": recalls, + "mAP": precisions.mean(), + "mAR": recalls.mean(), + } + + +def evaluate_assembly( + ass_pred_dict, + ass_true_dict, + oks_sigma=0.072, + oks_thresholds=np.linspace(0.5, 0.95, 10), + margin=0, + symmetric_kpts=None, + greedy_matching=False, + with_tqdm: bool = True, +): + if greedy_matching: + return evaluate_assembly_greedy( + ass_true_dict, + ass_pred_dict, + oks_sigma=oks_sigma, + oks_thresholds=oks_thresholds, + margin=margin, + symmetric_kpts=symmetric_kpts, + ) + + # sigma is taken as the median of all COCO keypoint standard deviations + all_matched = [] + total_gt_assemblies = 0 + + gt_assemblies = ass_true_dict.items() + if with_tqdm: + gt_assemblies = tqdm(gt_assemblies) + + for ind, gt_assembly in gt_assemblies: + pred_assemblies = ass_pred_dict.get(ind, []) + num_gt, matched = match_assemblies( + pred_assemblies, + gt_assembly, + oks_sigma, + margin, + symmetric_kpts, + greedy_matching, + ) + all_matched.extend(matched) + total_gt_assemblies += num_gt + + if not all_matched: + return { + "precisions": np.array([]), + "recalls": np.array([]), + "mAP": 0.0, + "mAR": 0.0, + } + + conf_pred = np.asarray([match.score for match in all_matched]) + idx = np.argsort(-conf_pred, kind="mergesort") + # Sort matching score (OKS) in descending order of assembly affinity + oks = np.asarray([match.oks for match in all_matched])[idx] + recall_thresholds = np.linspace(0, 1, 101) + precisions = [] + recalls = [] + for t in oks_thresholds: + p, r = _compute_precision_and_recall( + total_gt_assemblies, oks, t, recall_thresholds + ) + precisions.append(p) + recalls.append(r) + + precisions = np.asarray(precisions) + recalls = np.asarray(recalls) + return { + "precisions": precisions, + "recalls": recalls, + "mAP": precisions.mean(), + "mAR": recalls.mean(), + } diff --git a/dlclive/core/runner.py b/dlclive/core/runner.py new file mode 100644 index 0000000..00295d2 --- /dev/null +++ b/dlclive/core/runner.py @@ -0,0 +1,96 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""Base runner for DeepLabCut-Live""" +import abc +from pathlib import Path + +import numpy as np + + +class BaseRunner(abc.ABC): + """Base runner for live pose estimation using DeepLabCut-Live. + + Args: + path: The path to the model to run inference with. + + Attributes: + cfg: The pose configuration data. + path: The path to the model to run inference with. + """ + + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + self.cfg = None + + @abc.abstractmethod + def close(self) -> None: + """Clears any resources used by the runner.""" + pass + + @abc.abstractmethod + def get_pose(self, frame: np.ndarray | None, **kwargs) -> np.ndarray | None: + """ + Abstract method to calculate and retrieve the pose of an object or system + based on the given input frame of data. This method must be implemented + by any subclass inheriting from this abstract base class to define the + specific approach for pose estimation. + + Parameters + ---------- + frame : np.ndarray + The input data or image frame used for estimating the pose. Typically + represents visual data such as video or image frames. + kwargs : dict, optional + Additional keyword arguments that may be required for specific pose + estimation techniques implemented in the subclass. + + Returns + ------- + np.ndarray + The estimated pose resulting from the pose estimation process. The + structure of the array may depend on the specific implementation + but typically represents transformations or coordinates. + """ + pass + + @abc.abstractmethod + def init_inference(self, frame: np.ndarray | None, **kwargs) -> np.ndarray | None: + """ + Initializes inference process on the provided frame. + + This method serves as an abstract base method, meant to be implemented by + subclasses. It takes an input image frame and optional additional parameters + to set up and perform inference. The method must return a processed result + as a numpy array. + + Parameters + ---------- + frame : np.ndarray + The input image frame for which inference needs to be set up. + kwargs : dict, optional + Additional parameters that may be required for specific implementation + of the inference initialization. + + Returns + ------- + np.ndarray + The result of the inference after being initialized and processed. + """ + pass + + @abc.abstractmethod + def read_config(self): + """Reads the pose configuration file. + + Raises: + FileNotFoundError: if the pose configuration file does not exist + """ + pass diff --git a/dlclive/display.py b/dlclive/display.py index 4d73257..e349c2c 100644 --- a/dlclive/display.py +++ b/dlclive/display.py @@ -8,27 +8,22 @@ from tkinter import Label, Tk import colorcet as cc -import numpy as np from PIL import Image, ImageDraw, ImageTk -from dlclive import utils - -class Display(object): +class Display: """ Simple object to display frames with DLC labels. Parameters ----------- - cmap : string - string indicating the Matoplotlib colormap to use. + cmap: string + The Matplotlib colormap to use. pcutoff : float likelihood threshold to display points """ def __init__(self, cmap="bmy", radius=3, pcutoff=0.5): - """Constructor method""" - self.cmap = cmap self.colors = None self.radius = radius @@ -66,48 +61,46 @@ def display_frame(self, frame, pose=None): pose :class:`numpy.ndarray` the pose estimated by DeepLabCut for the image """ - frame = np.squeeze(frame) - frame = frame.astype(np.uint8) - pose = pose["poses"].squeeze() im_size = (frame.shape[1], frame.shape[0]) - if pose is not None: - if self.window is None: self.set_display(im_size, pose.shape[0]) img = Image.fromarray(frame) draw = ImageDraw.Draw(img) + if len(pose.shape) == 2: + pose = pose[None] for i in range(pose.shape[0]): - if pose[i, 2] > self.pcutoff: - try: - x0 = ( - pose[i, 0] - self.radius - if pose[i, 0] - self.radius > 0 - else 0 - ) - x1 = ( - pose[i, 0] + self.radius - if pose[i, 0] + self.radius < im_size[0] - else im_size[1] - ) - y0 = ( - pose[i, 1] - self.radius - if pose[i, 1] - self.radius > 0 - else 0 - ) - y1 = ( - pose[i, 1] + self.radius - if pose[i, 1] + self.radius < im_size[1] - else im_size[0] - ) - coords = [x0, y0, x1, y1] - draw.ellipse( - coords, fill=self.colors[i], outline=self.colors[i] - ) - except Exception as e: - print(e) + for j in range(pose.shape[1]): + if pose[i, j, 2] > self.pcutoff: + try: + x0 = ( + pose[i, j, 0] - self.radius + if pose[i, j, 0] - self.radius > 0 + else 0 + ) + x1 = ( + pose[i, j, 0] + self.radius + if pose[i, j, 0] + self.radius < im_size[0] + else im_size[1] + ) + y0 = ( + pose[i, j, 1] - self.radius + if pose[i, j, 1] - self.radius > 0 + else 0 + ) + y1 = ( + pose[i, j, 1] + self.radius + if pose[i, j, 1] + self.radius < im_size[1] + else im_size[0] + ) + coords = [x0, y0, x1, y1] + draw.ellipse( + coords, fill=self.colors[j], outline=self.colors[j] + ) + except Exception as e: + print(e) img_tk = ImageTk.PhotoImage(image=img, master=self.window) self.lab.configure(image=img_tk) @@ -117,5 +110,4 @@ def destroy(self): """ Destroys the opencv image window """ - self.window.destroy() diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index f2eebf8..68b348b 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -4,79 +4,88 @@ Licensed under GNU Lesser General Public License v3.0 """ +from __future__ import annotations -import glob -import os -import time -import typing -import warnings from pathlib import Path -from typing import List, Optional, Tuple +from typing import Any import numpy as np -import onnxruntime as ort -import ruamel.yaml -import torch -from deeplabcut.pose_estimation_pytorch.models import PoseModel -from dlclive import utils +import dlclive.factory as factory +import dlclive.utils as utils +from dlclive.core.runner import BaseRunner from dlclive.display import Display -from dlclive.exceptions import DLCLiveError, DLCLiveWarning -from dlclive.predictor import HeatmapPredictor +from dlclive.exceptions import DLCLiveError +from dlclive.processor import Processor -if typing.TYPE_CHECKING: - from dlclive.processor import Processor -class DLCLive(object): +class DLCLive: """ - Object that loads a DLC network and performs inference on single images (e.g. images captured from a camera feed) + Class that loads a DLC network and performs inference on single images (e.g. + images captured from a camera feed) Parameters ----------- - path : string - Full path to exported model directory + model_path: Path + Full path to exported model file model_type: string, optional which model to use: 'pytorch' or 'onnx' for exported snapshot - precision : string, optional - precision of model weights, only for model_type='onnx'. Can be 'FP32' (default) or 'FP16' + tf_config: - cropping : list of int - cropping parameters in pixel number: [x1, x2, y1, y2] #A: Maybe this is the dynamic cropping of each frame to speed of processing, so instead of analyzing the whole frame, it analyses only the part of the frame where the animal is - dynamic: triple containing (state, detectiontreshold, margin) #A: margin adds some space so the 'bbox' isn't too narrow around the animal'. First key points are predicted, then dynamic cropping is performed to 'single out' the animal, and then pose is estimated, we think. - If the state is true, then dynamic cropping will be performed. That means that if an object is detected (i.e. any body part > detectiontreshold), - then object boundaries are computed according to the smallest/largest x position and smallest/largest y position of all body parts. This window is - expanded by the margin and from then on only the posture within this crop is analyzed (until the object is lost, i.e. detectiontreshold), then object + boundaries are computed according to the smallest/largest x position and + smallest/largest y position of all body parts. This window is expanded by the + margin and from then on only the posture within this crop is analyzed (until the + object is lost, i.e. dict | None: + return self.runner.cfg - def read_config(self): + def read_config(self) -> None: """Reads configuration yaml file Raises ------ FileNotFoundError - error thrown if pose configuration file does nott exist + error thrown if pose configuration file does not exist """ - - cfg_path = Path(self.path).resolve() / "pytorch_config.yaml" - if not cfg_path.exists(): - raise FileNotFoundError( - f"The pose configuration file for the exported model at {str(cfg_path)} was not found. Please check the path to the exported model directory" - ) - - ruamel_file = ruamel.yaml.YAML() - self.cfg = ruamel_file.load(open(str(cfg_path), "r")) + self.runner.read_config() @property def parameterization( self, - ) -> ( - dict - ): + ) -> dict: return {param: getattr(self, param) for param in self.PARAMETERS} - def process_frame(self, frame): + def process_frame(self, frame: np.ndarray) -> np.ndarray: """ Crops an image according to the object's cropping and dynamic properties. @@ -185,35 +180,45 @@ def process_frame(self, frame): frame :class:`numpy.ndarray` processed frame: convert type, crop, convert color """ - if self.cropping: frame = frame[ self.cropping[2] : self.cropping[3], self.cropping[0] : self.cropping[1] ] - if self.dynamic[0]: + if self.dynamic[0]: if self.pose is not None: - detected = self.pose["poses"][0][0][:, 2] > self.dynamic[1] + # Deal with PyTorch multi-animal models + if len(self.pose.shape) == 3: + if len(self.pose) == 0: + pose = np.zeros((1, 3)) + elif len(self.pose) == 1: + pose = self.pose[0] + else: + raise ValueError( + "Cannot use Dynamic Cropping - more than 1 individual found" + ) - if torch.any(detected): + else: + pose = self.pose - x = self.pose["poses"][0][0][detected, 0] - y = self.pose["poses"][0][0][detected, 1] + detected = pose[:, 2] >= self.dynamic[1] + if np.any(detected): + h, w = frame.shape[0], frame.shape[1] - x1 = int(max([0, int(torch.amin(x)) - self.dynamic[2]])) - x2 = int( - min([frame.shape[1], int(torch.amax(x)) + self.dynamic[2]]) - ) - y1 = int(max([0, int(torch.amin(y)) - self.dynamic[2]])) - y2 = int( - min([frame.shape[0], int(torch.amax(y)) + self.dynamic[2]]) - ) - self.dynamic_cropping = [x1, x2, y1, y2] + x = pose[detected, 0] + y = pose[detected, 1] + xmin, xmax = int(np.min(x)), int(np.max(x)) + ymin, ymax = int(np.min(y)), int(np.max(y)) + + x1 = max([0, xmin - self.dynamic[2]]) + x2 = min([w, xmax + self.dynamic[2]]) + y1 = max([0, ymin - self.dynamic[2]]) + y2 = min([h, ymax + self.dynamic[2]]) + self.dynamic_cropping = [x1, x2, y1, y2] frame = frame[y1:y2, x1:x2] else: - self.dynamic_cropping = None if self.resize != 1: @@ -224,62 +229,10 @@ def process_frame(self, frame): return frame - def load_model(self): - if self.model_type == "pytorch": - # Requires DLC 3.0 to be imported ! - model_path = os.path.join(self.path, self.snapshot) - if not os.path.isfile(model_path): - raise FileNotFoundError( - "The model file {} does not exist.".format(model_path) - ) - weights = torch.load(model_path, map_location=torch.device(self.device)) - self.pose_model = PoseModel.build(self.cfg["model"]) - self.pose_model.load_state_dict(weights["model"]) - self.pose_model = self.pose_model.to(self.device) - self.pose_model.eval() - - elif self.model_type == "onnx": - model_paths = glob.glob(os.path.normpath(self.path + "/*.onnx")) - if self.precision == "FP16": - model_path = [model_paths[i] for i in range(len(model_paths)) if "fp16" in model_paths[i]][0] - else: - model_path = model_paths[0] - opts = ort.SessionOptions() - opts.enable_profiling = False - if self.device == "cuda": - self.sess = ort.InferenceSession( - model_path, opts, providers=["CUDAExecutionProvider"] - ) - elif self.device == "cpu": - self.sess = ort.InferenceSession( - model_path, opts, providers=["CPUExecutionProvider"] - ) - - elif self.device == "tensorrt": - provider = [("TensorrtExecutionProvider", { - "trt_engine_cache_enable": True, - "trt_engine_cache_path": "./trt_engines" - })] - self.sess = ort.InferenceSession( - model_path, opts, providers=provider - ) - self.predictor = HeatmapPredictor.build(self.cfg) - - if not os.path.isfile(model_path): - raise FileNotFoundError( - "The model file {} does not exist.".format(model_path) - ) - - else: - raise DLCLiveError( - "model_type = {} is not supported. model_type must be 'pytorch' or 'onnx'".format( - self.model_type - ) - ) - - def init_inference(self, frame=None, **kwargs): + def init_inference(self, frame=None, **kwargs) -> np.ndarray: """ - Load model and perform inference on first frame -- the first inference is usually very slow. + Load model and perform inference on first frame -- the first inference is + usually very slow. Parameters ----------- @@ -288,25 +241,20 @@ def init_inference(self, frame=None, **kwargs): Returns -------- - pose :class:`numpy.ndarray` - the pose estimated by DeepLabCut for the input image - inf_time:class: `float` - the pose inference time + pose: the pose estimated by DeepLabCut for the input image """ + if frame is None: + raise DLCLiveError("No frame provided to initialize inference.") - # load model - self.load_model() - - inf_time = 0. - # get pose of first frame (first inference is very slow) - if frame is not None: - pose, inf_time = self.get_pose(frame, **kwargs) - else: - pose = None + if frame.ndim >= 2: + self.convert2rgb = True - return pose, inf_time + processed_frame = self.process_frame(frame) + self.pose = self.runner.init_inference(processed_frame) + self.is_initialized = True + return self._post_process_pose(processed_frame, **kwargs) - def get_pose(self, frame=None, **kwargs): + def get_pose(self, frame: np.ndarray | None = None, **kwargs) -> np.ndarray: """ Get the pose of an image @@ -322,81 +270,42 @@ def get_pose(self, frame=None, **kwargs): inf_time:class: `float` the pose inference time """ - - inf_time = 0. if frame is None: raise DLCLiveError("No frame provided for live pose estimation") - if frame is not None: - if frame.ndim >= 2: - self.convert2rgb = True - - processed_frame = self.process_frame(frame) - - if self.model_type == "pytorch": - frame = torch.Tensor(processed_frame) - frame = frame.permute(2, 0, 1).unsqueeze(0) - frame = frame.to(self.device) - - with torch.no_grad(): - start = time.time() - outputs = self.pose_model(frame) - end = time.time() - inf_time = end - start - - self.pose = self.pose_model.get_predictions(outputs) - self.pose = self.pose["bodypart"] + if frame.ndim >= 2: + self.convert2rgb = True - elif self.model_type == "onnx": - if self.precision == "FP32": - frame = processed_frame.astype(np.float32) - elif self.precision == "FP16": - frame = processed_frame.astype(np.float16) - - frame = np.transpose(frame, (2, 0, 1)) - frame = np.expand_dims(frame, axis=0) - - ort_inputs = {self.sess.get_inputs()[0].name: frame} - - start = time.time() - outputs = self.sess.run(None, ort_inputs) - end = time.time() - inf_time = end - start - - outputs = { - "heatmap": torch.Tensor(outputs[0]), - "locref": torch.Tensor(outputs[1]), - } - - self.pose = self.predictor(outputs=outputs) - - else: - raise DLCLiveError( - "model_type = {} is not supported. model_type must be 'pytorch' or 'onnx'".format( - self.model_type - ) - ) + processed_frame = self.process_frame(frame) + self.pose = self.runner.get_pose(processed_frame) + return self._post_process_pose(processed_frame, **kwargs) + def _post_process_pose(self, processed_frame: np.ndarray, **kwargs) -> np.ndarray: + """Post-processes the frame and pose.""" # display image if display=True before correcting pose for cropping/resizing - if self.display is not None: self.display.display_frame(processed_frame, self.pose) # if frame is cropped, convert pose coordinates to original frame coordinates - if self.resize is not None: - self.pose["poses"][0][0][:, :2] *= 1 / self.resize + self.pose[..., :2] *= 1 / self.resize if self.cropping is not None: - self.pose["poses"][0][0][0] += self.cropping[0] - self.pose["poses"][0][0][0] += self.cropping[2] + self.pose[..., 0] += self.cropping[0] + self.pose[..., 1] += self.cropping[2] if self.dynamic_cropping is not None: - self.pose["poses"][0][0][:, 0] += self.dynamic_cropping[0] - self.pose["poses"][0][0][:, 1] += self.dynamic_cropping[2] + self.pose[..., 0] += self.dynamic_cropping[0] + self.pose[..., 1] += self.dynamic_cropping[2] # process the pose if self.processor: self.pose = self.processor.process(self.pose, **kwargs) - return self.pose, inf_time + return self.pose + + def close(self) -> None: + self.is_initialized = False + self.runner.close() + if self.display is not None: + self.display.destroy() diff --git a/dlclive/factory.py b/dlclive/factory.py new file mode 100644 index 0000000..caedc75 --- /dev/null +++ b/dlclive/factory.py @@ -0,0 +1,33 @@ + +"""Factory to build runners for DeepLabCut-Live inference""" +from pathlib import Path + +from dlclive.core.runner import BaseRunner + + +def build_runner( + model_type: str, + model_path: str | Path, + **kwargs, +) -> BaseRunner: + """ + + Parameters + ---------- + model_type + model_path + kwargs + + Returns + ------- + + """ + if model_type.lower() == "pytorch": + from dlclive.pose_estimation_pytorch.runner import PyTorchRunner + return PyTorchRunner(model_path, **kwargs) + + elif model_type.lower() in ("tensorflow", "base", "tensorrt", "lite"): + from dlclive.pose_estimation_tensorflow.runner import TensorFlowRunner + return TensorFlowRunner(model_path, model_type, **kwargs) + + raise ValueError(f"Unknown model type: {model_type}") diff --git a/dlclive/graph.py b/dlclive/graph.py index 56ec6c3..4cc3d40 100644 --- a/dlclive/graph.py +++ b/dlclive/graph.py @@ -5,6 +5,7 @@ Licensed under GNU Lesser General Public License v3.0 """ + import tensorflow as tf vers = (tf.__version__).split(".") @@ -110,7 +111,7 @@ def get_input_tensor(graph): return input_tensor -def extract_graph(graph, tf_config=None): +def extract_graph(graph, tf_config=None) -> tuple[tf.Session, tf.Tensor, list[tf.Tensor]]: """ Initializes a tensorflow session with the specified graph and extracts the model's inputs and outputs diff --git a/dlclive/live_inference.py b/dlclive/live_inference.py index 8a3fddf..3763fee 100644 --- a/dlclive/live_inference.py +++ b/dlclive/live_inference.py @@ -1,3 +1,10 @@ +""" +DeepLabCut Toolbox (deeplabcut.org) +© A. & M. Mathis Labs + +Licensed under GNU Lesser General Public License v3.0 +""" + import csv import os import platform @@ -8,7 +15,6 @@ import colorcet as cc import cv2 import h5py -import numpy as np import torch from PIL import ImageColor from pip._internal.operations import freeze @@ -119,8 +125,6 @@ def analyze_live_video( Prefix to label generated pose and video files precision : str, optional, default='FP32' Precision type for the model ('FP32' or 'FP16'). - snapshot : str, optional - Snapshot to use for the model, if using pytorch as model type. display : bool, optional, default=True Whether to display frame with labelled key points. pcutoff : float, optional, default=0.5 @@ -155,7 +159,7 @@ def analyze_live_video( """ # Create the DLCLive object with cropping dlc_live = DLCLive( - path=model_path, + model_path=model_path, model_type=model_type, device=device, display=False, @@ -163,7 +167,6 @@ def analyze_live_video( cropping=cropping, # Pass the cropping parameter dynamic=dynamic, precision=precision, - snapshot=snapshot, ) # Ensure save directory exists diff --git a/dlclive/pose.py b/dlclive/pose.py index 30cdb5b..3e69bb9 100644 --- a/dlclive/pose.py +++ b/dlclive/pose.py @@ -5,6 +5,7 @@ Licensed under GNU Lesser General Public License v3.0 """ + import numpy as np @@ -35,12 +36,11 @@ def extract_cnn_output(outputs, cfg): scmap = outputs[0] scmap = np.squeeze(scmap) locref = None - if cfg["model"]["heads"]["bodypart"]["predictor"]["location_refinement"]: + if cfg["location_refinement"]: locref = np.squeeze(outputs[1]) shape = locref.shape - print(shape, scmap.shape) - locref = np.reshape(locref, (shape[1], shape[2], -1, 2)) - locref *= cfg["model"]["heads"]["bodypart"]["predictor"]["locref_std"] + locref = np.reshape(locref, (shape[0], shape[1], -1, 2)) + locref *= cfg["locref_stdev"] if len(scmap.shape) == 2: # for single body part! scmap = np.expand_dims(scmap, axis=2) return scmap, locref @@ -67,31 +67,15 @@ def argmax_pose_predict(scmap, offmat, stride): pose as a numpy array """ - num_joints = scmap.shape[0] - # debug - print("joints", num_joints) + num_joints = scmap.shape[2] pose = [] for joint_idx in range(num_joints): maxloc = np.unravel_index( - np.argmax(scmap[joint_idx, :, :]), scmap[joint_idx, :, :].shape - ) - # debug - print("maxloc", maxloc) - # print(offmat.shape) - # offset = np.array(offmat[maxloc][joint_idx])[::-1] - # print(offmat[maxloc][joint_idx]) - offset = np.array(offmat[maxloc])[::-1] - print(offset[:, 0].shape) - # print(np.array(offmat[maxloc]).shape) - print("offset", offset.shape) - print( - "offmat*stride+offset", - (offmat * stride + offset).shape, - (offmat * stride + offset), + np.argmax(scmap[:, :, joint_idx]), scmap[:, :, joint_idx].shape ) - pos_f8 = np.array(offmat).astype("float") * stride + 0.5 * stride + offset - print("pos_f8", pos_f8[::-1].shape) - pose.append(np.hstack((pos_f8[::-1], [scmap[joint_idx][maxloc]]))) + offset = np.array(offmat[maxloc][joint_idx])[::-1] + pos_f8 = np.array(maxloc).astype("float") * stride + 0.5 * stride + offset + pose.append(np.hstack((pos_f8[::-1], [scmap[maxloc][joint_idx]]))) return np.array(pose) diff --git a/dlclive/pose_estimation_pytorch/__init__.py b/dlclive/pose_estimation_pytorch/__init__.py new file mode 100644 index 0000000..ae45b8d --- /dev/null +++ b/dlclive/pose_estimation_pytorch/__init__.py @@ -0,0 +1,15 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from dlclive.pose_estimation_pytorch.dynamic_cropping import ( + DynamicCropper, + TopDownDynamicCropper, +) +from dlclive.pose_estimation_pytorch.models import PoseModel diff --git a/dlclive/pose_estimation_pytorch/data/__init__.py b/dlclive/pose_estimation_pytorch/data/__init__.py new file mode 100644 index 0000000..2fabb6e --- /dev/null +++ b/dlclive/pose_estimation_pytorch/data/__init__.py @@ -0,0 +1,10 @@ +""" +DeepLabCut Toolbox (deeplabcut.org) +© A. & M. Mathis Labs + +Licensed under GNU Lesser General Public License v3.0 +""" +from dlclive.pose_estimation_pytorch.data.image import ( + top_down_crop, + top_down_crop_torch, +) diff --git a/dlclive/pose_estimation_pytorch/data/image.py b/dlclive/pose_estimation_pytorch/data/image.py new file mode 100644 index 0000000..a9d71dd --- /dev/null +++ b/dlclive/pose_estimation_pytorch/data/image.py @@ -0,0 +1,140 @@ +""" +DeepLabCut Toolbox (deeplabcut.org) +© A. & M. Mathis Labs + +Licensed under GNU Lesser General Public License v3.0 +""" + +import cv2 +import numpy as np +import torch +from torchvision.transforms import functional as F + + +def fix_bbox_aspect_ratio( + bbox: tuple[float, float, float, float] | np.ndarray | torch.Tensor, + margin: int | float, + out_w: int, + out_h: int, +) -> tuple[int, int, int, int]: + x, y, w, h = bbox + cx = x + w / 2 + cy = y + h / 2 + w += 2 * margin + h += 2 * margin + + input_ratio = w / h + output_ratio = out_w / out_h + if input_ratio > output_ratio: # h/w < h0/w0 => h' = w * h0/w0 + h = w / output_ratio + elif input_ratio < output_ratio: # w/h < w0/h0 => w' = h * w0/h0 + w = h * output_ratio + + # cx,cy,w,h will now give the right ratio -> check if padding is needed + x1, y1 = int(round(cx - (w / 2))), int(round(cy - (h / 2))) + x2, y2 = int(round(cx + (w / 2))), int(round(cy + (h / 2))) + + return x1, y1, x2, y2 + + +def crop_corners( + bbox: tuple[int, int, int, int], + image_size: tuple[int, int], + center_padding: bool = True, +) -> tuple[int, int, int, int, int, int, int, int]: + """""" + x1, y1, x2, y2 = bbox + img_w, img_h = image_size + + # pad symmetrically - compute total padding across axis + pad_left, pad_right, pad_top, pad_bottom = 0, 0, 0, 0 + if x1 < 0: + pad_left = -x1 + x1 = 0 + if x2 > img_w: + pad_right = x2 - img_w + x2 = img_w + if y1 < 0: + pad_top = -y1 + y1 = 0 + if y2 > img_h: + pad_bottom = y2 - img_h + y2 = img_h + + pad_x = pad_left + pad_right + pad_y = pad_top + pad_bottom + if center_padding: + pad_left = pad_x // 2 + pad_top = pad_y // 2 + + return x1, y1, x2, y2, pad_left, pad_top, pad_x, pad_y + + +def top_down_crop( + image: np.ndarray | torch.Tensor, + bbox: tuple[float, float, float, float] | np.ndarray | torch.Tensor, + output_size: tuple[int, int], + margin: int = 0, + center_padding: bool = False, +) -> tuple[np.array, tuple[int, int], tuple[float, float]]: + """ + Crops images around bounding boxes for top-down pose estimation. Computes offsets so + that coordinates in the original image can be mapped to the cropped one; + + x_cropped = (x - offset_x) / scale_x + x_cropped = (y - offset_y) / scale_y + + Bounding boxes are expected to be in COCO-format (xywh). + + Args: + image: (h, w, c) the image to crop + bbox: (4,) the bounding box to crop around + output_size: the (width, height) of the output cropped image + margin: a margin to add around the bounding box before cropping + center_padding: whether to center the image in the padding if any is needed + + Returns: + cropped_image, (offset_x, offset_y), (scale_x, scale_y) + """ + image_h, image_w, c = image.shape + img_size = (image_w, image_h) + out_w, out_h = output_size + + bbox = fix_bbox_aspect_ratio(bbox, margin, out_w, out_h) + x1, y1, x2, y2, pad_left, pad_top, pad_x, pad_y = crop_corners( + bbox, img_size, center_padding + ) + w, h = x2 - x1, y2 - y1 + crop_w, crop_h = w + pad_x, h + pad_y + + # crop the pixels we care about + image_crop = np.zeros((crop_h, crop_w, c), dtype=image.dtype) + image_crop[pad_top:pad_top + h, pad_left:pad_left + w] = image[y1:y2, x1:x2] + + # resize the cropped image + image = cv2.resize(image_crop, (out_w, out_h), interpolation=cv2.INTER_LINEAR) + + # compute scale and offset + offset = x1 - pad_left, y1 - pad_top + scale = crop_w / out_w, crop_h / out_h + return image, offset, scale + + +def top_down_crop_torch( + image: torch.Tensor, + bbox: tuple[float, float, float, float] | torch.Tensor, + output_size: tuple[int, int], + margin: int = 0, +) -> tuple[torch.Tensor, tuple[int, int], tuple[float, float]]: + """""" + out_w, out_h = output_size + + x1, y1, x2, y2 = fix_bbox_aspect_ratio(bbox, margin, out_w, out_h) + h, w = x2 - x1, y2 - y1 + + F.resized_crop(image, y1, x1, h, w, [out_h, out_w]) + + scale = w / out_w, h / out_h + offset = x1, y1 + crop = F.resized_crop(image, y1, x1, h, w, [out_h, out_w]) + return crop, offset, scale diff --git a/dlclive/pose_estimation_pytorch/dynamic_cropping.py b/dlclive/pose_estimation_pytorch/dynamic_cropping.py new file mode 100644 index 0000000..2853aab --- /dev/null +++ b/dlclive/pose_estimation_pytorch/dynamic_cropping.py @@ -0,0 +1,543 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""Modules to dynamically crop individuals out of videos to improve video analysis""" +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Optional + +import torch +import torchvision.transforms.functional as F + + +@dataclass +class DynamicCropper: + """ + If the state is true, then dynamic cropping will be performed. That means that + if an object is detected (i.e. any body part > detection threshold), then object + boundaries are computed according to the smallest/largest x position and + smallest/largest y position of all body parts. This window is expanded by the + margin and from then on only the posture within this crop is analyzed (until the + object is lost, i.e. < detection threshold). The current position is utilized for + updating the crop window for the next frame (this is why the margin is important + and should be set large enough given the movement of the animal). + + Attributes: + threshold: float + The threshold score for bodyparts above which an individual is deemed to + have been detected. + margin: int + The margin used to expand an individuals bounding box before cropping it. + + Examples: + >>> import deeplabcut.pose_estimation_pytorch.models as models + >>> + >>> model: models.PoseModel + >>> frames: torch.Tensor # shape (num_frames, 3, H, W) + >>> + >>> dynamic = DynamicCropper(threshold=0.6, margin=25) + >>> predictions = [] + >>> for image in frames: + >>> image = dynamic.crop(image) + >>> + >>> outputs = model(image) + >>> preds = model.get_predictions(outputs) + >>> pose = preds["bodypart"]["poses"] + >>> + >>> dynamic.update(pose) + >>> predictions.append(pose) + >>> + """ + threshold: float + margin: int + _crop: tuple[int, int, int, int] | None = field(default=None, repr=False) + _shape: tuple[int, int] | None = field(default=None, repr=False) + + def crop(self, image: torch.Tensor) -> torch.Tensor: + """Crops an input image according to the dynamic cropping parameters. + + Args: + image: The image to crop, of shape (1, C, H, W). + + Returns: + The cropped image of shape (1, C, H', W'), where [H', W'] is the size of + the crop. + + Raises: + RuntimeError: if there is not exactly one image in the batch to crop, or if + `crop` was previously called with an image of a different width or + height. + """ + if len(image) != 1: + raise RuntimeError( + "DynamicCropper can only be used with batch size 1 (found image " + f"shape: {image.shape})" + ) + + if self._shape is None: + self._shape = image.shape[3], image.shape[2] + + if image.shape[3] != self._shape[0] or image.shape[2] != self._shape[1]: + raise RuntimeError( + "All frames must have the same shape; The first frame had (W, H) " + f"{self._shape} but the current frame has shape {image.shape}." + ) + + if self._crop is None: + return image + + x0, y0, x1, y1 = self._crop + return image[:, :, y0:y1, x0:x1] + + def update(self, pose: torch.Tensor) -> torch.Tensor: + """Updates the dynamic crop according to the pose model output. + + Uses the pose predicted by the model to update the dynamic crop parameters for + the next frame. Scales the pose predicted in the cropped image back to the + original image space and returns it. + + Args: + pose: The pose that was predicted by the pose estimation model in the + cropped image coordinate space. + + Returns: + The pose, with coordinates updated to the full image space. + """ + if self._shape is None: + raise RuntimeError(f"You must call `crop` before calling `update`.") + + # offset the pose to the original image space + offset_x, offset_y = 0, 0 + if self._crop is not None: + offset_x, offset_y = self._crop[:2] + pose[..., 0] = pose[..., 0] + offset_x + pose[..., 1] = pose[..., 1] + offset_y + + # check whether keypoints can be used for dynamic cropping + keypoints = pose[..., :3].reshape(-1, 3) + keypoints = keypoints[~torch.any(torch.isnan(keypoints), dim=1)] + if len(keypoints) == 0: + self.reset() + return pose + + mask = keypoints[:, 2] >= self.threshold + if torch.all(~mask): + self.reset() + return pose + + # set the crop coordinates + x0 = self._min_value(keypoints[:, 0], self._shape[0]) + x1 = self._max_value(keypoints[:, 0], self._shape[0]) + y0 = self._min_value(keypoints[:, 1], self._shape[1]) + y1 = self._max_value(keypoints[:, 1], self._shape[1]) + crop_w, crop_h = x1 - x0, y1 - y0 + if crop_w == 0 or crop_h == 0: + self.reset() + else: + self._crop = x0, y0, x1, y1 + + return pose + + def reset(self) -> None: + """Resets the DynamicCropper to not crop the next frame""" + self._crop = None + + @staticmethod + def build( + dynamic: bool, threshold: float, margin: int + ) -> Optional["DynamicCropper"]: + """Builds the DynamicCropper based on the given parameters + + Args: + dynamic: Whether dynamic cropping should be used + threshold: The threshold score for bodyparts above which an individual is + deemed to have been detected. + margin: The margin used to expand an individuals bounding box before + cropping it. + + Returns: + None if dynamic is False + DynamicCropper to use if dynamic is True + """ + if not dynamic: + return None + + return DynamicCropper(threshold, margin) + + def _min_value(self, coordinates: torch.Tensor, maximum: int) -> int: + """Returns: min(coordinates - margin), clipped to [0, maximum]""" + return self._clip( + int(math.floor(torch.min(coordinates).item() - self.margin)), + maximum, + ) + + def _max_value(self, coordinates: torch.Tensor, maximum: int) -> int: + """Returns: max(coordinates + margin), clipped to [0, maximum]""" + return self._clip( + int(math.ceil(torch.max(coordinates).item() + self.margin)), + maximum, + ) + + def _clip(self, value: int, maximum: int) -> int: + """Returns: The value clipped to [0, maximum]""" + return min(max(value, 0), maximum) + + +class TopDownDynamicCropper(DynamicCropper): + """Dynamic cropping for top-down models used on single animal videos. + + The `TopDownDynamicCropper` can be used instead of an object detector to analyze + videos **containing a single animal** with top-down models. + + At frame 0, the full frame is split into (n, m) image patches, with a given overlap + between the patches. Patches are then + - Resized to the input size required by the model with a top-down crop. + - Stacked into a batch and given to the pose estimation model + - The output poses for each patch are post-processed: the patch containing the + highest average score prediction is selected as the patch containing the + individual, and the pose from that patch is selected as the predicted pose. + + At frame n, one of two things can happen: + - If the individual was successfully detected at frame n - 1, a bounding box + is generated from the predicted pose and used as the bounding box for the + next frame. + - If the individual was not detected at frame n - 1, patches are cropped as in + frame 0 and the pose selected as in frame 0 + + An individual is considered to be successfully detected if: + - at least `min_hq_keypoints` keypoint have scores above the `threshold` + + The bounding box is generated from the keypoints (either from all keypoints or only + the ones above the threshold) with a margin around the keypoints. If the bounding + box is smaller than a set minimum size, it is expanded to that size. + + Args: + top_down_crop_size: The (width, height) of to resize crops to. + patch_counts: The number of patches along the (width, height) of the images when + no crop is found. + patch_overlap: The amount of overlapping pixels between adjacent patches. + min_bbox_size: The minimum (width, height) for a detected bounding box. If the + bounding box computed from the keypoints is smaller than this value, it + will be expanded to these values. + threshold: The threshold score for bodyparts above which an individual is + considered to be detected. + margin: The margin to add around keypoints when generating bounding boxes. + min_hq_keypoints: The minimum number of keypoints above the threshold required + for the individual to be considered detected and a bounding box to be + computed from the pose. + bbox_from_hq: If True, only keypoints above the score threshold will be used + to compute the bounding boxes. + store_crops: Useful for debugging. When True, all crops are stored in the + `crop_history` attribute. + **kwargs: Key-word arguments passed to the DynamicCropper base class. + + Attributes: + min_bbox_size: tuple[int, int]. The minimum (width, height) for a detected + bounding box. If the bounding box computed from the keypoints is smaller + than this value, it will be expanded to these values. + min_hq_keypoints: int. The minimum number of keypoints above the threshold + required for the individual to be considered detected and a bounding box to + be computed from the pose. + bbox_from_hq: bool. If True, only keypoints above the score threshold will be + used to compute the bounding boxes. + store_crops: bool. Useful for debugging. When True, all crops are stored in the + `crop_history` attribute. + crop_history: list[list[tuple[int, int, int, int]]. Empty list if `store_crops` + is False. Every time `crop` is called, a list is appended to the + `crop_history` attribute. This list is empty if no crop was used for the + frame, otherwise a list containing a single (x, y, w, h) tuple is appended. + """ + + def __init__( + self, + top_down_crop_size: tuple[int, int], + patch_counts: tuple[int, int], + patch_overlap: int, + min_bbox_size: tuple[int, int], + threshold: float, + margin: int, + min_hq_keypoints: int = 2, + bbox_from_hq: bool = False, + store_crops: bool = False, + **kwargs, + ) -> None: + super().__init__(threshold=threshold, margin=margin, **kwargs) + self.min_bbox_size = min_bbox_size + self.min_hq_keypoints = min_hq_keypoints + self.bbox_from_hq = bbox_from_hq + + self._patch_counts = patch_counts + self._patch_overlap = patch_overlap + self._patches = [] + self._patch_offsets = [] + self._td_crop_size = top_down_crop_size + self._td_ratio = self._td_crop_size[0] / self._td_crop_size[1] + + self.crop_history = [] + self.store_crops = store_crops + + def patch_counts(self) -> tuple[int, int]: + """Returns: the number of patches created for an image.""" + return self._patch_counts + + def num_patches(self) -> int: + """Returns: the total number of patches created for an image.""" + return self._patch_counts[0] * self._patch_counts[1] + + def crop(self, image: torch.Tensor) -> torch.Tensor: + """Crops an input image according to the dynamic cropping parameters. + + Args: + image: The image to crop, of shape (1, C, H, W). + + Returns: + The cropped image of shape (B, C, H', W'), where [H', W'] is the size of + the crop. + + Raises: + RuntimeError: if there is not exactly one image in the batch to crop, or if + `crop` was previously called with an image of a different W or H. + """ + if len(image) != 1: + raise RuntimeError( + "DynamicCropper can only be used with batch size 1 (found image " + f"shape: {image.shape})" + ) + + if self._shape is None: + self._shape = image.shape[3], image.shape[2] + self._patches = self.generate_patches() + + if image.shape[3] != self._shape[0] or image.shape[2] != self._shape[1]: + raise RuntimeError( + "All frames must have the same shape; The first frame had (W, H) " + f"{self._shape} but the current frame has shape {image.shape}." + ) + + if self._crop is None: + if self.store_crops: + self.crop_history.append([]) + return self._crop_patches(image) + + if self.store_crops: + self.crop_history.append([self._crop]) + + return self._crop_bounding_box(image, self._crop) + + def update(self, pose: torch.Tensor) -> torch.Tensor: + """Updates the dynamic crop according to the pose model output. + + Uses the pose predicted by the model to update the dynamic crop parameters for + the next frame. Scales the pose predicted in the cropped image back to the + original image space and returns it. + + Args: + pose: The pose that was predicted by the pose estimation model in the + cropped image coordinate space. + + Returns: + The pose, with coordinates updated to the full image space. + """ + if self._shape is None: + raise RuntimeError(f"You must call `crop` before calling `update`.") + + # check whether this was a patched crop + batch_size = pose.shape[0] + if batch_size > 1: + pose = self._extract_best_patch(pose) + + if self._crop is None: + raise RuntimeError( + "The _crop should never be `None` when `update` is called. Ensure you " + "always alternate between `crop` and `update`." + ) + + # offset and rescale the pose to the original image space + out_w, out_h = self._td_crop_size + offset_x, offset_y, w, h = self._crop + scale_x, scale_y = w / out_w, h / out_h + pose[..., 0] = (pose[..., 0] * scale_x) + offset_x + pose[..., 1] = (pose[..., 1] * scale_y) + offset_y + pose[..., 0] = torch.clip(pose[..., 0], 0, self._shape[0]) + pose[..., 1] = torch.clip(pose[..., 1], 0, self._shape[1]) + + # check whether keypoints can be used for dynamic cropping + keypoints = pose[..., :3].reshape(-1, 3) + keypoints = keypoints[~torch.any(torch.isnan(keypoints), dim=1)] + if len(keypoints) == 0: + self.reset() + return pose + + mask = keypoints[:, 2] >= self.threshold + if torch.sum(mask) < self.min_hq_keypoints: + self.reset() + return pose + + if self.bbox_from_hq: + keypoints = keypoints[mask] + + # set the crop coordinates + x0 = self._min_value(keypoints[:, 0], self._shape[0]) + x1 = self._max_value(keypoints[:, 0], self._shape[0]) + y0 = self._min_value(keypoints[:, 1], self._shape[1]) + y1 = self._max_value(keypoints[:, 1], self._shape[1]) + crop_w, crop_h = x1 - x0, y1 - y0 + if crop_w == 0 or crop_h == 0: + self.reset() + else: + self._crop = self._prepare_bounding_box(x0, y0, x1, y1) + + return pose + + def _prepare_bounding_box( + self, x1: int, y1: int, x2: int, y2: int + ) -> tuple[int, int, int, int]: + """Prepares the bounding box for cropping. + + Adds a margin around the bounding box, then transforms it into the target aspect + ratio required for crops given as inputs to the model. + + Args: + x1: The x coordinate for the top-left corner of the bounding box. + y1: The y coordinate for the top-left corner of the bounding box. + x2: The x coordinate for the bottom-right corner of the bounding box. + y2: The y coordinate for the bottom-right corner of the bounding box. + + Returns: + The (x, y, w, h) coordinates for the prepared bounding box. + """ + x1 -= self.margin + x2 += self.margin + y1 -= self.margin + y2 += self.margin + w, h = x2 - x1, y2 - y1 + cx, cy = x1 + w / 2, y1 + h / 2 + + input_ratio = w / h + if input_ratio > self._td_ratio: # h/w < h0/w0 => h' = w * h0/w0 + h = w / self._td_ratio + elif input_ratio < self._td_ratio: # w/h < w0/h0 => w' = h * w0/h0 + w = h * self._td_ratio + + x1, y1 = int(round(cx - (w / 2))), int(round(cy - (h / 2))) + w, h = max(int(w), self.min_bbox_size[0]), max(int(h), self.min_bbox_size[1]) + return x1, y1, w, h + + def _crop_bounding_box( + self, image: torch.Tensor, bbox: tuple[int, int, int, int], + ) -> torch.Tensor: + """Applies a top-down crop to an image given a bounding box. + + Args: + image: The image to crop, of shape (1, C, H, W). + bbox: The bounding box to crop out of the image. + + Returns: + The cropped and resized image. + """ + x1, y1, w, h = bbox + out_w, out_h = self._td_crop_size + return F.resized_crop(image, y1, x1, h, w, [out_h, out_w]) + + def _crop_patches(self, image: torch.Tensor) -> torch.Tensor: + """Crops patches from the image. + + Args: + image: The image to crop patches from, of shape (1, C, H, W). + + Returns: + The patches, of shape (B, C, H', W'), where [H', W'] is the crop size. + """ + patches = [self._crop_bounding_box(image, patch) for patch in self._patches] + return torch.cat(patches, dim=0) + + def _extract_best_patch(self, pose: torch.Tensor) -> torch.Tensor: + """Extracts the best pose prediction from patches. + + Args: + pose: The predicted pose, of shape (b, num_idv, num_kpt, 3). The number of + individuals must be 1. + + Returns: + The selected pose, of shape [1, N, K, 3] + """ + # check that only 1 prediction was made in each image + if pose.shape[1] != 1: + raise ValueError( + "The TopDownDynamicCropper can only be used with models predicting " + f"a single individual per image. Found {pose.shape[0]} " + f"predictions." + ) + + # compute the score for each individual + idv_scores = torch.mean(pose[:, 0, :, 2], dim=1) + + # get the index of the best patch + best_patch = torch.argmax(idv_scores) + + # set the crop to the one used for the best patch + self._crop = self._patches[best_patch] + + return pose[best_patch:best_patch + 1] + + def generate_patches(self) -> list[tuple[int, int, int, int]]: + """Generates patch coordinates for splitting an image. + + Returns: + A list of patch coordinates as tuples (x0, y0, x1, y1). + """ + patch_xs = self.split_array( + self._shape[0], self._patch_counts[0], self._patch_overlap + ) + patch_ys = self.split_array( + self._shape[1], self._patch_counts[1], self._patch_overlap + ) + + patches = [] + for y0, y1 in patch_ys: + for x0, x1 in patch_xs: + patches.append(self._prepare_bounding_box(x0, y0, x1, y1)) + + return patches + + @staticmethod + def split_array(size: int, n: int, overlap: int) -> list[tuple[int, int]]: + """ + Splits an array into n segments of equal size, where the overlap between each + segment is at least a given value. + + Args: + size: The size of the array. + n: The number of segments to split the array into. + overlap: The minimum overlap between each segment. + + Returns: + (start_index, end_index) pairs for each segment. The end index is exclusive. + """ + if n < 1: + raise ValueError(f"Array must be split into at least 1 segment. Found {n}.") + + # FIXME - auto-correct the overlap to spread it out more evenly + padded_size = size + (n - 1) * overlap + segment_size = (padded_size // n) + (padded_size % n > 0) + segments = [] + end = overlap + for i in range(n): + start = end - overlap + end = start + segment_size + if end > size: + end = size + start = end - segment_size + + segments.append((start, end)) + + return segments diff --git a/dlclive/pose_estimation_pytorch/models/__init__.py b/dlclive/pose_estimation_pytorch/models/__init__.py new file mode 100644 index 0000000..edd4e27 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/__init__.py @@ -0,0 +1,9 @@ +""" +DeepLabCut Toolbox (deeplabcut.org) +© A. & M. Mathis Labs + +Licensed under GNU Lesser General Public License v3.0 +""" + +from dlclive.pose_estimation_pytorch.models.model import PoseModel +from dlclive.pose_estimation_pytorch.models.detectors import DETECTORS, BaseDetector diff --git a/dlclive/pose_estimation_pytorch/models/backbones/__init__.py b/dlclive/pose_estimation_pytorch/models/backbones/__init__.py new file mode 100644 index 0000000..55bd515 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/backbones/__init__.py @@ -0,0 +1,14 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from dlclive.pose_estimation_pytorch.models.backbones.base import BACKBONES, BaseBackbone +from dlclive.pose_estimation_pytorch.models.backbones.cspnext import CSPNeXt +from dlclive.pose_estimation_pytorch.models.backbones.hrnet import HRNet +from dlclive.pose_estimation_pytorch.models.backbones.resnet import DLCRNet, ResNet diff --git a/dlclive/pose_estimation_pytorch/models/backbones/base.py b/dlclive/pose_estimation_pytorch/models/backbones/base.py new file mode 100644 index 0000000..4bc6b4f --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/backbones/base.py @@ -0,0 +1,141 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +import logging +import shutil +from abc import ABC, abstractmethod +from pathlib import Path + +import torch +import torch.nn as nn +from huggingface_hub import hf_hub_download + +from dlclive.pose_estimation_pytorch.models.registry import build_from_cfg, Registry + +BACKBONES = Registry("backbones", build_func=build_from_cfg) + + +class BaseBackbone(ABC, nn.Module): + """Base Backbone class for pose estimation. + + Attributes: + stride: the stride for the backbone + freeze_bn_weights: freeze weights of batch norm layers during training + freeze_bn_stats: freeze stats of batch norm layers during training + """ + + def __init__( + self, + stride: int | float, + freeze_bn_weights: bool = True, + freeze_bn_stats: bool = True, + ): + super().__init__() + self.stride = stride + self.freeze_bn_weights = freeze_bn_weights + self.freeze_bn_stats = freeze_bn_stats + + @abstractmethod + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Abstract method for the forward pass through the backbone. + + Args: + x: Input tensor of shape (batch_size, channels, height, width). + + Returns: + a feature map for the input, of shape (batch_size, c', h', w') + """ + pass + + def freeze_batch_norm_layers(self) -> None: + """Freezes batch norm layers + + Running mean + var are always given to F.batch_norm, except when the layer is + in `train` mode and track_running_stats is False, see + https://pytorch.org/docs/stable/_modules/torch/nn/modules/batchnorm.html + So to 'freeze' the running stats, the only way is to set the layer to "eval" + mode. + """ + for module in self.modules(): + if isinstance(module, nn.BatchNorm2d): + if self.freeze_bn_weights: + module.weight.requires_grad = False + module.bias.requires_grad = False + if self.freeze_bn_stats: + module.eval() + + def train(self, mode: bool = True) -> None: + """Sets the module in training or evaluation mode. + + Args: + mode: whether to set training mode (True) or evaluation mode (False) + """ + super().train(mode) + if self.freeze_bn_weights or self.freeze_bn_stats: + self.freeze_batch_norm_layers() + + +class HuggingFaceWeightsMixin: + """Mixin for backbones where the pretrained weights are stored on HuggingFace""" + + def __init__( + self, + backbone_weight_folder: str | Path | None = None, + repo_id: str = "DeepLabCut/DeepLabCut-Backbones", + *args, + **kwargs, + ) -> None: + super().__init__(*args, **kwargs) + if backbone_weight_folder is None: + backbone_weight_folder = Path(__file__).parent / "pretrained_weights" + else: + backbone_weight_folder = Path(backbone_weight_folder).resolve() + + self.backbone_weight_folder = backbone_weight_folder + self.repo_id = repo_id + + def download_weights(self, filename: str, force: bool = False) -> Path: + """Downloads the backbone weights from the HuggingFace repo + + Args: + filename: The name of the model file to download in the repo. + force: Whether to re-download the file if it already exists locally. + + Returns: + The path to the model snapshot. + """ + model_path = self.backbone_weight_folder / filename + if model_path.exists(): + if not force: + return model_path + model_path.unlink() + + logging.info(f"Downloading the pre-trained backbone to {model_path}") + self.backbone_weight_folder.mkdir(exist_ok=True, parents=False) + output_path = Path( + hf_hub_download( + self.repo_id, filename, cache_dir=self.backbone_weight_folder + ) + ) + + # resolve gets the actual path if the output path is a symlink + output_path = output_path.resolve() + # move to the target path + output_path.rename(model_path) + + # delete downloaded artifacts + uid, rid = self.repo_id.split("/") + artifact_dir = self.backbone_weight_folder / f"models--{uid}--{rid}" + if artifact_dir.exists(): + shutil.rmtree(artifact_dir) + + return model_path diff --git a/dlclive/pose_estimation_pytorch/models/backbones/cspnext.py b/dlclive/pose_estimation_pytorch/models/backbones/cspnext.py new file mode 100644 index 0000000..f0595cf --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/backbones/cspnext.py @@ -0,0 +1,207 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""Implementation of the CSPNeXt Backbone + +Based on the ``mmdetection`` CSPNeXt implementation. For more information, see: + + +For more details about this architecture, see `RTMDet: An Empirical Study of Designing +Real-Time Object Detectors`: https://arxiv.org/abs/1711.05101. +""" +from dataclasses import dataclass + +import torch +import torch.nn as nn + +from dlclive.pose_estimation_pytorch.models.backbones.base import ( + BACKBONES, + BaseBackbone, + HuggingFaceWeightsMixin, +) +from dlclive.pose_estimation_pytorch.models.modules.csp import ( + CSPConvModule, + CSPLayer, + SPPBottleneck, +) + + +@dataclass(frozen=True) +class CSPNeXtLayerConfig: + """Configuration for a CSPNeXt layer""" + in_channels: int + out_channels: int + num_blocks: int + add_identity: bool + use_spp: bool + + +@BACKBONES.register_module +class CSPNeXt(HuggingFaceWeightsMixin, BaseBackbone): + """CSPNeXt Backbone + + Args: + model_name: The model variant to build. If ``pretrained==True``, must be one of + the variants for which weights are available on HuggingFace (in the + `DeepLabCut/DeepLabCut-Backbones` hub, e.g. `cspnext_m`). + pretrained: Whether to load pretrained weights for the model. + arch: The model architecture to build. Must be one of the keys of the + ``CSPNeXt.ARCH`` attribute (e.g. `P5`, `P6`, ...). + expand_ratio: Ratio used to adjust the number of channels of the hidden layer. + deepen_factor: Number of blocks in each CSP layer is multiplied by this value. + widen_factor: Number of channels in each layer is multiplied by this value. + out_indices: The branch indices to output. If a tuple of integers, the outputs + are returned as a list of tensors. If a single integer, a tensor is returned + containing the configured index. + channel_attention: Add chanel attention to all stages + norm_layer: The type of normalization layer to use. + activation_fn: The type of activation function to use. + **kwargs: BaseBackbone kwargs. + """ + + ARCH: dict[str, list[CSPNeXtLayerConfig]] = { + "P5": [ + CSPNeXtLayerConfig(64, 128, 3, True, False), + CSPNeXtLayerConfig(128, 256, 6, True, False), + CSPNeXtLayerConfig(256, 512, 6, True, False), + CSPNeXtLayerConfig(512, 1024, 3, False, True), + ], + "P6": [ + CSPNeXtLayerConfig(64, 128, 3, True, False), + CSPNeXtLayerConfig(128, 256, 6, True, False), + CSPNeXtLayerConfig(256, 512, 6, True, False), + CSPNeXtLayerConfig(512, 768, 3, True, False), + CSPNeXtLayerConfig(768, 1024, 3, False, True), + ] + } + + def __init__( + self, + model_name: str = "cspnext_m", + pretrained: bool = False, + arch: str = "P5", + expand_ratio: float = 0.5, + deepen_factor: float = 0.67, + widen_factor: float = 0.75, + out_indices: int | tuple[int, ...] = -1, + channel_attention: bool = True, + norm_layer: str = "SyncBN", + activation_fn: str = "SiLU", + **kwargs, + ) -> None: + super().__init__(stride=32, **kwargs) + if arch not in self.ARCH: + raise ValueError( + f"Unknown `CSPNeXT` architecture: {arch}. Must be one of " + f"{self.ARCH.keys()}" + ) + + self.model_name = model_name + self.layer_configs = self.ARCH[arch] + self.stem_out_channels = self.layer_configs[0].in_channels + self.spp_kernel_sizes = (5, 9, 13) + + # stem has stride 2 + self.stem = nn.Sequential( + CSPConvModule( + in_channels=3, + out_channels=int(self.stem_out_channels * widen_factor // 2), + kernel_size=3, + padding=1, + stride=2, + norm_layer=norm_layer, + activation_fn=activation_fn, + ), + CSPConvModule( + in_channels=int(self.stem_out_channels * widen_factor // 2), + out_channels=int(self.stem_out_channels * widen_factor // 2), + kernel_size=3, + padding=1, + stride=1, + norm_layer=norm_layer, + activation_fn=activation_fn, + ), + CSPConvModule( + in_channels=int(self.stem_out_channels * widen_factor // 2), + out_channels=int(self.stem_out_channels * widen_factor), + kernel_size=3, + padding=1, + stride=1, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + ) + self.layers = ["stem"] + + for i, layer_cfg in enumerate(self.layer_configs): + layer_cfg: CSPNeXtLayerConfig + in_channels = int(layer_cfg.in_channels * widen_factor) + out_channels = int(layer_cfg.out_channels * widen_factor) + num_blocks = max(round(layer_cfg.num_blocks * deepen_factor), 1) + stage = [] + conv_layer = CSPConvModule( + in_channels, + out_channels, + 3, + stride=2, + padding=1, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + stage.append(conv_layer) + if layer_cfg.use_spp: + spp = SPPBottleneck( + out_channels, + out_channels, + kernel_sizes=self.spp_kernel_sizes, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + stage.append(spp) + + csp_layer = CSPLayer( + out_channels, + out_channels, + num_blocks=num_blocks, + add_identity=layer_cfg.add_identity, + expand_ratio=expand_ratio, + channel_attention=channel_attention, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + stage.append(csp_layer) + self.add_module(f'stage{i + 1}', nn.Sequential(*stage)) + self.layers.append(f'stage{i + 1}') + + self.single_output = isinstance(out_indices, int) + if self.single_output: + if out_indices == -1: + out_indices = len(self.layers) - 1 + out_indices = (out_indices,) + self.out_indices = out_indices + + if pretrained: + weights_filename = f"{model_name}.pt" + weights_path = self.download_weights(weights_filename, force=False) + snapshot = torch.load(weights_path, map_location="cpu", weights_only=True) + self.load_state_dict(snapshot["state_dict"]) + + def forward(self, x: torch.Tensor) -> torch.Tensor | tuple[torch.Tensor]: + outs = [] + for i, layer_name in enumerate(self.layers): + layer = getattr(self, layer_name) + x = layer(x) + if i in self.out_indices: + outs.append(x) + + if self.single_output: + return outs[-1] + + return tuple(outs) diff --git a/dlclive/pose_estimation_pytorch/models/backbones/hrnet.py b/dlclive/pose_estimation_pytorch/models/backbones/hrnet.py new file mode 100644 index 0000000..7d99b77 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/backbones/hrnet.py @@ -0,0 +1,119 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +import timm +import torch +import torch.nn as nn +import torch.nn.functional as F + +from dlclive.pose_estimation_pytorch.models.backbones.base import BACKBONES, BaseBackbone + + +@BACKBONES.register_module +class HRNet(BaseBackbone): + """HRNet backbone. + + This version returns high-resolution feature maps of size 1/4 * original_image_size. + This is obtained using bilinear interpolation and concatenation of all the outputs + of the HRNet stages. + + The model outputs 4 branches, with strides 4, 8, 16 and 32. + + Args: + stride: The stride of the HRNet. Should always be 4, except for custom models. + model_name: Any HRNet variant available through timm (e.g., 'hrnet_w32', + 'hrnet_w48'). See timm for more options. + pretrained: If True, loads the backbone with ImageNet pretrained weights from + timm. + interpolate_branches: Needed for DEKR. Instead of returning features from the + high-resolution branch, interpolates all other branches to the same shape + and concatenates them. + increased_channel_count: As described by timm, it "allows grabbing increased + channel count features using part of the classification head" (otherwise, + the default features are returned). + kwargs: BaseBackbone kwargs + + Attributes: + model: the HRNet model + """ + + def __init__( + self, + stride: int = 4, + model_name: str = "hrnet_w32", + pretrained: bool = False, + interpolate_branches: bool = False, + increased_channel_count: bool = False, + **kwargs, + ) -> None: + super().__init__(stride=stride, **kwargs) + self.model = _load_hrnet(model_name, pretrained, increased_channel_count) + self.interpolate_branches = interpolate_branches + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward pass through the HRNet backbone. + + Args: + x: Input tensor of shape (batch_size, channels, height, width). + + Returns: + the feature map + + Example: + >>> import torch + >>> from dlclive.models.backbones import HRNet + >>> backbone = HRNet(model_name='hrnet_w32', pretrained=False) + >>> x = torch.randn(1, 3, 256, 256) + >>> y = backbone(x) + """ + y_list = self.model(x) + if not self.interpolate_branches: + return y_list[0] + + x0_h, x0_w = y_list[0].size(2), y_list[0].size(3) + x = torch.cat( + [ + y_list[0], + F.interpolate(y_list[1], size=(x0_h, x0_w), mode="bilinear"), + F.interpolate(y_list[2], size=(x0_h, x0_w), mode="bilinear"), + F.interpolate(y_list[3], size=(x0_h, x0_w), mode="bilinear"), + ], + 1, + ) + return x + + +def _load_hrnet( + model_name: str, + pretrained: bool, + increased_channel_count: bool, +) -> nn.Module: + """Loads a TIMM HRNet model. + + Args: + model_name: Any HRNet variant available through timm (e.g., 'hrnet_w32', + 'hrnet_w48'). See timm for more options. + pretrained: If True, loads the backbone with ImageNet pretrained weights from + timm. + increased_channel_count: As described by timm, it "allows grabbing increased + channel count features using part of the classification head" (otherwise, + the default features are returned). + + Returns: + the HRNet model + """ + # First stem conv is used for stride 2 features, so only return branches 1-4 + return timm.create_model( + model_name, + pretrained=pretrained, + features_only=True, + feature_location="incre" if increased_channel_count else "", + out_indices=(1, 2, 3, 4), + ) diff --git a/dlclive/pose_estimation_pytorch/models/backbones/resnet.py b/dlclive/pose_estimation_pytorch/models/backbones/resnet.py new file mode 100644 index 0000000..a71276b --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/backbones/resnet.py @@ -0,0 +1,148 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +import timm +import torch +import torch.nn as nn +from torchvision.transforms.functional import resize + +from dlclive.pose_estimation_pytorch.models.backbones.base import BACKBONES, BaseBackbone + + +@BACKBONES.register_module +class ResNet(BaseBackbone): + """ResNet backbone. + + This class represents a typical ResNet backbone for pose estimation. + + Attributes: + model: the ResNet model + """ + + def __init__( + self, + model_name: str = "resnet50", + output_stride: int = 32, + pretrained: bool = False, + drop_path_rate: float = 0.0, + drop_block_rate: float = 0.0, + **kwargs, + ) -> None: + """Initialize the ResNet backbone. + + Args: + model_name: Name of the ResNet model to use, e.g., 'resnet50', 'resnet101' + output_stride: Output stride of the network, 32, 16, or 8. + pretrained: If True, initializes with ImageNet pretrained weights. + drop_path_rate: Stochastic depth drop-path rate + drop_block_rate: Drop block rate + kwargs: BaseBackbone kwargs + """ + super().__init__(stride=output_stride, **kwargs) + self.model = timm.create_model( + model_name, + output_stride=output_stride, + pretrained=pretrained, + drop_path_rate=drop_path_rate, + drop_block_rate=drop_block_rate, + ) + self.model.fc = nn.Identity() # remove the FC layer + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward pass through the ResNet backbone. + + Args: + x: Input tensor. + + Returns: + torch.Tensor: Output tensor. + Example: + >>> import torch + >>> from dlclive.models.backbones import ResNet + >>> backbone = ResNet(model_name='resnet50', pretrained=False) + >>> x = torch.randn(1, 3, 256, 256) + >>> y = backbone(x) + + Expected Output Shape: + If input size is (batch_size, 3, shape_x, shape_y), the output shape + will be (batch_size, 3, shape_x//16, shape_y//16) + """ + return self.model.forward_features(x) + + +@BACKBONES.register_module +class DLCRNet(ResNet): + def __init__( + self, + model_name: str = "resnet50", + output_stride: int = 32, + pretrained: bool = True, + **kwargs, + ) -> None: + super().__init__(model_name, output_stride, pretrained, **kwargs) + self.interm_features = {} + self.model.layer1[2].register_forward_hook(self._get_features("bank1")) + self.model.layer2[2].register_forward_hook(self._get_features("bank2")) + self.conv_block1 = self._make_conv_block( + in_channels=512, out_channels=512, kernel_size=3, stride=2 + ) + self.conv_block2 = self._make_conv_block( + in_channels=512, out_channels=128, kernel_size=1, stride=1 + ) + self.conv_block3 = self._make_conv_block( + in_channels=256, out_channels=256, kernel_size=3, stride=2 + ) + self.conv_block4 = self._make_conv_block( + in_channels=256, out_channels=256, kernel_size=3, stride=2 + ) + self.conv_block5 = self._make_conv_block( + in_channels=256, out_channels=128, kernel_size=1, stride=1 + ) + + def _make_conv_block( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int, + momentum: float = 0.001, # (1 - decay) + ) -> torch.nn.Sequential: + return nn.Sequential( + nn.Conv2d( + in_channels, out_channels, kernel_size=kernel_size, stride=stride + ), + nn.BatchNorm2d(out_channels, momentum=momentum), + nn.ReLU(), + ) + + def _get_features(self, name): + def hook(model, input, output): + self.interm_features[name] = output.detach() + + return hook + + def forward(self, x): + out = super().forward(x) + + # Fuse intermediate features + bank_2_s8 = self.interm_features["bank2"] + bank_1_s4 = self.interm_features["bank1"] + bank_2_s16 = self.conv_block1(bank_2_s8) + bank_2_s16 = self.conv_block2(bank_2_s16) + bank_1_s8 = self.conv_block3(bank_1_s4) + bank_1_s16 = self.conv_block4(bank_1_s8) + bank_1_s16 = self.conv_block5(bank_1_s16) + # Resizing here is required to guarantee all shapes match, as + # Conv2D(..., padding='same') is invalid for strided convolutions. + h, w = out.shape[-2:] + bank_1_s16 = resize(bank_1_s16, [h, w], antialias=True) + bank_2_s16 = resize(bank_2_s16, [h, w], antialias=True) + + return torch.cat((bank_1_s16, bank_2_s16, out), dim=1) diff --git a/dlclive/pose_estimation_pytorch/models/detectors/__init__.py b/dlclive/pose_estimation_pytorch/models/detectors/__init__.py new file mode 100644 index 0000000..e9a99a6 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/detectors/__init__.py @@ -0,0 +1,16 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from dlclive.pose_estimation_pytorch.models.detectors.base import ( + DETECTORS, + BaseDetector, +) +from dlclive.pose_estimation_pytorch.models.detectors.fasterRCNN import FasterRCNN +from dlclive.pose_estimation_pytorch.models.detectors.ssd import SSDLite diff --git a/dlclive/pose_estimation_pytorch/models/detectors/base.py b/dlclive/pose_estimation_pytorch/models/detectors/base.py new file mode 100644 index 0000000..bcd9fb0 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/detectors/base.py @@ -0,0 +1,56 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +import logging +from abc import ABC, abstractmethod + +import torch +import torch.nn as nn + +from dlclive.pose_estimation_pytorch.models.registry import Registry, build_from_cfg + +DETECTORS = Registry("detectors", build_func=build_from_cfg) + + +class BaseDetector(ABC, nn.Module): + """ + Definition of the class BaseDetector object. + This is an abstract class defining the common structure and inference for detectors. + """ + + def __init__( + self, + freeze_bn_stats: bool = False, + freeze_bn_weights: bool = False, + pretrained: bool = False, + ) -> None: + super().__init__() + self.freeze_bn_stats = freeze_bn_stats + self.freeze_bn_weights = freeze_bn_weights + self._pretrained = pretrained + + @abstractmethod + def forward( + self, x: torch.Tensor, targets: list[dict[str, torch.Tensor]] | None = None + ) -> list[dict[str, torch.Tensor]]: + """ + Forward pass of the detector + + Args: + x: images to be processed + targets: ground-truth boxes present in each images + + Returns: + losses: {'loss_name': loss_value} + detections: for each of the b images, {"boxes": bounding_boxes} + """ + pass diff --git a/dlclive/pose_estimation_pytorch/models/detectors/fasterRCNN.py b/dlclive/pose_estimation_pytorch/models/detectors/fasterRCNN.py new file mode 100644 index 0000000..1656402 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/detectors/fasterRCNN.py @@ -0,0 +1,72 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +import torchvision.models.detection as detection + +from dlclive.pose_estimation_pytorch.models.detectors.base import DETECTORS +from dlclive.pose_estimation_pytorch.models.detectors.torchvision import TorchvisionDetectorAdaptor + + +@DETECTORS.register_module +class FasterRCNN(TorchvisionDetectorAdaptor): + """A FasterRCNN detector + + Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks + Ren, Shaoqing, Kaiming He, Ross Girshick, and Jian Sun. "Faster r-cnn: Towards + real-time object detection with region proposal networks." Advances in neural + information processing systems 28 (2015). + + This class is a wrapper of the torchvision implementation of a FasterRCNN (source: + https://github.com/pytorch/vision/blob/main/torchvision/models/detection/faster_rcnn.py). + + Some of the available FasterRCNN variants (from fastest to most powerful): + - fasterrcnn_mobilenet_v3_large_fpn + - fasterrcnn_resnet50_fpn + - fasterrcnn_resnet50_fpn_v2 + + Args: + variant: The FasterRCNN variant to use (see all options at + https://pytorch.org/vision/stable/models.html#object-detection). + pretrained: Whether to load model weights pretrained on COCO + box_score_thresh: during inference, only return proposals with a classification + score greater than box_score_thresh + """ + + def __init__( + self, + freeze_bn_stats: bool = False, + freeze_bn_weights: bool = False, + variant: str = "fasterrcnn_mobilenet_v3_large_fpn", + pretrained: bool = False, + box_score_thresh: float = 0.01, + ) -> None: + if not variant.lower().startswith("fasterrcnn"): + raise ValueError( + "The version must start with `fasterrcnn`. See available models at " + "https://pytorch.org/vision/stable/models.html#object-detection" + ) + + super().__init__( + model=variant, + weights=("COCO_V1" if pretrained else None), + num_classes=None, + freeze_bn_stats=freeze_bn_stats, + freeze_bn_weights=freeze_bn_weights, + box_score_thresh=box_score_thresh, + ) + + # Modify the base predictor to output the correct number of classes + num_classes = 2 + in_features = self.model.roi_heads.box_predictor.cls_score.in_features + self.model.roi_heads.box_predictor = detection.faster_rcnn.FastRCNNPredictor( + in_features, num_classes + ) diff --git a/dlclive/pose_estimation_pytorch/models/detectors/ssd.py b/dlclive/pose_estimation_pytorch/models/detectors/ssd.py new file mode 100644 index 0000000..e1c9da8 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/detectors/ssd.py @@ -0,0 +1,68 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +import torchvision.models.detection as detection + +from dlclive.pose_estimation_pytorch.models.detectors.base import DETECTORS +from dlclive.pose_estimation_pytorch.models.detectors.torchvision import TorchvisionDetectorAdaptor + + +@DETECTORS.register_module +class SSDLite(TorchvisionDetectorAdaptor): + """An SSD object detection model""" + + def __init__( + self, + freeze_bn_stats: bool = False, + freeze_bn_weights: bool = False, + pretrained: bool = False, + pretrained_from_imagenet: bool = False, + box_score_thresh: float = 0.01, + ) -> None: + model_kwargs = dict(weights_backbone=None) + if pretrained_from_imagenet: + model_kwargs["weights_backbone"] = "IMAGENET1K_V2" + + super().__init__( + model="ssdlite320_mobilenet_v3_large", + weights=None, + num_classes=2, + freeze_bn_stats=freeze_bn_stats, + freeze_bn_weights=freeze_bn_weights, + box_score_thresh=box_score_thresh, + model_kwargs=model_kwargs, + ) + + if pretrained and not pretrained_from_imagenet: + weights = detection.SSDLite320_MobileNet_V3_Large_Weights.verify("COCO_V1") + state_dict = weights.get_state_dict(progress=False, check_hash=True) + for k, v in state_dict.items(): + key_parts = k.split(".") + if ( + len(key_parts) == 6 + and key_parts[0] == "head" + and key_parts[1] == "classification_head" + and key_parts[2] == "module_list" + and key_parts[4] == "1" + and key_parts[5] in ("weight", "bias") + ): + # number of COCO classes: 90 + background (91) + # number of DLC classes: 1 + background (2) + # -> only keep weights for the background + first class + + # future improvement: find best-suited class for the project + # and use those weights, instead of naively taking the first + all_classes_size = v.shape[0] + two_classes_size = 2 * (all_classes_size // 91) + state_dict[k] = v[:two_classes_size] + + self.model.load_state_dict(state_dict) diff --git a/dlclive/pose_estimation_pytorch/models/detectors/torchvision.py b/dlclive/pose_estimation_pytorch/models/detectors/torchvision.py new file mode 100644 index 0000000..72dd54b --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/detectors/torchvision.py @@ -0,0 +1,96 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""Module to adapt torchvision detectors for DeepLabCut""" +from __future__ import annotations + +import torch +import torchvision.models.detection as detection + +from dlclive.pose_estimation_pytorch.models.detectors.base import BaseDetector + + +class TorchvisionDetectorAdaptor(BaseDetector): + """An adaptor for torchvision detectors + + This class is an adaptor for torchvision detectors to DeepLabCut detectors. Some of + the models (from fastest to most powerful) available are: + - ssdlite320_mobilenet_v3_large + - fasterrcnn_mobilenet_v3_large_fpn + - fasterrcnn_resnet50_fpn_v2 + + This class should not be used out-of-the-box. Subclasses (such as FasterRCNN or + SSDLite) should be used instead. + + The torchvision implementation does not allow to get both predictions and losses + with a single forward pass. Therefore, during evaluation only bounding box metrics + (mAP, mAR) are available for the test set. See validation loss issue: + - https://discuss.pytorch.org/t/compute-validation-loss-for-faster-rcnn/62333/12 + - https://stackoverflow.com/a/65347721 + + Args: + model: The torchvision model to use (see all options at + https://pytorch.org/vision/stable/models.html#object-detection). + weights: The weights to load for the model. If None, no pre-trained weights are + loaded. + num_classes: Number of classes that the model should output. If None, the number + of classes the model is pre-trained on is used. + freeze_bn_stats: Whether to freeze stats for BatchNorm layers. + freeze_bn_weights: Whether to freeze weights for BatchNorm layers. + box_score_thresh: during inference, only return proposals with a classification + score greater than box_score_thresh + """ + + def __init__( + self, + model: str, + weights: str | None = None, + num_classes: int | None = 2, + freeze_bn_stats: bool = False, + freeze_bn_weights: bool = False, + box_score_thresh: float = 0.01, + model_kwargs: dict | None = None, + ) -> None: + super().__init__( + freeze_bn_stats=freeze_bn_stats, + freeze_bn_weights=freeze_bn_weights, + pretrained=weights is not None, + ) + + # Load the model + model_fn = getattr(detection, model) + if model_kwargs is None: + model_kwargs = {} + + self.model = model_fn( + weights=weights, + box_score_thresh=box_score_thresh, + num_classes=num_classes, + **model_kwargs, + ) + + # See source: https://stackoverflow.com/a/65347721 + self.model.eager_outputs = lambda losses, detections: (losses, detections) + + def forward( + self, x: torch.Tensor, targets: list[dict[str, torch.Tensor]] | None = None + ) -> list[dict[str, torch.Tensor]]: + """ + Forward pass of the torchvision detector + + Args: + x: images to be processed, of shape (b, c, h, w) + targets: ground-truth boxes present in the images + + Returns: + losses: {'loss_name': loss_value} + detections: for each of the b images, {"boxes": bounding_boxes} + """ + return self.model(x, targets)[1] diff --git a/dlclive/pose_estimation_pytorch/models/heads/__init__.py b/dlclive/pose_estimation_pytorch/models/heads/__init__.py new file mode 100644 index 0000000..5bf207e --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/heads/__init__.py @@ -0,0 +1,16 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from dlclive.pose_estimation_pytorch.models.heads.base import HEADS, BaseHead +from dlclive.pose_estimation_pytorch.models.heads.dekr import DEKRHead +from dlclive.pose_estimation_pytorch.models.heads.dlcrnet import DLCRNetHead +from dlclive.pose_estimation_pytorch.models.heads.rtmcc_head import RTMCCHead +from dlclive.pose_estimation_pytorch.models.heads.simple_head import HeatmapHead +from dlclive.pose_estimation_pytorch.models.heads.transformer import TransformerHead diff --git a/dlclive/pose_estimation_pytorch/models/heads/base.py b/dlclive/pose_estimation_pytorch/models/heads/base.py new file mode 100644 index 0000000..56e1f01 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/heads/base.py @@ -0,0 +1,57 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +from abc import ABC, abstractmethod + +import torch +import torch.nn as nn + +from dlclive.pose_estimation_pytorch.models.predictors import BasePredictor +from dlclive.pose_estimation_pytorch.models.registry import Registry, build_from_cfg + +HEADS = Registry("heads", build_func=build_from_cfg) + + +class BaseHead(ABC, nn.Module): + """A head for pose estimation models + + Attributes: + stride: The stride for the head (or neck + head pair), where positive values + indicate an increase in resolution while negative values a decrease. + Assuming that H and W are divisible by `stride`, this is the value such + that if a backbone outputs an encoding of shape (C, H, W), the head will + output heatmaps of shape: + (C, H * stride, W * stride) if stride > 0 + (C, -H/stride, -W/stride) if stride < 0 + predictor: an object to generate predictions from the head outputs + """ + + def __init__(self, stride: int | float, predictor: BasePredictor) -> None: + super().__init__() + if stride == 0: + raise ValueError(f"Stride must not be 0. Found {stride}.") + + self.stride = stride + self.predictor = predictor + + @abstractmethod + def forward(self, x: torch.Tensor) -> dict[str, torch.Tensor]: + """ + Given the feature maps for an image () + + Args: + x: the feature maps, of shape (b, c, h, w) + + Returns: + the head outputs (e.g. "heatmap", "locref") + """ + pass diff --git a/dlclive/pose_estimation_pytorch/models/heads/dekr.py b/dlclive/pose_estimation_pytorch/models/heads/dekr.py new file mode 100644 index 0000000..d0e3d47 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/heads/dekr.py @@ -0,0 +1,412 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +import torch +import torch.nn as nn + +from dlclive.pose_estimation_pytorch.models.heads.base import HEADS, BaseHead +from dlclive.pose_estimation_pytorch.models.modules.conv_block import AdaptBlock, BaseBlock, BasicBlock +from dlclive.pose_estimation_pytorch.models.predictors import BasePredictor + + +@HEADS.register_module +class DEKRHead(BaseHead): + """ + DEKR head based on: + Bottom-Up Human Pose Estimation Via Disentangled Keypoint Regression + Zigang Geng, Ke Sun, Bin Xiao, Zhaoxiang Zhang, Jingdong Wang, CVPR 2021 + Code based on: + https://github.com/HRNet/DEKR + """ + + def __init__( + self, + predictor: BasePredictor, + heatmap_config: dict, + offset_config: dict, + stride: int | float = 1, # head stride - should always be 1 for DEKR + ) -> None: + super().__init__(stride, predictor) + self.heatmap_head = DEKRHeatmap(**heatmap_config) + self.offset_head = DEKROffset(**offset_config) + + def forward(self, x: torch.Tensor) -> dict[str, torch.Tensor]: + return {"heatmap": self.heatmap_head(x), "offset": self.offset_head(x)} + + +class DEKRHeatmap(nn.Module): + """ + DEKR head to compute the heatmaps corresponding to keypoints based on: + Bottom-Up Human Pose Estimation Via Disentangled Keypoint Regression + Zigang Geng, Ke Sun, Bin Xiao, Zhaoxiang Zhang, Jingdong Wang, CVPR 2021 + Code based on: + https://github.com/HRNet/DEKR + """ + + def __init__( + self, + channels: tuple[int], + num_blocks: int, + dilation_rate: int, + final_conv_kernel: int, + block: type(BaseBlock) = BasicBlock, + ) -> None: + """Summary: + Constructor of the HeatmapDEKRHead. + Loads the data. + + Args: + channels: tuple containing the number of channels for the head. + num_blocks: number of blocks in the head + dilation_rate: dilation rate for the head + final_conv_kernel: kernel size for the final convolution + block: type of block to use in the head. Defaults to BasicBlock. + + Returns: + None + + Examples: + channels = (64,128,17) + num_blocks = 3 + dilation_rate = 2 + final_conv_kernel = 3 + block = BasicBlock + """ + super().__init__() + self.bn_momentum = 0.1 + self.inp_channels = channels[0] + self.num_joints_with_center = channels[ + 2 + ] # Should account for the center being a joint + self.final_conv_kernel = final_conv_kernel + + self.transition_heatmap = self._make_transition_for_head( + self.inp_channels, channels[1] + ) + self.head_heatmap = self._make_heatmap_head( + block, num_blocks, channels[1], dilation_rate + ) + + def _make_transition_for_head( + self, in_channels: int, out_channels: int + ) -> nn.Sequential: + """Summary: + Construct the transition layer for the head. + + Args: + in_channels: number of input channels + out_channels: number of output channels + + Returns: + Transition layer consisting of Conv2d, BatchNorm2d, and ReLU + """ + transition_layer = [ + nn.Conv2d(in_channels, out_channels, 1, 1, 0, bias=False), + nn.BatchNorm2d(out_channels), + nn.ReLU(True), + ] + return nn.Sequential(*transition_layer) + + def _make_heatmap_head( + self, + block: type(BaseBlock), + num_blocks: int, + num_channels: int, + dilation_rate: int, + ) -> nn.ModuleList: + """Summary: + Construct the heatmap head + + Args: + block: type of block to use in the head. + num_blocks: number of blocks in the head. + num_channels: number of input channels for the head. + dilation_rate: dilation rate for the head. + + Returns: + List of modules representing the heatmap head layers. + """ + heatmap_head_layers = [] + + feature_conv = self._make_layer( + block, num_channels, num_channels, num_blocks, dilation=dilation_rate + ) + heatmap_head_layers.append(feature_conv) + + heatmap_conv = nn.Conv2d( + in_channels=num_channels, + out_channels=self.num_joints_with_center, + kernel_size=self.final_conv_kernel, + stride=1, + padding=1 if self.final_conv_kernel == 3 else 0, + ) + heatmap_head_layers.append(heatmap_conv) + + return nn.ModuleList(heatmap_head_layers) + + def _make_layer( + self, + block: type(BaseBlock), + in_channels: int, + out_channels: int, + num_blocks: int, + stride: int = 1, + dilation: int = 1, + ) -> nn.Sequential: + """Summary: + Construct a layer in the head. + + Args: + block: type of block to use in the head. + in_channels: number of input channels for the layer. + out_channels: number of output channels for the layer. + num_blocks: number of blocks in the layer. + stride: stride for the convolutional layer. Defaults to 1. + dilation: dilation rate for the convolutional layer. Defaults to 1. + + Returns: + Sequential layer containing the specified num_blocks. + """ + downsample = None + if stride != 1 or in_channels != out_channels * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + in_channels, + out_channels * block.expansion, + kernel_size=1, + stride=stride, + bias=False, + ), + nn.BatchNorm2d( + out_channels * block.expansion, momentum=self.bn_momentum + ), + ) + + layers = [ + block(in_channels, out_channels, stride, downsample, dilation=dilation) + ] + in_channels = out_channels * block.expansion + for _ in range(1, num_blocks): + layers.append(block(in_channels, out_channels, dilation=dilation)) + + return nn.Sequential(*layers) + + def forward(self, x): + heatmap = self.head_heatmap[1](self.head_heatmap[0](self.transition_heatmap(x))) + + return heatmap + + +class DEKROffset(nn.Module): + """ + DEKR module to compute the offset from the center corresponding to each keypoints: + Bottom-Up Human Pose Estimation Via Disentangled Keypoint Regression + Zigang Geng, Ke Sun, Bin Xiao, Zhaoxiang Zhang, Jingdong Wang, CVPR 2021 + Code based on: + https://github.com/HRNet/DEKR + """ + + def __init__( + self, + channels: tuple[int, ...], + num_offset_per_kpt: int, + num_blocks: int, + dilation_rate: int, + final_conv_kernel: int, + block: type(BaseBlock) = AdaptBlock, + ) -> None: + """Args: + channels: tuple containing the number of input, offset, and output channels. + num_offset_per_kpt: number of offset values per keypoint. + num_blocks: number of blocks in the head. + dilation_rate: dilation rate for convolutional layers. + final_conv_kernel: kernel size for the final convolution. + block: type of block to use in the head. Defaults to AdaptBlock. + """ + super().__init__() + self.inp_channels = channels[0] + self.num_joints = channels[2] + self.num_joints_with_center = self.num_joints + 1 + + self.bn_momentum = 0.1 + self.offset_perkpt = num_offset_per_kpt + self.num_joints_without_center = self.num_joints + self.offset_channels = self.offset_perkpt * self.num_joints_without_center + assert self.offset_channels == channels[1] + + self.num_blocks = num_blocks + self.dilation_rate = dilation_rate + self.final_conv_kernel = final_conv_kernel + + self.transition_offset = self._make_transition_for_head( + self.inp_channels, self.offset_channels + ) + ( + self.offset_feature_layers, + self.offset_final_layer, + ) = self._make_separete_regression_head( + block, + num_blocks=num_blocks, + num_channels_per_kpt=self.offset_perkpt, + dilation_rate=self.dilation_rate, + ) + + def _make_layer( + self, + block: type(BaseBlock), + in_channels: int, + out_channels: int, + num_blocks: int, + stride: int = 1, + dilation: int = 1, + ) -> nn.Sequential: + """Summary: + Create a sequential layer with the specified block and number of num_blocks. + + Args: + block: block type to use in the layer. + in_channels: number of input channels. + out_channels: number of output channels. + num_blocks: number of blocks to be stacked in the layer. + stride: stride for the first block. Defaults to 1. + dilation: dilation rate for the blocks. Defaults to 1. + + Returns: + A sequential layer containing stacked num_blocks. + + Examples: + input: + block=BasicBlock + in_channels=64 + out_channels=128 + num_blocks=3 + stride=1 + dilation=1 + """ + downsample = None + if stride != 1 or in_channels != out_channels * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + in_channels, + out_channels * block.expansion, + kernel_size=1, + stride=stride, + bias=False, + ), + nn.BatchNorm2d( + out_channels * block.expansion, momentum=self.bn_momentum + ), + ) + + layers = [] + layers.append( + block(in_channels, out_channels, stride, downsample, dilation=dilation) + ) + in_channels = out_channels * block.expansion + for _ in range(1, num_blocks): + layers.append(block(in_channels, out_channels, dilation=dilation)) + + return nn.Sequential(*layers) + + def _make_transition_for_head( + self, in_channels: int, out_channels: int + ) -> nn.Sequential: + """Summary: + Create a transition layer for the head. + + Args: + in_channels: number of input channels + out_channels: number of output channels + + Returns: + Sequential layer containing the transition operations. + """ + transition_layer = [ + nn.Conv2d(in_channels, out_channels, 1, 1, 0, bias=False), + nn.BatchNorm2d(out_channels), + nn.ReLU(True), + ] + return nn.Sequential(*transition_layer) + + def _make_separete_regression_head( + self, + block: type(BaseBlock), + num_blocks: int, + num_channels_per_kpt: int, + dilation_rate: int, + ) -> tuple: + """Summary: + + Args: + block: type of block to use in the head + num_blocks: number of blocks in the regression head + num_channels_per_kpt: number of channels per keypoint + dilation_rate: dilation rate for the regression head + + Returns: + A tuple containing two ModuleList objects. + The first ModuleList contains the feature convolution layers for each keypoint, + and the second ModuleList contains the final offset convolution layers. + """ + offset_feature_layers = [] + offset_final_layer = [] + + for _ in range(self.num_joints): + feature_conv = self._make_layer( + block, + num_channels_per_kpt, + num_channels_per_kpt, + num_blocks, + dilation=dilation_rate, + ) + offset_feature_layers.append(feature_conv) + + offset_conv = nn.Conv2d( + in_channels=num_channels_per_kpt, + out_channels=2, + kernel_size=self.final_conv_kernel, + stride=1, + padding=1 if self.final_conv_kernel == 3 else 0, + ) + offset_final_layer.append(offset_conv) + + return nn.ModuleList(offset_feature_layers), nn.ModuleList(offset_final_layer) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Summary: + Perform forward pass through the OffsetDEKRHead. + + Args: + x: input tensor to the head. + + Returns: + offset: Computed offsets from the center corresponding to each keypoint. + The tensor will have the shape (N, num_joints * 2, H, W), where N is the batch size, + num_joints is the number of keypoints, and H and W are the height and width of the output tensor. + """ + final_offset = [] + offset_feature = self.transition_offset(x) + + for j in range(self.num_joints): + final_offset.append( + self.offset_final_layer[j]( + self.offset_feature_layers[j]( + offset_feature[ + :, j * self.offset_perkpt : (j + 1) * self.offset_perkpt + ] + ) + ) + ) + + offset = torch.cat(final_offset, dim=1) + + return offset diff --git a/dlclive/pose_estimation_pytorch/models/heads/dlcrnet.py b/dlclive/pose_estimation_pytorch/models/heads/dlcrnet.py new file mode 100644 index 0000000..44e7664 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/heads/dlcrnet.py @@ -0,0 +1,134 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +import torch +import torch.nn as nn + +from dlclive.pose_estimation_pytorch.models.heads.base import HEADS +from dlclive.pose_estimation_pytorch.models.heads.simple_head import DeconvModule, HeatmapHead +from dlclive.pose_estimation_pytorch.models.predictors import BasePredictor + + +@HEADS.register_module +class DLCRNetHead(HeatmapHead): + """A head for DLCRNet models using Part-Affinity Fields to predict individuals""" + + def __init__( + self, + predictor: BasePredictor, + heatmap_config: dict, + locref_config: dict, + paf_config: dict, + num_stages: int = 5, + features_dim: int = 128, + ) -> None: + self.num_stages = num_stages + # FIXME Cleaner __init__ to avoid initializing unused layers + in_channels = heatmap_config["channels"][0] + num_keypoints = heatmap_config["channels"][-1] + num_limbs = paf_config["channels"][-1] # Already has the 2x multiplier + in_refined_channels = features_dim + num_keypoints + num_limbs + if num_stages > 0: + heatmap_config["channels"][0] = paf_config["channels"][0] = ( + in_refined_channels + ) + locref_config["channels"][0] = locref_config["channels"][-1] + + super().__init__(predictor, heatmap_config, locref_config) + if num_stages > 0: + self.stride *= 2 # extra deconv layer where it's multi-stage + + self.paf_head = DeconvModule(**paf_config) + + self.convt1 = self._make_layer_same_padding( + in_channels=in_channels, out_channels=num_keypoints + ) + self.convt2 = self._make_layer_same_padding( + in_channels=in_channels, out_channels=locref_config["channels"][-1] + ) + self.convt3 = self._make_layer_same_padding( + in_channels=in_channels, out_channels=num_limbs + ) + self.convt4 = self._make_layer_same_padding( + in_channels=in_channels, out_channels=features_dim + ) + self.hm_ref_layers = nn.ModuleList() + self.paf_ref_layers = nn.ModuleList() + for _ in range(num_stages): + self.hm_ref_layers.append( + self._make_refinement_layer( + in_channels=in_refined_channels, out_channels=num_keypoints + ) + ) + self.paf_ref_layers.append( + self._make_refinement_layer( + in_channels=in_refined_channels, out_channels=num_limbs + ) + ) + + def _make_layer_same_padding( + self, in_channels: int, out_channels: int + ) -> nn.ConvTranspose2d: + # FIXME There is no consensual solution to emulate TF behavior in pytorch + # see https://github.com/pytorch/pytorch/issues/3867 + return nn.ConvTranspose2d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=3, + stride=2, + padding=1, + output_padding=1, + ) + + def _make_refinement_layer(self, in_channels: int, out_channels: int) -> nn.Conv2d: + """Summary: + Helper function to create a refinement layer. + + Args: + in_channels: number of input channels + out_channels: number of output channels + + Returns: + refinement_layer: the refinement layer. + """ + return nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=1, padding="same" + ) + + def forward(self, x: torch.Tensor) -> dict[str, torch.Tensor]: + if self.num_stages > 0: + stage1_hm_out = self.convt1(x) + stage1_paf_out = self.convt3(x) + features = self.convt4(x) + stage2_in = torch.cat((stage1_hm_out, stage1_paf_out, features), dim=1) + stage_in = stage2_in + stage_paf_out = stage1_paf_out + stage_hm_out = stage1_hm_out + for i, (hm_ref_layer, paf_ref_layer) in enumerate( + zip(self.hm_ref_layers, self.paf_ref_layers) + ): + pre_stage_hm_out = stage_hm_out + stage_hm_out = hm_ref_layer(stage_in) + stage_paf_out = paf_ref_layer(stage_in) + if i > 0: + stage_hm_out += pre_stage_hm_out + stage_in = torch.cat((stage_hm_out, stage_paf_out, features), dim=1) + return { + "heatmap": self.heatmap_head(stage_in), + "locref": self.locref_head(self.convt2(x)), + "paf": self.paf_head(stage_in), + } + return { + "heatmap": self.heatmap_head(x), + "locref": self.locref_head(x), + "paf": self.paf_head(x), + } diff --git a/dlclive/pose_estimation_pytorch/models/heads/rtmcc_head.py b/dlclive/pose_estimation_pytorch/models/heads/rtmcc_head.py new file mode 100644 index 0000000..53c112d --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/heads/rtmcc_head.py @@ -0,0 +1,139 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""Modified SimCC head for the RTMPose model + +Based on the official ``mmpose`` RTMCC head implementation. For more information, see +. +""" +from __future__ import annotations + +import torch +import torch.nn as nn + +from dlclive.pose_estimation_pytorch.models.heads.base import ( + BaseHead, + HEADS, +) +from dlclive.pose_estimation_pytorch.models.modules import ( + GatedAttentionUnit, + ScaleNorm, +) +from dlclive.pose_estimation_pytorch.models.predictors import BasePredictor + + +@HEADS.register_module +class RTMCCHead(BaseHead): + """RTMPose Coordinate Classification head + + The RTMCC head is itself adapted from the SimCC head. For more information, see + "SimCC: a Simple Coordinate Classification Perspective for Human Pose Estimation" + () and "RTMPose: Real-Time Multi-Person Pose + Estimation based on MMPose" (). + + Args: + input_size: The size of images given to the pose estimation model. + in_channels: The number of input channels for the head. + out_channels: Number of channels output by the head (number of bodyparts). + in_featuremap_size: The size of the input feature map for the head. This is + equal to the input_size divided by the backbone stride. + simcc_split_ratio: The split ratio of pixels, as described in SimCC. + final_layer_kernel_size: Kernel size of the final convolutional layer. + gau_cfg: Configuration for the GatedAttentionUnit. + predictor: The predictor for the head. Should usually be a `SimCCPredictor`. + """ + + def __init__( + self, + input_size: tuple[int, int], + in_channels: int, + out_channels: int, + in_featuremap_size: tuple[int, int], + simcc_split_ratio: float, + final_layer_kernel_size: int, + gau_cfg: dict, + predictor: BasePredictor, + ) -> None: + super().__init__(1, predictor) + + self.input_size = input_size + self.in_channels = in_channels + self.out_channels = out_channels + + self.in_featuremap_size = in_featuremap_size + self.simcc_split_ratio = simcc_split_ratio + + flatten_dims = self.in_featuremap_size[0] * self.in_featuremap_size[1] + out_w = int(self.input_size[0] * self.simcc_split_ratio) + out_h = int(self.input_size[1] * self.simcc_split_ratio) + + self.gau = GatedAttentionUnit( + num_token=self.out_channels, + in_token_dims=gau_cfg["hidden_dims"], + out_token_dims=gau_cfg["hidden_dims"], + expansion_factor=gau_cfg["expansion_factor"], + s=gau_cfg["s"], + eps=1e-5, + dropout_rate=gau_cfg["dropout_rate"], + drop_path=gau_cfg["drop_path"], + attn_type="self-attn", + act_fn=gau_cfg["act_fn"], + use_rel_bias=gau_cfg["use_rel_bias"], + pos_enc=gau_cfg["pos_enc"], + ) + + self.final_layer = nn.Conv2d( + in_channels, + out_channels, + kernel_size=final_layer_kernel_size, + stride=1, + padding=final_layer_kernel_size // 2, + ) + self.mlp = nn.Sequential( + ScaleNorm(flatten_dims), + nn.Linear(flatten_dims, gau_cfg["hidden_dims"], bias=False), + ) + + self.cls_x = nn.Linear(gau_cfg["hidden_dims"], out_w, bias=False) + self.cls_y = nn.Linear(gau_cfg["hidden_dims"], out_h, bias=False) + + def forward(self, x: torch.Tensor) -> dict[str, torch.Tensor]: + feats = self.final_layer(x) # -> B, K, H, W + feats = torch.flatten(feats, start_dim=2) # -> B, K, hidden=HxW + feats = self.mlp(feats) # -> B, K, hidden + feats = self.gau(feats) + x, y = self.cls_x(feats), self.cls_y(feats) + return dict(x=x, y=y) + + @staticmethod + def update_input_size(model_cfg: dict, input_size: tuple[int, int]) -> None: + """Updates an RTMPose model configuration file for a new image input size + + Args: + model_cfg: The model configuration to update in-place. + input_size: The updated input (width, height). + """ + _sigmas = {192: 4.9, 256: 5.66, 288: 6, 384: 6.93} + + def _sigma(size: int) -> float: + sigma = _sigmas.get(size) + if sigma is None: + return 2.87 + 0.01 * size + + return sigma + + w, h = input_size + model_cfg["data"]["inference"]["top_down_crop"] = dict(width=w, height=h) + model_cfg["data"]["train"]["top_down_crop"] = dict(width=w, height=h) + head_cfg = model_cfg["model"]["heads"]["bodypart"] + head_cfg["input_size"] = input_size + head_cfg["in_featuremap_size"] = h // 32, w // 32 + head_cfg["target_generator"]["input_size"] = input_size + head_cfg["target_generator"]["sigma"] = (_sigma(w), _sigma(h)) diff --git a/dlclive/pose_estimation_pytorch/models/heads/simple_head.py b/dlclive/pose_estimation_pytorch/models/heads/simple_head.py new file mode 100644 index 0000000..545d854 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/heads/simple_head.py @@ -0,0 +1,224 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +import torch +import torch.nn as nn + +from dlclive.pose_estimation_pytorch.models.heads.base import HEADS, BaseHead +from dlclive.pose_estimation_pytorch.models.predictors import BasePredictor + + +@HEADS.register_module +class HeatmapHead(BaseHead): + """Deconvolutional head to predict maps from the extracted features. + + This class implements a simple deconvolutional head to predict maps from the + extracted features. + + Args: + predictor: The predictor used to transform heatmaps into keypoints. + heatmap_config: The configuration for the heatmap outputs of the head. + locref_config: The configuration for the location refinement outputs (None if + no location refinement should be used). + """ + + def __init__( + self, + predictor: BasePredictor, + heatmap_config: dict, + locref_config: dict | None = None, + ) -> None: + heatmap_head = DeconvModule(**heatmap_config) + locref_head = None + if locref_config is not None: + locref_head = DeconvModule(**locref_config) + + # check that the heatmap and locref modules have the same stride + if heatmap_head.stride != locref_head.stride: + raise ValueError( + f"Invalid model config: Your heatmap and locref need to have the " + f"same stride (found {heatmap_head.stride}, " + f"{locref_head.stride}). Please check your config (found " + f"heatmap_config={heatmap_config}, locref_config={locref_config}" + ) + + super().__init__(heatmap_head.stride, predictor) + self.heatmap_head = heatmap_head + self.locref_head = locref_head + + def forward(self, x: torch.Tensor) -> dict[str, torch.Tensor]: + outputs = {"heatmap": self.heatmap_head(x)} + if self.locref_head is not None: + outputs["locref"] = self.locref_head(x) + return outputs + + @staticmethod + def convert_weights( + state_dict: dict[str, torch.Tensor], + module_prefix: str, + conversion: torch.Tensor, + ) -> dict[str, torch.Tensor]: + """Converts pre-trained weights to be fine-tuned on another dataset + + Args: + state_dict: the state dict for the pre-trained model + module_prefix: the prefix for weights in this head (e.g., 'heads.bodypart.') + conversion: the mapping of old indices to new indices + """ + state_dict = DeconvModule.convert_weights( + state_dict, + f"{module_prefix}heatmap_head.", + conversion, + ) + + locref_conversion = torch.stack( + [2 * conversion, 2 * conversion + 1], + dim=1, + ).reshape(-1) + state_dict = DeconvModule.convert_weights( + state_dict, + f"{module_prefix}locref_head.", + locref_conversion, + ) + return state_dict + + +class DeconvModule(nn.Module): + """ + Deconvolutional module to predict maps from the extracted features. + """ + + def __init__( + self, + channels: list[int], + kernel_size: list[int], + strides: list[int], + final_conv: dict | None = None, + ) -> None: + """ + Args: + channels: List containing the number of input and output channels for each + deconvolutional layer. + kernel_size: List containing the kernel size for each deconvolutional layer. + strides: List containing the stride for each deconvolutional layer. + final_conv: Configuration for a conv layer after the deconvolutional layers, + if one should be added. Must have keys "out_channels" and "kernel_size". + """ + super().__init__() + if not (len(channels) == len(kernel_size) + 1 == len(strides) + 1): + raise ValueError( + "Incorrect DeconvModule configuration: there should be one more number" + f" of channels than kernel_sizes and strides, found {len(channels)} " + f"channels, {len(kernel_size)} kernels and {len(strides)} strides." + ) + + in_channels = channels[0] + head_stride = 1 + self.deconv_layers = nn.Identity() + if len(kernel_size) > 0: + self.deconv_layers = nn.Sequential( + *self._make_layers(in_channels, channels[1:], kernel_size, strides) + ) + for s in strides: + head_stride *= s + + self.stride = head_stride + self.final_conv = nn.Identity() + if final_conv: + self.final_conv = nn.Conv2d( + in_channels=channels[-1], + out_channels=final_conv["out_channels"], + kernel_size=final_conv["kernel_size"], + stride=1, + ) + + @staticmethod + def _make_layers( + in_channels: int, + out_channels: list[int], + kernel_sizes: list[int], + strides: list[int], + ) -> list[nn.Module]: + """ + Helper function to create the deconvolutional layers. + + Args: + in_channels: number of input channels to the module + out_channels: number of output channels of each layer + kernel_sizes: size of the deconvolutional kernel + strides: stride for the convolution operation + + Returns: + the deconvolutional layers + """ + layers = [] + for out_channels, k, s in zip(out_channels, kernel_sizes, strides): + layers.append( + nn.ConvTranspose2d(in_channels, out_channels, kernel_size=k, stride=s) + ) + layers.append(nn.ReLU()) + in_channels = out_channels + return layers[:-1] + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass of the HeatmapHead + + Args: + x: input tensor + + Returns: + out: output tensor + """ + x = self.deconv_layers(x) + x = self.final_conv(x) + return x + + @staticmethod + def convert_weights( + state_dict: dict[str, torch.Tensor], + module_prefix: str, + conversion: torch.Tensor, + ) -> dict[str, torch.Tensor]: + """Converts pre-trained weights to be fine-tuned on another dataset + + Args: + state_dict: the state dict for the pre-trained model + module_prefix: the prefix for weights in this head (e.g., 'heads.bodypart') + conversion: the mapping of old indices to new indices + """ + if f"{module_prefix}final_conv.weight" in state_dict: + # has final convolution + weight_key = f"{module_prefix}final_conv.weight" + bias_key = f"{module_prefix}final_conv.bias" + state_dict[weight_key] = state_dict[weight_key][conversion] + state_dict[bias_key] = state_dict[bias_key][conversion] + return state_dict + + # get the last deconv layer of the net + next_index = 0 + while f"{module_prefix}deconv_layers.{next_index}.weight" in state_dict: + next_index += 1 + last_index = next_index - 1 + + # if there are deconv layers for this module prefix (there might not be, + # e.g., when there are no location refinement layers in a heatmap head) + if last_index >= 0: + weight_key = f"{module_prefix}deconv_layers.{last_index}.weight" + bias_key = f"{module_prefix}deconv_layers.{last_index}.bias" + + # for ConvTranspose2d, the weight shape is (in_channels, out_channels, ...) + # while it's (out_channels, in_channels, ...) for Conv2d + state_dict[weight_key] = state_dict[weight_key][:, conversion] + state_dict[bias_key] = state_dict[bias_key][conversion] + + return state_dict diff --git a/dlclive/pose_estimation_pytorch/models/heads/transformer.py b/dlclive/pose_estimation_pytorch/models/heads/transformer.py new file mode 100644 index 0000000..cd64677 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/heads/transformer.py @@ -0,0 +1,94 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +import torch +from einops import rearrange +from timm.layers import trunc_normal_ +from torch import nn as nn + +from dlclive.pose_estimation_pytorch.models.heads import HEADS, BaseHead +from dlclive.pose_estimation_pytorch.models.predictors import BasePredictor + + +@HEADS.register_module +class TransformerHead(BaseHead): + """ + Transformer Head module to predict heatmaps using a transformer-based approach + """ + + def __init__( + self, + predictor: BasePredictor, + dim: int, + hidden_heatmap_dim: int, + heatmap_dim: int, + apply_multi: bool, + heatmap_size: tuple[int, int], + apply_init: bool, + head_stride: int, + ): + """ + Args: + dim: Dimension of the input features. + hidden_heatmap_dim: Dimension of the hidden features in the MLP head. + heatmap_dim: Dimension of the output heatmaps. + apply_multi: If True, apply a multi-layer perceptron (MLP) with LayerNorm + to generate heatmaps. If False, directly apply a single linear + layer for heatmap prediction. + heatmap_size: Tuple (height, width) representing the size of the output + heatmaps. + apply_init: If True, apply weight initialization to the module's layers. + head_stride: The stride for the head (or neck + head pair), where positive + values indicate an increase in resolution while negative values a + decrease. Assuming that H and W are divisible by head_stride, this is + the value such that if a backbone outputs an encoding of shape + (C, H, W), the head will output heatmaps of shape: + (C, H * head_stride, W * head_stride) if head_stride > 0 + (C, -H/head_stride, -W/head_stride) if head_stride < 0 + """ + super().__init__(head_stride, predictor) + self.mlp_head = ( + nn.Sequential( + nn.LayerNorm(dim * 3), + nn.Linear(dim * 3, hidden_heatmap_dim), + nn.LayerNorm(hidden_heatmap_dim), + nn.Linear(hidden_heatmap_dim, heatmap_dim), + ) + if (dim * 3 <= hidden_heatmap_dim * 0.5 and apply_multi) + else nn.Sequential(nn.LayerNorm(dim * 3), nn.Linear(dim * 3, heatmap_dim)) + ) + self.heatmap_size = heatmap_size + + def forward(self, x: torch.Tensor) -> dict[str, torch.Tensor]: + x = self.mlp_head(x) + x = rearrange( + x, + "b c (p1 p2) -> b c p1 p2", + p1=self.heatmap_size[0], + p2=self.heatmap_size[1], + ) + return {"heatmap": x} + + def _init_weights(self, m: nn.Module) -> None: + """ + Custom weight initialization for linear and layer normalization layers. + + Args: + m: module to initialize + """ + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) diff --git a/dlclive/pose_estimation_pytorch/models/model.py b/dlclive/pose_estimation_pytorch/models/model.py new file mode 100644 index 0000000..1d8fce8 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/model.py @@ -0,0 +1,127 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +import copy + +import torch +import torch.nn as nn + +from dlclive.pose_estimation_pytorch.models.backbones import BACKBONES, BaseBackbone +from dlclive.pose_estimation_pytorch.models.heads import HEADS, BaseHead +from dlclive.pose_estimation_pytorch.models.necks import NECKS, BaseNeck +from dlclive.pose_estimation_pytorch.models.predictors import PREDICTORS + + +class PoseModel(nn.Module): + """A pose estimation model + + A pose estimation model is composed of a backbone, optionally a neck, and an + arbitrary number of heads. Outputs are computed as follows: + """ + + def __init__( + self, + cfg: dict, + backbone: BaseBackbone, + heads: dict[str, BaseHead], + neck: BaseNeck | None = None, + ) -> None: + """ + Args: + cfg: configuration dictionary for the model. + backbone: backbone network architecture. + heads: the heads for the model + neck: neck network architecture (default is None). Defaults to None. + """ + super().__init__() + self.cfg = cfg + self.backbone = backbone + self.heads = nn.ModuleDict(heads) + self.neck = neck + + self._strides = { + name: _model_stride(self.backbone.stride, head.stride) + for name, head in heads.items() + } + + def forward(self, x: torch.Tensor) -> dict[str, dict[str, torch.Tensor]]: + """ + Forward pass of the PoseModel. + + Args: + x: input images + + Returns: + Outputs of head groups + """ + if x.dim() == 3: + x = x[None, :] + features = self.backbone(x) + if self.neck: + features = self.neck(features) + + outputs = {} + for head_name, head in self.heads.items(): + outputs[head_name] = head(features) + return outputs + + def get_predictions(self, outputs: dict[str, dict[str, torch.Tensor]]) -> dict: + """Abstract method for the forward pass of the Predictor. + + Args: + outputs: outputs of the model heads + + Returns: + A dictionary containing the predictions of each head group + """ + return { + name: head.predictor(self._strides[name], outputs[name]) + for name, head in self.heads.items() + } + + @staticmethod + def build(cfg: dict) -> "PoseModel": + """ + Args: + cfg: The configuration of the model to build. + + Returns: + the built pose model + """ + cfg["backbone"]["pretrained"] = False + backbone = BACKBONES.build(dict(cfg["backbone"])) + + neck = None + if cfg.get("neck"): + neck = NECKS.build(dict(cfg["neck"])) + + heads = {} + for name, head_cfg in cfg["heads"].items(): + head_cfg = copy.deepcopy(head_cfg) + + # Remove keys not needed for DLCLive inference + for k in ("target_generator", "criterion", "aggregator", "weight_init"): + if k in head_cfg: + head_cfg.pop(k) + + head_cfg["predictor"] = PREDICTORS.build(head_cfg["predictor"]) + heads[name] = HEADS.build(head_cfg) + + return PoseModel(cfg=cfg, backbone=backbone, neck=neck, heads=heads) + + +def _model_stride(backbone_stride: int | float, head_stride: int | float) -> float: + """Computes the model stride from a backbone and a head""" + if head_stride > 0: + return backbone_stride / head_stride + + return backbone_stride * -head_stride diff --git a/dlclive/pose_estimation_pytorch/models/modules/__init__.py b/dlclive/pose_estimation_pytorch/models/modules/__init__.py new file mode 100644 index 0000000..4974948 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/modules/__init__.py @@ -0,0 +1,24 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from dlclive.pose_estimation_pytorch.models.modules.conv_block import ( + AdaptBlock, + BasicBlock, + Bottleneck, +) +from dlclive.pose_estimation_pytorch.models.modules.conv_module import ( + HighResolutionModule, +) +from dlclive.pose_estimation_pytorch.models.modules.gated_attention_unit import ( + GatedAttentionUnit, +) +from dlclive.pose_estimation_pytorch.models.modules.norm import ( + ScaleNorm, +) diff --git a/dlclive/pose_estimation_pytorch/models/modules/conv_block.py b/dlclive/pose_estimation_pytorch/models/modules/conv_block.py new file mode 100644 index 0000000..f3fbb02 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/modules/conv_block.py @@ -0,0 +1,307 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""The code is based on DEKR: https://github.com/HRNet/DEKR/tree/main""" +from __future__ import annotations + +from abc import ABC, abstractmethod + +import torch +import torch.nn as nn +import torchvision.ops as ops + +from dlclive.pose_estimation_pytorch.models.registry import Registry, build_from_cfg + + +BLOCKS = Registry("blocks", build_func=build_from_cfg) + + +class BaseBlock(ABC, nn.Module): + """Abstract Base class for defining custom blocks. + + This class defines an abstract base class for creating custom blocks used in the HigherHRNet for Human Pose Estimation. + + Attributes: + bn_momentum: Batch normalization momentum. + + Methods: + forward(x): Abstract method for defining the forward pass of the block. + """ + + def __init__(self): + super().__init__() + self.bn_momentum = 0.1 + + @abstractmethod + def forward(self, x: torch.Tensor): + """Abstract method for defining the forward pass of the block. + + Args: + x: Input tensor. + + Returns: + Output tensor. + """ + pass + + def _init_weights(self, pretrained: str | None): + """Method for initializing block weights from pretrained models. + + Args: + pretrained: Path to pretrained model weights. + """ + if pretrained: + self.load_state_dict(torch.load(pretrained)) + + +@BLOCKS.register_module +class BasicBlock(BaseBlock): + """Basic Residual Block. + + This class defines a basic residual block used in HigherHRNet. + + Attributes: + expansion: The expansion factor used in the block. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + stride: Stride value for the convolutional layers. Default is 1. + downsample: Downsample layer to be used in the residual connection. Default is None. + dilation: Dilation rate for the convolutional layers. Default is 1. + """ + + expansion: int = 1 + + def __init__( + self, + in_channels: int, + out_channels: int, + stride: int = 1, + downsample: nn.Module | None = None, + dilation: int = 1, + ): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=dilation, + bias=False, + dilation=dilation, + ) + self.bn1 = nn.BatchNorm2d(out_channels, momentum=self.bn_momentum) + self.relu = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=dilation, + bias=False, + dilation=dilation, + ) + self.bn2 = nn.BatchNorm2d(out_channels, momentum=self.bn_momentum) + self.downsample = downsample + self.stride = stride + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward pass through the BasicBlock. + + Args: + x: Input tensor. + + Returns: + Output tensor. + """ + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +@BLOCKS.register_module +class Bottleneck(BaseBlock): + """Bottleneck Residual Block. + + This class defines a bottleneck residual block used in HigherHRNet. + + Attributes: + expansion: The expansion factor used in the block. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + stride: Stride value for the convolutional layers. Default is 1. + downsample: Downsample layer to be used in the residual connection. Default is None. + dilation: Dilation rate for the convolutional layers. Default is 1. + """ + + expansion: int = 4 + + def __init__( + self, + in_channels: int, + out_channels: int, + stride: int = 1, + downsample: nn.Module | None = None, + dilation: int = 1, + ): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(out_channels, momentum=self.bn_momentum) + self.conv2 = nn.Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=dilation, + bias=False, + dilation=dilation, + ) + self.bn2 = nn.BatchNorm2d(out_channels, momentum=self.bn_momentum) + self.conv3 = nn.Conv2d( + out_channels, out_channels * self.expansion, kernel_size=1, bias=False + ) + self.bn3 = nn.BatchNorm2d( + out_channels * self.expansion, momentum=self.bn_momentum + ) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward pass through the Bottleneck block. + + Args: + x : Input tensor. + + Returns: + Output tensor. + """ + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +@BLOCKS.register_module +class AdaptBlock(BaseBlock): + """Adaptive Residual Block with Deformable Convolution. + + This class defines an adaptive residual block with deformable convolution used in HigherHRNet. + + Attributes: + expansion: The expansion factor used in the block. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + stride: Stride value for the convolutional layers. Default is 1. + downsample: Downsample layer to be used in the residual connection. Default is None. + dilation: Dilation rate for the convolutional layers. Default is 1. + deformable_groups: Number of deformable groups in the deformable convolution. Default is 1. + """ + + expansion: int = 1 + + def __init__( + self, + in_channels: int, + out_channels: int, + stride: int = 1, + downsample: nn.Module | None = None, + dilation: int = 1, + deformable_groups: int = 1, + ): + super(AdaptBlock, self).__init__() + regular_matrix = torch.tensor( + [[-1, -1, -1, 0, 0, 0, 1, 1, 1], [-1, 0, 1, -1, 0, 1, -1, 0, 1]] + ) + self.register_buffer("regular_matrix", regular_matrix.float()) + self.downsample = downsample + self.transform_matrix_conv = nn.Conv2d(in_channels, 4, 3, 1, 1, bias=True) + self.translation_conv = nn.Conv2d(in_channels, 2, 3, 1, 1, bias=True) + self.adapt_conv = ops.DeformConv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=dilation, + dilation=dilation, + bias=False, + groups=deformable_groups, + ) + self.bn = nn.BatchNorm2d(out_channels, momentum=self.bn_momentum) + self.relu = nn.ReLU(inplace=True) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward pass through the AdaptBlock. + + Args: + x: Input tensor. + + Returns: + Output tensor. + """ + residual = x + + N, _, H, W = x.shape + transform_matrix = self.transform_matrix_conv(x) + transform_matrix = transform_matrix.permute(0, 2, 3, 1).reshape( + (N * H * W, 2, 2) + ) + offset = torch.matmul(transform_matrix, self.regular_matrix) + offset = offset - self.regular_matrix + offset = offset.transpose(1, 2).reshape((N, H, W, 18)).permute(0, 3, 1, 2) + + translation = self.translation_conv(x) + offset[:, 0::2, :, :] += translation[:, 0:1, :, :] + offset[:, 1::2, :, :] += translation[:, 1:2, :, :] + + out = self.adapt_conv(x, offset) + out = self.bn(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out diff --git a/dlclive/pose_estimation_pytorch/models/modules/conv_module.py b/dlclive/pose_estimation_pytorch/models/modules/conv_module.py new file mode 100644 index 0000000..8f7241b --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/modules/conv_module.py @@ -0,0 +1,244 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""The code is based on DEKR: https://github.com/HRNet/DEKR/tree/main""" +import logging +from typing import List + +import torch.nn as nn + +from dlclive.pose_estimation_pytorch.models.modules import BasicBlock + + +BN_MOMENTUM = 0.1 +logger = logging.getLogger(__name__) + + +class HighResolutionModule(nn.Module): + """High-Resolution Module. + + This class implements the High-Resolution Module used in HigherHRNet for Human Pose Estimation. + + Args: + num_branches: Number of branches in the module. + block: The block type used in each branch of the module. + num_blocks: List containing the number of blocks in each branch. + num_inchannels: List containing the number of input channels for each branch. + num_channels: List containing the number of output channels for each branch. + fuse_method: The fusion method used in the module. + multi_scale_output: Whether to output multi-scale features. Default is True. + """ + + def __init__( + self, + num_branches: int, + block: BasicBlock, + num_blocks: int, + num_inchannels: int, + num_channels: int, + fuse_method: str, + multi_scale_output: bool = True, + ): + super(HighResolutionModule, self).__init__() + self._check_branches( + num_branches, block, num_blocks, num_inchannels, num_channels + ) + + self.num_inchannels = num_inchannels + self.fuse_method = fuse_method + self.num_branches = num_branches + + self.multi_scale_output = multi_scale_output + + self.branches = self._make_branches( + num_branches, block, num_blocks, num_channels + ) + self.fuse_layers = self._make_fuse_layers() + self.relu = nn.ReLU(True) + + def _check_branches( + self, + num_branches: int, + block: BasicBlock, + num_blocks: int, + num_inchannels: int, + num_channels: int, + ): + if num_branches != len(num_blocks): + error_msg = "NUM_BRANCHES({}) <> NUM_BLOCKS({})".format( + num_branches, len(num_blocks) + ) + logger.error(error_msg) + raise ValueError(error_msg) + + if num_branches != len(num_channels): + error_msg = "NUM_BRANCHES({}) <> NUM_CHANNELS({})".format( + num_branches, len(num_channels) + ) + logger.error(error_msg) + raise ValueError(error_msg) + + if num_branches != len(num_inchannels): + error_msg = "NUM_BRANCHES({}) <> NUM_INCHANNELS({})".format( + num_branches, len(num_inchannels) + ) + logger.error(error_msg) + raise ValueError(error_msg) + + def _make_one_branch( + self, + branch_index: int, + block: BasicBlock, + num_blocks: int, + num_channels: int, + stride: int = 1, + ) -> nn.Sequential: + downsample = None + if ( + stride != 1 + or self.num_inchannels[branch_index] + != num_channels[branch_index] * block.expansion + ): + downsample = nn.Sequential( + nn.Conv2d( + self.num_inchannels[branch_index], + num_channels[branch_index] * block.expansion, + kernel_size=1, + stride=stride, + bias=False, + ), + nn.BatchNorm2d( + num_channels[branch_index] * block.expansion, momentum=BN_MOMENTUM + ), + ) + + layers = [] + layers.append( + block( + self.num_inchannels[branch_index], + num_channels[branch_index], + stride, + downsample, + ) + ) + self.num_inchannels[branch_index] = num_channels[branch_index] * block.expansion + for i in range(1, num_blocks[branch_index]): + layers.append( + block(self.num_inchannels[branch_index], num_channels[branch_index]) + ) + + return nn.Sequential(*layers) + + def _make_branches( + self, num_branches: int, block: BasicBlock, num_blocks: int, num_channels: int + ) -> nn.ModuleList: + branches = [] + + for i in range(num_branches): + branches.append(self._make_one_branch(i, block, num_blocks, num_channels)) + + return nn.ModuleList(branches) + + def _make_fuse_layers(self) -> nn.ModuleList: + if self.num_branches == 1: + return None + + num_branches = self.num_branches + num_inchannels = self.num_inchannels + fuse_layers = [] + for i in range(num_branches if self.multi_scale_output else 1): + fuse_layer = [] + for j in range(num_branches): + if j > i: + fuse_layer.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_inchannels[i], + 1, + 1, + 0, + bias=False, + ), + nn.BatchNorm2d(num_inchannels[i]), + nn.Upsample(scale_factor=2 ** (j - i), mode="nearest"), + ) + ) + elif j == i: + fuse_layer.append(None) + else: + conv3x3s = [] + for k in range(i - j): + if k == i - j - 1: + num_outchannels_conv3x3 = num_inchannels[i] + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_outchannels_conv3x3, + 3, + 2, + 1, + bias=False, + ), + nn.BatchNorm2d(num_outchannels_conv3x3), + ) + ) + else: + num_outchannels_conv3x3 = num_inchannels[j] + conv3x3s.append( + nn.Sequential( + nn.Conv2d( + num_inchannels[j], + num_outchannels_conv3x3, + 3, + 2, + 1, + bias=False, + ), + nn.BatchNorm2d(num_outchannels_conv3x3), + nn.ReLU(True), + ) + ) + fuse_layer.append(nn.Sequential(*conv3x3s)) + fuse_layers.append(nn.ModuleList(fuse_layer)) + + return nn.ModuleList(fuse_layers) + + def get_num_inchannels(self) -> int: + return self.num_inchannels + + def forward(self, x) -> List: + """Forward pass through the HighResolutionModule. + + Args: + x: List of input tensors for each branch. + + Returns: + List of output tensors after processing through the module. + """ + if self.num_branches == 1: + return [self.branches[0](x[0])] + + for i in range(self.num_branches): + x[i] = self.branches[i](x[i]) + + x_fuse = [] + + for i in range(len(self.fuse_layers)): + y = x[0] if i == 0 else self.fuse_layers[i][0](x[0]) + for j in range(1, self.num_branches): + if i == j: + y = y + x[j] + else: + y = y + self.fuse_layers[i][j](x[j]) + x_fuse.append(self.relu(y)) + + return x_fuse diff --git a/dlclive/pose_estimation_pytorch/models/modules/csp.py b/dlclive/pose_estimation_pytorch/models/modules/csp.py new file mode 100644 index 0000000..3099eeb --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/modules/csp.py @@ -0,0 +1,387 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""Implementation of modules needed for the CSPNeXt Backbone. Used in CSP-style models. + +Based on the building blocks used for the ``mmdetection`` CSPNeXt implementation. For +more information, see . +""" +import torch +import torch.nn as nn + + +def build_activation(activation_fn: str, *args, **kwargs) -> nn.Module: + if activation_fn == "SiLU": + return nn.SiLU(*args, **kwargs) + elif activation_fn == "ReLU": + return nn.ReLU(*args, **kwargs) + + raise NotImplementedError( + f"Unknown `CSPNeXT` activation: {activation_fn}. Must be one of 'SiLU', 'ReLU'" + ) + + +def build_norm(norm: str, *args, **kwargs) -> nn.Module: + if norm == "SyncBN": + return nn.SyncBatchNorm(*args, **kwargs) + elif norm == "BN": + return nn.BatchNorm2d(*args, **kwargs) + + raise NotImplementedError( + f"Unknown `CSPNeXT` norm_layer: {norm}. Must be one of 'SyncBN', 'BN'" + ) + + +class SPPBottleneck(nn.Module): + """Spatial pyramid pooling layer used in YOLOv3-SPP and (among others) CSPNeXt + + Args: + in_channels: input channels to the bottleneck + out_channels: output channels of the bottleneck + kernel_sizes: kernel sizes for the pooling layers + norm_layer: norm layer for the bottleneck + activation_fn: activation function for the bottleneck + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_sizes: tuple[int, ...] = (5, 9, 13), + norm_layer: str | None = "SyncBN", + activation_fn: str | None = "SiLU", + ): + super().__init__() + mid_channels = in_channels // 2 + self.conv1 = CSPConvModule( + in_channels, + mid_channels, + kernel_size=1, + stride=1, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + + self.poolings = nn.ModuleList( + [ + nn.MaxPool2d(kernel_size=ks, stride=1, padding=ks // 2) + for ks in kernel_sizes + ] + ) + conv2_channels = mid_channels * (len(kernel_sizes) + 1) + self.conv2 = CSPConvModule( + conv2_channels, + out_channels, + kernel_size=1, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + + def forward(self, x): + x = self.conv1(x) + with torch.amp.autocast("cuda", enabled=False): + x = torch.cat([x] + [pooling(x) for pooling in self.poolings], dim=1) + x = self.conv2(x) + return x + + +class ChannelAttention(nn.Module): + """Channel attention Module. + + Args: + channels: Number of input/output channels of the layer. + """ + + def __init__(self, channels: int) -> None: + super().__init__() + self.global_avgpool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Conv2d(channels, channels, 1, 1, 0, bias=True) + self.act = nn.Hardsigmoid(inplace=True) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + with torch.amp.autocast("cuda", enabled=False): + out = self.global_avgpool(x) + out = self.fc(out) + out = self.act(out) + return x * out + + +class CSPConvModule(nn.Module): + """Configurable convolution module used for CSPNeXT. + + Applies sequentially + - a convolution + - (optional) a norm layer + - (optional) an activation function + + Args: + in_channels: Input channels of the convolution. + out_channels: Output channels of the convolution. + kernel_size: Convolution kernel size. + stride: Convolution stride. + padding: Convolution padding. + dilation: Convolution dilation. + groups: Number of blocked connections from input to output channels. + norm_layer: Norm layer to apply, if any. + activation_fn: Activation function to apply, if any. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int | tuple[int, int], + stride: int | tuple[int, int] = 1, + padding: int | tuple[int, int] = 0, + dilation: int | tuple[int, int] = 1, + groups: int = 1, + norm_layer: str | None = None, + activation_fn: str | None = "ReLU", + ): + super().__init__() + + self.with_activation = activation_fn is not None + self.with_bias = norm_layer is None + self.with_norm = norm_layer is not None + + self.conv = nn.Conv2d( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=self.with_bias, + ) + self.activate = None + self.norm = None + + if self.with_norm: + self.norm = build_norm(norm_layer, out_channels) + + if self.with_activation: + # Careful when adding activation functions: some should not be in-place + self.activate = build_activation(activation_fn, inplace=True) + + self._init_weights() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.conv(x) + if self.with_norm: + x = self.norm(x) + if self.with_activation: + x = self.activate(x) + return x + + def _init_weights(self) -> None: + """Same init as in convolutions""" + nn.init.kaiming_normal_(self.conv.weight, a=0, nonlinearity="relu") + if self.with_bias: + nn.init.constant_(self.conv.bias, 0) + + if self.with_norm: + nn.init.constant_(self.norm.weight, 1) + nn.init.constant_(self.norm.bias, 0) + + +class DepthwiseSeparableConv(nn.Module): + """Depth-wise separable convolution module used for CSPNeXT. + + Applies sequentially + - a depth-wise conv + - a point-wise conv + + Args: + in_channels: Input channels of the convolution. + out_channels: Output channels of the convolution. + kernel_size: Convolution kernel size. + stride: Convolution stride. + padding: Convolution padding. + dilation: Convolution dilation. + norm_layer: Norm layer to apply, if any. + activation_fn: Activation function to apply, if any. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int | tuple[int, int], + stride: int | tuple[int, int] = 1, + padding: int | tuple[int, int] = 0, + dilation: int | tuple[int, int] = 1, + norm_layer: str | None = None, + activation_fn: str | None = "ReLU", + ): + super().__init__() + + # depthwise convolution + self.depthwise_conv = CSPConvModule( + in_channels, + in_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=in_channels, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + + self.pointwise_conv = CSPConvModule( + in_channels, + out_channels, + kernel_size=1, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.depthwise_conv(x) + x = self.pointwise_conv(x) + return x + + +class CSPNeXtBlock(nn.Module): + """Basic bottleneck block used in CSPNeXt. + + Args: + in_channels: input channels for the block + out_channels: output channels for the block + expansion: expansion factor for the hidden channels + add_identity: add a skip-connection to the block + kernel_size: kernel size for the DepthwiseSeparableConv + norm_layer: Norm layer to apply, if any. + activation_fn: Activation function to apply, if any. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + expansion: float = 0.5, + add_identity: bool = True, + kernel_size: int = 5, + norm_layer: str | None = None, + activation_fn: str | None = "ReLU", + ) -> None: + super().__init__() + hidden_channels = int(out_channels * expansion) + self.conv1 = CSPConvModule( + in_channels, + hidden_channels, + 3, + stride=1, + padding=1, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + self.conv2 = DepthwiseSeparableConv( + hidden_channels, + out_channels, + kernel_size, + stride=1, + padding=kernel_size // 2, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + self.add_identity = add_identity and in_channels == out_channels + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward function.""" + identity = x + out = self.conv1(x) + out = self.conv2(out) + + if self.add_identity: + return out + identity + else: + return out + + +class CSPLayer(nn.Module): + """Cross Stage Partial Layer. + + Args: + in_channels: input channels for the layer + out_channels: output channels for the block + expand_ratio: expansion factor for the mid-channels + num_blocks: the number of blocks to use + add_identity: add a skip-connection to the blocks + channel_attention: whether to apply channel attention + norm_layer: Norm layer to apply, if any. + activation_fn: Activation function to apply, if any. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + expand_ratio: float = 0.5, + num_blocks: int = 1, + add_identity: bool = True, + channel_attention: bool = False, + norm_layer: str | None = None, + activation_fn: str | None = "ReLU", + ) -> None: + super().__init__() + mid_channels = int(out_channels * expand_ratio) + self.channel_attention = channel_attention + self.main_conv = CSPConvModule( + in_channels, + mid_channels, + 1, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + self.short_conv = CSPConvModule( + in_channels, + mid_channels, + 1, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + self.final_conv = CSPConvModule( + 2 * mid_channels, + out_channels, + 1, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + + self.blocks = nn.Sequential( + *[ + CSPNeXtBlock( + mid_channels, + mid_channels, + 1.0, + add_identity, + norm_layer=norm_layer, + activation_fn=activation_fn, + ) + for _ in range(num_blocks) + ] + ) + if channel_attention: + self.attention = ChannelAttention(2 * mid_channels) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward function.""" + x_short = self.short_conv(x) + + x_main = self.main_conv(x) + x_main = self.blocks(x_main) + + x_final = torch.cat((x_main, x_short), dim=1) + + if self.channel_attention: + x_final = self.attention(x_final) + return self.final_conv(x_final) diff --git a/dlclive/pose_estimation_pytorch/models/modules/gated_attention_unit.py b/dlclive/pose_estimation_pytorch/models/modules/gated_attention_unit.py new file mode 100644 index 0000000..f26aa20 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/modules/gated_attention_unit.py @@ -0,0 +1,237 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""Gated Attention Unit + +Based on the building blocks used for the ``mmdetection`` CSPNeXt implementation. For +more information, see . +""" +from __future__ import annotations + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +import timm.layers as timm_layers + +from dlclive.pose_estimation_pytorch.models.modules.norm import ScaleNorm + + +def rope(x, dim): + """Applies Rotary Position Embedding to input tensor.""" + shape = x.shape + if isinstance(dim, int): + dim = [dim] + + spatial_shape = [shape[i] for i in dim] + total_len = 1 + for i in spatial_shape: + total_len *= i + + position = torch.reshape( + torch.arange(total_len, dtype=torch.int, device=x.device), spatial_shape + ) + + for i in range(dim[-1] + 1, len(shape) - 1, 1): + position = torch.unsqueeze(position, dim=-1) + + half_size = shape[-1] // 2 + freq_seq = -torch.arange(half_size, dtype=torch.int, device=x.device) / float( + half_size + ) + inv_freq = 10000**-freq_seq + + sinusoid = position[..., None] * inv_freq[None, None, :] + + sin = torch.sin(sinusoid) + cos = torch.cos(sinusoid) + x1, x2 = torch.chunk(x, 2, dim=-1) + + return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1) + + +class Scale(nn.Module): + """Scale vector by element multiplications. + + Args: + dim: The dimension of the scale vector. + init_value: The initial value of the scale vector. + trainable: Whether the scale vector is trainable. + """ + + def __init__(self, dim, init_value=1.0, trainable=True): + super().__init__() + self.scale = nn.Parameter(init_value * torch.ones(dim), requires_grad=trainable) + + def forward(self, x): + return x * self.scale + + +class GatedAttentionUnit(nn.Module): + """Gated Attention Unit (GAU) in RTMBlock""" + + def __init__( + self, + num_token, + in_token_dims, + out_token_dims, + expansion_factor=2, + s=128, + eps=1e-5, + dropout_rate=0.0, + drop_path=0.0, + attn_type="self-attn", + act_fn="SiLU", + bias=False, + use_rel_bias=True, + pos_enc=False, + ): + super(GatedAttentionUnit, self).__init__() + self.s = s + self.num_token = num_token + self.use_rel_bias = use_rel_bias + self.attn_type = attn_type + self.pos_enc = pos_enc + + if drop_path > 0.0: + self.drop_path = timm_layers.DropPath(drop_path) + else: + self.drop_path = nn.Identity() + + self.e = int(in_token_dims * expansion_factor) + if use_rel_bias: + if attn_type == "self-attn": + self.w = nn.Parameter( + torch.rand([2 * num_token - 1], dtype=torch.float) + ) + else: + self.a = nn.Parameter(torch.rand([1, s], dtype=torch.float)) + self.b = nn.Parameter(torch.rand([1, s], dtype=torch.float)) + self.o = nn.Linear(self.e, out_token_dims, bias=bias) + + if attn_type == "self-attn": + self.uv = nn.Linear(in_token_dims, 2 * self.e + self.s, bias=bias) + self.gamma = nn.Parameter(torch.rand((2, self.s))) + self.beta = nn.Parameter(torch.rand((2, self.s))) + else: + self.uv = nn.Linear(in_token_dims, self.e + self.s, bias=bias) + self.k_fc = nn.Linear(in_token_dims, self.s, bias=bias) + self.v_fc = nn.Linear(in_token_dims, self.e, bias=bias) + nn.init.xavier_uniform_(self.k_fc.weight) + nn.init.xavier_uniform_(self.v_fc.weight) + + self.ln = ScaleNorm(in_token_dims, eps=eps) + + nn.init.xavier_uniform_(self.uv.weight) + + if act_fn == "SiLU" or act_fn == nn.SiLU: + self.act_fn = nn.SiLU(True) + elif act_fn == "ReLU" or act_fn == nn.ReLU: + self.act_fn = nn.ReLU(True) + else: + raise NotImplementedError + + if in_token_dims == out_token_dims: + self.shortcut = True + self.res_scale = Scale(in_token_dims) + else: + self.shortcut = False + + self.sqrt_s = math.sqrt(s) + + self.dropout_rate = dropout_rate + + if dropout_rate > 0.0: + self.dropout = nn.Dropout(dropout_rate) + + def rel_pos_bias(self, seq_len, k_len=None): + """Add relative position bias.""" + + if self.attn_type == "self-attn": + t = F.pad(self.w[: 2 * seq_len - 1], [0, seq_len]).repeat(seq_len) + t = t[..., :-seq_len].reshape(-1, seq_len, 3 * seq_len - 2) + r = (2 * seq_len - 1) // 2 + t = t[..., r:-r] + else: + a = rope(self.a.repeat(seq_len, 1), dim=0) + b = rope(self.b.repeat(k_len, 1), dim=0) + t = torch.bmm(a, b.permute(0, 2, 1)) + return t + + def _forward(self, inputs): + """GAU Forward function.""" + + if self.attn_type == "self-attn": + x = inputs + else: + x, k, v = inputs + + x = self.ln(x) + + # [B, K, in_token_dims] -> [B, K, e + e + s] + uv = self.uv(x) + uv = self.act_fn(uv) + + if self.attn_type == "self-attn": + # [B, K, e + e + s] -> [B, K, e], [B, K, e], [B, K, s] + u, v, base = torch.split(uv, [self.e, self.e, self.s], dim=2) + # [B, K, 1, s] * [1, 1, 2, s] + [2, s] -> [B, K, 2, s] + base = base.unsqueeze(2) * self.gamma[None, None, :] + self.beta + + if self.pos_enc: + base = rope(base, dim=1) + # [B, K, 2, s] -> [B, K, s], [B, K, s] + q, k = torch.unbind(base, dim=2) + + else: + # [B, K, e + s] -> [B, K, e], [B, K, s] + u, q = torch.split(uv, [self.e, self.s], dim=2) + + k = self.k_fc(k) # -> [B, K, s] + v = self.v_fc(v) # -> [B, K, e] + + if self.pos_enc: + q = rope(q, 1) + k = rope(k, 1) + + # [B, K, s].permute() -> [B, s, K] + # [B, K, s] x [B, s, K] -> [B, K, K] + qk = torch.bmm(q, k.permute(0, 2, 1)) + + if self.use_rel_bias: + if self.attn_type == "self-attn": + bias = self.rel_pos_bias(q.size(1)) + else: + bias = self.rel_pos_bias(q.size(1), k.size(1)) + qk += bias[:, : q.size(1), : k.size(1)] + # [B, K, K] + kernel = torch.square(F.relu(qk / self.sqrt_s)) + + if self.dropout_rate > 0.0: + kernel = self.dropout(kernel) + # [B, K, K] x [B, K, e] -> [B, K, e] + x = u * torch.bmm(kernel, v) + + # [B, K, e] -> [B, K, out_token_dims] + x = self.o(x) + + return x + + def forward(self, x): + if self.shortcut: + if self.attn_type == "cross-attn": + res_shortcut = x[0] + else: + res_shortcut = x + main_branch = self.drop_path(self._forward(x)) + return self.res_scale(res_shortcut) + main_branch + else: + return self.drop_path(self._forward(x)) diff --git a/dlclive/pose_estimation_pytorch/models/modules/norm.py b/dlclive/pose_estimation_pytorch/models/modules/norm.py new file mode 100644 index 0000000..1cbc0f4 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/modules/norm.py @@ -0,0 +1,41 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""Normalization layers""" +from __future__ import annotations + +import torch +import torch.nn as nn + + +class ScaleNorm(nn.Module): + """Implementation of ScaleNorm + + ScaleNorm was introduced in "Transformers without Tears: Improving the Normalization + of Self-Attention". + + Code based on the `mmpose` implementation. See https://github.com/open-mmlab/mmpose + for more details. + + Args: + dim: The dimension of the scale vector. + eps: The minimum value in clamp. + """ + + def __init__(self, dim: int, eps: float = 1e-5): + super().__init__() + self.scale = dim ** -0.5 + self.eps = eps + self.g = nn.Parameter(torch.ones(1)) + + def forward(self, x): + norm = torch.linalg.norm(x, dim=-1, keepdim=True) + norm = norm * self.scale + return x / norm.clamp(min=self.eps) * self.g diff --git a/dlclive/pose_estimation_pytorch/models/necks/__init__.py b/dlclive/pose_estimation_pytorch/models/necks/__init__.py new file mode 100644 index 0000000..a0bcb2e --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/necks/__init__.py @@ -0,0 +1,12 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from dlclive.pose_estimation_pytorch.models.necks.base import NECKS, BaseNeck +from dlclive.pose_estimation_pytorch.models.necks.transformer import Transformer diff --git a/dlclive/pose_estimation_pytorch/models/necks/base.py b/dlclive/pose_estimation_pytorch/models/necks/base.py new file mode 100644 index 0000000..0b3cb6e --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/necks/base.py @@ -0,0 +1,48 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from abc import ABC, abstractmethod + +import torch + +from dlclive.pose_estimation_pytorch.models.registry import Registry, build_from_cfg + +NECKS = Registry("necks", build_func=build_from_cfg) + + +class BaseNeck(ABC, torch.nn.Module): + """Base Neck class for pose estimation""" + + def __init__(self): + super().__init__() + + @abstractmethod + def forward(self, x: torch.Tensor): + """Abstract method for the forward pass through the Neck. + + Args: + x: Input tensor. + + Returns: + Output tensor. + """ + pass + + def _init_weights(self, pretrained: str): + """Initialize the Neck with pretrained weights. + + Args: + pretrained: Path to the pretrained weights. + + Returns: + None + """ + if pretrained: + self.model.load_state_dict(torch.load(pretrained)) diff --git a/dlclive/pose_estimation_pytorch/models/necks/layers.py b/dlclive/pose_estimation_pytorch/models/necks/layers.py new file mode 100644 index 0000000..a25ad50 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/necks/layers.py @@ -0,0 +1,287 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# + +import torch +import torch.nn.functional as F +from einops import rearrange, repeat + + +class Residual(torch.nn.Module): + """Residual block module. + + This module implements a residual block for the transformer layers. + + Attributes: + fn: The function to apply in the residual block. + """ + + def __init__(self, fn: torch.nn.Module): + """Initialize the Residual block. + + Args: + fn: The function to apply in the residual block. + """ + super().__init__() + self.fn = fn + + def forward(self, x: torch.Tensor, **kwargs): + """Forward pass through the Residual block. + + Args: + x: Input tensor. + **kwargs: Additional keyword arguments for the function. + + Returns: + Output tensor. + """ + return self.fn(x, **kwargs) + x + + +class PreNorm(torch.nn.Module): + """PreNorm block module. + + This module implements pre-normalization for the transformer layers. + + Attributes: + dim: Dimension of the input tensor. + fn: The function to apply after normalization. + fusion_factor: Fusion factor for layer normalization. + Defaults to 1. + """ + + def __init__(self, dim: int, fn: torch.nn.Module, fusion_factor: int = 1): + """Initialize the PreNorm block. + + Args: + dim: Dimension of the input tensor. + fn: The function to apply after normalization. + fusion_factor: Fusion factor for layer normalization. + Defaults to 1. + """ + super().__init__() + self.norm = torch.nn.LayerNorm(dim * fusion_factor) + self.fn = fn + + def forward(self, x, **kwargs): + """Forward pass through the PreNorm block. + + Args: + x: Input tensor. + **kwargs: Additional keyword arguments for the function. + + Returns: + Output tensor. + """ + return self.fn(self.norm(x), **kwargs) + + +class FeedForward(torch.nn.Module): + """FeedForward block module. + + This module implements the feedforward layer in the transformer layers. + + Attributes: + dim: Dimension of the input tensor. + hidden_dim: Dimension of the hidden layer. + dropout: Dropout rate. Defaults to 0.0. + """ + + def __init__(self, dim: int, hidden_dim: int, dropout: float = 0.0): + """Initialize the FeedForward block. + + Args: + dim: Dimension of the input tensor. + hidden_dim: Dimension of the hidden layer. + dropout: Dropout rate. Defaults to 0.0. + """ + super().__init__() + self.net = torch.nn.Sequential( + torch.nn.Linear(dim, hidden_dim), + torch.nn.GELU(), + torch.nn.Dropout(dropout), + torch.nn.Linear(hidden_dim, dim), + torch.nn.Dropout(dropout), + ) + + def forward(self, x: torch.Tensor): + """Forward pass through the FeedForward block. + + Args: + x: Input tensor. + + Returns: + Output tensor. + """ + return self.net(x) + + +class Attention(torch.nn.Module): + """Attention block module. + + This module implements the attention mechanism in the transformer layers. + + Attributes: + dim: Dimension of the input tensor. + heads: Number of attention heads. Defaults to 8. + dropout: Dropout rate. Defaults to 0.0. + num_keypoints: Number of keypoints. Defaults to None. + scale_with_head: Scale attention with the number of heads. + Defaults to False. + """ + + def __init__( + self, + dim: int, + heads: int = 8, + dropout: float = 0.0, + num_keypoints: int = None, + scale_with_head: bool = False, + ): + """Initialize the Attention block. + + Args: + dim: Dimension of the input tensor. + heads: Number of attention heads. Defaults to 8. + dropout: Dropout rate. Defaults to 0.0. + num_keypoints: Number of keypoints. Defaults to None. + scale_with_head: Scale attention with the number of heads. + Defaults to False. + """ + super().__init__() + self.heads = heads + self.scale = (dim // heads) ** -0.5 if scale_with_head else dim**-0.5 + + self.to_qkv = torch.nn.Linear(dim, dim * 3, bias=False) + self.to_out = torch.nn.Sequential( + torch.nn.Linear(dim, dim), torch.nn.Dropout(dropout) + ) + self.num_keypoints = num_keypoints + + def forward(self, x: torch.Tensor, mask: torch.Tensor = None): + """Forward pass through the Attention block. + + Args: + x: Input tensor. + mask: Attention mask. Defaults to None. + + Returns: + Output tensor. + """ + b, n, _, h = *x.shape, self.heads + qkv = self.to_qkv(x).chunk(3, dim=-1) + q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=h), qkv) + + dots = torch.einsum("bhid,bhjd->bhij", q, k) * self.scale + mask_value = -torch.finfo(dots.dtype).max + + if mask is not None: + mask = F.pad(mask.flatten(1), (1, 0), value=True) + assert mask.shape[-1] == dots.shape[-1], "mask has incorrect dimensions" + mask = mask[:, None, :] * mask[:, :, None] + dots.masked_fill_(~mask, mask_value) + del mask + + attn = dots.softmax(dim=-1) + + out = torch.einsum("bhij,bhjd->bhid", attn, v) + + out = rearrange(out, "b h n d -> b n (h d)") + out = self.to_out(out) + return out + + +class TransformerLayer(torch.nn.Module): + """TransformerLayer block module. + + This module implements the Transformer layer in the transformer model. + + Attributes: + dim: Dimension of the input tensor. + depth: Depth of the transformer layer. + heads: Number of attention heads. + mlp_dim: Dimension of the MLP layer. + dropout: Dropout rate. + num_keypoints: Number of keypoints. Defaults to None. + all_attn: Apply attention to all keypoints. + Defaults to False. + scale_with_head: Scale attention with the number of heads. + Defaults to False. + """ + + def __init__( + self, + dim: int, + depth: int, + heads: int, + mlp_dim: int, + dropout: float, + num_keypoints: int = None, + all_attn: bool = False, + scale_with_head: bool = False, + ): + """Initialize the TransformerLayer block. + + Args: + dim: Dimension of the input tensor. + depth: Depth of the transformer layer. + heads: Number of attention heads. + mlp_dim: Dimension of the MLP layer. + dropout: Dropout rate. + num_keypoints: Number of keypoints. Defaults to None. + all_attn: Apply attention to all keypoints. Defaults to False. + scale_with_head: Scale attention with the number of heads. Defaults to False. + """ + super().__init__() + self.layers = torch.nn.ModuleList([]) + self.all_attn = all_attn + self.num_keypoints = num_keypoints + for _ in range(depth): + self.layers.append( + torch.nn.ModuleList( + [ + Residual( + PreNorm( + dim, + Attention( + dim, + heads=heads, + dropout=dropout, + num_keypoints=num_keypoints, + scale_with_head=scale_with_head, + ), + ) + ), + Residual( + PreNorm(dim, FeedForward(dim, mlp_dim, dropout=dropout)) + ), + ] + ) + ) + + def forward( + self, x: torch.Tensor, mask: torch.Tensor = None, pos: torch.Tensor = None + ): + """Forward pass through the TransformerLayer block. + + Args: + x: Input tensor. + mask: Attention mask. Defaults to None. + pos: Positional encoding. Defaults to None. + + Returns: + Output tensor. + """ + for idx, (attn, ff) in enumerate(self.layers): + if idx > 0 and self.all_attn: + x[:, self.num_keypoints :] += pos + x = attn(x, mask=mask) + x = ff(x) + return x diff --git a/dlclive/pose_estimation_pytorch/models/necks/transformer.py b/dlclive/pose_estimation_pytorch/models/necks/transformer.py new file mode 100644 index 0000000..939ddf7 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/necks/transformer.py @@ -0,0 +1,274 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from typing import Tuple + +import torch +from einops import rearrange, repeat +from timm.layers import trunc_normal_ + +from dlclive.pose_estimation_pytorch.models.necks.base import NECKS, BaseNeck +from dlclive.pose_estimation_pytorch.models.necks.layers import TransformerLayer +from dlclive.pose_estimation_pytorch.models.necks.utils import make_sine_position_embedding + +MIN_NUM_PATCHES = 16 +BN_MOMENTUM = 0.1 + + +@NECKS.register_module +class Transformer(BaseNeck): + """Transformer Neck for pose estimation. + title={TokenPose: Learning Keypoint Tokens for Human Pose Estimation}, + author={Yanjie Li and Shoukui Zhang and Zhicheng Wang and Sen Yang and Wankou Yang and Shu-Tao Xia and Erjin Zhou}, + booktitle={IEEE/CVF International Conference on Computer Vision (ICCV)}, + year={2021} + + Args: + feature_size: Size of the input feature map (height, width). + patch_size: Size of each patch used in the transformer. + num_keypoints: Number of keypoints in the pose estimation task. + dim: Dimension of the transformer. + depth: Number of transformer layers. + heads: Number of self-attention heads in the transformer. + mlp_dim: Dimension of the MLP used in the transformer. + Defaults to 3. + apply_init: Whether to apply weight initialization. + Defaults to False. + heatmap_size: Size of the heatmap. Defaults to [64, 64]. + channels: Number of channels in each patch. Defaults to 32. + dropout: Dropout rate for embeddings. Defaults to 0.0. + emb_dropout: Dropout rate for transformer layers. + Defaults to 0.0. + pos_embedding_type: Type of positional embedding. + Either 'sine-full', 'sine', or 'learnable'. + Defaults to "sine-full". + + Examples: + # Creating a Transformer neck with sine positional embedding + transformer = Transformer( + feature_size=(128, 128), + patch_size=(16, 16), + num_keypoints=17, + dim=256, + depth=6, + heads=8, + pos_embedding_type="sine" + ) + + # Creating a Transformer neck with learnable positional embedding + transformer = Transformer( + feature_size=(256, 256), + patch_size=(32, 32), + num_keypoints=17, + dim=512, + depth=12, + heads=16, + pos_embedding_type="learnable" + ) + """ + + def __init__( + self, + *, + feature_size: Tuple[int, int], + patch_size: Tuple[int, int], + num_keypoints: int, + dim: int, + depth: int, + heads: int, + mlp_dim: int = 3, + apply_init: bool = False, + heatmap_size: Tuple[int, int] = (64, 64), + channels: int = 32, + dropout: float = 0.0, + emb_dropout: float = 0.0, + pos_embedding_type: str = "sine-full" + ): + super().__init__() + + num_patches = (feature_size[0] // (patch_size[0])) * ( + feature_size[1] // (patch_size[1]) + ) + patch_dim = channels * patch_size[0] * patch_size[1] + + self.inplanes = 64 + self.patch_size = patch_size + self.heatmap_size = heatmap_size + self.num_keypoints = num_keypoints + self.num_patches = num_patches + self.pos_embedding_type = pos_embedding_type + self.all_attn = self.pos_embedding_type == "sine-full" + + self.keypoint_token = torch.nn.Parameter( + torch.zeros(1, self.num_keypoints, dim) + ) + h, w = ( + feature_size[0] // (self.patch_size[0]), + feature_size[1] // (self.patch_size[1]), + ) + + self._make_position_embedding(w, h, dim, pos_embedding_type) + + self.patch_to_embedding = torch.nn.Linear(patch_dim, dim) + self.dropout = torch.nn.Dropout(emb_dropout) + + self.transformer1 = TransformerLayer( + dim, + depth, + heads, + mlp_dim, + dropout, + num_keypoints=num_keypoints, + scale_with_head=True, + ) + self.transformer2 = TransformerLayer( + dim, + depth, + heads, + mlp_dim, + dropout, + num_keypoints=num_keypoints, + all_attn=self.all_attn, + scale_with_head=True, + ) + self.transformer3 = TransformerLayer( + dim, + depth, + heads, + mlp_dim, + dropout, + num_keypoints=num_keypoints, + all_attn=self.all_attn, + scale_with_head=True, + ) + + self.to_keypoint_token = torch.nn.Identity() + + if apply_init: + self.apply(self._init_weights) + + def _make_position_embedding( + self, w: int, h: int, d_model: int, pe_type="learnable" + ): + """Create position embeddings for the transformer. + + Args: + w: Width of the input feature map. + h: Height of the input feature map. + d_model: Dimension of the transformer encoder. + pe_type: Type of position embeddings. + Either "learnable" or "sine". Defaults to "learnable". + """ + with torch.no_grad(): + self.pe_h = h + self.pe_w = w + length = h * w + if pe_type != "learnable": + self.pos_embedding = torch.nn.Parameter( + make_sine_position_embedding(h, w, d_model), requires_grad=False + ) + else: + self.pos_embedding = torch.nn.Parameter( + torch.zeros(1, self.num_patches + self.num_keypoints, d_model) + ) + + def _make_layer( + self, block: torch.nn.Module, planes: int, blocks: int, stride: int = 1 + ) -> torch.nn.Sequential: + """Create a layer of the transformer encoder. + + Args: + block: The basic building block of the layer. + planes: Number of planes in the layer. + blocks: Number of blocks in the layer. + stride: Stride value. Defaults to 1. + + Returns: + The layer of the transformer encoder. + """ + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = torch.nn.Sequential( + torch.nn.Conv2d( + self.inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False, + ), + torch.nn.BatchNorm2d(planes * block.expansion, momentum=BN_MOMENTUM), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return torch.nn.Sequential(*layers) + + def _init_weights(self, m: torch.nn.Module): + """Initialize the weights of the model. + + Args: + m: A module of the model. + """ + print("Initialization...") + if isinstance(m, torch.nn.Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, torch.nn.Linear) and m.bias is not None: + torch.nn.init.constant_(m.bias, 0) + elif isinstance(m, torch.nn.LayerNorm): + torch.nn.init.constant_(m.bias, 0) + torch.nn.init.constant_(m.weight, 1.0) + + def forward(self, feature: torch.Tensor, mask=None) -> torch.Tensor: + """Forward pass through the Transformer neck. + + Args: + feature: Input feature map. + mask: Mask to apply to the transformer. + Defaults to None. + + Returns: + Output tensor from the transformer neck. + + Examples: + # Assuming feature is a torch.Tensor of shape (batch_size, channels, height, width) + output = transformer(feature) + """ + p = self.patch_size + + x = rearrange( + feature, "b c (h p1) (w p2) -> b (h w) (p1 p2 c)", p1=p[0], p2=p[1] + ) + x = self.patch_to_embedding(x) + + b, n, _ = x.shape + + keypoint_tokens = repeat(self.keypoint_token, "() n d -> b n d", b=b) + if self.pos_embedding_type in ["sine", "sine-full"]: + x += self.pos_embedding[:, :n] + x = torch.cat((keypoint_tokens, x), dim=1) + else: + x = torch.cat((keypoint_tokens, x), dim=1) + x += self.pos_embedding[:, : (n + self.num_keypoints)] + x = self.dropout(x) + + x1 = self.transformer1(x, mask, self.pos_embedding) + x2 = self.transformer2(x1, mask, self.pos_embedding) + x3 = self.transformer3(x2, mask, self.pos_embedding) + + x1_out = self.to_keypoint_token(x1[:, 0 : self.num_keypoints]) + x2_out = self.to_keypoint_token(x2[:, 0 : self.num_keypoints]) + x3_out = self.to_keypoint_token(x3[:, 0 : self.num_keypoints]) + + x = torch.cat((x1_out, x2_out, x3_out), dim=2) + return x diff --git a/dlclive/pose_estimation_pytorch/models/necks/utils.py b/dlclive/pose_estimation_pytorch/models/necks/utils.py new file mode 100644 index 0000000..028078b --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/necks/utils.py @@ -0,0 +1,60 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# + +import math + +import torch + + +def make_sine_position_embedding( + h: int, w: int, d_model: int, temperature: int = 10000, scale: float = 2 * math.pi +) -> torch.Tensor: + """Generate sine position embeddings for a given height, width, and model dimension. + + Args: + h: Height of the embedding. + w: Width of the embedding. + d_model: Dimension of the model. + temperature: Temperature parameter for position embedding calculation. + Defaults to 10000. + scale: Scaling factor for position embedding. Defaults to 2 * math.pi. + + Returns: + Sine position embeddings with shape (batch_size, d_model, h * w). + + Example: + >>> h, w, d_model = 10, 20, 512 + >>> pos_emb = make_sine_position_embedding(h, w, d_model) + >>> print(pos_emb.shape) # Output: torch.Size([1, 512, 200]) + """ + area = torch.ones(1, h, w) + y_embed = area.cumsum(1, dtype=torch.float32) + x_embed = area.cumsum(2, dtype=torch.float32) + one_direction_feats = d_model // 2 + eps = 1e-6 + y_embed = y_embed / (y_embed[:, -1:, :] + eps) * scale + x_embed = x_embed / (x_embed[:, :, -1:] + eps) * scale + + dim_t = torch.arange(one_direction_feats, dtype=torch.float32) + dim_t = temperature ** (2 * (dim_t // 2) / one_direction_feats) + + pos_x = x_embed[:, :, :, None] / dim_t + pos_y = y_embed[:, :, :, None] / dim_t + pos_x = torch.stack( + (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4 + ).flatten(3) + pos_y = torch.stack( + (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4 + ).flatten(3) + pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) + pos = pos.flatten(2).permute(0, 2, 1) + + return pos diff --git a/dlclive/pose_estimation_pytorch/models/predictors/__init__.py b/dlclive/pose_estimation_pytorch/models/predictors/__init__.py new file mode 100644 index 0000000..5220642 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/predictors/__init__.py @@ -0,0 +1,15 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from dlclive.pose_estimation_pytorch.models.predictors.base import PREDICTORS, BasePredictor +from dlclive.pose_estimation_pytorch.models.predictors.dekr_predictor import DEKRPredictor +from dlclive.pose_estimation_pytorch.models.predictors.sim_cc import SimCCPredictor +from dlclive.pose_estimation_pytorch.models.predictors.single_predictor import HeatmapPredictor +from dlclive.pose_estimation_pytorch.models.predictors.paf_predictor import PartAffinityFieldPredictor diff --git a/dlclive/pose_estimation_pytorch/models/predictors/base.py b/dlclive/pose_estimation_pytorch/models/predictors/base.py new file mode 100644 index 0000000..f8b8b98 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/predictors/base.py @@ -0,0 +1,64 @@ +""" +DeepLabCut Toolbox (deeplabcut.org) +© A. & M. Mathis Labs + +Licensed under GNU Lesser General Public License v3.0 +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +import torch +from torch import nn + +from dlclive.pose_estimation_pytorch.models.registry import Registry, build_from_cfg + +PREDICTORS = Registry("predictors", build_func=build_from_cfg) + + +class BasePredictor(ABC, nn.Module): + """The base Predictor class. + + This class is an abstract base class (ABC) for defining predictors used in the + DeepLabCut Toolbox. All predictor classes should inherit from this base class and + implement the forward method. Regresses keypoint coordinates from a models output + maps + + Attributes: + num_animals: Number of animals in the project. Should be set in subclasses. + + Example: + # Create a subclass that inherits from BasePredictor + class MyPredictor(BasePredictor): + def __init__(self, num_animals): + super().__init__() + self.num_animals = num_animals + + def forward(self, outputs): + # Implement the forward pass of your custom predictor here. + pass + """ + + def __init__(self): + super().__init__() + self.num_animals = None + + @abstractmethod + def forward( + self, stride: float, outputs: dict[str, torch.Tensor] + ) -> dict[str, torch.Tensor]: + """Abstract method for the forward pass of the Predictor. + + Args: + stride: the stride of the model + outputs: outputs of the model heads + + Returns: + A dictionary containing a "poses" key with the output tensor as value, and + optionally a "unique_bodyparts" with the unique bodyparts tensor as value. + + Raises: + NotImplementedError: This method must be implemented in subclasses. + """ + pass diff --git a/dlclive/pose_estimation_pytorch/models/predictors/dekr_predictor.py b/dlclive/pose_estimation_pytorch/models/predictors/dekr_predictor.py new file mode 100644 index 0000000..6f8fd05 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/predictors/dekr_predictor.py @@ -0,0 +1,405 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# + +from __future__ import annotations + +import torch +import torch.nn.functional as F + +from dlclive.pose_estimation_pytorch.models.predictors.base import PREDICTORS, BasePredictor + + +@PREDICTORS.register_module +class DEKRPredictor(BasePredictor): + """DEKR Predictor class for multi-animal pose estimation. + + This class regresses keypoints and assembles them (if multianimal project) + from the output of DEKR (Bottom-Up Human Pose Estimation Via Disentangled Keypoint Regression). + Based on: + Bottom-Up Human Pose Estimation Via Disentangled Keypoint Regression + Zigang Geng, Ke Sun, Bin Xiao, Zhaoxiang Zhang, Jingdong Wang + CVPR + 2021 + Code based on: + https://github.com/HRNet/DEKR + + Args: + num_animals (int): Number of animals in the project. + detection_threshold (float, optional): Threshold for detection. Defaults to 0.01. + apply_sigmoid (bool, optional): Apply sigmoid to heatmaps. Defaults to True. + use_heatmap (bool, optional): Use heatmap to refine keypoint predictions. Defaults to True. + keypoint_score_type (str): Type of score to compute for keypoints. "heatmap" applies the heatmap + score to each keypoint. "center" applies the score of the center of each individual to + all of its keypoints. "combined" multiplies the score of the heatmap and individual + center for each keypoint. + + Attributes: + num_animals (int): Number of animals in the project. + detection_threshold (float): Threshold for detection. + apply_sigmoid (bool): Apply sigmoid to heatmaps. + use_heatmap (bool): Use heatmap. + keypoint_score_type (str): Type of score to compute for keypoints. "heatmap" applies the heatmap + score to each keypoint. "center" applies the score of the center of each individual to + all of its keypoints. "combined" multiplies the score of the heatmap and individual + center for each keypoint. + + Example: + # Create a DEKRPredictor instance with 2 animals. + predictor = DEKRPredictor(num_animals=2) + + # Make a forward pass with outputs and scale factors. + outputs = (heatmaps, offsets) # tuple of heatmaps and offsets + scale_factors = (0.5, 0.5) # tuple of scale factors for the poses + poses_with_scores = predictor.forward(outputs, scale_factors) + """ + + default_init = {"apply_sigmoid": True, "detection_threshold": 0.01} + + def __init__( + self, + num_animals: int, + detection_threshold: float = 0.01, + apply_sigmoid: bool = True, + clip_scores: bool = False, + use_heatmap: bool = True, + keypoint_score_type: str = "combined", + max_absorb_distance: int = 75, + ): + """ + Args: + num_animals: Number of animals in the project. + detection_threshold: Threshold for detection + apply_sigmoid: Apply sigmoid to heatmaps + clip_scores: If a sigmoid is not applied, this can be used to clip scores + for predicted keypoints to values in [0, 1]. + use_heatmap: Use heatmap to refine the keypoint predictions. + keypoint_score_type: Type of score to compute for keypoints. "heatmap" + applies the heatmap score to each keypoint. "center" applies the score + of the center of each individual to all of its keypoints. "combined" + multiplies the score of the heatmap and individual for each keypoint. + """ + super().__init__() + self.num_animals = num_animals + self.detection_threshold = detection_threshold + self.apply_sigmoid = apply_sigmoid + self.clip_scores = clip_scores + self.use_heatmap = use_heatmap + self.keypoint_score_type = keypoint_score_type + if self.keypoint_score_type not in ("heatmap", "center", "combined"): + raise ValueError(f"Unknown keypoint score type: {self.keypoint_score_type}") + + # TODO: Set as in HRNet/DEKR configs. Define as a constant. + self.max_absorb_distance = max_absorb_distance + + def forward( + self, stride: float, outputs: dict[str, torch.Tensor] + ) -> dict[str, torch.Tensor]: + """Forward pass of DEKRPredictor. + + Args: + stride: the stride of the model + outputs: outputs of the model heads (heatmap, locref) + + Returns: + A dictionary containing a "poses" key with the output tensor as value, and + optionally a "unique_bodyparts" with the unique bodyparts tensor as value. + + Example: + # Assuming you have 'outputs' (heatmaps and offsets) and 'scale_factors' for poses + poses_with_scores = predictor.forward(outputs, scale_factors) + """ + heatmaps, offsets = outputs["heatmap"], outputs["offset"] + scale_factors = stride, stride + + if self.apply_sigmoid: + heatmaps = F.sigmoid(heatmaps) + + posemap = self.offset_to_pose(offsets) + + batch_size, num_joints_with_center, h, w = heatmaps.shape + num_joints = num_joints_with_center - 1 + + center_heatmaps = heatmaps[:, -1] + pose_ind, ctr_scores = self.get_top_values(center_heatmaps) + + posemap = posemap.permute(0, 2, 3, 1).view(batch_size, h * w, -1, 2) + poses = torch.zeros(batch_size, pose_ind.shape[1], num_joints, 2).to( + ctr_scores.device + ) + for i in range(batch_size): + pose = posemap[i, pose_ind[i]] + poses[i] = pose + + if self.use_heatmap: + poses = self._update_pose_with_heatmaps(poses, heatmaps[:, :-1]) + + if self.keypoint_score_type == "center": + score = ( + ctr_scores.unsqueeze(-1) + .expand(batch_size, -1, num_joints) + .unsqueeze(-1) + ) + elif self.keypoint_score_type == "heatmap": + score = self.get_heat_value(poses, heatmaps).unsqueeze(-1) + elif self.keypoint_score_type == "combined": + center_score = ( + ctr_scores.unsqueeze(-1) + .expand(batch_size, -1, num_joints) + .unsqueeze(-1) + ) + htmp_score = self.get_heat_value(poses, heatmaps).unsqueeze(-1) + score = center_score * htmp_score + else: + raise ValueError(f"Unknown keypoint score type: {self.keypoint_score_type}") + + poses[:, :, :, 0] = ( + poses[:, :, :, 0] * scale_factors[1] + 0.5 * scale_factors[1] + ) + poses[:, :, :, 1] = ( + poses[:, :, :, 1] * scale_factors[0] + 0.5 * scale_factors[0] + ) + + if self.clip_scores: + score = torch.clip(score, min=0, max=1) + + poses_w_scores = torch.cat([poses, score], dim=3) + # self.pose_nms(heatmaps, poses_w_scores) + return {"poses": poses_w_scores} + + def get_locations( + self, height: int, width: int, device: torch.device + ) -> torch.Tensor: + """Get locations for offsets. + + Args: + height: Height of the offsets. + width: Width of the offsets. + device: Device to use. + + Returns: + Offset locations. + + Example: + # Assuming you have 'height', 'width', and 'device' + locations = predictor.get_locations(height, width, device) + """ + shifts_x = torch.arange(0, width, step=1, dtype=torch.float32).to(device) + shifts_y = torch.arange(0, height, step=1, dtype=torch.float32).to(device) + shift_y, shift_x = torch.meshgrid(shifts_y, shifts_x, indexing="ij") + shift_x = shift_x.reshape(-1) + shift_y = shift_y.reshape(-1) + locations = torch.stack((shift_x, shift_y), dim=1) + return locations + + def get_reg_poses(self, offsets: torch.Tensor, num_joints: int) -> torch.Tensor: + """Get the regression poses from offsets. + + Args: + offsets: Offsets tensor. + num_joint: Number of joints. + + Returns: + Regression poses. + + Example: + # Assuming you have 'offsets' tensor and 'num_joints' + regression_poses = predictor.get_reg_poses(offsets, num_joints) + """ + batch_size, _, h, w = offsets.shape + offsets = offsets.permute(0, 2, 3, 1).reshape(batch_size, h * w, num_joints, 2) + locations = self.get_locations(h, w, offsets.device) + locations = locations[None, :, None, :].expand(batch_size, -1, num_joints, -1) + poses = locations - offsets + + return poses + + def offset_to_pose(self, offsets: torch.Tensor) -> torch.Tensor: + """Convert offsets to poses. + + Args: + offsets: Offsets tensor. + + Returns: + Poses from offsets. + + Example: + # Assuming you have 'offsets' tensor + poses = predictor.offset_to_pose(offsets) + """ + batch_size, num_offset, h, w = offsets.shape + num_joints = int(num_offset / 2) + reg_poses = self.get_reg_poses(offsets, num_joints) + + reg_poses = ( + reg_poses.contiguous() + .view(batch_size, h * w, 2 * num_joints) + .permute(0, 2, 1) + ) + reg_poses = reg_poses.contiguous().view(batch_size, -1, h, w).contiguous() + + return reg_poses + + def max_pool(self, heatmap: torch.Tensor) -> torch.Tensor: + """Apply max pooling to the heatmap. + + Args: + heatmap: Heatmap tensor. + + Returns: + Max pooled heatmap. + + Example: + # Assuming you have 'heatmap' tensor + max_pooled_heatmap = predictor.max_pool(heatmap) + """ + pool1 = torch.nn.MaxPool2d(3, 1, 1) + pool2 = torch.nn.MaxPool2d(5, 1, 2) + pool3 = torch.nn.MaxPool2d(7, 1, 3) + map_size = (heatmap.shape[1] + heatmap.shape[2]) / 2.0 + maxm = pool2( + heatmap + ) # Here I think pool 2 is a good match for default 17 pos_dist_tresh + + return maxm + + def get_top_values( + self, heatmap: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + """Get top values from the heatmap. + + Args: + heatmap: Heatmap tensor. + + Returns: + Position indices and scores. + + Example: + # Assuming you have 'heatmap' tensor + positions, scores = predictor.get_top_values(heatmap) + """ + maximum = self.max_pool(heatmap) + maximum = torch.eq(maximum, heatmap) + heatmap *= maximum + + batchsize, ny, nx = heatmap.shape + heatmap_flat = heatmap.reshape(batchsize, nx * ny) + + scores, pos_ind = torch.topk(heatmap_flat, self.num_animals, dim=1) + + return pos_ind, scores + + ########## WIP to take heatmap into account for scoring ########## + def _update_pose_with_heatmaps( + self, _poses: torch.Tensor, kpt_heatmaps: torch.Tensor + ): + """If a heatmap center is close enough from the regressed point, the final prediction is the center of this heatmap + + Args: + poses: poses tensor, shape (batch_size, num_animals, num_keypoints, 2) + kpt_heatmaps: heatmaps (does not contain the center heatmap), shape (batch_size, num_keypoints, h, w) + """ + poses = _poses.clone() + maxm = self.max_pool(kpt_heatmaps) + maxm = torch.eq(maxm, kpt_heatmaps).float() + kpt_heatmaps *= maxm + batch_size, num_keypoints, h, w = kpt_heatmaps.shape + kpt_heatmaps = kpt_heatmaps.view(batch_size, num_keypoints, -1) + val_k, ind = kpt_heatmaps.topk(self.num_animals, dim=2) + + x = ind % w + y = (ind / w).long() + heats_ind = torch.stack((x, y), dim=3) + + for b in range(batch_size): + for i in range(num_keypoints): + heat_ind = heats_ind[b, i].float() + pose_ind = poses[b, :, i] + pose_heat_diff = pose_ind[:, None, :] - heat_ind + pose_heat_diff.pow_(2) + pose_heat_diff = pose_heat_diff.sum(2) + pose_heat_diff.sqrt_() + keep_ind = torch.argmin(pose_heat_diff, dim=1) + + for p in range(keep_ind.shape[0]): + if pose_heat_diff[p, keep_ind[p]] < self.max_absorb_distance: + poses[b, p, i] = heat_ind[keep_ind[p]] + + return poses + + def get_heat_value( + self, pose_coords: torch.Tensor, heatmaps: torch.Tensor + ) -> torch.Tensor: + """Get heat values for pose coordinates and heatmaps. + + Args: + pose_coords: Pose coordinates tensor (batch_size, num_animals, num_joints, 2) + heatmaps: Heatmaps tensor (batch_size, 1+num_joints, h, w). + + Returns: + Heat values. + + Example: + # Assuming you have 'pose_coords' and 'heatmaps' tensors + heat_values = predictor.get_heat_value(pose_coords, heatmaps) + """ + h, w = heatmaps.shape[2:] + heatmaps_nocenter = heatmaps[:, :-1].flatten( + 2, 3 + ) # (batch_size, num_joints, h*w) + + # Predicted poses based on the offset can be outside of the image + x = torch.clamp(torch.floor(pose_coords[:, :, :, 0]), 0, w - 1).long() + y = torch.clamp(torch.floor(pose_coords[:, :, :, 1]), 0, h - 1).long() + keypoint_poses = (y * w + x).mT # (batch, num_joints, num_individuals) + heatscores = torch.gather(heatmaps_nocenter, 2, keypoint_poses) + return heatscores.mT # (batch, num_individuals, num_joints) + + def pose_nms(self, heatmaps: torch.Tensor, poses: torch.Tensor): + """Non-Maximum Suppression (NMS) for regressed poses. + + Args: + heatmaps: Heatmaps tensor. + poses: Pose proposals. + + Returns: + None + + Example: + # Assuming you have 'heatmaps' and 'poses' tensors + predictor.pose_nms(heatmaps, poses) + """ + pose_scores = poses[:, :, :, 2] + pose_coords = poses[:, :, :, :2] + + if pose_coords.shape[1] == 0: + return [], [] + + batch_size, num_people, num_joints, _ = pose_coords.shape + heatvals = self.get_heat_value(pose_coords, heatmaps) + heat_score = (torch.sum(heatvals, dim=1) / num_joints)[:, 0] + + # return heat_score + # pose_score = pose_score*heatvals + # poses = torch.cat([pose_coord.cpu(), pose_score.cpu()], dim=2) + + # keep_pose_inds = nms_core(cfg, pose_coord, heat_score) + # poses = poses[keep_pose_inds] + # heat_score = heat_score[keep_pose_inds] + + # if len(keep_pose_inds) > cfg.DATASET.MAX_NUM_PEOPLE: + # heat_score, topk_inds = torch.topk(heat_score, + # cfg.DATASET.MAX_NUM_PEOPLE) + # poses = poses[topk_inds] + + # poses = [poses.numpy()] + # scores = [i[:, 2].mean() for i in poses[0]] + + # return poses, scores diff --git a/dlclive/pose_estimation_pytorch/models/predictors/identity_predictor.py b/dlclive/pose_estimation_pytorch/models/predictors/identity_predictor.py new file mode 100644 index 0000000..e7e7d06 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/predictors/identity_predictor.py @@ -0,0 +1,66 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""Predictor to generate identity maps from head outputs""" +import torch +import torch.nn as nn +import torchvision.transforms.functional as F + +from dlclive.pose_estimation_pytorch.models.predictors.base import PREDICTORS, BasePredictor + + +@PREDICTORS.register_module +class IdentityPredictor(BasePredictor): + """Predictor to generate identity maps from head outputs + + Attributes: + apply_sigmoid: Apply sigmoid to heatmaps. Defaults to True. + """ + + def __init__(self, apply_sigmoid: bool = True): + """ + Args: + apply_sigmoid: Apply sigmoid to heatmaps. Defaults to True. + """ + super().__init__() + self.apply_sigmoid = apply_sigmoid + self.sigmoid = nn.Sigmoid() + + def forward( + self, stride: float, outputs: dict[str, torch.Tensor] + ) -> dict[str, torch.Tensor]: + """ + Swaps the dimensions so the heatmap are (batch_size, h, w, num_individuals), + optionally applies a sigmoid to the heatmaps, and rescales it to be the size + of the original image (so that the identity scores of keypoints can be computed) + + Args: + stride: the stride of the model + outputs: output of the model identity head, of shape (b, num_idv, w', h') + + Returns: + A dictionary containing a "heatmap" key with the identity heatmap tensor as + value. + """ + heatmaps = outputs["heatmap"] + h_out, w_out = heatmaps.shape[2:] + h_in, w_in = int(h_out * stride), int(w_out * stride) + heatmaps = F.resize( + heatmaps, + size=[h_in, w_in], + interpolation=F.InterpolationMode.BILINEAR, + antialias=True, + ) + if self.apply_sigmoid: + heatmaps = self.sigmoid(heatmaps) + + # permute to have shape (batch_size, h, w, num_individuals) + heatmaps = heatmaps.permute((0, 2, 3, 1)) + return {"heatmap": heatmaps} diff --git a/dlclive/pose_estimation_pytorch/models/predictors/paf_predictor.py b/dlclive/pose_estimation_pytorch/models/predictors/paf_predictor.py new file mode 100644 index 0000000..b6ba68a --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/predictors/paf_predictor.py @@ -0,0 +1,368 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +import numpy as np +import torch +import torch.nn.functional as F +from numpy.typing import NDArray + +from dlclive.pose_estimation_pytorch.models.predictors.base import ( + BasePredictor, + PREDICTORS, +) +from dlclive.core import inferenceutils + +Graph = list[tuple[int, int]] + + +@PREDICTORS.register_module +class PartAffinityFieldPredictor(BasePredictor): + """Predictor class for multiple animal pose estimation with part affinity fields. + + TODO: INSTALL scipy-1.14.1 + + Args: + num_animals: Number of animals in the project. + num_multibodyparts: Number of animal's body parts (ignoring unique body parts). + num_uniquebodyparts: Number of unique body parts. # FIXME - should not be needed here if we separate the unique bodypart head + graph: Part affinity field graph edges. + edges_to_keep: List of indices in `graph` of the edges to keep. + locref_stdev: Standard deviation for location refinement. + nms_radius: Radius of the Gaussian kernel. + sigma: Width of the 2D Gaussian distribution. + min_affinity: Minimal edge affinity to add a body part to an Assembly. + + Returns: + Regressed keypoints from heatmaps, locref_maps and part affinity fields, as in Tensorflow maDLC. + """ + + default_init = { + "locref_stdev": 7.2801, + "nms_radius": 5, + "sigma": 1, + "min_affinity": 0.05, + } + + def __init__( + self, + num_animals: int, + num_multibodyparts: int, + num_uniquebodyparts: int, + graph: Graph, + edges_to_keep: list[int], + locref_stdev: float, + nms_radius: int, + sigma: float, + min_affinity: float, + add_discarded: bool = False, + apply_sigmoid: bool = True, + clip_scores: bool = False, + force_fusion: bool = False, + return_preds: bool = False, + ): + """Initialize the PartAffinityFieldPredictor class. + + Args: + num_animals: Number of animals in the project. + num_multibodyparts: Number of animal's body parts (ignoring unique body parts). + num_uniquebodyparts: Number of unique body parts. + graph: Part affinity field graph edges. + edges_to_keep: List of indices in `graph` of the edges to keep. + locref_stdev: Standard deviation for location refinement. + nms_radius: Radius of the Gaussian kernel. + sigma: Width of the 2D Gaussian distribution. + min_affinity: Minimal edge affinity to add a body part to an Assembly. + return_preds: Whether to return predictions alongside the animals' poses + + Returns: + None + """ + super().__init__() + self.num_animals = num_animals + self.num_multibodyparts = num_multibodyparts + self.num_uniquebodyparts = num_uniquebodyparts + self.graph = graph + self.edges_to_keep = edges_to_keep + self.locref_stdev = locref_stdev + self.nms_radius = nms_radius + self.return_preds = return_preds + self.sigma = sigma + self.apply_sigmoid = apply_sigmoid + self.clip_scores = clip_scores + self.sigmoid = torch.nn.Sigmoid() + self.assembler = inferenceutils.Assembler.empty( + num_animals, + n_multibodyparts=num_multibodyparts, + n_uniquebodyparts=num_uniquebodyparts, + graph=graph, + paf_inds=edges_to_keep, + min_affinity=min_affinity, + add_discarded=add_discarded, + force_fusion=force_fusion, + ) + + def forward( + self, stride: float, outputs: dict[str, torch.Tensor] + ) -> dict[str, torch.Tensor]: + """Forward pass of PartAffinityFieldPredictor. Gets predictions from model output. + + Args: + stride: the stride of the model + outputs: Output tensors from previous layers. + output = heatmaps, locref, pafs + heatmaps: torch.Tensor([batch_size, num_joints, height, width]) + locref: torch.Tensor([batch_size, num_joints, height, width]) + + Returns: + A dictionary containing a "poses" key with the output tensor as value. + + Example: + >>> predictor = PartAffinityFieldPredictor(num_animals=3, location_refinement=True, locref_stdev=7.2801) + >>> output = (torch.rand(32, 17, 64, 64), torch.rand(32, 34, 64, 64), torch.rand(32, 136, 64, 64)) + >>> stride = 8 + >>> poses = predictor.forward(stride, output) + """ + heatmaps = outputs["heatmap"] + locrefs = outputs["locref"] + pafs = outputs["paf"] + scale_factors = stride, stride + batch_size, n_channels, height, width = heatmaps.shape + + if self.apply_sigmoid: + heatmaps = self.sigmoid(heatmaps) + + # Filter predicted heatmaps with a 2D Gaussian kernel as in: + # https://openaccess.thecvf.com/content_CVPR_2020/papers/Huang_The_Devil_Is_in_the_Details_Delving_Into_Unbiased_Data_CVPR_2020_paper.pdf + kernel = self.make_2d_gaussian_kernel( + sigma=self.sigma, size=self.nms_radius * 2 + 1 + )[None, None] + kernel = kernel.repeat(n_channels, 1, 1, 1).to(heatmaps.device) + heatmaps = F.conv2d( + heatmaps, kernel, stride=1, padding="same", groups=n_channels + ) + + peaks = self.find_local_peak_indices_maxpool_nms( + heatmaps, self.nms_radius, threshold=0.01 + ) + if ~torch.any(peaks): + return { + "poses": -torch.ones( + (batch_size, self.num_animals, self.num_multibodyparts, 5) + ) + } + + locrefs = locrefs.reshape(batch_size, n_channels, 2, height, width) + locrefs = locrefs * self.locref_stdev + pafs = pafs.reshape(batch_size, -1, 2, height, width) + + graph = [self.graph[ind] for ind in self.edges_to_keep] + preds = self.compute_peaks_and_costs( + heatmaps, + locrefs, + pafs, + peaks, + graph, + self.edges_to_keep, + scale_factors, + n_id_channels=0, # FIXME Handle identity training + ) + poses = -torch.ones((batch_size, self.num_animals, self.num_multibodyparts, 5)) + poses_unique = -torch.ones((batch_size, 1, self.num_uniquebodyparts, 4)) + for i, data_dict in enumerate(preds): + assemblies, unique = self.assembler._assemble(data_dict, ind_frame=0) + if assemblies is not None: + for j, assembly in enumerate(assemblies): + poses[i, j, :, :4] = torch.from_numpy(assembly.data) + poses[i, j, :, 4] = assembly.affinity + if unique is not None: + poses_unique[i, 0, :, :4] = torch.from_numpy(unique) + + if self.clip_scores: + poses[..., 2] = torch.clip(poses[..., 2], min=0, max=1) + + out = {"poses": poses} + if self.return_preds: + out["preds"] = preds + return out + + @staticmethod + def find_local_peak_indices_maxpool_nms( + input_: torch.Tensor, radius: int, threshold: float + ) -> torch.Tensor: + pooled = F.max_pool2d(input_, kernel_size=radius, stride=1, padding=radius // 2) + maxima = input_ * torch.eq(input_, pooled).float() + peak_indices = torch.nonzero(maxima >= threshold, as_tuple=False) + return peak_indices.int() + + @staticmethod + def make_2d_gaussian_kernel(sigma: float, size: int) -> torch.Tensor: + k = torch.arange(-size // 2 + 1, size // 2 + 1, dtype=torch.float32) ** 2 + k = F.softmax(-k / (2 * (sigma ** 2)), dim=0) + return torch.einsum("i,j->ij", k, k) + + @staticmethod + def calc_peak_locations( + locrefs: torch.Tensor, + peak_inds_in_batch: torch.Tensor, + strides: tuple[float, float], + ) -> torch.Tensor: + s, b, r, c = peak_inds_in_batch.T + stride_y, stride_x = strides + strides = torch.Tensor((stride_x, stride_y)).to(locrefs.device) + off = locrefs[s, b, :, r, c] + loc = strides * peak_inds_in_batch[:, [3, 2]] + strides // 2 + off + return loc + + @staticmethod + def compute_edge_costs( + pafs: NDArray, + peak_inds_in_batch: NDArray, + graph: Graph, + paf_inds: list[int], + n_bodyparts: int, + n_points: int = 10, + n_decimals: int = 3, + ) -> list[dict[int, NDArray]]: + # Clip peak locations to PAFs dimensions + h, w = pafs.shape[-2:] + peak_inds_in_batch[:, 2] = np.clip(peak_inds_in_batch[:, 2], 0, h - 1) + peak_inds_in_batch[:, 3] = np.clip(peak_inds_in_batch[:, 3], 0, w - 1) + + n_samples = pafs.shape[0] + sample_inds = [] + edge_inds = [] + all_edges = [] + all_peaks = [] + for i in range(n_samples): + samples_i = peak_inds_in_batch[:, 0] == i + peak_inds = peak_inds_in_batch[samples_i, 1:] + if not np.any(peak_inds): + continue + peaks = peak_inds[:, 1:] + bpt_inds = peak_inds[:, 0] + idx = np.arange(peaks.shape[0]) + idx_per_bpt = {j: idx[bpt_inds == j].tolist() for j in range(n_bodyparts)} + edges = [] + for k, (s, t) in zip(paf_inds, graph): + inds_s = idx_per_bpt[s] + inds_t = idx_per_bpt[t] + if not (inds_s and inds_t): + continue + candidate_edges = ((i, j) for i in inds_s for j in inds_t) + edges.extend(candidate_edges) + edge_inds.extend([k] * len(inds_s) * len(inds_t)) + if not edges: + continue + sample_inds.extend([i] * len(edges)) + all_edges.extend(edges) + all_peaks.append(peaks[np.asarray(edges)]) + if not all_peaks: + return [dict() for _ in range(n_samples)] + + sample_inds = np.asarray(sample_inds, dtype=np.int32) + edge_inds = np.asarray(edge_inds, dtype=np.int32) + all_edges = np.asarray(all_edges, dtype=np.int32) + all_peaks = np.concatenate(all_peaks) + vecs_s = all_peaks[:, 0] + vecs_t = all_peaks[:, 1] + vecs = vecs_t - vecs_s + lengths = np.linalg.norm(vecs, axis=1).astype(np.float32) + lengths += np.spacing(1, dtype=np.float32) + xy = np.linspace(vecs_s, vecs_t, n_points, axis=1, dtype=np.int32) + y = pafs[ + sample_inds.reshape((-1, 1)), + edge_inds.reshape((-1, 1)), + :, + xy[..., 0], + xy[..., 1], + ] + integ = np.trapz(y, xy[..., ::-1], axis=1) + affinities = np.linalg.norm(integ, axis=1).astype(np.float32) + affinities /= lengths + np.round(affinities, decimals=n_decimals, out=affinities) + np.round(lengths, decimals=n_decimals, out=lengths) + + # Form cost matrices + all_costs = [] + for i in range(n_samples): + samples_i_mask = sample_inds == i + costs = dict() + for k in paf_inds: + edges_k_mask = edge_inds == k + idx = np.flatnonzero(samples_i_mask & edges_k_mask) + s, t = all_edges[idx].T + n_sources = np.unique(s).size + n_targets = np.unique(t).size + costs[k] = dict() + costs[k]["m1"] = affinities[idx].reshape((n_sources, n_targets)) + costs[k]["distance"] = lengths[idx].reshape((n_sources, n_targets)) + all_costs.append(costs) + + return all_costs + + @staticmethod + def _linspace(start: torch.Tensor, stop: torch.Tensor, num: int) -> torch.Tensor: + # Taken from https://github.com/pytorch/pytorch/issues/61292#issue-937937159 + steps = torch.linspace(0, 1, num, dtype=torch.float32, device=start.device) + steps = steps.reshape([-1, *([1] * start.ndim)]) + out = start[None] + steps * (stop - start)[None] + return out.swapaxes(0, 1) + + def compute_peaks_and_costs( + self, + heatmaps: torch.Tensor, + locrefs: torch.Tensor, + pafs: torch.Tensor, + peak_inds_in_batch: torch.Tensor, + graph: Graph, + paf_inds: list[int], + strides: tuple[float, float], + n_id_channels: int, + n_points: int = 10, + n_decimals: int = 3, + ) -> list[dict[str, NDArray]]: + n_samples, n_channels = heatmaps.shape[:2] + n_bodyparts = n_channels - n_id_channels + pos = self.calc_peak_locations(locrefs, peak_inds_in_batch, strides) + pos = np.round(pos.detach().cpu().numpy(), decimals=n_decimals) + heatmaps = heatmaps.detach().cpu().numpy() + pafs = pafs.detach().cpu().numpy() + peak_inds_in_batch = peak_inds_in_batch.detach().cpu().numpy() + costs = self.compute_edge_costs( + pafs, peak_inds_in_batch, graph, paf_inds, n_bodyparts, n_points, n_decimals + ) + s, b, r, c = peak_inds_in_batch.T + prob = np.round(heatmaps[s, b, r, c], n_decimals).reshape((-1, 1)) + if n_id_channels: + ids = np.round(heatmaps[s, -n_id_channels:, r, c], n_decimals) + + peaks_and_costs = [] + for i in range(n_samples): + xy = [] + p = [] + id_ = [] + samples_i_mask = peak_inds_in_batch[:, 0] == i + for j in range(n_bodyparts): + bpts_j_mask = peak_inds_in_batch[:, 1] == j + idx = np.flatnonzero(samples_i_mask & bpts_j_mask) + xy.append(pos[idx]) + p.append(prob[idx]) + if n_id_channels: + id_.append(ids[idx]) + dict_ = {"coordinates": (xy,), "confidence": p} + if costs is not None: + dict_["costs"] = costs[i] + if n_id_channels: + dict_["identity"] = id_ + peaks_and_costs.append(dict_) + + return peaks_and_costs diff --git a/dlclive/pose_estimation_pytorch/models/predictors/sim_cc.py b/dlclive/pose_estimation_pytorch/models/predictors/sim_cc.py new file mode 100644 index 0000000..1b4e007 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/predictors/sim_cc.py @@ -0,0 +1,163 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""SimCC predictor for the RTMPose model + +Based on the official ``mmpose`` SimCC codec and RTMCC head implementation. For more +information, see . +""" +from __future__ import annotations + +import numpy as np +import torch + +from dlclive.pose_estimation_pytorch.models.predictors.base import ( + BasePredictor, + PREDICTORS, +) + + +@PREDICTORS.register_module +class SimCCPredictor(BasePredictor): + """Class used to make pose predictions from RTMPose head outputs + + The RTMPose model uses coordinate classification for pose estimation. For more + information, see "SimCC: a Simple Coordinate Classification Perspective for Human + Pose Estimation" () and "RTMPose: Real-Time + Multi-Person Pose Estimation based on MMPose" (). + + Args: + simcc_split_ratio: The split ratio of pixels, as described in SimCC. + apply_softmax: Whether to apply softmax on the scores. + normalize_outputs: Whether to normalize the outputs before predicting maximums. + """ + + def __init__( + self, + simcc_split_ratio: float = 2.0, + apply_softmax: bool = False, + normalize_outputs: bool = False, + ) -> None: + super().__init__() + self.simcc_split_ratio = simcc_split_ratio + self.apply_softmax = apply_softmax + self.normalize_outputs = normalize_outputs + + def forward( + self, stride: float, outputs: dict[str, torch.Tensor] + ) -> dict[str, torch.Tensor]: + x, y = outputs["x"].detach(), outputs["y"].detach() + if self.normalize_outputs: + x = get_simcc_normalized(x) + y = get_simcc_normalized(y) + + keypoints, scores = get_simcc_maximum( + x.cpu().numpy(), y.cpu().numpy(), self.apply_softmax + ) + + if keypoints.ndim == 2: + keypoints = keypoints[None, :] + scores = scores[None, :] + + keypoints /= self.simcc_split_ratio + scores = scores.reshape((*scores.shape, -1)) + keypoints_with_score = np.concatenate([keypoints, scores], axis=-1) + keypoints_with_score = torch.tensor(keypoints_with_score).unsqueeze(1) + return dict(poses=keypoints_with_score) + + +def get_simcc_maximum( + simcc_x: np.ndarray, + simcc_y: np.ndarray, + apply_softmax: bool = False, +) -> tuple[np.ndarray, np.ndarray]: + """Get maximum response location and value from SimCC representations. + + Note: + instance number: N + num_keypoints: K + heatmap height: H + heatmap width: W + + Args: + simcc_x (np.ndarray): x-axis SimCC in shape (K, Wx) or (N, K, Wx) + simcc_y (np.ndarray): y-axis SimCC in shape (K, Wy) or (N, K, Wy) + apply_softmax (bool): whether to apply softmax on the heatmap. + Defaults to False. + + Returns: + tuple: + - locs (np.ndarray): locations of maximum heatmap responses in shape + (K, 2) or (N, K, 2) + - vals (np.ndarray): values of maximum heatmap responses in shape + (K,) or (N, K) + """ + + assert isinstance(simcc_x, np.ndarray), "simcc_x should be numpy.ndarray" + assert isinstance(simcc_y, np.ndarray), "simcc_y should be numpy.ndarray" + assert simcc_x.ndim == 2 or simcc_x.ndim == 3, f"Invalid shape {simcc_x.shape}" + assert simcc_y.ndim == 2 or simcc_y.ndim == 3, f"Invalid shape {simcc_y.shape}" + assert simcc_x.ndim == simcc_y.ndim, f"{simcc_x.shape} != {simcc_y.shape}" + + if simcc_x.ndim == 3: + N, K, Wx = simcc_x.shape + simcc_x = simcc_x.reshape(N * K, -1) + simcc_y = simcc_y.reshape(N * K, -1) + else: + N = None + + if apply_softmax: + simcc_x = simcc_x - np.max(simcc_x, axis=1, keepdims=True) + simcc_y = simcc_y - np.max(simcc_y, axis=1, keepdims=True) + ex, ey = np.exp(simcc_x), np.exp(simcc_y) + simcc_x = ex / np.sum(ex, axis=1, keepdims=True) + simcc_y = ey / np.sum(ey, axis=1, keepdims=True) + + x_locs = np.argmax(simcc_x, axis=1) + y_locs = np.argmax(simcc_y, axis=1) + locs = np.stack((x_locs, y_locs), axis=-1).astype(np.float32) + max_val_x = np.amax(simcc_x, axis=1) + max_val_y = np.amax(simcc_y, axis=1) + + mask = max_val_x > max_val_y + max_val_x[mask] = max_val_y[mask] + vals = max_val_x + locs[vals <= 0.0] = -1 + + if N: + locs = locs.reshape(N, K, 2) + vals = vals.reshape(N, K) + + return locs, vals + + +def get_simcc_normalized(pred: torch.Tensor) -> torch.Tensor: + """Normalize the predicted SimCC. + + See: + github.com/open-mmlab/mmpose/blob/main/mmpose/codecs/utils/post_processing.py#L12 + + Args: + pred: The predicted output. + + Returns: + The normalized output. + """ + b, k, _ = pred.shape + pred = pred.clamp(min=0) + + # Compute the binary mask + mask = (pred.amax(dim=-1) > 1).reshape(b, k, 1) + + # Normalize the tensor using the maximum value + norm = (pred / pred.amax(dim=-1).reshape(b, k, 1)) + + # return the normalized tensor + return torch.where(mask, norm, pred) diff --git a/dlclive/pose_estimation_pytorch/models/predictors/single_predictor.py b/dlclive/pose_estimation_pytorch/models/predictors/single_predictor.py new file mode 100644 index 0000000..96f31d9 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/predictors/single_predictor.py @@ -0,0 +1,161 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +from __future__ import annotations + +from typing import Tuple + +import torch + +from dlclive.pose_estimation_pytorch.models.predictors.base import BasePredictor, PREDICTORS + + +@PREDICTORS.register_module +class HeatmapPredictor(BasePredictor): + """Predictor class for pose estimation from heatmaps (and optionally locrefs). + + Args: + location_refinement: Enable location refinement. + locref_std: Standard deviation for location refinement. + apply_sigmoid: Apply sigmoid to heatmaps. Defaults to True. + + Returns: + Regressed keypoints from heatmaps and locref_maps of baseline DLC model (ResNet + Deconv). + """ + + def __init__( + self, + apply_sigmoid: bool = True, + clip_scores: bool = False, + location_refinement: bool = True, + locref_std: float = 7.2801, + ): + """ + Args: + apply_sigmoid: Apply sigmoid to heatmaps. Defaults to True. + clip_scores: If a sigmoid is not applied, this can be used to clip scores + for predicted keypoints to values in [0, 1]. + location_refinement : Enable location refinement. + locref_std: Standard deviation for location refinement. + """ + super().__init__() + self.apply_sigmoid = apply_sigmoid + self.clip_scores = clip_scores + self.sigmoid = torch.nn.Sigmoid() + self.location_refinement = location_refinement + self.locref_std = locref_std + + def forward( + self, stride: float, outputs: dict[str, torch.Tensor] + ) -> dict[str, torch.Tensor]: + """Forward pass of SinglePredictor. Gets predictions from model output. + + Args: + stride: the stride of the model + outputs: output of the model heads (heatmap, locref) + + Returns: + A dictionary containing a "poses" key with the output tensor as value. + + Example: + >>> predictor = HeatmapPredictor(location_refinement=True, locref_std=7.2801) + >>> stride = 8 + >>> output = {"heatmap": torch.rand(32, 17, 64, 64), "locref": torch.rand(32, 17, 64, 64)} + >>> poses = predictor.forward(stride, output) + """ + heatmaps = outputs["heatmap"] + scale_factors = stride, stride + + if self.apply_sigmoid: + heatmaps = self.sigmoid(heatmaps) + + heatmaps = heatmaps.permute(0, 2, 3, 1) + batch_size, height, width, num_joints = heatmaps.shape + + locrefs = None + if self.location_refinement: + locrefs = outputs["locref"] + locrefs = locrefs.permute(0, 2, 3, 1).reshape( + batch_size, height, width, num_joints, 2 + ) + locrefs = locrefs * self.locref_std + + poses = self.get_pose_prediction(heatmaps, locrefs, scale_factors) + + if self.clip_scores: + poses[..., 2] = torch.clip(poses[..., 2], min=0, max=1) + + return {"poses": poses} + + def get_top_values( + self, heatmap: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Get the top values from the heatmap. + + Args: + heatmap: Heatmap tensor. + + Returns: + Y and X indices of the top values. + + Example: + >>> predictor = HeatmapPredictor(location_refinement=True, locref_std=7.2801) + >>> heatmap = torch.rand(32, 17, 64, 64) + >>> Y, X = predictor.get_top_values(heatmap) + """ + batchsize, ny, nx, num_joints = heatmap.shape + heatmap_flat = heatmap.reshape(batchsize, nx * ny, num_joints) + heatmap_top = torch.argmax(heatmap_flat, dim=1) + y, x = heatmap_top // nx, heatmap_top % nx + return y, x + + def get_pose_prediction( + self, heatmap: torch.Tensor, locref: torch.Tensor | None, scale_factors + ) -> torch.Tensor: + """Gets the pose prediction given the heatmaps and locref. + + Args: + heatmap: Heatmap tensor of shape (batch_size, height, width, num_joints) + locref: Locref tensor of shape (batch_size, height, width, num_joints, 2) + scale_factors: Scale factors for the poses. + + Returns: + Pose predictions of the format: (batch_size, num_people = 1, num_joints, 3) + + Example: + >>> predictor = HeatmapPredictor( + >>> location_refinement=True, locref_std=7.2801 + >>> ) + >>> heatmap = torch.rand(32, 17, 64, 64) + >>> locref = torch.rand(32, 17, 64, 64, 2) + >>> scale_factors = (0.5, 0.5) + >>> poses = predictor.get_pose_prediction(heatmap, locref, scale_factors) + """ + y, x = self.get_top_values(heatmap) + + batch_size, num_joints = x.shape + + dz = torch.zeros((batch_size, 1, num_joints, 3)).to(x.device) + for b in range(batch_size): + for j in range(num_joints): + dz[b, 0, j, 2] = heatmap[b, y[b, j], x[b, j], j] + if locref is not None: + dz[b, 0, j, :2] = locref[b, y[b, j], x[b, j], j, :] + + x, y = torch.unsqueeze(x, 1), torch.unsqueeze(y, 1) + + x = x * scale_factors[1] + 0.5 * scale_factors[1] + dz[:, :, :, 0] + y = y * scale_factors[0] + 0.5 * scale_factors[0] + dz[:, :, :, 1] + + pose = torch.empty((batch_size, 1, num_joints, 3)) + pose[:, :, :, 0] = x + pose[:, :, :, 1] = y + pose[:, :, :, 2] = dz[:, :, :, 2] + return pose diff --git a/dlclive/pose_estimation_pytorch/models/registry.py b/dlclive/pose_estimation_pytorch/models/registry.py new file mode 100644 index 0000000..45ed735 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/models/registry.py @@ -0,0 +1,330 @@ +""" +DeepLabCut Toolbox (deeplabcut.org) +© A. & M. Mathis Labs + +Licensed under GNU Lesser General Public License v3.0 +""" + +import inspect +from functools import partial +from typing import Any, Dict, Optional + + +def build_from_cfg( + cfg: Dict, registry: "Registry", default_args: Optional[Dict] = None +) -> Any: + """Builds a module from the configuration dictionary when it represents a class configuration, + or call a function from the configuration dictionary when it represents a function configuration. + + Args: + cfg: Configuration dictionary. It should at least contain the key "type". + registry: The registry to search the type from. + default_args: Default initialization arguments. + Defaults to None. + + Returns: + Any: The constructed object. + + Example: + >>> from dlclive.models.registry import Registry, build_from_cfg + >>> class Model: + >>> def __init__(self, param): + >>> self.param = param + >>> cfg = {"type": "Model", "param": 10} + >>> registry = Registry("models") + >>> registry.register_module(Model) + >>> obj = build_from_cfg(cfg, registry) + >>> assert isinstance(obj, Model) + >>> assert obj.param == 10 + """ + + args = cfg.copy() + + if default_args is not None: + for name, value in default_args.items(): + args.setdefault(name, value) + + obj_type = args.pop("type") + if isinstance(obj_type, str): + obj_cls = registry.get(obj_type) + if obj_cls is None: + raise KeyError(f"{obj_type} is not in the {registry.name} registry") + elif inspect.isclass(obj_type) or inspect.isfunction(obj_type): + obj_cls = obj_type + else: + raise TypeError(f"type must be a str or valid type, but got {type(obj_type)}") + try: + return obj_cls(**args) + except Exception as e: + # Normal TypeError does not print class name. + raise type(e)(f"{obj_cls.__name__}: {e}") + + +class Registry: + """A registry to map strings to classes or functions. + Registered objects could be built from the registry. Meanwhile, registered + functions could be called from the registry. + + Args: + name: Registry name. + build_func: Builds function to construct an instance from + the Registry. If neither ``parent`` nor + ``build_func`` is specified, the ``build_from_cfg`` + function is used. If ``parent`` is specified and + ``build_func`` is not given, ``build_func`` will be + inherited from ``parent``. Default: None. + parent: Parent registry. The class registered in + children's registry could be built from the parent. + Default: None. + scope: The scope of the registry. It is the key to search + for children's registry. If not specified, scope will be the + name of the package where the class is defined, e.g. mmdet, mmcls, mmseg. + Default: None. + + Attributes: + name: Registry name. + module_dict: The dictionary containing registered modules. + children: The dictionary containing children registries. + scope: The scope of the registry. + """ + + def __init__(self, name, build_func=None, parent=None, scope=None): + self._name = name + self._module_dict = dict() + self._children = dict() + self._scope = "." + + if build_func is None: + if parent is not None: + self.build_func = parent.build_func + else: + self.build_func = build_from_cfg + else: + self.build_func = build_func + if parent is not None: + assert isinstance(parent, Registry) + parent._add_children(self) + self.parent = parent + else: + self.parent = None + + def __len__(self): + return len(self._module_dict) + + def __contains__(self, key): + return self.get(key) is not None + + def __repr__(self): + format_str = ( + self.__class__.__name__ + f"(name={self._name}, " + f"items={self._module_dict})" + ) + return format_str + + @staticmethod + def split_scope_key(key): + """Split scope and key. + The first scope will be split from key. + Examples: + >>> Registry.split_scope_key('mmdet.ResNet') + 'mmdet', 'ResNet' + >>> Registry.split_scope_key('ResNet') + None, 'ResNet' + Return: + tuple[str | None, str]: The former element is the first scope of + the key, which can be ``None``. The latter is the remaining key. + """ + split_index = key.find(".") + if split_index != -1: + return key[:split_index], key[split_index + 1 :] + else: + return None, key + + @property + def name(self): + return self._name + + @property + def scope(self): + return self._scope + + @property + def module_dict(self): + return self._module_dict + + @property + def children(self): + return self._children + + def get(self, key): + """Get the registry record. + + Args: + key: The class name in string format. + + Returns: + class: The corresponding class. + + Example: + >>> from dlclive.models.registry import Registry + >>> registry = Registry("models") + >>> class Model: + >>> pass + >>> registry.register_module(Model, "Model") + >>> assert registry.get("Model") == Model + """ + scope, real_key = self.split_scope_key(key) + if scope is None or scope == self._scope: + # get from self + if real_key in self._module_dict: + return self._module_dict[real_key] + else: + # get from self._children + if scope in self._children: + return self._children[scope].get(real_key) + else: + # goto root + parent = self.parent + while parent.parent is not None: + parent = parent.parent + return parent.get(key) + + def build(self, *args, **kwargs): + """Builds an instance from the registry. + + Args: + *args: Arguments passed to the build function. + **kwargs: Keyword arguments passed to the build function. + + Returns: + Any: The constructed object. + + Example: + >>> from dlclive.models.registry import Registry, build_from_cfg + >>> class Model: + >>> def __init__(self, param): + >>> self.param = param + >>> cfg = {"type": "Model", "param": 10} + >>> registry = Registry("models") + >>> registry.register_module(Model) + >>> obj = registry.build(cfg, param=20) + >>> assert isinstance(obj, Model) + >>> assert obj.param == 20 + """ + return self.build_func(*args, **kwargs, registry=self) + + def _add_children(self, registry): + """Add children for a registry. + + Args: + registry: The registry to be added as children based on its scope. + + Returns: + None + + Example: + >>> from dlclive.models.registry import Registry + >>> models = Registry('models') + >>> mmdet_models = Registry('models', parent=models) + >>> class Model: + >>> pass + >>> mmdet_models.register_module(Model) + >>> obj = models.build(dict(type='mmdet.Model')) + >>> assert isinstance(obj, Model) + """ + assert isinstance(registry, Registry) + assert registry.scope is not None + assert ( + registry.scope not in self.children + ), f"scope {registry.scope} exists in {self.name} registry" + self.children[registry.scope] = registry + + def _register_module(self, module, module_name=None, force=False): + """Register a module. + + Args: + module: Module class or function to be registered. + module_name: The module name(s) to be registered. + If not specified, the class name will be used. + force: Whether to override an existing class with the same name. + Default: False. + + Returns: + None + + Example: + >>> from dlclive.models.registry import Registry + >>> registry = Registry("models") + >>> class Model: + >>> pass + >>> registry._register_module(Model, "Model") + >>> assert registry.get("Model") == Model + """ + if not inspect.isclass(module) and not inspect.isfunction(module): + raise TypeError( + "module must be a class or a function, " f"but got {type(module)}" + ) + + if module_name is None: + module_name = module.__name__ + if isinstance(module_name, str): + module_name = [module_name] + for name in module_name: + if not force and name in self._module_dict: + raise KeyError(f"{name} is already registered " f"in {self.name}") + self._module_dict[name] = module + + def deprecated_register_module(self, cls=None, force=False): + """Decorator to register a class in the registry. + + Args: + cls: The class to be registered. + force: Whether to override an existing class with the same name. + Default: False. + + Returns: + type: The input class. + + Example: + >>> from dlclive.models.registry import Registry + >>> registry = Registry("models") + >>> @registry.deprecated_register_module() + >>> class Model: + >>> pass + >>> assert registry.get("Model") == Model + """ + if cls is None: + return partial(self.deprecated_register_module, force=force) + self._register_module(cls, force=force) + return cls + + def register_module(self, name=None, force=False, module=None): + """Register a module. + A record will be added to `self._module_dict`, whose key is the class + name or the specified name, and value is the class itself. + It can be used as a decorator or a normal function. + Args: + name: The module name to be registered. If not + specified, the class name will be used. + force: Whether to override an existing class with + the same name. Default: False. + module: Module class or function to be registered. + """ + if not isinstance(force, bool): + raise TypeError(f"force must be a boolean, but got {type(force)}") + # NOTE: This is a walkaround to be compatible with the old api, + # while it may introduce unexpected bugs. + if isinstance(name, type): + return self.deprecated_register_module(name, force=force) + + # use it as a normal method: x.register_module(module=SomeClass) + if module is not None: + self._register_module(module=module, module_name=name, force=force) + return module + + # use it as a decorator: @x.register_module() + def _register(module): + self._register_module(module=module, module_name=name, force=force) + return module + + return diff --git a/dlclive/pose_estimation_pytorch/runner.py b/dlclive/pose_estimation_pytorch/runner.py new file mode 100644 index 0000000..c7029d5 --- /dev/null +++ b/dlclive/pose_estimation_pytorch/runner.py @@ -0,0 +1,326 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""PyTorch and ONNX runners for DeepLabCut-Live""" +import copy +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +import numpy as np +import torch +from torchvision.transforms import v2 + +import dlclive.pose_estimation_pytorch.data as data +import dlclive.pose_estimation_pytorch.models as models +import dlclive.pose_estimation_pytorch.dynamic_cropping as dynamic_cropping +from dlclive.core.runner import BaseRunner + + +@dataclass +class SkipFrames: + """Configuration for skip frames. + + Skip-frames can be used for top-down models running with a detector. If skip > 0, + then the detector will only be run every `skip` frames. Between frames where the + detector is run, bounding boxes will be computed from the pose estimated in the + previous frame. + """ + skip: int + margin: int + _age: int = 0 + _detections: dict[str, torch.Tensor] | None = None + + def get_detections(self) -> dict[str, torch.Tensor] | None: + return self._detections + + def update(self, pose: torch.Tensor, w: int, h: int) -> None: + """Generates bounding boxes from a pose. + + Args: + pose: The pose from which to generate bounding boxes. + w: The width of the image. + h: The height of the image. + + Returns: + A dictionary containing the bounding boxes and scores for each detection. + """ + if self._age >= self.skip: + self._age = 0 + self._detections = None + return + + num_det, num_kpts = pose.shape[:2] + size = max(w, h) + + bboxes = torch.zeros((num_det, 4)) + bboxes[:, :2] = ( + torch.min(torch.nan_to_num(pose, size)[..., :2], dim=1)[0] - self.margin + ) + bboxes[:, 2:4] = ( + torch.max(torch.nan_to_num(pose, 0)[..., :2], dim=1)[0] + self.margin + ) + bboxes = torch.clip(bboxes, min=torch.zeros(4), max=torch.tensor([w, h, w, h])) + self._detections = dict(boxes=bboxes, scores=torch.ones(num_det)) + self._age += 1 + + +@dataclass +class TopDownConfig: + """Configuration for top-down models. + + Attributes: + skip_frames: If defined, the detector will only be run every + `skip_frames.skip` frames. + """ + bbox_cutoff: float + max_detections: int + crop_size: tuple[int, int] = (256, 256) + skip_frames: SkipFrames | None = None + + def read_config(self, detector_cfg: dict) -> None: + crop = detector_cfg.get("data", {}).get("inference", {}).get("top_down_crop") + if crop is not None: + self.crop_size = (crop["width"], crop["height"]) + + +class PyTorchRunner(BaseRunner): + """PyTorch runner for live pose estimation using DeepLabCut-Live. + + Args: + path: The path to the model to run inference with. + device: The device on which to run inference, e.g. "cpu", "cuda", "cuda:0" + precision: The precision of the model. One of "FP16" or "FP32". + single_animal: This option is only available for single-animal pose estimation + models. It makes the code behave in exactly the same way as DeepLabCut-Live + with version < 3.0.0. This ensures backwards compatibility with any + Processors that were implemented. + dynamic: Whether to use dynamic cropping. + top_down_config: Only for top-down models running with a detector. + """ + + def __init__( + self, + path: str | Path, + device: str = "auto", + precision: Literal["FP16", "FP32"] = "FP32", + single_animal: bool = True, + bbox_cutoff: float = 0.6, # FIXME(niels) + max_detections: int | None = None, + dynamic: dynamic_cropping.DynamicCropper | None = None, + top_down_config: TopDownConfig | None = None, + ) -> None: + super().__init__(path) + self.device = _parse_device(device) + self.precision = precision + self.single_animal = single_animal + + self.cfg = None + self.detector = None + self.model = None + self.transform = None + + self.dynamic = dynamic + self.top_down_config = top_down_config + + def close(self) -> None: + """Clears any resources used by the runner.""" + pass + + @torch.inference_mode() + def get_pose(self, frame: np.ndarray) -> np.ndarray: + c, h, w = frame.shape + frame = ( + self.transform(torch.from_numpy(frame).permute(2, 0, 1)) + .unsqueeze(0) + .to(self.device) + ) + if self.precision == "FP16": + frame = frame.half() + + offsets_and_scales = None + if self.detector is not None: + detections = None + if self.top_down_config.skip_frames is not None: + detections = self.top_down_config.skip_frames.get_detections() + + if detections is None: + detections = self.detector(frame)[0] + + frame_batch, offsets_and_scales = self._prepare_top_down(frame, detections) + if len(frame_batch) == 0: + offsets_and_scales = [(0, 0), 1] + else: + frame = frame_batch.to(self.device) + + if self.dynamic is not None: + frame = self.dynamic.crop(frame) + + outputs = self.model(frame) + batch_pose = self.model.get_predictions(outputs)["bodypart"]["poses"] + + if self.dynamic is not None: + batch_pose = self.dynamic.update(batch_pose) + + if self.detector is None: + pose = batch_pose[0] + else: + pose = self._postprocess_top_down(batch_pose, offsets_and_scales) + if self.top_down_config.skip_frames is not None: + self.top_down_config.skip_frames.update(pose, w, h) + + if self.single_animal: + if len(pose) == 0: + bodyparts, coords = pose.shape[-2:] + return np.zeros((bodyparts, coords)) + + pose = pose[0] + + return pose.cpu().numpy() + + def init_inference(self, frame: np.ndarray, **kwargs) -> np.ndarray: + """ + Initializes inference process on the provided frame. + + This method serves as an abstract base method, meant to be implemented by + subclasses. It takes an input image frame and optional additional parameters + to set up and perform inference. The method must return a processed result + as a numpy array. + + Parameters + ---------- + frame : np.ndarray + The input image frame for which inference needs to be set up. + kwargs : dict, optional + Additional parameters that may be required for specific implementation + of the inference initialization. + + Returns + ------- + np.ndarray + The result of the inference after being initialized and processed. + """ + self.load_model() + return self.get_pose(frame) + + def load_model(self) -> None: + """Loads the model from the provided path.""" + raw_data = torch.load(self.path, map_location="cpu", weights_only=True) + + self.cfg = raw_data["config"] + self.model = models.PoseModel.build(self.cfg["model"]) + self.model.load_state_dict(raw_data["pose"]) + self.model = self.model.to(self.device) + self.model.eval() + + if self.precision == "FP16": + self.model = self.model.half() + + self.detector = None + if raw_data.get("detector") is not None: + self.detector = models.DETECTORS.build(self.cfg["detector"]["model"]) + self.detector.to(self.device) + self.detector.load_state_dict(raw_data["detector"]) + self.detector.eval() + + if self.precision == "FP16": + self.detector = self.detector.half() + + if self.cfg["method"] == "td" and self.detector is None: + crop_cfg = self.cfg["data"]["inference"]["top_down_crop"] + top_down_crop_size = crop_cfg["width"], crop_cfg["height"] + self.dynamic = dynamic_cropping.TopDownDynamicCropper( + top_down_crop_size, + patch_counts=(4, 3), + patch_overlap=50, + min_bbox_size=(250, 250), + threshold=0.6, + margin=25, + min_hq_keypoints=2, + bbox_from_hq=True, + ) + + self.transform = v2.Compose( + [ + v2.ToDtype(torch.float32, scale=True), + v2.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ] + ) + + def read_config(self) -> dict: + """Reads the configuration file""" + if self.cfg is not None: + return copy.deepcopy(self.cfg) + + raw_data = torch.load(self.path, map_location="cpu", weights_only=True) + return raw_data["config"] + + def _prepare_top_down( + self, frame: torch.Tensor, detections: dict[str, torch.Tensor] + ): + bboxes, scores = detections["boxes"], detections["scores"] + bboxes = bboxes[scores >= self.top_down_config.bbox_cutoff] + if len(bboxes) > 0: + bboxes = bboxes[: self.top_down_config.max_detections] + + crops = [] + offsets_and_scales = [] + for bbox in bboxes: + x1, y1, x2, y2 = bbox + cropped_frame, offset, scale = data.top_down_crop_torch( + frame[0], + (x1, y1, x2 - x1, y2 - y1), + output_size=self.top_down_config.crop_size, + margin=0, + ) + crops.append(cropped_frame) + offsets_and_scales.append((offset, scale)) + + if len(crops) > 0: + frame_batch = torch.stack(crops, dim=0) + else: + crop_w, crop_h = self.top_down_config.crop_size + frame_batch = torch.zeros((0, 3, crop_h, crop_w), device=frame.device) + offsets_and_scales = [(0, 0), 1] + + return frame_batch, offsets_and_scales + + def _postprocess_top_down( + self, + batch_pose: torch.Tensor, + offsets_and_scales: list[tuple[tuple[int, int], tuple[float, float]]], + ) -> torch.Tensor: + """Post-processes pose for top-down models.""" + if len(batch_pose) == 0: + bodyparts, coords = batch_pose.shape[-2:] + return torch.zeros((0, bodyparts, coords)) + + poses = [] + for pose, (offset, scale) in zip(batch_pose, offsets_and_scales): + poses.append( + torch.cat( + [pose[..., :2] * torch.tensor(scale) + torch.tensor(offset), pose[..., 2:3]], + dim=-1, + ) + ) + + return torch.cat(poses) + + +def _parse_device(device: str | None) -> str: + if device is None: + device = "auto" + + if device == "auto": + if torch.cuda.is_available(): + return "cuda" + return "cpu" + + return device diff --git a/dlclive/pose_estimation_tensorflow/__init__.py b/dlclive/pose_estimation_tensorflow/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dlclive/pose_estimation_tensorflow/graph.py b/dlclive/pose_estimation_tensorflow/graph.py new file mode 100644 index 0000000..4cc3d40 --- /dev/null +++ b/dlclive/pose_estimation_tensorflow/graph.py @@ -0,0 +1,138 @@ +""" +DeepLabCut Toolbox (deeplabcut.org) +© A. & M. Mathis Labs + +Licensed under GNU Lesser General Public License v3.0 +""" + + +import tensorflow as tf + +vers = (tf.__version__).split(".") +if int(vers[0]) == 2 or int(vers[0]) == 1 and int(vers[1]) > 12: + tf = tf.compat.v1 +else: + tf = tf + + +def read_graph(file): + """ + Loads the graph from a protobuf file + + Parameters + ----------- + file : string + path to the protobuf file + + Returns + -------- + graph_def :class:`tensorflow.tf.compat.v1.GraphDef` + The graph definition of the DeepLabCut model found at the object's path + """ + + with tf.io.gfile.GFile(file, "rb") as f: + graph_def = tf.GraphDef() + graph_def.ParseFromString(f.read()) + return graph_def + + +def finalize_graph(graph_def): + """ + Finalize the graph and get inputs to model + + Parameters + ----------- + graph_def :class:`tensorflow.compat.v1.GraphDef` + The graph of the DeepLabCut model, read using the :func:`read_graph` method + + Returns + -------- + graph :class:`tensorflow.compat.v1.GraphDef` + The finalized graph of the DeepLabCut model + inputs :class:`tensorflow.Tensor` + Input tensor(s) for the model + """ + + graph = tf.Graph() + with graph.as_default(): + tf.import_graph_def(graph_def, name="DLC") + graph.finalize() + + return graph + + +def get_output_nodes(graph): + """ + Get the output node names from a graph + + Parameters + ----------- + graph :class:`tensorflow.Graph` + The graph of the DeepLabCut model + + Returns + -------- + output : list + the output node names as a list of strings + """ + + op_names = [str(op.name) for op in graph.get_operations()] + if "concat_1" in op_names[-1]: + output = [op_names[-1]] + else: + output = [op_names[-1], op_names[-2]] + + return output + + +def get_output_tensors(graph): + """ + Get the names of the output tensors from a graph + + Parameters + ----------- + graph :class:`tensorflow.Graph` + The graph of the DeepLabCut model + + Returns + -------- + output : list + the output tensor names as a list of strings + """ + + output_nodes = get_output_nodes(graph) + output_tensor = [out + ":0" for out in output_nodes] + return output_tensor + + +def get_input_tensor(graph): + + input_tensor = str(graph.get_operations()[0].name) + ":0" + return input_tensor + + +def extract_graph(graph, tf_config=None) -> tuple[tf.Session, tf.Tensor, list[tf.Tensor]]: + """ + Initializes a tensorflow session with the specified graph and extracts the model's inputs and outputs + + Parameters + ----------- + graph :class:`tensorflow.Graph` + a tensorflow graph containing the desired model + tf_config :class:`tensorflow.ConfigProto` + + Returns + -------- + sess :class:`tensorflow.Session` + a tensorflow session with the specified graph definition + outputs :class:`tensorflow.Tensor` + the output tensor(s) for the model + """ + + input_tensor = get_input_tensor(graph) + output_tensor = get_output_tensors(graph) + sess = tf.Session(graph=graph, config=tf_config) + inputs = graph.get_tensor_by_name(input_tensor) + outputs = [graph.get_tensor_by_name(out) for out in output_tensor] + + return sess, inputs, outputs diff --git a/dlclive/pose_estimation_tensorflow/pose.py b/dlclive/pose_estimation_tensorflow/pose.py new file mode 100644 index 0000000..3e69bb9 --- /dev/null +++ b/dlclive/pose_estimation_tensorflow/pose.py @@ -0,0 +1,120 @@ +""" +DeepLabCut Toolbox (deeplabcut.org) +© A. & M. Mathis Labs + +Licensed under GNU Lesser General Public License v3.0 +""" + + +import numpy as np + + +def extract_cnn_output(outputs, cfg): + """ + Extract location refinement and score map from DeepLabCut network + + Parameters + ----------- + outputs : list + List of outputs from DeepLabCut network. + Requires 2 entries: + index 0 is output from Sigmoid + index 1 is output from pose/locref_pred/block4/BiasAdd + + cfg : dict + Dictionary read from the pose_cfg.yaml file for the network. + + Returns + -------- + scmap : ? + score map + + locref : ? + location refinement + """ + + scmap = outputs[0] + scmap = np.squeeze(scmap) + locref = None + if cfg["location_refinement"]: + locref = np.squeeze(outputs[1]) + shape = locref.shape + locref = np.reshape(locref, (shape[0], shape[1], -1, 2)) + locref *= cfg["locref_stdev"] + if len(scmap.shape) == 2: # for single body part! + scmap = np.expand_dims(scmap, axis=2) + return scmap, locref + + +def argmax_pose_predict(scmap, offmat, stride): + """ + Combines score map and offsets to the final pose + + Parameters + ----------- + scmap : ? + score map + + offmat : ? + offsets + + stride : ? + ? + + Returns + -------- + pose :class:`numpy.ndarray` + pose as a numpy array + """ + + num_joints = scmap.shape[2] + pose = [] + for joint_idx in range(num_joints): + maxloc = np.unravel_index( + np.argmax(scmap[:, :, joint_idx]), scmap[:, :, joint_idx].shape + ) + offset = np.array(offmat[maxloc][joint_idx])[::-1] + pos_f8 = np.array(maxloc).astype("float") * stride + 0.5 * stride + offset + pose.append(np.hstack((pos_f8[::-1], [scmap[maxloc][joint_idx]]))) + return np.array(pose) + + +def get_top_values(scmap, n_top=5): + batchsize, ny, nx, num_joints = scmap.shape + scmap_flat = scmap.reshape(batchsize, nx * ny, num_joints) + if n_top == 1: + scmap_top = np.argmax(scmap_flat, axis=1)[None] + else: + scmap_top = np.argpartition(scmap_flat, -n_top, axis=1)[:, -n_top:] + for ix in range(batchsize): + vals = scmap_flat[ix, scmap_top[ix], np.arange(num_joints)] + arg = np.argsort(-vals, axis=0) + scmap_top[ix] = scmap_top[ix, arg, np.arange(num_joints)] + scmap_top = scmap_top.swapaxes(0, 1) + + Y, X = np.unravel_index(scmap_top, (ny, nx)) + return Y, X + + +def multi_pose_predict(scmap, locref, stride, num_outputs): + Y, X = get_top_values(scmap[None], num_outputs) + Y, X = Y[:, 0], X[:, 0] + num_joints = scmap.shape[2] + DZ = np.zeros((num_outputs, num_joints, 3)) + for m in range(num_outputs): + for k in range(num_joints): + x = X[m, k] + y = Y[m, k] + DZ[m, k, :2] = locref[y, x, k, :] + DZ[m, k, 2] = scmap[y, x, k] + + X = X.astype("float32") * stride + 0.5 * stride + DZ[:, :, 0] + Y = Y.astype("float32") * stride + 0.5 * stride + DZ[:, :, 1] + P = DZ[:, :, 2] + + pose = np.empty((num_joints, num_outputs * 3), dtype="float32") + pose[:, 0::3] = X.T + pose[:, 1::3] = Y.T + pose[:, 2::3] = P.T + + return pose diff --git a/dlclive/pose_estimation_tensorflow/runner.py b/dlclive/pose_estimation_tensorflow/runner.py new file mode 100644 index 0000000..3fa4eec --- /dev/null +++ b/dlclive/pose_estimation_tensorflow/runner.py @@ -0,0 +1,212 @@ +# +# DeepLabCut Toolbox (deeplabcut.org) +# © A. & M.W. Mathis Labs +# https://github.com/DeepLabCut/DeepLabCut +# +# Please see AUTHORS for contributors. +# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS +# +# Licensed under GNU Lesser General Public License v3.0 +# +"""TensorFlow runners for DeepLabCut-Live""" +import glob +import os +from pathlib import Path +from typing import Any + +import numpy as np +import tensorflow as tf + +from dlclive.core.config import read_yaml +from dlclive.core.runner import BaseRunner +from dlclive.exceptions import DLCLiveError +from dlclive.pose_estimation_tensorflow.graph import ( + extract_graph, + finalize_graph, + get_output_nodes, + get_output_tensors, + read_graph, +) +from dlclive.pose_estimation_tensorflow.pose import ( + argmax_pose_predict, + extract_cnn_output, + multi_pose_predict, +) + + +class TensorFlowRunner(BaseRunner): + """TensorFlow runner for live pose estimation using DeepLabCut-Live. + + Args: + path: The path to the model to run inference with. + + Attributes: + path: The path to the model to run inference with. + """ + + def __init__( + self, + path: str | Path, + model_type: str = "base", + tf_config: Any = None, + ) -> None: + super().__init__(path) + self.cfg = self.read_config() + self.model_type = model_type + self.tf_config = tf_config + self.precision = "FP32" + self.sess = None + self.inputs = None + self.outputs = None + self.tflite_interpreter = None + + def close(self) -> None: + """Clears any resources used by the runner.""" + if self.sess is not None: + self.sess.close() + self.sess = None + + def get_pose(self, frame: np.ndarray, **kwargs) -> np.ndarray: + if self.model_type in ["base", "tensorrt"]: + pose_output = self.sess.run( + self.outputs, feed_dict={self.inputs: np.expand_dims(frame, axis=0)} + ) + + elif self.model_type == "tflite": + self.tflite_interpreter.set_tensor( + self.inputs[0]["index"], + np.expand_dims(frame, axis=0).astype(np.float32), + ) + self.tflite_interpreter.invoke() + + if len(self.outputs) > 1: + pose_output = [ + self.tflite_interpreter.get_tensor(self.outputs[0]["index"]), + self.tflite_interpreter.get_tensor(self.outputs[1]["index"]), + ] + else: + pose_output = self.tflite_interpreter.get_tensor( + self.outputs[0]["index"] + ) + + else: + raise DLCLiveError( + f"model_type={self.model_type} is not supported. model_type must be " + f"'base', 'tflite', or 'tensorrt'" + ) + + # check if using TFGPUinference flag + # if not, get pose from network output + if len(pose_output) > 1: + scmap, locref = extract_cnn_output(pose_output, self.cfg) + num_outputs = self.cfg.get("num_outputs", 1) + if num_outputs > 1: + pose = multi_pose_predict( + scmap, locref, self.cfg["stride"], num_outputs + ) + else: + pose = argmax_pose_predict(scmap, locref, self.cfg["stride"]) + else: + pose = np.array(pose_output[0]) + pose = pose[:, [1, 0, 2]] + + return pose + + def init_inference(self, frame: np.ndarray, **kwargs) -> np.ndarray: + model_file = glob.glob(os.path.normpath(str(self.path) + "/*.pb"))[0] + + tf_ver = tf.__version__ + tf_version_2 = tf_ver[0] == "2" + + # load model + if self.model_type == "base": + graph_def = read_graph(model_file) + graph = finalize_graph(graph_def) + self.sess, self.inputs, self.outputs = extract_graph( + graph, tf_config=self.tf_config + ) + + elif self.model_type == "tflite": + ### + # the frame size needed to initialize the tflite model as + # tflite does not support saving a model with dynamic input size + ### + + # get input and output tensor names from graph_def + graph_def = read_graph(model_file) + graph = finalize_graph(graph_def) + output_nodes = get_output_nodes(graph) + output_nodes = [on.replace("DLC/", "") for on in output_nodes] + placeholder_shape = [1, frame.shape[0], frame.shape[1], 3] + + if tf_version_2: + converter = tf.compat.v1.lite.TFLiteConverter.from_frozen_graph( + model_file, + ["Placeholder"], + output_nodes, + input_shapes={"Placeholder": placeholder_shape}, + ) + else: + converter = tf.lite.TFLiteConverter.from_frozen_graph( + model_file, + ["Placeholder"], + output_nodes, + input_shapes={"Placeholder": placeholder_shape}, + ) + + try: + tflite_model = converter.convert() + except Exception: + raise DLCLiveError( + ( + "This model cannot be converted to tensorflow lite format. " + "To use tensorflow lite for live inference, " + "make sure to set TFGPUinference=False " + "when exporting the model from DeepLabCut" + ) + ) + + self.tflite_interpreter = tf.lite.Interpreter(model_content=tflite_model) + self.tflite_interpreter.allocate_tensors() + self.inputs = self.tflite_interpreter.get_input_details() + self.outputs = self.tflite_interpreter.get_output_details() + + elif self.model_type == "tensorrt": + + graph_def = read_graph(model_file) + graph = finalize_graph(graph_def) + output_tensors = get_output_tensors(graph) + output_tensors = [ot.replace("DLC/", "") for ot in output_tensors] + + if (tf_ver[0] > 1) | (tf_ver[0] == 1 & tf_ver[1] >= 14): + converter = trt.TrtGraphConverter( + input_graph_def=graph_def, + nodes_blacklist=output_tensors, + is_dynamic_op=True, + ) + graph_def = converter.convert() + else: + graph_def = trt.create_inference_graph( + input_graph_def=graph_def, + outputs=output_tensors, + max_batch_size=1, + precision_mode=self.precision, + is_dynamic_op=True, + ) + + graph = finalize_graph(graph_def) + self.sess, self.inputs, self.outputs = extract_graph( + graph, tf_config=self.tf_config + ) + + else: + raise DLCLiveError( + f"model_type={self.model_type} is not supported. model_type must be " + "'base', 'tflite', or 'tensorrt'" + ) + + return self.get_pose(frame, **kwargs) + + def read_config(self) -> dict: + """Reads the configuration file""" + return read_yaml(self.path / "pose_cfg.yaml") diff --git a/dlclive/predictor/base.py b/dlclive/predictor/base.py index 8a15194..f8b8b98 100644 --- a/dlclive/predictor/base.py +++ b/dlclive/predictor/base.py @@ -1,37 +1,35 @@ -# -# DeepLabCut Toolbox (deeplabcut.org) -# © A. & M.W. Mathis Labs -# https://github.com/DeepLabCut/DeepLabCut -# -# Please see AUTHORS for contributors. -# https://github.com/DeepLabCut/DeepLabCut/blob/main/AUTHORS -# -# Licensed under GNU Lesser General Public License v3.0 -# +""" +DeepLabCut Toolbox (deeplabcut.org) +© A. & M. Mathis Labs + +Licensed under GNU Lesser General Public License v3.0 +""" + from __future__ import annotations from abc import ABC, abstractmethod import torch -from deeplabcut.pose_estimation_pytorch.registry import (Registry, - build_from_cfg) from torch import nn +from dlclive.pose_estimation_pytorch.models.registry import Registry, build_from_cfg + PREDICTORS = Registry("predictors", build_func=build_from_cfg) class BasePredictor(ABC, nn.Module): """The base Predictor class. - This class is an abstract base class (ABC) for defining predictors used in the DeepLabCut Toolbox. - All predictor classes should inherit from this base class and implement the forward method. - Regresses keypoint coordinates from a models output maps + This class is an abstract base class (ABC) for defining predictors used in the + DeepLabCut Toolbox. All predictor classes should inherit from this base class and + implement the forward method. Regresses keypoint coordinates from a models output + maps Attributes: num_animals: Number of animals in the project. Should be set in subclasses. Example: - # Create a subclass that inherits from BasePredictor and implements the forward method. + # Create a subclass that inherits from BasePredictor class MyPredictor(BasePredictor): def __init__(self, num_animals): super().__init__() diff --git a/dlclive/predictor/single_predictor.py b/dlclive/predictor/single_predictor.py index 81b443e..96f31d9 100644 --- a/dlclive/predictor/single_predictor.py +++ b/dlclive/predictor/single_predictor.py @@ -13,10 +13,11 @@ from typing import Tuple import torch -from deeplabcut.pose_estimation_pytorch.models.predictors.base import \ - BasePredictor +from dlclive.pose_estimation_pytorch.models.predictors.base import BasePredictor, PREDICTORS + +@PREDICTORS.register_module class HeatmapPredictor(BasePredictor): """Predictor class for pose estimation from heatmaps (and optionally locrefs). @@ -35,7 +36,6 @@ def __init__( clip_scores: bool = False, location_refinement: bool = True, locref_std: float = 7.2801, - stride: float = 8.0, ): """ Args: @@ -51,9 +51,10 @@ def __init__( self.sigmoid = torch.nn.Sigmoid() self.location_refinement = location_refinement self.locref_std = locref_std - self.stride = stride - def forward(self, outputs: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: + def forward( + self, stride: float, outputs: dict[str, torch.Tensor] + ) -> dict[str, torch.Tensor]: """Forward pass of SinglePredictor. Gets predictions from model output. Args: @@ -70,7 +71,7 @@ def forward(self, outputs: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: >>> poses = predictor.forward(stride, output) """ heatmaps = outputs["heatmap"] - scale_factors = self.stride, self.stride + scale_factors = stride, stride if self.apply_sigmoid: heatmaps = self.sigmoid(heatmaps) @@ -121,15 +122,17 @@ def get_pose_prediction( """Gets the pose prediction given the heatmaps and locref. Args: - heatmap: Heatmap tensor with the following format (batch_size, height, width, num_joints) - locref: Locref tensor with the following format (batch_size, height, width, num_joints, 2) + heatmap: Heatmap tensor of shape (batch_size, height, width, num_joints) + locref: Locref tensor of shape (batch_size, height, width, num_joints, 2) scale_factors: Scale factors for the poses. Returns: Pose predictions of the format: (batch_size, num_people = 1, num_joints, 3) Example: - >>> predictor = HeatmapPredictor(location_refinement=True, locref_std=7.2801) + >>> predictor = HeatmapPredictor( + >>> location_refinement=True, locref_std=7.2801 + >>> ) >>> heatmap = torch.rand(32, 17, 64, 64) >>> locref = torch.rand(32, 17, 64, 64, 2) >>> scale_factors = (0.5, 0.5) @@ -147,45 +150,12 @@ def get_pose_prediction( dz[b, 0, j, :2] = locref[b, y[b, j], x[b, j], j, :] x, y = torch.unsqueeze(x, 1), torch.unsqueeze(y, 1) + x = x * scale_factors[1] + 0.5 * scale_factors[1] + dz[:, :, :, 0] y = y * scale_factors[0] + 0.5 * scale_factors[0] + dz[:, :, :, 1] + pose = torch.empty((batch_size, 1, num_joints, 3)) pose[:, :, :, 0] = x pose[:, :, :, 1] = y pose[:, :, :, 2] = dz[:, :, :, 2] return pose - - @staticmethod - def build(cfg: dict) -> HeatmapPredictor: - # if cfg["method"] == "bu": - apply_sigmoid = cfg["model"]["heads"]["bodypart"]["predictor"]["apply_sigmoid"] - clip_scores = cfg["model"]["heads"]["bodypart"]["predictor"]["clip_scores"] - loc_ref = cfg["model"]["heads"]["bodypart"]["predictor"]["location_refinement"] - loc_ref_std = cfg["model"]["heads"]["bodypart"]["predictor"]["locref_std"] - if len(cfg["model"]["heads"]["bodypart"]["heatmap_config"]["strides"]) > 0: - if cfg["model"]["heads"]["bodypart"]["heatmap_config"]["strides"][0] > 0: - stride = float(cfg["model"]["backbone"]["output_stride"]) / float( - cfg["model"]["heads"]["bodypart"]["heatmap_config"]["strides"][0] - ) - else: - stride = float(cfg["model"]["backbone"]["output_stride"]) * -float( - cfg["model"]["heads"]["bodypart"]["heatmap_config"]["strides"][0] - ) - else: - stride = float(cfg["model"]["backbone"]["output_stride"]) - predictor = HeatmapPredictor( - apply_sigmoid=apply_sigmoid, - stride=stride, - clip_scores=clip_scores, - location_refinement=loc_ref, - locref_std=loc_ref_std, - ) - - # elif cfg["method"] == "td": - # apply_sigmoid = cfg["model"]["heads"]["bodypart"]["predictor"]["apply_sigmoid"] - # clip_scores = cfg["model"]["heads"]["bodypart"]["predictor"]["clip_scores"] - # loc_ref = cfg["model"]["heads"]["bodypart"]["predictor"]["location_refinement"] - # heatmap_stride = cfg[] - # predictor = HeatmapPredictor(apply_sigmoid=apply_sigmoid, clip_scores=clip_scores, location_refinement=loc_ref) - - return predictor diff --git a/dlclive/processor/__init__.py b/dlclive/processor/__init__.py index a360448..657b405 100644 --- a/dlclive/processor/__init__.py +++ b/dlclive/processor/__init__.py @@ -4,3 +4,4 @@ Licensed under GNU Lesser General Public License v3.0 """ +from dlclive.processor.processor import Processor diff --git a/dlclive/processor/processor.py b/dlclive/processor/processor.py index 8a52f5f..8bd28de 100644 --- a/dlclive/processor/processor.py +++ b/dlclive/processor/processor.py @@ -12,7 +12,7 @@ """ -class Processor(object): +class Processor: def __init__(self, **kwargs): pass diff --git a/dlclive/utils.py b/dlclive/utils.py index 657011b..615758a 100644 --- a/dlclive/utils.py +++ b/dlclive/utils.py @@ -13,16 +13,14 @@ try: import skimage - SK_IM = True -except Exception: +except ImportError as e: SK_IM = False try: import cv2 - OPEN_CV = True -except Exception: +except ImportError as e: from PIL import Image OPEN_CV = False @@ -32,18 +30,18 @@ ) -def convert_to_ubyte(frame): +def convert_to_ubyte(frame: np.ndarray) -> np.ndarray: """Converts an image to unsigned 8-bit integer numpy array. If scikit-image is installed, uses skimage.img_as_ubyte, otherwise, uses a similar custom function. Parameters ---------- - image : :class:`numpy.ndarray` + frame: an image as a numpy array Returns ------- - :class:`numpy.ndarray` + :class: `numpy.ndarray` image converted to uint8 """ @@ -53,21 +51,21 @@ def convert_to_ubyte(frame): return _img_as_ubyte_np(frame) -def resize_frame(frame, resize=None): +def resize_frame(frame: np.ndarray, resize=None) -> np.ndarray: """Resizes an image. Uses OpenCV if installed, otherwise, uses pillow Parameters ---------- - image : :class:`numpy.ndarray` + frame: an image as a numpy array """ if (resize is not None) and (resize != 1): + new_x = int(frame.shape[0] * resize) + new_y = int(frame.shape[1] * resize) if OPEN_CV: - new_x = int(frame.shape[0] * resize) - new_y = int(frame.shape[1] * resize) return cv2.resize(frame, (new_y, new_x)) else: @@ -81,7 +79,7 @@ def resize_frame(frame, resize=None): return frame -def img_to_rgb(frame): +def img_to_rgb(frame: np.ndarray) -> np.ndarray: """Convert an image to RGB. Uses OpenCV is installed, otherwise uses pillow. Parameters @@ -107,7 +105,7 @@ def img_to_rgb(frame): return frame -def gray_to_rgb(frame): +def gray_to_rgb(frame: np.ndarray) -> np.ndarray: """Convert an image from grayscale to RGB. Uses OpenCV is installed, otherwise uses pillow. Parameters @@ -127,7 +125,7 @@ def gray_to_rgb(frame): return np.asarray(img) -def bgr_to_rgb(frame): +def bgr_to_rgb(frame: np.ndarray) -> np.ndarray: """Convert an image from BGR to RGB. Uses OpenCV is installed, otherwise uses pillow. Parameters @@ -147,7 +145,7 @@ def bgr_to_rgb(frame): return np.asarray(img) -def _img_as_ubyte_np(frame): +def _img_as_ubyte_np(frame: np.ndarray) -> np.ndarray: """Converts an image as a numpy array to unsinged 8-bit integer. As in scikit-image img_as_ubyte, converts negative pixels to 0 and converts range to [0, 255] diff --git a/dlclive/version.py b/dlclive/version.py index a35486f..a2047a2 100644 --- a/dlclive/version.py +++ b/dlclive/version.py @@ -6,5 +6,5 @@ Licensed under GNU Lesser General Public License v3.0 """ -__version__ = "1.0.4" +__version__ = "3.0.0a0" VERSION = __version__ diff --git a/docs/DLC Live Benchmark.md b/docs/DLC Live Benchmark.md new file mode 100755 index 0000000..583e9f2 --- /dev/null +++ b/docs/DLC Live Benchmark.md @@ -0,0 +1,32 @@ +## Inference time + +| System | Model type | Runtime | Device type | Precision | Video | Video length (s) - # Frames | FPS | Frame size | Display settings | Pose model backbone | Avg Inference time ± Std
*(including 1st inference)* | Avg Inference time ± Std | Average FPS ± Std | Model size | +| ------ | ---------- | -------- | ----------- | -------------------------------------- | ------------ | --------------------------- | --- | ---------- | ---------------- | ---------------------------------- | -------------------------------------------------------- | ------------------------ | ----------------- | ---------- | +| Linux | ONNX | ONNX | CUDA | Full precision (FP32) | Ventral gait | 10s - 1.5k | 150 | (658,302) | None | `ResNet50` (bu) | 29.02ms ± 47.59ms | 27.8ms ± 2.32ms | 36 ± 3 | 92.12 MB | +| Linux | ONNX | ONNX | CPU | Full precision (FP32) | Ventral gait | 10s - 1.5k | 150 | (658,302) | None | `ResNet50` (bu) | 146.12ms ± 13.26ms | 146.11 ± 13.25 | 7 ± 1 | 92.12 MB | +| Linux | PyTorch | PyTorch | CUDA | Full precision (FP32) | Ventral gait | 10s - 1.5k | 150 | (658,302) | None | `ResNet50` (bu) | 6.04ms ± 7.37ms | 5.97ms ± 6.8ms | 271 ± 112 | 96.5 MB | +| Linux | PyTorch | PyTorch | CPU | Full precision (FP32) | Ventral gait | 10s - 1.5k | 150 | (658,302) | None | `ResNet50` (bu) | 365.26ms ± 13.88ms | 365.17ms ± 13.44ms | 3 ± 0 | 96.5 MB | +| Linux | ONNX | TensorRT | CUDA | Full precision (FP32) - no caching | Ventral gait | 10s - 1.5k | 150 | (658,302) | None | `ResNet50` (bu) | 55.32ms ± 1254.16ms | 22.93ms ± 0.88 | 44 ± 2 | 92.12 MB | +| Linux | ONNX | TensorRT | CUDA | Full precision (FP32) - engine caching | Ventral gait | 10s - 1.5k | 150 | (658,302) | None | `ResNet50` (bu) | 20.8ms ± 3.4ms | 20.72ms ± 1.25ms | 48 ± 3 | 92.12 MB | +| Linux | ONNX | TensorRT | CUDA | FP16 | Ventral gait | 10s - 1.5k | 150 | (658,302) | None | `ResNet50` (bu) | 34.37ms ± 858.96ms | 12.19ms ± 0.87 | 82 ± 6 | 46.16 MB | +| Linux | ONNX | ONNX | CUDA | FP16 | Ventral gait | 10s - 1.5k | 150 | (658,302) | None | `ResNet50` (bu) | 21.74ms ± 43.24ms | 20.62ms ± 2.5ms | 49 ± 5 | 46.16 MB | +| Linux | PyTorch | PyTorch | CUDA | FP32 | Ventral gait | 10s - 1.5k | 150 | (164,75) | Resize=0.25 | `ResNet50` (bu) | 22.27ms ± 12.5ms | 22.16ms ± 11.65ms | 70 ± 68 | 96.5 MB | +| Linux | ONNX | ONNX | CUDA | (FP32) | Ventral gait | 10s - 1.5k | 150 | (164,75) | Resize=0.25 | `ResNet50` (bu) | 6.18ms ± 37.03ms | 5.22ms ± 0.86ms | 195 ± 25 | | +| Linux | ONNX | ONNX | CPU | (FP32) | Ventral gait | 10s - 1.5k | 150 | (164,75) | Resize=0.25 | `ResNet50` (bu) | 13.17ms ± 1.25ms | 13.17ms ± 1.23ms | 76 ± 4 | | +| Linux | ONNX | TensorRT | CUDA | (FP32) | Ventral gait | 10s - 1.5k | 150 | (164,75) | Resize=0.25 | `ResNet50` (bu) | 15.12ms ± 458.27ms | 3.28ms ± 0.24ms | 306 ± 23 | | +| Linux | ONNX | ONNX | CUDA | FP16 | Ventral gait | 10s - 1.5k | 150 | (164,75) | Resize=0.25 | `ResNet50` (bu) | 5.83ms ± 33.27ms | 4.97ms ± 1.5ms | 214 ± 45 | | +| Linux | ONNX | ONNX | CUDA | FP16 | Pigeon | 36s - ~1k | 30 | (480, 270) | Resize=0.25 | `ResNet50 + SSDLite detector` (td) | 17.08 ms ± 139.91ms | 12.82 ms ± 1.52 | 79 ± 8 | 45.50 MB | +| Linux | ONNX | ONNX | CUDA | FP32 | Pigeon | 36s - ~1k | 30 | (480, 270) | Resize=0.25 | `ResNet50 + SSDLite detector` (td) | 25.06 ms ± 129.74ms | 21.1 ms ± 0.82ms | 47 ± 2 | 90.79 MB | +| Linux | ONNX | TensorRT | CUDA | FP32 | Pigeon | 36s - ~1k | 30 | (480, 270) | Resize=0.25 | `ResNet50 + SSDLite detector` (td) | 6.18 ms ± 1376.44 ms | 14.22 ms ± 0.48ms | 70 ± 3 | | +| Linux | ONNX | TensorRT | CUDA | FP16 | Pigeon | 36s - ~1k | 30 | (480, 270) | Resize=0.25 | `ResNet50 + SSDLite detector` (td) | 49.81 ms ± 1361.7ms | 8.3 ms ± 0.75ms | 121 ± 11 | | +| Linux | PyTorch | PyTorch | CUDA | FP32 | Pigeon | 36s - ~1k | 30 | (480, 270) | Resize=0.25 | `ResNet50 + SSDLite detector` (td) | 7.7 ms ± 5.38 ms | 7.78 ms ± 6.0 ms | 185 ± 96 | | +| Linux | PyTorch | PyTorch | CPU | FP32 | Pigeon | 36s - ~1k | 30 | (480, 270) | Resize=0.25 | `ResNet50 + SSDLite detector` (td) | 167.33 ms ± 21.0 ms | 167.32 ms ± 21.01 ms | 6 ± 1 | | +| Linux | ONNX | ONNX | CPU | FP32 | Pigeon | 36s - ~1k | 30 | (480, 270) | Resize=0.25 | `ResNet50 + SSDLite detector` (td) | 85.64 ms ± 8.23 ms | 85.65 ms ± 8.23 | 12 ± 1 | | +| Linux | ONNX | ONNX | CPU | FP16 | Pigeon | 36s - ~1k | 30 | (480, 270) | Resize=0.25 | `ResNet50 + SSDLite detector` (td) | 161.32 ms ± 18.29ms | 161.3 ms ± 18.29ms | 6 ± 1 | | + +** **CUDA: NVIDIA GeForce RTX 3050 (6GB)** +** **CPU: 13th Gen Intel Core i7-13620H × 16** +** **Linux: Ubuntu 24.04 LTS** + +^ *Startup time at inference for a TensorRT engine takes between 30 and 50 seconds, +which skews the inference time measurement. Caching is used to reduce that time.* diff --git a/docs/assets/select_dlc.png b/docs/assets/select_dlc.png new file mode 100644 index 0000000000000000000000000000000000000000..1848884ba921a59d7f22ab6ab349612c6cacd9de GIT binary patch literal 201080 zcmZU)1z2275-yBuun;6bfZ)O1H8=!!cXxLNf&~a}!QBb&?ry=|B{&Q^Fu)yl_wIi8 z|IhQBIX!*4TB@t6tE=A=t|%{whC+Y>1qFpBEhVN51qB0y-0VmQkQ@N*#6A?%J3%W^ zQAKG{QBp-`2XiZ1Gbkvj@FWdHO_dScEFI;~_0xsH8*p(1!ZJt<0XJkzNYVj{#_!;; ze6&;@&y1l7TWRx_plit8aR);El*B}|@KD|Nl+d7nV({_J)53$l5>*jO31b6^r(qDfe~yje)fpR;LDL7FZH~3+K*cpIelpX3 zJ$w%WF(pL-K7mty#B=pDXP&E$O_qf#QY~!nE z8PZs+6(~+sGy=2)#tc7%kU-8pKa{MK+3A&xVhFB)%gh$Sc*YijTr@Do5v%AI`6Tl4Yq+Q75?-Vy zqQzt#M#;Ec`7o%Nlv$nyw>ZMZB=N4fg_KKmFkSRC*#N(aD3Thd8@_aUh^-%<1v!HI zYRA^z^Cuax@5di-wwQ4E!YDSpzu;;Es~eg+@ixAAe}%%65~jk0lB8tgCezv=|Gwy{ z`0gcG3JMrD2tVorpMwcB2-6&x$1)xELER^J?pFD6Eetmta<2@c`4ihjaS6xT3-CPb z9SyoV=%qb*y$6#%Wg3Z4DUV1Y3b4*0GKBZ^lN#f*1Y3p??)W!j5F=MXSGR6`TeEQ_ z-aJEAK(-4W3QiA3@68PFZHdp$BSPcN@&Cb!cO-NG(-fL^?n!!%I-kp08A%8uxrX*= zwEZ$&$7i}c+H9s}1TJ`P`Ut*cPjvrafggproecggq7rg& zSXU1<-I!^oOO#8@HJ-RZV2;#XG{UL$PU(9ib9Lw-_%0+E1j&9Ef=NdwcPCikJ3poN z&W--jF2W!7V;=S!Bv21oMELmJkC~`K$ld-5&CfBuhqkGgv$-(cA>{m}1&Te}wW!dY zlylqiY*19)FiX9lW+;p;SQueuB*Z_Y&;(B~x;2{?NK|X@_M=Fy(XoLBD}tiGltGKRYnaEcr9MrpA&U zXRMoI>08AIx}maa?FuZx!xK%!g-E~taPChQThWg2VcX%{7}-Ck`tf#8t};D8ZTvJC zBH8wP;DU)V6mul?#O4lW3I5jm;VW`(Ms=p+FGt+-LdIS`vbYl2`p@3I9lySgQ^}Jv zQrVN9Q)ZDhV)2DPQ@kGu*b@d)iYGJ|z$hbIeX`;@qBtTbjl2o0_{vGy7ArbjYKrB> z*pks6>HFO`3j7nCnkQ0|=c$tS#dlP7RC$zBhI}8TFcO$rCf$(xsEl2*JjXNFFsFU2 zb?kRch=YUUg2R-giIY2Oo76!2q{>*ruM8?FE3~c}RsU2>tEQLNDteoE{Ar$xonRsj zS6H?XXF7Sh%*w(N!-{0?ueH225l80=eFI7ZZ-b3B)BKlX^<(;5_ZEq=^qIp_Gp zfy_L@sb%KeTQHPpe{4a^u4m>wbHgJV)?>WKy1<(D(a&7M+{j$Hj=}u5dG>_eIQGP0 z+64!|P^7_7+f=in*0T0*9eV?j9nX60Ag~f=Ai^ritYPrDecs20@Wk{4e&Kduo%4~C zDBU=nApNP4T?eo_(x}$Rb^62oTN_aun!CoCl8?s|{S(R)9ePPnSkSalkm#youVdAA z-nLrb>L;~NTG1iV9MLfuu9;++2fXuq9YjVvWN!UHtV2m(7k7_Iw~Yf|hk}LS%QzQF z&S>8=9+E$I*!%xvj=OXN`R8e)Uf<_f+>)#974eW3*$46=bK_ zr(~wwi#XXrTYh=d!>!($Wf~$X2?;D*FnV`q0OdO%!zsttjAj&1c zBB>GFaVqdDps00X=jHXi8U1sgUy&!Kbo+-i4ZDN&QRJuG_};Jln*OCXz1^9}yQXEPb!|IqirNJfe^UCql5#SW<)Gy zS=UtfKaY7qvWHj!cId8Jbkq7i`fP=#GNn>2e)zsVa( zHcht1aWV6vDcdwN7Mquf%H`ZTkICH0k<{xa-NP2TgU-wp{wk-=EAa9)9{~Beut@B; znDR_#w?$7;$ivs|-Uac>$OVQ+^--n{)dO91{W~)jhpR#Z$(Po<#X!oBvf3%$^y|$( zo4?hcE;zsL^J!RC(&+zbalRRCRr{${SiVu2)%g`kTE>R~6(B3+QWzf+n=#`shHrdSg}Rczwo@t8-jdQ9j!Az6safua&`S z6}0)QOU;36O~3K^1pQBRbfyzudbvP(xyQ#dZS9~1DI0J1m{mVFh3TwN?V%M0AGiDD zGuWqsr>qNwRltdGe_&D2cu+%F1-cq>5eegO(c_|p+j)z55B37~`cKG0Gjo4uJs(PD z|9l!6k5(gE8#Ax!aU(ma1pK+W&u{K#PX~bRxS7mk* zyFRa>O-7Az<=Z!|%e1?-`RV!$*}a^d^R;;BuQRkifi_N`f}aX8*ohN`*u6Dx!`4ZB z`1pCFg;2cbfZQ)v9rF5zk<=iCPS**5jF0xq_>GoZlk9y>R%}-J)R5qX_hR)?-r6Jl$0Ps7_ zWQ8-_3$}U&*8Tu&fcC>H@ha-8{R;F_6B=<>PUj;xl(8igg(vjfyjux+*8JX~jMnE~ zm+pgysw1zXt<4Vusn=o60QZ=3fr1LNf`a=`8F|R_?TUrmZ#4gY!p4U}AwvGWhuoey zF#oqS3@``wziD>JJ17wqQE6$&Q^myD%*@`!(!n(`QN{+6f$S)y5eO;$X*UWa?mS#^`D1_*M@Tzb7vwX=mnYMCxg0YwyDADM0qG61A#A&+6a(o$|;hHIyjq=axyYAGLs3Skdl(}JDZyGDvL?{haB=xfXvd>)sdHp z$-~2g(SwcA!P$a|g@=cSiJ6s&m6ZWfg2Bbh-qpyH!QO@Z-|OpdEXV+v-rg{=Ffud!w{8e2|64AvqLrta zt(KUT9fW6)HUv31+4=uf{{Q#pe=YulQuBY5Tpa%o>y3L3jS}p z{?qI~U;c-XpXqJn|KP;G+5E3u2%iN}_?iAYXM!l-vup$)(@10`rl1OWLWJz?f(RoN z6wSY%kTi6@n{i0<7bqxUC}}YfRZr+sJ;Zmra{xUtPKRE1eG zvHCK3PSydh&nUE^e7EBP0Rb62Yo%evU&m!wQS;(c-i@74zWPSK?<#wFye-#=gN8+r5JwRs!-}JMflDC(dwr0`3YW-#xOQqD>_U3A`cUP&<%OWE zNPTTV+S$Dy$tn*^u7|7|8B6S^@e$SYk$#NMfUruJj~r`HHqQWaVzIM2%W|$Mq!4vB zM$9+M^A&a0BeCyhBnD=hVGiJ$%041%*LA{8_%oo9RdCQ?5Tq?DxjCQwU9enIXuadc zjg2el57$dNc=-W!3iV~}faDb{Yw_YyS31!Zk_E`nKLyoI2mxec0NhiFKf7FY2QK4= z0O%SpnpizF4qwdrOG_^elpa*BlLNXU3S-1}Ov28VaHC%-z^MVR{Eeo+D_`T@nP0k8 z##mHEWCL*IYjF|xq}fXn+-Qd3YjX4#yJ0527R5azGw7I}Ec~wZnl)dJZxaUsfj6x= zIl{o}Ypux3j8F88&8;G}!M@wPJJ>e-oNBuK4S(74r_yb~Tp&9Cj8I9%vYKRCM%|(k z7eWB%3k@OXi!B(3Ly}^sIL$_RfHFna&F(s8LMSn^c3ddpImqZ_pUHQelv}byr8xSs z-G03_J~j2r?a5+FMa3g`74_}$yd;4k02+!f#`m*g!P%9^W$k;P<TMR~wd*YzmzS4k znw?CnmZ}3y=N$qDEi8i|#5@7OcSM@(*gG%>7`l4DJ8zpQhVMC!W@Z9U zd{=?CuE}jAXAE{xY>;C=n@nBD zxzJ`&FR^5{`!Em_h)CeM&|h1ruXxz9P>I|_>I9QV@;O$XipBC8CfW+L#V}CVY5)() z8Td+%Th2H9A$qpGVi>Vd{IhuQj4O-|kzdNP+Sck&XXp{)gvS|%C8&Mi)_^eAPNu?O zEC=Q_8CSJ5EaJH9#AW39-g#@)@TRA(;OWAv&!iP-0rptS^TpwM`S!w{&>s{f1d)PQ z3PHebe&4~SYqn4*fQBO}4B$yp67Io`nFuvp>Q|*wY)}8P>d+N8CG@I{%b-mmm&P6^ zfkr63TO{{^&F19N&~-{OhS@~ABo@V{xF+Tu(R6n5>JQWau0R(kW#^^?naMIDttS89 zH^=ijcbg6del~`_jNfb)Dzx&qvb4R^7A_frCBOS1aCT?gh(%<3UMo|mJFD*O*2IV7 zsQ9@7bAFj?#XEI#8xk?2<0JlrT_r_)_o})fSI85ExMi&!WJ7$?Q%^-lCoAGJYd)b$ zZh%3>Nf?-GG2tzVORt%e%uw$*VjWAVRI;Cm*7f8qMH!7iS~CDTF#JDw7b(6XI4z&Q+!tOB!B!Ki| z5PCNN(yeL+$L>po+jVgm=d^it>p7`3j+Uz08G`!6R316O*WVS)2L*~0)BZ|h@1=z0~@%|<&N_?uEj1+ zyOmK5m&4K1WfIz7)Wi3IULBw|V@cg|t0=^H)sxwCzl)B$?{dBhTD5x?dS+t-l^cZx zRTcb_3RV1Ws$$E`^`&ov)U5Bc0r zX<9_0Pve-4CI2oE z4Z8!L`;ZKKZ2BScJ*q2r=@)To>ml|HQaO?{G5&-qW7iMx9|3^IE_^0~ijeB3(P_gL3UsWqyX6Mb9CtUHCMiR248A1y)`C$R`i>8~M ze)(g`3<;k+)|Br0VKmNpZwYamAvRx)O?^&54z8S%r8e(TUs-@7j*GHkc_!-rYJee8 zNhT39Sf%5h;nRx?wC&R!pA;u$qqV^E z4c3yfdmZMSOOK7XSMleXG!)fYBZ^gn<=i4*aqW}cE#d!kO!4-u{#$+wm2Y9MbL|$Q zpQUHdO*tE)K_aGCTg&Z#2F zGVFcrpo@-JS=aqjI1KEzHME{A_Wx^jAUx;!H_yrZUL+|Y){ociI9Sh+ZNF4Hq3~Kh z)5ux{gRieOQ=j~7wJCmO@|9ehP3_LC7w}w_2VM(VF#ND!4A4{i->R;*t1iJGRKEPXc&s z^Qt6t9Yw$a7aK+=qY8JQ@SN9n(OJG%q0b zi^O$IXFVRCtYn>&-0j|U0S2+;<94ObyR|FtzR)OcS%A#KcS>;g8Wy&@XfEnjdwctP z=f(CtqrOn-`}1`ZG@rk@VRN=@xJJ|vjS_Bf$uYM0UF%Uy16H{su#p_O0bMvZ+O4Xm z&H79XWgFKmsFl$;(GXWVUT*p23AA}6Y(niIAJCEASgk3dTmMnLv)NE3=utFSoHs_L zT>&a?U)X1?CRvO6caCd!$Zu#js2@}wiq-%{yC!eB6IX3f_`vCKajnav%Hoho`NtoAYM{NaJ?&2GiXTg>6D!BtUlq* zi4)-;tL>tzS>?o`&-MjM|DLv;)m!0M)P?mE^;J*jut{#eUC=qI?E`*l(4Q?IuZtLfy%9uSUS0$3-#*O8?B3baP@k?! z@z#%i%el}w94nR0rV-Q;LrK&UVwy^^*wkT~0w`WF5gLh1ZZw6nZ+3;x+30D2#E5Z7 zMT42X?%_>3ZL(kL^>uiNQFwsVM^YQ))w|jzJ$j-C1GR?&2_4#5&NvSb4o1UB{7Xn# z=3$=qTB+pHC^iAFek-?ZyPZ$xbciIrj^SoVe~abPi%x#Zj1!Ft#~4v(y2gKhP;|{z zxx0ws+UQ2@LkyW2C1f!j#?{mvM-9b6#D)(6N z$duuX7{TOBL$)ezoE&CnaLtgIPPfSf*%FKWimD)XO0J#Gs1;v*eCsqbmTP=FYSeKm}ngLGntRLCE*T}_&;pd9%lIz z&B9G2#8gGXH5&L{iEEn?aM{yaWcxKvCsM6nPV!w0kEgQDPGqTfqB6BJKYTl>o%+QB zYw+;^hjz8i!@=+95}Ezk>#_&&+3a+f3IFM=Df`am1q@>KxjWTN%Y5@UIf7u(byNX5 z{OA=g$*Oj!GlyMcBMp+7yk`XlR-2wKMW?mbjF!wZ0ea-r*{)v`G2$<}o+_87<|y5+ z{w_@Huc6yfsMuW~^NW1H3Bo4W_n9vJD~MpAC-_R~PKP1@yskxA99O{xY_P}`66@V= zo%MF8!*!snd|Ysp3@FIgl@gY)a!hSJ5Z~`fSx_%A&!V?Rm-9{EtrI`KM6rV-X|~Z| z^|Om;4EKs1rF!h9I*>Fwa^ZrwF=0Bjg-5uPDw1Aqz2t%5nCtKSG#^ zY894nLW>rAF;f`%6Q70b+t;ahh;+}$2vQ*U;V9I!c39b8f^+F0oQ)fIM)H|ge;HGQoR5Z(xfGSmB&&R zl+fCe8R=rNW8+}2u_27w%PqJZpwI2Ln=cPAo3z*ruAtPVI>4nrxc6yGnG;@O*eOL@ zk}uQGAes?B9E7;M^nHgF+a6zDZ)UY5gq^TWme(`mypH~KrxVGoNc3>*p z`}SC^L_xH%HRK8~kduP?|T%PwxW z{=({~e#>X&2mR^8E9 z=qqnE!AW`hvT_WuK(GO!@0K=#L*qzFjh=fh)ksK#&En4TbyO$jS^M>*H=w240|eav zYUl5Cf>=hGeR zG4XdY@)005!lP46BFr!5LmjIcz?OwOKY~4YN~f32QYve`#&Aj05(;NEa7N9s1_RT^16{kh zhWbaSv*`tV?!Nt%sZ|%uY<*9Q#E5h}Df9{srdatILn0(tNCp0760NlL7Zp==!)fS~ zJ^D@DmuL-rufj*fJ88H9_DpGXpRe_=k7AQ!&57{Fa@Z-lDe7C$4tDfw}3*3zn(w_o}cKY8fTYM#^ zLWDWOGw@|8Or828Ny5-})~Mr*rH3Rpo1>I3L0Rzpd;!4gw0&SU?1tjm`9WSlT(4US zb6rc!Sk`Drw1mqeOrwMiV>ssi@y&8c7s_hTFg708CzZ}-#SyRj(BM*DlUe*EBK{3q z@QOzK&J3h0i`QPtyZv0 z*#6nwsYOra?z_#%t3X>Jd$32EF!aSr2^uOF(xGK>T6r*qY~A+CYQ@vdZF-*b^KQB6 z%Z^x^Zhfa}AR`rD)}l7+aS+lGVxa(r$;~aCmg76uyWtaVe2SL%3aD?4OBqZ;uP!Yu zMfCIQ)#LLMfu@dHi9GM9IjqY{Zy`4RCMDipuLh|I3&1 zLWp<#3GoM^_1DW^#Z>i>?Lw^JtDbyXihGdGCJp-(c6Qonrwrm2lk-BVg=J+(^%%YE zgI`-RYncE^QRSfUVxUL49yQ#p6hW71%eU9xzNiKY z0+^_y(ZP*;M8dH*Y|WmkYG+dK{+=N3>dxwZ(cRJ7T^SjCFGt>&Usaj(#UaUI+h$C( zy4wxOc2{KvgG}IAlbACS@1Zw`i9|3^)_9p|abch7aF|x-LB?FGgpx=KAw?Y8%bhwY z3ba&XGVh;xR9G4b^5Z^l-ktLuGNA4veRu#8M{0kQW#EvrKq3W6`hfGkV*i?U?4gel zwCyQgBmsbx>`snmSZz&67BcFHpk0^R!`!%EAJDm9PYhmX8lZGj0cXsqScXZ5wxnSX&T7l3vmt1*Uq)t4PdPu0yJW)Og zb?zIff`oIRnpW>EpGRy*de6!CvIyu&k#BSq4-|g-beiS_EWAHhVc+{nu0_p>F`QJ2 zZ%k>?y*~MY>OVLAv}Js{3oVL>QhFbLGlGQBsal=6!X9OWrH@$6T~mXSznR*mWuJ-v zs!RBRZa%j!^>=}^(CKstK>Uvdp@0|VX=|RPp4S<9ob1VG*f@)IJi`4HYK_D)f#hXw z#Y`d^w1@N6rm76bUS#BnLOdLA6(brsy=|GlfULU}!D;=pwB-o&xxs9w!H>vy7bEQ4 z)YF5RjmH`J$TNN00kG(pFQD#}=pkYsdUJ`H78eU6K*_|_HpC&6UjkR<;k6~>4m0)2 zLg{%ZKgqj%Nm~wHgsZ;r%{7#|1&St6R2t+UWZF<$w)Pz=a(j|0456p!ds}f!+jl+R z{mMt&NT~kyCp6oDg&wj!QZysztMB zuhWQyT9fit^kZ5X4|m|o`}cpa!GComGa?jb@QZl*#}9@GT0Fh7GW-^Sp1G_UXTEkm zrHI*gov#(&<(z-NqrJK5LtZ&2UAO3qn=mm1;?`e;z0z$)#;ThO4F=Ng2wQb?iQrAB z^nPyNYG8K@Ig#S@+TNKu%mxT%6!uQ?_26Av6=xs0D>ZmXNDR5j{uH|IEX3{F{FWIyey z3B*&s%COhdgd+fjv&<44)5Aph!mVV1%y#x~palv9#|b}K4#lx6)v*DQ?6G#F#<{|3 zB1(gh7<##J=kv?$*lYts{Tne^%rK^2TQcXu?m3^81FXN}Qchc(MOO+vYQ%Eug%#>C zZVpc9h(~v{_@X-dfDI(WP2&who-ROzXMG_al1JCpC*8wR)mBMugAU!V;NME1gOzNK z75|^G4S}$Q(3{|KZr8ul!S-sw5?MurnS376^Mss|;o|CB$O0{nSTtSC19sl@LS ze6_ZttKLBr)-4x4eOo+pyw9$qZyU0>#X%NL9$d9d;Ve7KVRXc_E5++`SiB6u$G`c_ zKZn&c^ir$a&g-TG%gtE?nPN(D>3m41ZuVYSqXVhL+0F8_-gj(zD)2GR`$xg9;v)7w zOYhrQn65o z6wWtq0H5E(Awsj#XWIe=>|&yh1#CAqDtBc)Ra8*XvS@dH&I#dePFYNerFkAdSw;HT ze}_;-@olIua@KMLUQw_pL%8#4(>?JW_oif1ga3tbd_fYfzjTVhyCr(OAtVyazTun+ z!Q>%Jd)3ZPdnfME$tYNkGzb%cBj%K+_;_vJQ}l(UbmNH-e?aN+<_J47&ugILeuJ>B zava;fM=s?DBzTj;sMkud)alde&~ckQp2i`2Hd57br%jT1$D$RP`ObE%?X#-YH=AbS z095O~&tAJxD@rG96#=X3u(RW1POQ|yqL7G9g+G-?qC+Og3Nd5&wE zEu!?*NgmPkJ8)*5G|Sq^4OFvB9Fq2Hi&w`KeTS za*}Baf`so|_Li}Ww+JtL6ZM1FL(vUEZ&EnYcv??F!e&m-dyF7qb&TQ?My&MrBPCHU z6VkS47B(i8Ps`_}yA7lG>y(2u?0aeI%tXW8^DfNk^XJx+O4JWjmFh6lrgF6gNV2m; z2#;=n+j$K=Xz&6q=@bI?aG2i)wk7pNLHcNjYU8Y(pyS=Lf~#>yDOX+Ubf6{j-Mv{$lIBKPy+7rz_>jv(4B7}cD_d%Av{ zwEvjO5cI^3l+#odaKrCMau{uk2JHs@x9)63iYWXH^_G=z^!_XKsS|W$gP#okQ8- z)8#r*o}FSi;GFW4FxM~Fun}ROi|Ul5op$g8aAar&Zm(t49wNTRdO051UsSRN8?wKM z>#0z6{h);PGJtNl*R|=<^KMa6YqrlWF7P?JUjDV9Rm3Yw$9BGeBI^A zl-Th4eU(lY55P)WsKzsEh%0^?VOwc(|MmWsJ032Vy(!$LN;)VUMIsPHUUqA4&mMH; zeHKt6u}x{%TSaH84d0TjYNW(l`3Y{kECk~ZMu|IT8;)7`4dlo6aQ z|5~&)g~65g(=JRR&BCG}<%9@9j~nCs-ZSSj7qG$IjJ4M7f%c{;WaDGak0=APIhQq) z3@*u!rk%rkG{kU7R8`gsp+zElNV)`$F)2U|f6F`k;H`*^nAYt`Uc~(Cpt1l%;6{;K zC)Xv-gTg_Z-KZB1bX1)E9$3quyEP(SBRC!Ev(q2$EsY&_DPLN;Mmb}iads29-?Ced zF(RC0&IitmC5o5Rw{-4h!dZi&BcdIv4`oN^*~9$E@9Fp}qZ8_i^2gbhc2Kon$JF0I z{*Hy;JD z__;Fi=i%MF3VoaA&vSNqd@_B!YfH#E@$5!zU*QE}ygzj-0{VW1b3q%|yJ zd=J-zlzP=oq21h=vVRVp5d+slMuggM3A z@=`GWpVkFU*?4-QPbd*&$_r=LaeS0A?J&jfR+x z=j5YpQ_s3vBri);ps7}J0#tZDbl)px92iOGm=t@Ne-qQ!gLXiO96iL6kls0*!A=`x z@o`GPP!O^V805x01;!hV@9MMeT>dJ(V$bSg_6b_Mwgok;pW3bFGEn$r50)hJyF7d3 zWU+(PgUJn8+2@^pYKSCw7WFi&c8Kh^W5~_k%<(Bbg5O+k_O}p|>x$%W+I4JaC2Q`) z#>&O&+&>vSF5hWOkV#Uvj+u2mln(1lvsld{Ce6el9y=r&T73#}pk7#ilb@Yisqda` zft33OT8w7XcWCpdUIGz`-_ZVDCv&F7D7rsJ;Y;7$_`6&U;1!QjZR3@W!T~9=-Zdr= z_NSn9!i5AO0(}E_7L!feJUed`^A!?`9dj;Tj%JI)@=w>et0x?Voy0iqLM_dG(2uT! zvLc1ExMR+#eQr*nDXS_Uf}?zSJRo0S{GDyi{fY2vlZ6gj<9s2uJ}85z zsJ@T8w@^@#?1(10(i<6c_e|QfE{hJPxBKYNyo(7!S##qI-r{zhdPa|Bx9T#`vo~jb z4SEy)IcOLkc7WHT{4GxqKV&~Sa-N{7 z_yM}xgV-NmWaE9mWc9FHIhfNe$rdc=bdAqo1My@D#sg=*J6Km)0l%WTJJZ*5H!^9zA*IU5YBu-BCqY8n8M% zN&%JRXd=vR>Jg_60G^uP`hvqci0P5RxP z7xU7tsvj!RN!I7&!xeuW#vV7`>T4t{eDTTiK1#IWyBlIwQlpKb<7i~b-oK-wqt_&f zuaxAf9i|tnnkz&Y49cGrNF@CWN!?86)*YP&Y!=@~&2hG!kUIFQCJUsK^IY&?;LzLm za4YI?3p3G=6wz)sVpprcDx?JK1{^mGZ1cag7qt2c0^ZKR_7fBYuTs1z@YnqA*XQSo z)XLS7tzO8esU_`3yTCwM616fD$RfA*gFIT11ZZ@5M3>w6E6v|KO?qQ_{xCNjJ&-}f znzgyG2&D3BrX4nca3P6ynlt0M0Q5~=JvnW*ngKi`qW>f;$2p}XHu`+O z$<(eOsA!@v`{j;r8I7__n?4vcm+tu`-zo}zx4Us1dH{cZT4VFkZxzbLh*C zL4yC~# z#rUOZf1fZy&)ez-IgBa`c}%{yvi*8mwEZ>rixU6$MHZ&zL^>vV19&-KU7Ng*v=4{P z9fYG-L4>kH3vS=j^iFQ@Ac2ocv&sp$Nnb?@=0fsH)UR|H4gfvP0!yy`D#H~^4TSN)IVPLQw(NC?7bkkAO^%4*o1ux1?h)k@CR!PG|FqC zvMg&P(w5I3zZM)_l#i*z14JS=bS^z7q>d(1M>JLDg(~fWI7$(!_h&9rs1M&TO}K>^YA1N z$Y>y_7VSpcQbvlS$gXDLOF@&tpKz-WJ3yT@Cqth@;Ss}HUm{wxmoFrRD_X4-FZTs{ z{JjZqK<3}M;s}f0>djViXQl@2^OP*CS#`D+RKsSw=}B+L_&;Ht(8a1*G)d^y&wg;g z|90)6!z-G(rC7wF*J=EQx22wR=HIU8ZcbFv@ox-4CB@6^KE-dAcB?1{yAVeA+oXj% z6X|h!o7Z>!DM>Eslm7t5j2;ARXOA8@Uftpi0h&ImvfFD1l{HV!1^8m|GeoC~fB3?= zBJ5jpS|j!kl#4agzrX`>G!d`5rx8hUETk9hLNLP`gT{-;W+DShOcAMtg@&Y^^+a{x ze*#zV-pKF{Kdq~RizOLbWxbriaO=Nf%sF+Jj@W@THICcgkL}#hin1bdkJuNXe0g1NM$0X0YD-e+pb6xJG61!6q5Pw> zs{g_`VQfk#5rAOU0t+AQk#Lc(-Uq|17YlcKLS+E=R<~s!= z&I33a-GYKa@-{G0{PN=xot_BVW&dd7U%kxX<=)}|LSotUZ$}#P@2lip+W2|7R&!ni8q4zen=_jUX_FlD>UCFu^K5A z4xp=riT_^t(ImdsKji0Ml)@}TmjqsE3*e&&At$eL(`osaQ#Q$-Q7Y?m>PVw3%C}YH zAN<5eEQ^HQkno?`E8Yi=F%B6oBCRR)%5Q*&H5M6ITK}Jt5W>KC9?Uy!m*xGGqXWgV zxXri4{?p&Lh@LY;@;@+l|3>{m@HfinP*rptRh^GAj2$=1x0?CFFa6jGs-!~qXAN8n_qc=pj&sRVmcla zQPJW(O?R46^sLSmQ)zSk>?xmzR%|-`WHTT!{P#P+7&QKPK%?AhIP~6r~T9i;a!1kr#Fh&4?26(XeiZ8%2 zdAWG^T?i5#lOQ%ao-3`6>w;@0qDlYdw0V1i6QXAhPwcb4ZwitX^ z_v$tp4!OW0_;c=#m15F|t?x{*(Gi0xbX8P>KYkWkhLYlqAyWPnYIyb%0YtgfFlc#x#%s2QGNE z+yldrn{2j7Aw`z4rN1Vni89Fwjgs4*zOT5ZQ3NHRb$$z$D@@zfS{nZ$y+@+f>>VQv zlQj3ALAo1fE3kOR3ct!?P`9#&@vZjXFzh?*@5L6 zhZNFOOkZT{<0FyN5^8A^h5myGKg@Pedj{8mifykII0JpxTdf2D?_Tj=%B{hxbj-iK z9*>H*8xE8QHjn+C`yGER7$$2m1E2#i0^N|5$puk_t6;`ltheJPo7iA{wFw@gs|);D zDWbpfvwY}OhCiYgXB{69!@HsV;mmQ}`!_*lloGi3U{C;7BEJaarz~~_;tiW>?I{%2 zw*wt++zJR^-DK$J!4|;7EQ$=0FGA2lw&3fd@a{0l@F|0IfbxV81P!l@D7=x?B_I6` zFh>uTjpDz$pF5g0z4`m6EKXyJgZ=mRmO;QkrM>zd`2d%jLu&ijI;2qSK*CMENUeL2QpbYdJrEj5S|nw>yFzMFl6*BZyw9091g)3e%6P z`a=fp?k*!0=^y!x=m6ljTbaVN1EG9_wuj9&4Tq#(U#5p+I!){6MoRj0kY)+!-S>4C zd*SDX*|t)?4v>6z3`j3|TvFAX42E*GVO{HT8Mc82BOKBuRuagm1oV5yb-!{)4Pfv2 z{H|NDzxl-1L$eF00j3~Efz#&ePP7S6=7xc8aiaJnWA)ZM!aVDLlH@U|9uD` z--|reRh9~0<4klKL4v`$9C7;b#{VK+CuEBN_G~(>~Mlj zbsUMb6xlUiw+$LxO3rZGs|yY*-iu9ITM{}mbRZ%pXM2@y#z?n|ICt{_D%6-&fvmZ8 ztjd>dlq0yuniP0Q;~2r>AtT{Zw4kxEO5F%a+JaVcj-aRk z!mkLV_d{?5+-x&8`3jw}TJ7ZAgWs=&lEC}>JcM3{0$(P{YomzX@#}w~!`@?xT^#+7 z>bT5m<$^W`vnLffA<+Blu8M01_iOXxhQX|^P@x~>x<5=%&s|hAADS#KHX_oC`=0m2 z@$?IXC7*|5@L4kcDjgVEXiQdFT0ni~`Jf9Ibp$es3|H0f7i|wXFegy>8kq8+NffD` zF-n&?p85x8X4BJ61R2i=Voyr{p zl;FH=Ns7!sTMK8dFu}YHf@0Z4$7Sy4yk;4QnVV3Gt!B{3lW3A6+~BDDO1HYv>FE?x zSsX{fFCM5l@EPI-RE1w_q04UDCp({Z5pRIsh&~1~3FHvr+8TyoH?3SArSBT`9)nnv zS@i<(Ezu!b@kS5xAd!F|18}}i!|vyv8N4ahfw5b>gXttn8?1pdx5@`-qz<+#T%^3>1_HMcdcGkjCb|mM2$DX*5zd`^CXFl zeCJ0o`f-6A@bp69n^xgS299i!N%3VVjx}flHURx^nF~WgeAj7so=K0uMzUqW=xGwU zovsG5_mh%7ZWvx0{Z2*&S4T{bmU=X?@i@ehSFELWFM`?P`}aj`5=h78Hk<;}7bE&U zVa-5&YoUIk-Lm)dl8pQ`bKY-`pixQI77X1EEx9naoz8eym_^%LO1(J6%q|&`4TuLi zeKj~)m^$qLUB~|}x7smaJ9d?YleuFJnm^X<0cVECitiVtnpeZ9i)rt?WlYeB1aaRi zc_M|Ds=TSK@K!lkcg=3PX@kFakt+RET>1UeSD%>j1BrU2;}`{o`b2L}UQzZr)282| z)4ybMINkW5&<8MViGk!=tIX=jk3zqd`5Nq^*Gsk77{?4~vGxzD9^=RfWjb|094`oo z1>(5}*|s!e!B$}`1=;(lcK9g?456g_PYI;vcEzP~Lf*(ksOKH>cRlL?M*_}l_ygAc z{iigg(#)2hlKu)acK;xBC2Lfh;*7_{Yc|bL4(zi&+4}9Shs0p0!r=+SM&>9^ku;&4 z)N|L$!2F58M$Fxj9d=+U6=sn!`*Xm|$PHUmxnLD=@POc`?rqCCX+B?Z33iAF7%@btFP9rEuA))Q5;^G4eL z5(?>R^P|q~yqDtqxV&jPZV$3)D`y5qqE~*E z#yp@MjiQ(Ryy1LT{uy__gB;twQi#NAL!&gTX-nLlP8;9jky~3NI1+e8>}q61Y^oKW zPlrQ{g2O%k>#SSr5ucFB&&j9#OXBL0xe_`X35x#F!?{9 z?~TeW*`rH89z}K@2}&L9EFcMStbg1_5^xJ*ERTP|0~*+{Gs&8L^DL)5)_HT_lFVX% zu(SD_%w0bCU61&o`qHO6G!tYhKF|H$L@PCk)0stqS~3&H82?25pfZ%Wd=rTRD`184 zue6A;=s^SqwXxMn3VP|W93(@8O%?KBFI*47J!Gp^3g!xtr_iR$xs`5(_nEh)@Hk@jl)eVu^h-sN?`0O~MEPf&@aM#@cupd8*)397lJDt+2APuLHyI0oR~p~lNJ0d& zAnQSz3HV@&eSR6lr(rfv@HM`mkJ4Bca5SD2c{Fkxk zzno;e*SW$Y0eIBuHt;DAvO5fEC%H&D<%Qs}NL6BZYlv(%o1fy7+&MO>uaPGlx|i%p ztdgbLl9WEd!FjPA@9F|iM&{f~k-82xugEPt!(&csq;rn|(ms{ zv+JmgSSS(CkuC&UkuXwOb^ngP{c&Q_t*+4DX`%4d$aO>TQliff2(1sd?Q(dT@>8HmOcxtOh&$HdI|?G{f^o}?sw9F(_Msz+@<^1 z0M`HP41U(5x*U+_4=`FRkblkGsxnQii6iWNV@U) zuBX+PF)-6#pa3H%@pjmhDcj>BKBe`j<*%-O0p5f!DEZ+RKploYgrQ*GaO>K)Bll}X zVeudD6skzf;Q9yhd~O%qI)SZmUwq@ErU624TBR}Mfy=k59mn$E+=g*D!m3pJF88TY z-zjf`CmCA!`5l*WGi#!IE2)=fwNn4GKfqF_^tm6t_H6tunzg-S_<{O}^0@bG4#7jP zN+)lZ{ez6=B)D@0hN-ZUCEp-_R5n4!;hXn0_z^Azq&AsABW%(->wMa%i$`vEt8=ld zL~MywUg@)fh*3@Iih`}A@ahotyfXt{#r@kI-@1h#jK4LqFl-q{&(&(^`yJHJ8HB?J zjfJ*yYR8fYww#yM4*?ps#|+M{7l;p#2d9rl0c15qq!!m&O;rWbV1$!$?6H8(tiNpV z&tPJD$vDM60A4tL6k^J!RQj-_=cH9|xBuNQWn~R}kF9;+QJ_=4y!evjAy>1@lBIvu zrzNxc6Y#7_Ls{5Wb@x6JxOWJf5X0f(P{la}^s?-D4U92e!p#Y{HcxLw8W6+m`I!bF z|1B1S(U9i(R7`lOv@>r4^~`&r#D2mQt|ZZDKGTTIYV-PHg*35xtlYBa^G(KM9=51A z{0pIhdpx!gG9+;5r;jXf2xa%NYxrc&H{VeSoW1qS1?F-4Hxfa7PFp&+(9NVS?p{t@ zxOY(&b!i{L@ud@pa||>+9Yknan5gl^9e|!I+hb?pD2|UcpH`gsmz$3Wyl2V+K@}Y` zsnVQ=AL+&6s9y4o!+V6u*z6)}oODg`rdO*$=V^xuE!qUz^Hw7wt-J4$F@}2_2CIA4A?-3d$PW$v>DPVj z*iK4uEAfs<){t``Cw6~o%mNZf>VE42)`ZfZ%Cg1ycj7cQdY!tuBPO}GH3IgV%cgrc zPJ5Bifih^I;ND!dhLM)6BEESvU-gPMnfd`~DroxWLHpdwWVN-mu-$0N)gg@gYn1%% zN@Z)^Q4KHpebNOYNdKS1OnPW5?KkD7Bl>yv;EQV?G{KiUfOZ%nrP|l*9H#^R9**ms zbGB9QH*LiKlU+qV&{n!oFhidtn|i0Y&Q*!{F2AM)<0ysm6pi;T#~^rKMt;STN~qii ztD0ywt(_MWC)_a-;T4L_fUBJc`e>{A37#+l0nSAsBR0JHQ*4~nPt8$98{o&sVL}#r zl-}iP3E&=XLIX()_)le(Pn8a0k+~G_<%c^Tp(xm}(Zktb&~G@!9Wwnb7MLD^kY)1& z;KE7;)T46Nir7uY%|Fg&@@LCsK2!~G8kr>l0+&KEdhbymjE)Tpl2&}oVhT&M4gP*C zYggyZe>`yN$EoI2n5y}ISpbR-WR9$NLWi`&;$EI|a>_SIsW$QBmv%k~;SgVQEsc5g zn+;^UiIWR!WIq83VP?aejS2s;bT_aTcp7$@0ZX41vf3=tHvlPUJlSHu#hL8=CTU!o zwZPY>p9e*KAn+F{4vhw3jzVC6gM z{&?*aKpipgJ^iK67!?At0eJ|8)obE^uhsK0viX4lV>rf08U4ce6ldbJF}4#<`j;FC zI18O)TSYic5pUY^>SJR&X$X7+p@}(7&AE;g*vK(2tx?aDKoLqq`SQV2`C~X2Gso>_ z!%*B8^BFSc$-1|+Xr8#UFrK`g$p&4Qv+0R?7n5?Rv3|$Zc7WOO8Ogia;8$7ZPD)@& zNu%?5dk`6b=&nOHmS{Vq=lY~6Tr8<(b$bI*N07vYT5w{$%dZ`JvYS|+xE+uY*ky8ts?n;vN4F6yuBGu~oyDm_B@>E;dA8=gZdTA;UFH)D1n1j6l2pXAg7 zp^Ac)Co|@K5qDf^#HmbL%j4vw(1|ed#-0uB)%w0{f6Nl+)4Ft4!2i`^{d@K$wlQG7 zWOBo9M$Qd@-N1?+;^zjw3ta%$!It|2R2<{2a^IzX)pv-9PE6^y0V|`3a(_dv2#z3Y zhqkoc`8VqXr{1*p6$&62*)<82m&0<4*Cg{I%Jrj|$N z(GP7n3-3_+=M;Thdm!#+hsx(}f@~)GUymov_p2BQYYHG|_kFzmucEq-dRpx}GcyJ{ z49Fd^30q3#aWXbB^Umy0+DN_M|IId-h<)79orv1bHD}r%fGRDgrp_W?Vx0ka=LA&y)&z$cz#e*^}^Y#R-!$-mJ~MU_Z_!Oqc!B zL@C@~q0Svf{0SJd#^1X>F1OC!`dA>Eq>(Qhv|0cm*=fz<3@VMb@Y_l?&A$qGH@^d( zCEFH+Ck#rC>7)QX{1E>Y!?3PUIcH9<&7?_oB@`I&Z)&^;NAAw*j{Rv)fW0~k$=Gq&)9n6ubYw3{=(_1 z=l%o9)|)7Pu5^7xyry@b_I9Fud{Q{*scA*YMP^WeEx;sJ?i(fY32;gn5WBCNEtM2X z>XT~d82-8cvWPgBRk$n2~R=_FHNQNI(5{L0Z6fQ68_hg*K;f`B>B`?eOy~ zDYu_|!i29%d~CrZ0*6M>n~eJ865oj9aDVl9VczYh2Xpw&(i`|Kj?` zz&FrW`x{c|{Q((kh={(vpE*p*ZNAB{1G6u=1x#gqaQ_*1)YJCwS3lPjGv2B_AiPh+ z_n}k8DIyEo5F56=oB-QMkb8C5J{Imq49!af+VCx5Cy49AN3tiwk_%FmudSu50u9+7 zy9!kPt?Sb6y7_T;3-Z!UCsRTVkLpzQ@3{T-3xT^QD-q}tZIb=tF)}rm)t<=v`soEs z3g>9W9jl0B+<+|Pfo-*(LvT^r#_2dzW|wSp;3?w-4U#ti&yFvr#LZR6NTKNj7UBI; z=tnn=dGtjl!pNu-^NU8+$grq!kA zNJP4?7Kd2-u)PeS##x8!11@tv!_VFOeS39QQxts_udxka(hvpa=ghJoZ*^qXOZZem zyVd7+@}J3~K;Tmz=aW<3kEg#hhO7;^zaV>FUG3aeebf8V`Jr)rK85P%TOwn2<4a_@2J=Lq#w%EV?t-_4C_XQa^BA!$o}DG}QRh2mI`;drR790`0iA zU{2YAJYi?UdLK*!csC5&t5;aq25km08JG7HkUN*zujGb*Re;>VwAu>hDEkGodGr~( zj3zknZoVuvyEJz9p@i>;jNuRf8dg@8!11M;(j^sGL&>o6_`4HfoIXI<_Ge|>mVSED z^mFfJgteIRE1%~D9MBg5>)K)!Lk6SZiw&}H>Mzo~5hOB+)YpWKvHaG->klK}BuAg{ zszdYI`qoRn+Ab+vbh_x)%rnhB#t$fF1FraLJkr{8xc6VM#5`&Lf3sNJNt8zJzmz

=jp$kMWp>QsZ0e;3o3 z@tlHEO`WzXus^M$fW!hmS`QyC#=NHZS(`UC6%qy?w%`|in_JQ~isSq-_i zw!Yo!pdccN@YlZb9w)vUdRX?+Ba1rF2`h7_9ChcHh>3+?>Mih@l+PHQ=Riq1JYf9e zWk0=_Y@m`0F;L2|WUbF#jtY^Qsc|9*SooB*ioW`VDU&sou*0G)m#(w(XRr`9Z2d|d zxsK*Pz-!JDUZ(#QRNGuJ7O6#bl9svBKaD*b-(M;y*=94nhkNaUlmM=v1C1LVWbW{Dmj-k#6trPWf#0wiAo@=cnY|F z3}g~##HW8#3T@bYTo|*(l(GD7dC`WU>n2`5^K)SM#q1uqTjy;b3}3P(&AO0rV*e9x znkg*F^Vyp<{`hO-!?pj}>5!7bu!(j}jn{FL3nBg1BS{itxpBxZb|d=IeN^L8_^%_P zOX6Rw;iYEg+4*1ev{8yuZm}}uuQ_X<+#K8$!sXI8@OQ@r{^~T4&ed%%pBI`tB!*(kC#WH#25;x|!Ow7&=jxD)b7zromCLmGKl+ z{^e4lvn_#ibD1LG?c399*#7UKty_VdHcJF^GJg^AZ2dr!__b}v$#Dey|22TnycJHeI$MI1M3NeTab);&2OcZdde`70!%lMlm+eU^~$4Jz!dU)iGDf%xW-fC#h@g~7U zjk||mFK6V5{Bjbakfo$GN@;7K8TCG!&GZYHkpP^{DyL%#R~EwE@2S}F{*%G|@kB)d z+7)S)b#5!|6;*BKrMqEfuZU6@VjI1WGYXl9nU)}gI4=iUxl;oyt(6~(r`)jdJVoiU z_gQ|1@H zduDiUTt8`d@6gY!W5l)I9Z?)xnJaL+K8e4wy(akW?xcgcqYgg&Fj5*#(d7u9o3HWO zG$sytB6rHrVR{s)3+&rsa<1!X-G^|mChlzZQ8R)Yw^qj!oDo^!tJ?9?ANvPpQ( z;eSmtC#!oOZRu_cccIZB_p!W%(A!+AO*!4&K1B6%&27%v`uLAX&%FyEkgQLfH0LR(q9pf>ZrX<%B%h&DSizRGFRm`XznQHNt6lMD z@7KXHJ%;y_vaqT%>GkI$`k z+u8;~ip?LBTfjpXnYD_!=bx={w|u#!ukcC6Gr$H&Aw&)J;;M&H~jiW zo0KGuRtfJi-!1uFUGdm_s@fWou~2z)F!_$kBm5J6dYjs~Ya78jw`=oUvF?o{);5e> z?f37;Mtmb>dcAYagw47_-_uG-+(zs6hPwj8^ANMzm06auBn`juc4VFhEdi21LNYq4 z?A>N3a$ceR5igO}~#g_94s~hDiP1ZRV;+S3M9*1mo5$4mP(% zvdQo}#P1*macx<*d3wdXA5eVK$)9}czA#GxJ-j;Kqxie}cCuK@$SGETBKB{&r(@;O z*&M)YV(nQe5zp44LBa#J?~_E!WON$<&t}nUCfvdKHy^W{V3E+F1xILvWRS4niP_zH zFUwZ1sfHu+{Jyq!qJ2I+j%q*4yaE4FDe}h7ThKo!B1aCa_zmylA)?Ek)0<0Ju_U{7 zCZXeN6M1D4Hu-c9dL6a_+#H8+f)9`4sRFQY@5MsiuIKx1=Xp5uHaLh`&Gz}(V++;8 z;+z#uvCqLY;V|gtU5722Cae9HA<-vsHmcv3KdB0o#*!s+v!PuSWyaw!SwdLIQc28g zD|6s);F7(*3T2Mf?O$dcQDOLXHFn~wr(x}hy-J{t{|}I)X}DUhi*U3h-p~FpI(-G% z8)?*}Zj+Z#MzVY>(rJE*Vee)B0fP%-x*ZSj4xf>bCrAt7(NEj9ZRgj>1=Eb6Tb%!t zPSuz#Z=X;~Qd_w^UE9kh!Q!oWkNj7+K0I!l9S;CWqopEFu3@ay8oS|E2ha(=av-Y$i3&nD=j}75=DMj&{?S7sNAm1xWQZz22t^AxIv2?WVsR zDTNchKJlJTQ>;lUOYl3_R*eRx6cRO5Kp98xSK74no}%tIeTmKPaj!yMC&W2TgNd@P zL>?Wzj+SOLu?%9G^gQMnE&{75Q`Ydf!^3+lVPtq^$$UBQS!|5K`gb3w?6 zt+|l~bL&3?uZ^1N4Kb7xT^_{FGa}x9%z@T7$G)%@2{u&@4u9~31sI4*#G$Cgp~ZyQ z*}y;UW+49Cu&xNNu(-C5Bifuwywx+;_v_|X>eLTpc;0*2yg)y3UD;hpu5kB9a0x!D8ZLu=IBk`X&s&UPBp%)M@=tdD9QP)@9tq?O2i zC&jz4iJrZcNC=*`VSUTf0wM=jHOO50`86k#jtDVBk`o55|HDDHz8AfVyn9)wW%8A+ zQ*jIuoRpbRZ8*yRWJ#aoxY~j?PJVt!O!uD}8MfVt#w_?K=mL)FHxlAVl^lDj{g&iS zSz&cp=W8aaQ@W|&YTCYk%|xck^yWTh<$MB~Pmlw3zciZ2R<8-s8SWvLM__wW+wm)C z3oxt#V9RcZN;Ugl1ZVsVefJZv!dv;G$aX) zroE^J6x3eL92i$G>V-nrJSQiCyZjzYo`$FsF5*2Ito`BCME@!^rFX#@wjCE0inGP2 zoR2gjcCO2;u|ec=Q~AUO3_s0cBC1?98G_E-!pSOgAmpd)A%Wvdl7GmNG>k)bZ{N1p`U9>rmVvVc6rErGV$z1)pM;eL#x5C^b~5zAE6{-g5s ziu({UZ~}6XxGZS~#8*!KV65Na%O9fCX{rM)bUut(^Op%WCYpc&Haw9AromVjQ(=-| z^&BEG``P1wkic?Fyfd1|z zMke9e-I2Z{^>YH(mOk9Y4Lh!&AfL;9M-c}s(36Ad>=7B?Z^oP7^+6F#MXS9Wq8JKfb-$(+5t z4L0+>d8hcJt>74crxm_0+fCpPWHFqOjD_IzO!_dV?4~N9jK;9?GEpa0wc*t=$qX2svT$KA@8FHQpv9OVi&GLr-MOp_IWNo!oV*L-c zstEyg=pzVB)B>*c;Zbn~bZ;C=P-vON_S#d=JBS9s!0%^!XQvc4Yr7}zy;>hRhH!;l z{t7|y;Fm?G^E>OCP-{@+dP#Wiz>(iHoPPj3@}fyJbj>HKt8SY;YxY@KTpmeECjNRl zLga6v4g_6XMv$oKc4Gqf-h~xOH|c+Km7AOXqwC_|Q}>paZ}^t5cW%Yu{fC|u`!Lv{ zcST%p0t`4^Nr^bS<2P@c*4pNO*<-^y7SM$~(t`JhG_I`owMaxML{B*atb>yHp>E*i z2}A;@hiwWiA`dGXq-G&vfI9S+pfIF&0EU8&$??AyG^VThkp^B~U>f{_oi@tQoAKT})v8tsWxlPwC*Q$64=bfj@g)T3PCi^VQzDmP44AXb*9Q=iWa1!9% z$pUN(+hcT~>34OmnCF?|RY`NMmp$o}B!1+M=AI1%bH*&)Oq6`%TgNhQ3IlsfxuYer z7|Jps0@>6PwyZ6tw*Z+fvIdQ-=f1IHGc3JNt|7v~oZ#Vz^J3bmca#u{jI_$+x_zP5 zCUtpZAOZ`u;RPwma#7s+v$m>Ao5Ww!G)2+k8Hnmddfe>l7YphDN0G(qvPpYK&?5=! zVL}@zdXx~3Z6o_f#Jl&Ps3FaJTG3sBf+DSKEPVk2JI|^MZNAwPSK|kL5-k5OcBcx8 z?EVuG8q=X+?O(p>-sQG~hlZz5WFHrHUb4{2abzJQ5cdb6{M)~GKCB7M!xNf0t(kI~ zsU%TCi*_){tQ7yF$xX>llRc8s-SYG>^tP*qJa?f z0RJ!9zj@}~ObYck>@9dXm=wI;Z+w_QLnzv2G150R=;5BaBr2x(BbvPVY_Y_ouEGt0xpF@5eZZRX~`DsS3`fB?!XO*g1BpWs^TjmE&H(OEnoa1xH zz0mDvytaDq^gwQMS`GpIo)acyaU~HtXqC&8dYG0~@6S;GleW)~g#VaBd5^s&6;|mD z(@;lGi_fsoZbKA{PamQXdvrB+iMyWj4{H z-ZU#h8Vf)?z-YH~fn-I-WEG3WAeH~DaQfDv_P@&HC{JlV# zoX_D;g|DQ3OBdKjaklOFwE?Wg)?(>{E|3!=h{EGiDP}&AHeCl$$At*@W{J&9nAE1 zKgj>UyX!a;v5z?QS~pB}{3^Bl-6-k!SSb@HNzPL?*)-8j*)_%8u+QpgdN$B^so2EtiAoos-sEU z%@)T;9JSoF0IxQ{BNWLyC5XCI;|u55E@mQeFwRALvRob`x{-czHP6i+kfXihDrZf> zl=AWVBwO>n-~L|l`|TH2HMJ|lcXrL`+)7KsL35|ZdX0$)$VIzUc>j&dCkRl?>$8>h z3+XnxackF>3-A);`5@=9<)K1XqT*jXnbgV2s#^mQPL9r+?CGbS5L05z@Qk8|jFA|m zNIfnYyfKvem@8tpIl_8j=PK{5O72r{m6ED$sD5bKUP5uIFn~L}z40)b zW-Z2uXfqRWfpFSXjj=?9^<06SB%}^3_NnfjKB=X3(^}U2B5m2)srGBQzu^=_1Rm2u zG=8|0d{oI=M`1^pd)t{za@XuB1=+d~c`8Q!mqWDDwZ2BfKC4wR`~J9O^~MvTPDelV zP3wm{h2KKY4nvtq7gth%TTlo#Vz@ifXsLhIWnYoA)H)BPZ)b%!Izvq9_d*bmBK=W< z($f0c1G}+&AJj7jE8OPZfTc7vS?rD*q|Ls>dOk+~dFapzoa6crdf$s3;Es?Fzzwkg z!6Rc*o%a+lC|@QBPzwCQFXQARKHTNn=Wq{B;@4o$yyRNaf7>mcw+QDB?XA*Lq6F#r zyRF$tI^XMAQz7zfDXlOntHSwgLrUnqW#3d5X;|;$oL%0#Z_PtOM2U*n6=$!=rO?wK zlkYSJLUxiM|FHMxyH;e+^&YJ4&P9RtFIK!7#_E#5S2?xzyKVy}#8yP5(axKOa=I=A zo|-p7o{DB&>3T^YcEW7so8-1Js=s(E=<3w?D)1Z(`rA6SW#hJL_nY|g;|e#PsQN+B zZP$?u;)9Nc7ctIY`D@cQRbElm=y2Rp>0v`s@-QTA#>e-YL+uL`ROJY98a~)aO;Eck z&1fVk<{&;}$ybPCCq_Sf1hPaMDY_|2fFvyS+-}#?Qgh{MYrxqFa(^|!`T{ZnrM+PH zn|HmMtfPT&$c0VH_w_fkZNPqBO=^pU&K1j8ym1E-x(iXQhi%j=iEMZCE|?T31#dPU zH#T*|~09*Q4Lpa`F_NbTgka<+|@^J%hr zNWcB%PUo1sfSO+KJ*P(1`l(Js)=o7XCw85YYNKzN2K0DC&<58UuwAij`~xpEIH%LG zrH3bUA1py*0bFrFy%9c(xaW<^8hmxTW@_OB$`A!V5Y`C*g;?9k-c; zf{*jVxAZcGKKGCX{}Mv4$ZqS&?yQD?>ymXZxp*ocA#C18Nz|Gh{I-fhE>Ir|YICU} zW)#{cmt1<0M~0QP-OsgW*f}BI)pb!MO)N80aW(J=EZ|^n@(OpNc~5uE@7Z^;`&Z-O zXaW4zq7CDS>By^eqk7$)F2nqMHgr3Wrl98k2!JY4aXe6bYIOO;y!LY>H_vPdM@!9;7Ef1%qqzJjGpZ*mw`O>@E2(w?>2BwCMZ;kbP?_@Y9>Ll01 zBaONJkME+FFTesH+Ho>}hB&U9#>`}X)SzZyBn-FbysFP$S;;q~-J_4g|X`|!I_`4b(^ z3XOsUm>iC>nU1S$-uv;FO7qBSlBGu5Zp=+(Ne*4zXEEhL$hW!yU579;$#X&=yJP3u zoWPR)Y*9<^K3^;0tD{Enix+U~7r?^G8!U`Q?g1g<=)D?Nl~(p;2e1|}ojMNy*9f4i zdn;*2pLppbSH88$e3k}RkPfZfzJV91oVyJK$?)x_vI$2>?~vRqiZ1pfBCxZHv?kH3 zX!}c?&L;GchrHDy=$lCM>teC(Xp5zkM?k_8oM!VrTKkuGz8K{nQr(y)u605vR_|Q< zSyzxoO9OA1?DvYpd%7l#r`JQW30$sCU6vUgKLE4nt>jDn6@Tnhz+uJ3^|Xq^28?0a zKWs3z0|k*w0lR%Qk*SSmHy%J&+y9Q;7|R1Ptn6xYvOT-3dk7zX@hEM=GNTDPs&_#@ z*m<+{)8TsF+N%AMi!Qt_lCeeRjvsSt>f5ef-ca1ye6$)9o@cwxxA-5IG;5QV%!)5{ zokRbXV?tA|CHT)vx(WTEjlQ_>htyzg3J{gZLGY~21gkDJ%g)8}U-(gr(2~C5x{eDU zapK;PO0;5VniO$z-pk3l60{ci3}p!ZyFl`p!yl*YsPb?$YHIzO;&Y}0yGX-d|qHy6HzgeBpSmk;@u>k2unT$Xy;eIh{d1gNOFr!H@|M^>$7%Pf{1n=v>7=b3w=#b zm&b0+Sr_wwt+kl=A%-H2jDPBhvi8qfQiP~wJ%loW4l$0DU>pG_tB47Xxif-#P$ptU z{&=I;R%DjG@{^^oEzV)KLdfjwcfQVmfv4gM1^3qtSRCX(?Xll<4KjR&d*0WH__Pff z8HUh)Xd5DN2=-Fx+rEyxhzNhJ5oS{KG~6?5 zx|39i(&P`}paS+5ud70}n3{myFQS~V!0`ON>~Tr>!6j8q`$l|Sa~%A;p(ETRlv?(c zmgD&xBCzPw(W1imC)Eq`rv&Zcr@X@ZABX3Ix~1=^JpPI_3SQ?Oi-GCooFwEUmrVft z1!SkW44P@C|8_K|Jo{C9!bpXB0$1E8&c!EBN2cvfS2aeE>KRY>A*;lSPvOk}87MSI zL*qK0{+(OQyd7ckRoXeNXN*NEg)&M{&MTRM%U8+rW-$2&+0*us-VV|Gla4tb?@Y9m zw7T4}%N#@HTx<10dP|4XZU+KAxCF-q#T~}LN%D4g9xK za}n44;@i?<*g5LiK_~AY!GoLmrc|8l31Kne*ldjTwXZ(y==_CKRZg#^?!@kfj-|8m(T&M^qGx{dTL=_lXHQcGzZYou13R`kHj?`o zKX{6($3}5zDgcu%G;?4S{U~%j4{+!O|IF&j18h9Zwjtjhg7BY~s&jE=>Ur`-UeA%E zxyB14_@>^fCuaYHv#s*>-t-$cX4-LQ+ZcfGf00*Kx`DVwbv%~pDBkZucAC{+a+_#b zWvxqO@V(hmwmwLWHI$MZdCr?$RYCV{Au=F&5|@wC0jRyS%EdQoVWI z!v8PQL?AgMpb%q8;H{&~jO7pNDH&Y2Mjm+JD59vE0zM~UgWz#(q&izS|p(GSH* zX^)Az_z34tP|R}r`jlT*NF3ZLYks{;BuD3KW8d?m=}DS$#e?oHX=@@=JzmS-+a6_~ z(Zf+-veRfat2pD;BOyS2`ZIG z#ETg)LdM$m!}JOMqA)sSB?>GGSz1Vaj4t0yV7TpzdtvxI5j^;0tBPT3 z?0;S?%8*CXPfDVYkpIrNnI6Z=wE9V&ncHK+N#`hfxT!P-BAxqSpYSS6>xAGR&W|yc z8n^m6V|MuXa;(eCeHs%D{-}*Buz_90lT78fEOPk7Wu{DpZ~L-dScbgWG!%)}+aBJb z+7%0q?bwL6FSd_Hf$MR*kp^{1{MiCpg3rE{`wArB%4t;`6s3bo3jYsN<9>UC_On3^ zwrqb4_?H3M4Ni`+JizX8yZ#AOQvlXODkZid7dA4Nw)l(BNR7yDD*}y=`yV!KNp7at z#T#U>E+b4p(|CFHgKTr0{$taGTuoGc-#L6I^Fpm5ZroM!&xxGyX%oj^$+R`OKKs;Y z=|;Jw!jsUX`3*csH@SM%_F_|v53UX^%I(5{GO4#^xpW9sXaQ#=w~e?_E&bG69Ezfm z9)A=G1wTbJ6CZl6XUGGF@0~DZlbaB^4anF^O;S{>;eQ_Fygfv865+S}QDme+Y{4)( zzdMGp{p$m975i>;@3_qbN5KR(gA~OHi~w)(#a%ch8}6mDKO153=PpdDA~3Ps6zIf! zrJw6zuRh}p`DA(fH^qc~cc&c7Kiiy=Zu?cS07jz_KEg3o1eQJNaq=O7gkfqO{i#KGZ`Ej> zdT3zC2UB{_@pBeh2acLamJ4rI#*Ya15pKX%_A4HEoBy9Yw;w}Ht|!b#8vJ*nlnE*7!;w`GQA&}7*T0vnk6kl63h~4N58oD0@=@?|kGg*=e9CbFTzQ|EBdqpaC zdlCQ#m=M1+{#B18DGbxpV*Q-YFi9S^T*pAp8TS2?=64&Su*v98NVmOMj9Cc(4+{2&fr@X_f?Iomw z@q=eFLkr_@C^@^|kW~_naR34cU>j?$e(!yvL=%pL5_g(1-L>HUTzFelIS<6+Uc)qW z-IK{czdl{qDnJ%IsCRxwHI+Gr?jWizr%0P@%zwEsgPKWXUQ5^k08*jI7{&1JL8O41`Dc4$h+p`( zaQ9NJl*mK=huM#syyTN1Brl$!xiyK;G5T=}-&cA0q>^LJZsDs>>E@;2EhocPPzr1!~+MFntMmcr4#O=k|iXJ-b~jR5s2fJJWX$b2t~m^}yE)IPzJ zdYMLY2&Ralxu1tc=#M}I4pn0Dit46!@d?M_iS*Y8ugHET+2QX#TnE$hWkF0;PjNAX zbE6h7v+<{=Nw{#Y!9kC}K9rv$#E+T!<``Ji7sEXOiMQhEhpFO|c|gyn>PkH}`sP2G zH=D8f6Mxtm#}N zOxsf#q=byOo!F z#haU4;-N3aaXVeAglNTG|mLRP#;5R;%m@J#V>!~ zAWeTxzNCSkgTQcRP42b>14d+#X7V(sWMk+m^>M!qC67`@n5xdvGoTT z+dDrfL?3)~G9SY`f9KZJCVp&z55I3P#e@4@vnE+x-pUZp8J}E44s#Mq3oYYa$SD=sE zTPN;gI&i>Z*izHwZ23 zpQs+YXuZ_@Dd1PLGV6WyQ;IJnVbSiGFBq{b$iyXMCCoyx2nm`a5lzBzD?lmUWYZajDTIz?o@&&E{B@wf}g03aSzAsVA%(_lW=DQ-B3rWQY&nZ z?d$Wq%E2^$$h77e*!S%X|B1oRedRALp#gd|20F99zbFoYUyxCPZ!7LJD4u&f6s-*0 zl<24EdGFPRhsP-PG%H9Ujp{2^+>flymdG$R#l(WlSDii*^ zO(G7QbAyt@Y5&H9-uZ90VXr#Z>f-;a>RxzBr^(D(>Seddw}Z-haT^sjmx<2u<%=ecuTmtKs8E)0DLWW>b0s zsPY)?G)H=FgT$?$+nmx0_L3^LUsJmfA0L*oThp7oEn=zI;p+xTJUi9jgxx}>J$Bp{ zy{TBxz#+do|6D#i!2d&H(POlE20%id;#>7Ige?4{onq%R*3O$%o0^>`uEU!D_iV?^ za<`)KGF7nYLGHf|OA6{u+b-E96x17p66#gBO^(Ts8?u^%dWG z+32Q_uir;eFuSu?;Tro#gG%&!BDs+9Zy8rdvNUf78n*{akCKEB&+|W5wbR;=4?YQsqc#oUw|^}(p%r64=!3Q2ybv#hZGkGLqyJyIus1Q` zzXhnySvQD_Kt-Z+FuRGem|h{%xvus=rQ?KzSWP^=f~L^baLgtQrj1V}=#D{x9s7%f zIV{zKSG?BMR+i=yN{QAk8WBYs;r6C0&MUoYfp>k#kSTM`ADH?ny!;Y6GrOusWo-x2 zy2OfKVY^O~4L)0MPU-->GhtB1H9FtJ#eFBYhWFul?0<%pE#~28iHyWT zeBAIt3bAO7F>CGY5RDgu+!1aE-xK!pc{F{y_pMPRMIGt_g`4yGr?@Y9U`Wr2MwtAu z4pebjieR$2zh-Z68U1d#JZo8wDl3~ zjpMAXt0KynO$oVHxby1&qv_6oMY#cnS^ zW7c`^{yQ;HCo&TDV6?wWA}h~bGlaUVtWrHI3wnr{Mb5jkyFEp30~APJq<)wK@kbRT zn$mebb&stfO&(|jjInAp!1g1Q{tI;?{8%gw9XP;Iwbj>BrzM2>E2~RnQJNBAQwp7Bli4IHp>mvs-ZMW|oL|NnR8YOVqj=IloATcuyU&|m@1 zQAONer40hfGWS(_oYz;jUi=y_*#R_?dJBF0l#uy}_O!&jTu~5eJ_VA5Gh^h1Xx@IN3>P2nk99UXr0c45R}*txi*wjC-P%lLKiVn6Ser^rfLh zS}G^fOmu!25`laUcm5f%$JFlSi zgs#Tf!he=6KKH)UJo!q?2G1kFQ+baFxZaDLS9);wplvL9P`H#@(VnEvDUyg~kg8{p z8B!H3LocnE4d76#NP;1~U z3SOc9f)RO@(Fy$SUz$hZ{OdeiIOUHhw61}hSh_rQTgv)EruawWrI0zIDkixKqXdwT z$HJpQtfH~{0>;z{3PTj+aHXZ-{$aKqrwxv&)tMNd|AB7pM7L|sj$ZIb{1S)Wm(eTi zQ3IPq-HZ)Mh2@JS$B+JxN9*84pE53isaher0)B*@Bk$LhdBfjysaC|Kb0#U_{7dJb z6sHs%hov>Eg00!$sj@k0zh;rNPc^B|W=^!7q+AzY+_wCL%kVEuYrgk(3Og^5=QECg zdlQ?Cw_SHg!JC?xn)*|MHrvL9RmptAw7)Km^FE+=DeMJ@kt>-~ZFC$*o-frYi%0JZ zoSm7!RGTLX^o?KV;EZ6tPRT%$H0;xE+=2t4mmQ52Fc*=QX?78;? zQRkLAuk5fUrGA0rkrNKY8EO7Mh+^S5cH1(<{QY!Oj9~dR8Xhl~}>Z!Q<)6HB4 zgLCUdGPgn3_Grpf!J9M_m?bdz-+96MVo--|O!T2*HudP08;F>oI5PB2=vKQ~DDtcT znaHL-g38%|>bnYx-Gz{j!0)f1DBSn+odUHST>VZ}qEyn&o!m;meP1lSIRsys8JV;n z_+z;SCvf}YN6Ql&;{?a-7Il2?$X80rIo-C!zyDcG$Zh<*321)%1uw;kWwME{0BuZE zIc})sHPjvLIlKT1?-KYHS!#V~Iq?Qp)+{=*7X>u5VC@$ju{UJPsk)vlNGNdfjS~0Z zFaGKubgoz=P}PK0Ok_e1bFIS9k#hT(skLj7N6|b_w6Vz^buMS*RoeiFaMB&sg%iAbUd#ei zK7eq7{?r>G>Gs?YP}zu5CyNbI_m{gbpFUN7a@EdOB0u^yJi)ko( zPwcBuBv2c9*8Vt4L@zw>rKx24?7kVEOd=SS-~*)Fg6yW`CEoE-hT08(hEZAOvlCuXTOa4M zSkF5?XM4V^fl!O_hFkNz6~u8S0nb*o<`Q-t?FRFr%4{6+>}0mGS4xw@|CMfDrwL-m zP>UYXSK>&1tyjcv=|_H}c036L>Sl`CX$Fh*qrVs;)gf15`+OZR9uLA5Z9@iBV|?H& zn<3y!b3;|??X=In^C^q4j@odJl0?$^ zSRxEoV>CK0Rd%duT=aze_c)w=M%Gb#38mT(vSDr5j!hzQE6?HNPuiUz@iGxZyZi6l z21OF)2nQVH9Qc=p?9_-9rPLP*!f}5y*(tg{Hw)&El!SDqOl9tHTM$N{;b%2uW0H`T z8#}V0euNG2d*Im!P*L^w4Q})S(CR4wi{7`sa1L{P!14)jLn3}j&w9>_Fi_69y?`p( z-$KuCt#k5IJ3LJfi>b`W5H9PjTG*fh*j>#Ff6vhTK1X z`E!p+N_PQ%hp@$QH9)@eAZ4xrQqHY(vQ~IK47K=RET?^rP|nDm|6>6N)wBsx@}TG} zc$C|{w)_$CDjYLqm{auATWOyM8Qu6_`jGu$uM~@WZJfh_T+i`W3%6Yl`_9!s+2$Mr-F*zTCr*Ej zYw)pOB)#!F2mbKxBj3F?nn%@6#tUh~ae)v+lUvUW0a$3{A#CE~%zx`8pgJGXDy<4v z7QFD^t4MVlaJA}=M80Bs&;MTuex+Xlq1f&;Y%??LzL$D%!n@4;tEZN4Pu%uPBRpI! z@XYpE;LSbZ0@F;=2)_6==X}@Q*;;P4LoS!a<_8zcm+-AY>gd9jMfUd2 zu(L1sofrHdfBXG>%RtZu=C{Aby!#2z;br1=_7;IRyf!_+wxF3MvVu}nbatfEZ!7tp zJGjlyXz&`UonZD7m*l8QI;VZgV(2#*u!->1TO!y=?c~{bmB%I7_fDg@@gh^{CL;#v zwb|qP0;WOuhLntWg4QX0R4#XfdkdvTjp;QfT2lW*Uw{QVZdf7D{`v4kvVx#~UJ2uRAjAN5HCZ`u;(ZS($GY?g1$eMBy?PP)pGNfZ zYHTFR?&*GMZcRO6Z2H&okuu(89+GyiPHp%Z^^uCyR)`MSc9X@R12IX!YO-^7!Rn<{_uzL4|FcxZ}V2RnTEL0`0r8Fl$XAiMM`XI zf85xT^jg1Lc)HD}7@m1LlqA}ARKT!-#&&2yih^_v_tc6m18nx`XS6XKlr$EEYIk3- zHlBxoN{8{LO6u+m@BZ7wT>C0eki*#y!I-eOK?_o2`LXhJaXf=X28BN&gGsSDQB&Sj z)&n5}x>S_Brd9QWM*HvmAGbn%J)vT|X}KUu)3l~tFWtru*yp<%SF>+IMm7MqY)n)k z%~n>?N%oE*3SwRL1#u8@^Hf2hK+q8?9R57G4V<`>Enn0ZK+0f}2F>}B+K$Z3EDlG@ z{3*OYoZC}T#C4q;$zNs+nD<(eV6PM+M}RP%w1Eo0OIDL_TMnR7M|zossBHC!dm(yI z0{gW4ov44lf2E$Kl$qvqrZSfw_a3Tk5A>U=rvJBMXFag54%SHZM+Y?Lyt|`zb#f|Ap()EE6_MsK8$CD!M+~N$C z^9$vbe$vOB*LV(X0x6#cQmD{xew!2SG@94?QaegJ>s}9ArN*5{L{dZ}ruNqg9rUkG zj~8eKTj?ys?w8eLP}=N<@Y^ie5pnQbX6?f+dO>`%=MgJ#!M~2$#DTrk2z5Lw2wnW` zw#TqmwjdpPF~chQv-$2LJDL(pLXw}J8jjHhIQI3mGJ$B*53H;(JW$>f#|rH$mDt!q z#)xZB3c>)T7OI_gp5NkDC`ye61AG$(t^lMhhUk>F6f3WQT;m~QrQ>G@=~kysPX$et zuk0!2iH<+4F`|Y>;h*XPjdZKKMhe>Lrdnbb?oVubKAD=`Fr=#q_iIU}%1|Nx-WX;5 z*l{6`scJLK`neVmo7uPg>33Dr;jA3_8`1uFfYj6CYFKQq(9knADFlcbG83L;ScM?O z2aN}Q`SU_^9KS}#fIM*g5TJyF8n~zHwe>q_N|rv%Ycj0r%5y#AEI=?L9_9KWFXQ*L z!fVDPaqmt=EOVV5_Oy7fw3)ibh`l9+3wf8rziQYFt@lIY&LnxF;d7f&)7SyIjS9vuTY=ST}g!@kIJw(so1q=yy`{PUkJU*kv$ z7_yo725LR0PtiW8+LHCtXr4JfSK_$up`rkJU~46Q2UJustj`UU|B`dIO5`reft)DY z2+@v}u%GJoqj`Afs`&}c`D`@9Sue7T^fOX4zubnr8cf0k5;O;v1+&iLot z{>+*nLrg5E3~}kdy!vx3BqukksUl7a_~h|n41j^|@#7<8yGeeJMpSC0JgCFEcknUHGa$Mu6W7E35G^517 zY4;v{4Nv+D_fc z9nB*0ZjvR~42#GWW>yt0{i^H9GzzGFDqpnpdu4or{c5%4EK_Rr+m-g#n2(;?RE@Vw z5_3ylydq-06}A~&KZxu=L=Qod5E2g4!cwaq&t}V>?M4;n2>W@d>K?0ocNO13ux{nn zX|l!g7 zuy!F);E1`W_;h}mt8~-61;(|8-#fiPWuzwAwh>SvN7YiY95JsMfZ$Pjw1nSz$AQvV zE7CoR3#Ix=0OFXkvri)keW&TsWNz;yX{X6N^^rLT)&#bM!=Knk9xIi|9IgzPv(LPO zgloPFP>@5lwdJ|!LWTr0vut;^7rhzF@dLs^tlS0H9W~M3JNJ#3;~do6ETEO);H-PE zfjN=m`Hvj85-5WtpcJ%rE?N07HW9z(H&bf(5Ulpj6}@GpMo1%%bMu{c-85&|#W}pC z(p5vZSWBDpAM#9#$vkx=#OUQ+$4s`GE{d}3B1@c3Q1)4N0`zJx+=s9c5KwuA{C@N= zJBF$+16EE}jm?pI&Rt* zA;oy_7_rdvCAq@O{!}l|u8$!U^*W(QBU4b@s4$A!R_ki_u9}a3eg5@BY_Mu`?-MAY+MCPg?R<5+bcQ#IsuQ&yZ%;759Gm*AaN)MsbzgQb zm=zo;&d-n=Xw6_ejABtBStYz?$ENp^%4T9SY&-}alMS2$p?m9aQ&7;Wet9}uZHp+; zYiqtdoc$MaMRD|r0y)tKN<{SZDse-?atmJ(*AwqkHc04hj=lezuL$l?xnu*i zfhAX#3*Mzuv_~CnCvK?ki~zLO5K)$e&bd$keA0B^(eh5O5w#Y{1zqMZH2ePCCtLrj z^Wj2!{m$vRMC-WJKHVN)z+LJU3nWl`x~x}(gPnEh0tt|BNEn&YhK|AgJcKN3|H?GXPX z;cDWs;`Ne0FQUS@qL3$`j%wS<@3^n4A!-h#9&uDO8yunWIZTPTQ)C^_Cb8@OUg$1* zwcwC^eM*V9f3>D3FOPICUFb3Pd_!D#WxKUE44pRDe!ffJ>x`+4=;F{^sc)OMZm*+# z`oNd60f#hhQ=vW<<{D2|h{F=~)kr4Qcn&~v;c8pr<**6yY&FJ7DzRy18jc6&xS3|P z)v&dpJVL&2-Z}2}bpFe%e{%<;Y~xyX*4gt?cc~>$`j71f$fgYOJU)wpPlsDBS#fZP z+0zF7a9yhT#F8$wHA4tr?7|2IZokL)^pM zBeqNU{^NT!+Q43hsGfL*D(vSO-Dl3}uikUJR$9O;4+I%#jGlV|;uP+;lZtSy2*D21 zf2zWccnj411O@b<$^4%elLd79tOgP~G1!72c%Vjs;_v-+xl%-^_5iL_^RGlU%C(_1 z0i)T1ZXCvz*C|(*E*o7U6%weiTqpUV{Yt**a)XfW@))#q$y<^U+g$UFZB@LOHqWEG zq6em#lK>rB?6BdFoOmy;RAHc<6#eAEY~QQhj9VtR=h?AZcz+LM6C>Al4d55PL+GMt zmeUQmBAY^PB~;rPPXs2a@=qQvtyD@883|xfqcCZtH*VLi&n4vr;29K2!%i12JV==kkj=E!9Z?5qJzQLxlr3zsJUY4Sa@7 z*(n!iTJ%IKu)6AcBb3=-W5}S4?hRCE_LiFYYi^8Q{q|?G%}H}z*YGhgdQEW2hegT# z!uDVcVJk2-j(@pYTLd`@D^#y}4QHXm#Q&mtZb2ab@qyp4j?wV} zfPCKTj|`v0Xjfx`=`@%5?DM-e`F#F-dG}2E^;I}42ZEP4f9DSDFO+`tYS?A?FXwLh zwgUTtYGZjDcqdUfejO=~_BgK5k`;t*xuV#P982a?L$@}Ihu*;!z72il)d z4{0i#f>?l0D20q{^=uCU3^9$>cOiRZg71a7Wd0>S{51A;zTz4}fbHAhb_=fYq4??V zgI9ZSpe?0Ns@$@Mp9RK(+gcCW@Xq)S7=PF#a1$GO|Ih&8y452Tmc>*v>@fR?vj~KU zS7t%o-ftm?ZkWL=pq8QPWvQd}m2J}NwKR^P^M~&HvoL5FnJCSxg!mC)=IX~_eCF=V zooJ}T?+3%hlAQv)WU6>#37IjzJZVs1UsDd^R&+o7qlVb7X24saaqtKCcNfE&C%DXE^@?RsIF8cb#nwbOzu-e!TYYuY z(<-h9#Az5u)t$o`Lmu&_qH*+t3v`cUmlB%$;N8e2Jdx=2>2p-wN6ROg5#S~HTA(Pu z&!csX>+W_CvJ~Kphf2XQUsBtsrQA{ zwjmDtfS%lEVASx4n(}th}lKAKXWFkwcE@v8GdE zTPYYnkjXsJ(LrCPnZ$J$G~G0R85=#zyLiM9I+wzhTAt zD>#y1i8&gE=&#TF>iVe>&#yAAWdLrECyx%VgtkxR2H}<7T#kAT&q+SH!D0u0C^QV_ zFjRQFbAFU!fWBLV@y#Q2OHv`O_DO7a*qo|G&>^nUk@5BmmI}lEmlz_w_rGs>!QwGL z*{r|7+QbJpHh(1nDxSuI)r8pGl+PCl4P8uH-Zvg&YQ%~)jY7t=u`}HeXAAYcjLQ%? zY{arJA^KZtiM_`W2M+E64ZlntG{Y)Szxn_NlG7#2>g@q*LGBMwt8Vj+T#zj1<7%v_ zKkw4VF;Ex%d9o8`ljvD5hbBp^DfNs3+B1yFK0<*FM6pKJEXu!-oXjrQ5MW7S-s-NL zH{`A1wqOcsvnjNDDPTRa`FzdKuWmJFo=LdS*^Wj8QpqUo9WhJO` zup?=tHUP)aLb)TJaM^KRAY2=5`rAnkj>}{;jgPAxY0beK?j1C~Cpz2A+p)KO&F=I$ zBK^?yQDW#tZhY@e?zP>uE6{y4EEP~h3K}}sN-za|SzV(F;1QZHAl1qT+wf8`_b}60 zeEqDWTqu1-&9 z7y_ERD_8hKsAL=6hYjV(s{&|RVOkp%L^IS`>=d1 z|LT>{izgp-R$J$0D)@Q-rqpk?YP}e-q82c=d!1ZRP53FetIGX5b>Hrnzh#8y#ifzS zkSZjy3>Izrjc3)XuzPv0NR(F3mMy-!92H1rJgC`ROr+_+KG09hPjhaS{*7P6Hv1I& z-m1n7p)ys@BtF4tO7e+!%O3{|X;5R6r73;99X%4E)~*^6S5MJ|ixF;39YNV}5o3Mf zu5c|p5Mw9#x0j>Z#@MVysI4}X4rmTOoXq*7ovX;b#qtylbKeVRLAg-#av^5H_GbrV z5lo+ga(D5$i=1b1JC~%D+fhh!n}_KGwP;ZfH=y`MG}s#OepZady&L(e{S!Xd7P6hg7Ikq z=~&2+)Ugj%u&Nzw$@Q=tH%6hyDT4_8-`8ql>*8Zv)6t1GQ#9t~0s+Vgejj&=+Ka+- zNa>{=8z^1!67QqyJj%GtsEU6ne@yUAr$r4-W>m=Ri}I)8)GK4^ctWng`vvGX-%ta5 zt+dZ7_+dp7)SSU_o#-7uMdviab&SYJ+d99@4HddWjf6r27wHM`yn5OJaFBZ`RJ zCtD=9Ftsv(&ITfniotXicS5>1N!ko1s>>Isz|qG6w`ZaiG5gk}R1{Gm#iu1KMsF#h zRvn-JAekZBW3h_Uj;q)ve3yBpXw9bUU20Z+k*hYip;Yx52@b&N6=P%g*OV3X#kiv- z6I*flz|1$xDEk8^G0ANeQxa;E|8_!cA95~|s+SvN!cuf@myblW4{+q}z=lA8tKCYkmb-1SO zKMm84bqT-ntB+o=ErH6WJx*r24%5q_EF$~fCCo>!H|y4& z9A~4#fOg4=POb2g3Hze0pb|2i$v5N2Yp2fcj2K+LF=OL=ZjCt=r*~%yHnaRjk>AkL zJ3z63z}f8RSf;PgXyMoeaCb%E)4yn$SDhJfks@ygpT~ymy4rNhp#S<)eojH+$c|NVy_wgXi=5 zNM8cNcE^CE4%-3tOX_~M?|kGjGJTSMnU$W@*Ym9pZy5fqof$|IGF|02aij`upT$3a zupLg#C}wqUMJzg?C0aFmouFvshbK|-;-vhFUoLug`UqgWh(khL{j80eCkfW2mGG~s z?@T2X2UqT2fNsJ<{*9b3ATkq9oK>6h{|GVUZlC)9f;@C}&e*^lL0SV|4f)GD*YDHA ziz8Snf;X2x5M64bC_+v4Q$-pH+PFz7qn!n^o`>dIp=u}xiFpa3k^zXk_Dn>X`PQtG zVQbW7ZH2!)@{FnJ>Fh{eGeEf8=+XkWHHP8)^oNsmC4fq7cF2zM~^f zlf`dbyv`N`%Fb!h8Rd!IJhIH1o$CvZIhs{<^=eLR_G*qY%0O4!{64zef2Rts;N*l82}0>n>3JocXAueZh}&!nUXA%d9l23S*u^sVH@Rp1oUY$_&$8MW}t>fl<455 z6BJH@cI6iTbrLh6g@_$jRLv>&%969m+ZJCuNsleq3>tlXWLt@Y?=CJ)9i`R)(zpJjrF zm7V_9qxyR51D(%Y}+n#Gh4^wvM;XosHvqzPBL1Btxhko1S^?;)(yX zpq=eB-F5$}vNXTePOjv;N!k(C(S9|lQ{zuAWm!LtO+yvJver*UOl76mr7r!+duK!f zOUKhuG|*|ms`O_YpeH;dLvu3h7qbiMHfd`s$8~PFm6xw4Pu*n+!^d^xQk+uu8Ux3P z@Wq7`%(&(yn~z?<+f8b6;eCs^f*pLVd(VV-`D(l<0o*!lo0*wC1am+%dB$92u$n}Z zD4P&IXPNIS-iM6-r&-=7;ple1Q8xJ?A}dYET&xxtv_y$V2nrh8s#tAo;|bg$Rv~9Z z``FDofMP;-rvqw=h{WoQ#ub+(x9TPWMl#2!`!0Z5o~3HcCv=gqy$>_s;d^yOm&rn4 z`5vCyCTWWgt-KTo{H=ef`x6tfR@jn5%dZ?fYG8{9awHA<%fLsFfn{Is$yU^N(EYnb`05)Es*>L)|%vY>@?oqv%_)%Oj)@0_c)M)7$O&40bv z22k&LjQ<{CjNGLG)@8i-r4MGFpWXJI3LA*`KI9`5pF8X=+_4?cep^7&<{=@SqJ`8# zakHZR3$MOW4ZC36=P+*C_+2ud zVVh(AecjdC!fb*qf;7Vg@<@`k2^eXj$>#FyZa0V_b}P-nJ=L|iQ--R;XK1u`zC`yB zoX$5ryByO)@lk2AG;OWF-|+ll;B!dI%j@BEJeoa&-k!8*I>Qh^Wb0Mcy;hdt7ACji zkI1#+M~CP2$x494Vq1+k%n@Bpj@Z7G=h4M4 zouQi?Z&(M4`vnNxywE1*-7KN!dl7P4d{c>kuve*l%r^;(XibPiL!@4s%awSf4s!1j zN{rQgnX3+e`n&!O9|LA!5 zFjkx_#Uyz-+y5^UuzogQ5>oQ{u{<;k!_>Z;1p>yb4yZ~ZmL-Ny4;)qUEo$dC3TQf9 z(`xQR%C<7Y-h|+c4>Z`9g(?O-(mM0cgC=5M)LDt88NZ*`%z;ygP2c&QwW+(+KM>(f z3z*M(j44$5TKQtFS@!9d8RIWK-0tRy5!+8S6E8oLFw-+W_f(5zv{s7)doedQze$J) z3LY;r(r;A!Hg7dTkc5N1B;C$?N_bq9@sfw|r zJeXQ(S?J>NLi-p<$a$F?7G~%Sw>Ru2rkOerFq?YBB(qjz<;t+3iYAo%Tp|&cnx!Ju zP3q^Rq$)M#7&eGDqRFkZ*V`TE=}(In3x9{Qm$CkxtW{Hy(sD3k6S_>O*nt{N`WEZ;6Btu6v?M~L;ulx`g1D#N z`1rSo+)T9QY=`*+I>$_cKM3Y@oB;Vf&!)TIgRgxz!CpVj_w;-SgI!g%+8FkS-cw#HcyS-z5%AnJnD#rvP&eqygnSm&*P>1(H%G?HvnT+xt zOskW%Ys5iOcct}w@*Myy|LRwVBEP9HL*~$o=-^_L6QQKpK~+wHlvXnd(JF?*+EZXB zTdciG4{lB^H#O1v!lUVLLK0@Mz}Es??*GONa)PWA~EbU-$HPIEg!d zI^KmN?n$pqvL4&o+nK$$087Qp$L2G&pj^p~T4LKLX=RfuK0E$uft>~1)EjUDGkb^9 zrB7#uiNixTqnZQtcfVkfD&#GTG?s6&5312q-;*&5Mbpxj>W}+GDqXTsY@id}uikaN z?O10|u-jE61oRAucH$>7pOP(bU`rpZ1>a*!&1me-PpEFTsggA-AoWvOKv|bu#Nb$Y zNot~ljyesBOoQbNRXLX-YT_qyxH~E`UHVDdV3(ID zPe{IvyY39WEyw)pN1GUWU%N@CnvXdq4%gQDmy)wF0itE**-vWZ&%@xId`r;1O}A{V zyh#zgAZ%UI(sJXM;1(V8+&;b#Rr(3kPgR%OMxJ?qV6#<|M(jp zDAe~(p|#Arv7?5Dn%iu#-U-)7;GNnW-^1)L{UXD1=TMl$TIj5NGkmY%D;Q zI{&P4YY;P5+`E6;U;pn1I>PC(KU=bY`P$__)(S6+zNhUC+0Am%M8NOKf2-w4?l6(j zVAS$yc&*2>Y(IpdXYh2P(rvU*17pxm)rinen*FfO#;1BCH-^45_mhvLgzNe1%M2K` zIY1K774==#W0q9I)(*4gMV-i?y7D}e%=XbNo)5Y<8hiiFVM_m6nl{2K^T?>9M|n zFjw~-Dt&I|M{^S*;K64|V1RsjTqdqaKE0G9J3O>=CxwGb$4T08r$2^{4X(iK9q~CJ zkXPU0*N9nz;OCL-yS4tG^e(*mqm-s8XmF31w6wCT5Q}K>#pvi4^S%xBDKd=gU z$aj=qUC8D(k9b6VC^{^(h%GHL@$6Z()t%>MGLYl%4=*bkXO|}$e?Nd2z49y{Ytx=& zmPT$OT<9~E{^hV~E3z*@2FTd!Z+G?waUm0$3XJzdyhz;L3{rZi?UF8jlY=apcN71L_2M+=!~0 zIT4vlZW)k$Lv!q^IjQPg$g6iFO2_+kDGK_~AeUoL5s?(>I|C!~f{`;X!tAmG-epMmOEYvcCEA{2PG zIv3b&69At&fttY9XW=ZC>va!pt{UH1;SOS@pR0OWF-Eu^bLZ^0Q-X8n`V=uURg;Fl zg}%jd6c1|f|Hi0EI!cbb37?~P(H&N4mwxk4@X!}t{0qeC_{vC*!FTp|+ZmQ~$GO=h z%snU9)a83CSK@*2PeaHd8(Kuk#z(2lL*ffpRUel{yDNOjTp|=FM5RbVF98(MW*$8Q zs{650BXJQ;yT5r6@rV>YS-enUe=-nsdol+qG@&Vd!lDLVfLp1$rYu5wv^tgSw!2|T z!Y2N|J(pHUKLu0Y@IXld3Zf(@tQ~Us z2TZJRM$?AsgpT(lk!@U{Bt=Z9g-JNzpmuS9-Get8Qw*wCAnw_n569RjeCLMs(yiJfrs<%&bAnlIT}mHi5b4xSh3C543svs z>U%S zSgMz*z3olFDWvi7e>gY!#Ww6yAyELaJJw8%L?&+Ngy&(RoKCb)lB@8>Js~mn0e9UH z>7z4K5#^nE+oiyj!?+>z$sY{YV-OS<7)yT^sSg99k$o@d8G~BQ52z;eo6l9Mt?FOF z;y1NWxY(;q!0n0DwWCA>U5>1_+YUt>wP2yghcF`!(MO=FfDo0$`Y|Y`GU^ZmEB3O z_MdTq(3z47unI;Xq|=$X?r*(nK*+$U&0yR-`9Kqcr3RNY$J%cU=OpQr+j1%)0^jlj%M2yV zD|T=g?_5x^G8s}X%urmCOi}CMj-KBnwOtF5L#3f5P*eemN#xNH^bw*QFjY?+X2-A^ zbpE)pObVU&-48`|v^#=Fnx&UL!^gr|P**e9Vdf&<;YwUZ<;`JO)ES00+Lf@N?G7q} z!uGU%k9j+RSK4l-FxnS1%DCb?MXzV+%j~6f40LqTp$egHUVoh|Y}p^*5ccjHB}tj!=VnUD55*Z@cZRZB&e{kg zti|l!vF7wfr|zF0%gBs&hX4CvPqIkPd9ux^nawTjSNTqQSANSQVvc@cpry}ozR{#y zZLS->yL>e&#$>LNECdRiF>3d3vY#aa@OCm$J!6vf5WSmj>c7QBL9QC6(GOga-8t{! zu-`IZ<-B%e%oQgTq)9OJd=uMOFfU$xjseh|rrUwkJm6$QN&B4zmcT%n_mI=uqX<4nbKneopXU z_2xK-M4sb4b?c9{=kYYPE%8nycHR&s!@%3vMTLa{wo@$!_;m5zvu2U)UrJifvB_~3 z+r9Rs=;*tq3O$*wT?6c|Baw?ko-wt7-0FUhAETQtEX=gY$H@b+!kv$4Z6@j!UETW~ z75YBu+`~C?kMu(O70NXsI5{HaB^C;n&>kGCnaR6QH3@uN?gQSlqRd!qzR@@UGyy#; zYIh}~6Uq^N*+e(+$47KksnrR_2}X;Gy)G6OD3gA#&4{GS38&vAJ7%RMSF0t*yU?fO zd|p`5_E~r>bHHnaJp1VHjF`UuCcH>VZfJZoXFPy~l9|96HR;)rhT}ER0(D=>!~-eX z;alg%i(XeK91h~9eZ zQf$m1B?etNLGJwrQ^alNc#$Lp0fESs**7J;UTe0sxR2&2Yy7!vU!S6}KhjsJQdE4V zRM>CmYPZJsEU&iLMQW_kb4y!VvcDYpO`-TjJ$Ba}8Kak0ZVpegzPVr8X4(onKlU=p zU<~lu)DbNv^zSGRBOC3IGCSnG{1g!ZolD!_q}aZh4|!yJGZM8mY6(H#rJhG^nUd^& zlxk_(k$UkLE&KXUSB)sqtl+Behbru8lIPD+oD;I{tN2yh+YMsEHRlz)XH4O9thH0y z{NAh4nn})>A6;%HNk_7TuG75QraDUA>*+8k(gK>_#3JD^_n)o!l5m2&G^ZD$lFh)b zTE}`UUs)3~WWf0qFcB%*?Ta}wE4p!l*-h^oXyI5C4JYVIZDZnxE8v`eOn10W#bfwhTQH*R7uSnE*Uj#E9? zR%BK3qp{l^Bhx?J1@bhTT%lva*$U6JlN(S(rIv{AZ4cP*21?J#OIiI{Xmoq_np4$_ zmek3OO;60!EJXL0^v`GPTaN4Ex<}=UulL4KV8$MhVO}+&@x9+MD^h3==PdNdz9y&4>u#u$L6Yb_w3Z zlBgHDQ>Q8UUf_>CMc?@u)P%Re^gvC}jbV3Kd;>MTC5i(*`|sgnWjFopvapXLU6`SU z4c&3+02c3%7rGj!|EjN~V*3l$2VwDj@99Qe6=5-+^!f<1zkB?FMIAVBi&kgz^`O_4 zIL7hF>S4h&YFPnlgZkam7Kf}62J_;W+qQ?rqiyiOyiBx8Zxo+2*T3DOn;*~)i>T* z8mBi>6gYX-jI+bFdIjhIevD8+T;#&?sX|?G-G;>{O~KxHkf9QPN4UYlRzu59(!4Z# z27e)aP?K8+W)SQ(&$%I{aRD+w`uMf%j6G&7C1jp}dlCaw6E`mqsNBcYSGAQ#_w7XM z8k3bsH-UY(Qcixc=4`(CohvHYj$7M_q0KK(I(Xx|WC_!u^n}RZE;*;Tn+LF8bgOkm z;SxnL%dAhSOaw7>No`ae68k8aAZ}Y`I0@>86?-B~6Z9J$QJEDjzpx~mE4KpuCdbi` z3Zx3m>35F7-nb8uHf$Q&fzVJkeKLAmQ~`A^(6J=6NUX(^{%pcBp1#6(cKhd(bwqop zf=J!O@y-hA4O;o|$4jI2Us5z6mbfP?iMVc=rZJk(nf*1*>v)4hwo4&iHyKU*X2S6P zI#T9OYIi-XvW%~1VV14g065p91k z7bzQSOV71~7P@CE=GY9fIfdcorU0L>A6yd>0phX*XMDGLPA+wRy}!Ii7VE{5`s^E`YgWW!x-ACcOcSN*WE~?>Dt$?#Y8d4vn|SNTEh?#NM3)xR zux1etp(KPi-jq3e=i$-;mdK+8$F0zIWze7Y8sWfLT+sao!rDd6)0h#PV;7hX$8y&$ z68U$_H`_9#5Bo=`Ld4sev=01L&qtb5HX$u;uUZmF>{T0s6Ncj77|?%pJb^>VOny~| ztkKymUQmx3N^^=7vRLVbGZ=1~hGLdqt<#H0ApQl7?B2|Kh_=eORT?}^FzNy_zc_Ph zq@n*j>}uU*vCt?Y#xl%2Ip-fxx;}x!)6&&`KGyJRx*y+D#OH3Eq3nk>C*sj9q0DxZ zX3w%j2wS_eT`o6A(nyzU<4&fi0pnnMB3`@S6*60|E0u#hIiXqfW|l9m6mT*Zw0iVy z;|KZQhV_7=V9y>=jXBV~=OUiZu)%#)A^EN{E$E+#rMWCVu~1h~+}&pE2y2PQGven5}0TEWq@yThz^-1X97^Q-8*r}fipwrDKO8s+JN2`0J>U)vQ+~L>eE=_R^buc*~oKW^?FF8QA%+Sh+$TT-JQaH#M=a z6EaBCCQZ!IW5;B&{U9{?gvInc=bo!EEe^%#;zf%9FH=yFZyCy0s9@H>{D&qjhYlYx zr#nT2ZH@gEF~KLlKe&3RpcsC9|5JluOXCeHed=3z6E53t#D6fBp4{(fuS2yYhojphb7lRBaa*3 z|1f^Svmr@#XRTpc!$9BqQ()gDLg44fGm`t>EkOd+}Ca^VeUa1P~Kr z{6%=NhUiMhoPb?0i_z?*3wO0H&_8^S9*+seXy>&?3lm-^D&P{+$S32q(+D$q^q2sj zk4&5}5zJ*DB6n1;j)}Qd+6eYP8~VUkWZ!K08a|DWJIwxq0`_sO_US=FP7dLZl}yQh z9>4Fu{K%7~f%q4^cBR*vF#Vg|UP5@Nt>MFs;n}veq+Qa$;fJi^>3?6>K@g-1soWqu z9I3$~yqIZC>|iR1P4{JVo5=7?o7&j4z-=4S#ee9Q6*`eABy0*2?bB2KqpjUNOl#!Q z5*iG&g@CdK-MsjCYn%W-=@GwvIqs-BYUbV3@MtK}g|hK&N{{6>a|$$G<9s zTkCNq3H0U%p;iGZDNTk#>_hug~q7OppjfrieI)Zfa-+5>c53BQT)zZ1K5_{sn2adx0yyI!E<#RALo22^zL4T@+@bU?$O@cNq?erqFNT;+#%Zw}M9 zZLTVN8^eFM6c%NfVqU@;NE>?cpZv(XGdA+o4;z0K0)Cm+pnOkgYA}SxgBOOc2 zYYjIexM8DMATHR%d+)v1)U8+7RE9}(ID|Fws~{YgC|Sy82Z(H>Y?-oVInKXbS+1P< z2-W}$8h=k36dN{t7_|Q_l}CH{qVAw<)22#xcIl!w4tBu#74Z+=m~2D7b9R<=8MRU;Y{7rJORyq1C{t=ASA77|%9VsS1x_ z0I*=eB6IaM6>L`McjlXKz+LwpO8MmDPt4_4T#h&~l7=>f?-KhTD&~u)QY#@V?luNp z?kAyRoiri zxe0S$etC*{7P~Wb6F)RzDNvw*@@mwmfkUdhA+RsPy)pA3$am-fjS&9vAS_KhNYo|J z__v~KMj|6lz`?qpI6qU{3pQ5_vU$%|GM(Pg$Du4j*=XsAWqWiorx>9?FOu_$BRc zzUd~U7;_g62yfcFQMo}G6g*Y`Ip)&50V3n9!~Y&Xl57GJ!{Z;o_61Nb8@}wy*ZeF; z8~hD(V*E$^&Ed6%ZxJ4RaO6Dd!w5s`wdyg|MgpZuW1;5_(?M-bjgWr79 z=Cv_|sXPz@*alKfW+AX`wHqYx3^GVpY6`85JUlPKFE#DJQ=>m$&C`!I!zTEYfBCs5 z5KA@>LDQZUE<=fD2uuzRALg4Uwnbed65+4Y`YDqiZ!Opso`cwm4G3*=y;%=UwP>Ou z4IxVbfbFjxe}23vFs*?`-X08#(f@)%Ma|{cRuUfz2^)c+wvd1TwgqlSSImcb3+~T` zA9-(DvlAv+qn~b#YNU&uQyxDxl&yM?6|6nmzcVABfJO@SFTD~r@{QUeJ;}UB(;8^x zJ^g+ErICjwnti4q|0!IVvJ5v*`~VPfsDZwn#E*Y0|1H}38QY3Gj*}Oh z{YSw#{MF<-(z<({O!t#G?CSa7ADlf34fMbcby*d8w%_$}9wjem$-u+E+ytaH{~p%H z;xu`Y8H72+;QP<{d>_uJP{Bl6ES^2hWWkTMZTPOkwr|#A!#67EzutI3$P}Kppe=rTlHvhr9shA`+xDLXZDl5$ zIoYW>!?ytp^l-Cq))bVm<6opS-p<{RjVi+2SH{bCynjnJ7#jM+@EG-5^iRak3Pmoz z&eUj%O*aDjGQMv+5?=S7{^UQNg}A$A5BYj$CS9P?UFEK=-{1~CMW6l%er^b;ktdDD zU}M5V`mC>U4dLfpr1=qB#IreE$2bsl8yx%S7TkKi1@k2W1H?z2X-!Xft>O5On(YK| z=HoxzPtxv~75MIQKip4haG6O&?sAe5gp()W9D`n7Qn8U&0r$hwoGliJz-T%NW~gAS zgew!ui;vopC1F#(njeHh+vwh#eRupvT)r(o;}AB5YkISx6OBQG1_imk49hS3^iz2NWPBpzJ1Qp`ct)}IU3ZZU zbbgMCzpvJ=HN_y5((~TleS6I%7heJ|GF#2-Z@eK7dTf6Vc;4H)XOEPyVVQcI@VGKBQg^Kk!DFK1H|OY1(B%07vvHR+Hewc^vE$Oq@6}S$*lQm z4a%{CUu;zsc0(|&!NEY9)--M06t6!sn)hbEE6w;Jn9o$Kc)K}v>?eH!c+_aCIsD|~ zkIe%QJ*bb6EMBxwfUF?jUhr!|li%$TEMev?W9Q*x|3ihzuer{ zw3(?*ojut=+~)R$E;tA!&w1Di@#UTCo$HQluAKy`OBF3ypke{3*z@PgcWQK_Di>7xCt=R#}oD z&+~a&ys~?E&~zOtV{0hzABBem6oHltW-#(vLydepnbt7<4m{W{z3e*SpYojZi~;<% z{Y2;bN%FgPX($l4UyFt23}xsEXZv?w3s)Wp@=YEYq?)UA*mbaNC1K!SnQJP+Eueix zID5))yh8s${bR1Rvwd0Rh?%g%pLyqjNFFX_ThtDe^!;ZHx5A5yB6`^1Mx`8nJ-q#s z9zenmi(2p<&o=`MGBw(~4lxTfvHz#Se|5#Xv zgQjkHkLJoxr2l0&z;~NFz_%D~{XTpHpJ@kzJMQabij}=qZJacSzUjdiw#9M%Zvz{7 zzIi!`gILeBy+erSEsV>H01Hs{)&5dNE?4yGI@_C2Z1* zn*y5F9F=yzZXEs#3enE}S=Hmu)L~DVjra(VSUWWTta%P>*IF_54Jn(|W4=2$i zW+cvWh>oH8pB-ML?As=MlK^km_U-(hT}y#K3qG#ZvKKb1!2jcsL-;x?-mCyvc>L*e z{m%P9lQQKA0ZN8r_{wWHd>C&-iuKQr=Wne)^>~ogxt@Jvd_VY~pAWofAsZRO-|ks# zbZCwRg-&122PiDqiIU}Sia&?&7B)++|BnoRQ)Of$kFuj>TI1($5)QkzH))|L!jlas z>9uA8(9);@6PWzOM3`ntj($6ih9s6UgvaTPFT4T+gowPsv^YWp?S<%*btB=t=qIVg9i;}AstrD0bBYL_yX=L zufAex+*t#gfr0oQQ3sq?yba$Rnq`_bhv^P10Umtt0W)maFnPG4Fw3xf#Y%YR;d5O` zVXE~byxCNT-9KCdBgsE!XfJ;4wbz+OjT&gKE$p}Z4R>^ zS|>1{{I7ZG@CQ|55tIuD4=bGJ<37({ZF>Bj!FP~~%W8yeH9CB4`L9T^BI3W7r%Z;) z()sE?<*7K+8g+`ys>%S3xIgSl*F;U4Hc`{NVOLiRhZKnXsVAR=o%$QtHT5>P+R;dT0@OI()mEERH>rr+4Cv&V8$yrh}zp~&JyvLDp4A)F0FQnhdWzB zTU-g+apGj4nM!YXud%@soQ+R5x4;~Rt{xfWN1moNcnwhr!st^^+4{`;a?O`Q%Z95a z`;Slz1pE1?<2b;vQC?X#;K~!%{>!e^$n(I5EzKJ7_(dF-*$`f9TDNK=1Uq)@l-C*| z)hH!H+rLwjjGk)jLlfY%a9IF65JH-V!@-02iXcpj4j(*(U1=7%iXy|!g+{(JHS&%# zs#T*#zPsR#9y8huA3jop&&`gT1rp@_K@$3rU$<^;9Dd^Mi8mt^@rHzfyX(!HHhC0E zBSOpYY%_TW7uSC+lUf83b7;f(gMzAuN~{;^zeAu4{m9cj7!mM;7n_1FyITDBLy%|e znvnZ79(;QT8b)m(0{qlCUF$SXwxNR_hG`8BTeIm^8_@lv)t>9YiNmhMdHGG%1Cxl= zaCta$`~cTh!uz@ITS&7a`#+y;16urT?-8n~ZCIl2b4o^W7V<39)m3_O@B z8}d;9`7tXVzNA|$e$ntK4&)XsdktVFo97>E4ik#)qPNFi?lv4=yQ8t_NI`+NxEc;1 z_ViB+!H8O3Ry>T20l)01LLj~^i(dj%P>RYj#tu-M#q9F8;sD;Ajg&VZny*E0Q)fql z)lmTDD1>Fol0)5is6Bc3ihz;EQd`kLYzHA2#tRte%`Z`KkRD1 z##%@9ey#u2Ajs>>u0X&KLTy8=C!WBMd;euR^F~%hJ>de*N7C zx>Nx_b|V7(n8+R<@$f#7U6`)pNFIESppo6#oM@-2X)8C%BU0sP6BoG591duKyt&Y#)a|M zrA8iWC<#c{vNQ13O~hm!#E747VfZ$Ncy@2c!Sg3@@EkYfKf3s(hcIR?!}|LkXzcNK zdw!<++6y-FEbj$*f@MPr-P}oLyBbqGUwGIx90W%RXbWyA%u`cMk1qE4=Ss&4B7p_j zIF?uQLm=Y{s;D|;TO2BvGRm6#){FI8bCo{c2^2QOfyLX)wd+mcq7VX+-W)>nNPXcQ z8hLo{8wpp`Iq)5xT@>VldO-wVh+T&7{9GZpG|#>7zWWr9mEdi%If}#4XJt4GeLQJe zwQL2O{2?mGyz*C7FiSuFNOI8N!2=~r%zk&a88UdV+(o|r`fK>!O%1rQyby2uH{pEb zF!K-;sJY=OaXa2DSO#_AaN*g6_vg+vFF^A@d-iO3Qnb0GK(Z-!WjQlz_AGeg$)fv; zw^uT=-+Nd6yByyQI+gsfY}u!V`Bke`fx@@D1$m0!#Qy)4<4-fB)6$*>83NB*J7Lrc zZ(75%4Egfq6J6NjkRZQ?w?UIRWnhH}L4J|d$U~5C3PGMl3Kc9Q8yyspCch0l$jqAc zj*Rnp;E}I6PM$c44+>a8o*kymc{?0-g;9jbSD=9Wyc63>_&;yjv?*dK$x^FEEokn$ zswLyb+C!=Q5YkkferBNWblUCPcH0G;P}uf|9(>5b#X}FKHGQ$s;4Zt+g#}>eKSS7P z>&4B14Jq-n{~U1piWh$5bLBK|QzMTcuQfO^&H0%ce(vFXXyCr=N(|3pb1@ZOYwVyi z#fMYlA3bKQDTAFi!`!*=oBsX!o1K_+Ue@F8ZQ{fU%76d;Ip$&5$b0-0(((S@=SMA!EY?}l8ztw;CjN?xF z^Ge)d`~r#;3&-EFL_GaDvf0Ws6f~CNVWbVPkq_giA9+H$2nW*mU}oU`>tp)h@Zh3g zJ;;{p0@D;9b)jE)8tozA9O82HUq!O7+Vi921yboV+EonN*dHj4*l(91Rjdg%i>Ig-c3^ARywJ23*wgY4Jg!wCU2pAAWn`Vx7g%1aou2(d9P? zn&ON?j`Q-dwt0WXSRG7l_UKR?7Ca9nlHmXTRWk?PfjoX{31<_qJ{@BOfY_xOundA-KfLk)k(C^P!gviAgFgTRH2)G%xSk3z`w_%A96 zw~7xu=E38mN$_L@uQi_kJpI*NAn-U?-{V(h*Z(d0ubor5oMqY4G@KpV`Hu9Vs zR&f5BHu8@D9|kP?=O?TI@SRSCVf;K;dVh~$fX(%SUTdC*e}8O3Tp?b%z<)SB-yDHP zo*!3Bjs84S!?~N`tjx$)3DmTH?deCJeTd_CtUu!S&zA-LqRCn{e3+`(W!JJPF%vOD zL_x9$VQ05q613wZ;|h_7&fJ&d0pAiL{<9`Njy5A~DG+Rqkau0Y{N zq(DF%H^GC>Rqi8K+-&m=ohj z$3wlGe*|Jp!QR9jAR#FuT-j=l@?&?n|NJ9(>%rHI6+q--pfOz!GtjhfFjUAPuv|z!4FmXI&vcuyM&3M+Y-(Vhjv>QHpGy~>2^;@iXw$};0kK&3M<10*(P=PGK6v;r%*D@0fa!p#CcEaT9*eoG z*d@M7ke-qY$Vl8Zz ztf+-5$n-wI7ca4agUPF=&kw|Hj6G zz{O2g0|ySk38*#DraU(stN!#_!@yP2F))kaOU451u%=1L0<3ClwMeR_^}jrrie>n= zJqP|zseo3*#>3}d#Aa9E88rr*ccI9u)51%l5rMGaXEh_j-?m*__{{I5TOKk+`=)dt z{d4gL`WG!$)Qy2#sUkdW4#1Cq>&zeKfl+1PMJW01?6c06pH<%3=#N7nfO1Z%l`D0N%ANFRg-IS@|1qd-~lQHrQ_=0jIzGkqCEodV@!&X7p zrT_dAqNi9cVt@Zz{U>%#sqg|&ra?Qq!yvGYUyMM1M0cZq`=hub5AuLqEza)=?<@^9 zCOn;x51IK;{7PUGbG6zHT?u@CW-$ElS_6e+F&-=&U{kseQ$9DQ(nAV)iSoQD6mqS4 zBA34GsZi5Kn1}XMtJ46RKg+5f53srGvNPQBmCJ-ge&qdWv34NTbm6M#SLE5#30RST z{!y6Lpc7&8aw7j<-A3?VeYJH}{l!ZJykXFL3R#XxY#iqR&+<&;C(Rj?Kh`G6 zJSf&w;KLE{RKD-O=6v_V+*xX)aFVt;`1R=V{U`sE4n)Op;#uABlCd8W>GyesT=mVy zsvJM{*J*fwlyVisc-@D?oqqW4YnA27=H6K0Brf{Y=V4KNc=DhwD%;{ev~M6bs?Nqn zHX@Z*7TUfCLIc1;v5w)ZP+ zkfcJ)lk}?BX=Ls%(WXUEba`0T&p*)zKkf5jcaOuU*#CD&Qjv!j2M-rXY9G-u-s+-W zuo=b|0knD9vc*v3-;#;b2UIXcKz0FzMQdJJ-#MT!X{!8?G zIUb0mw%!IW8ujdSLS#>!@D6-yUnenuBPs-&?n+DIfy}^_X}TQcmzZsK{6E;xABa?s zAF=diS9_2EX#wNEw*J#)q!f9WU*3W*U4i}@g#!9dcY*XYM4FhpN18l|DDVMqrZwC+ zdkbF*tSOf3=`Sxc@Pp4puH4^)gV>hhFf5)t&Hn3jSiB4@gl8ApjgycC(SPS%an3-8 z?#dSe&_Z7b{EG!?(d;aky!e-Ddb;@vW_z>Xy^5P}>){sSv(}3CfqBe}qMbQBpcHZL;>Ad+mML)4`Wo8}#8g(_kuiDa^k>f4Wd%Uurkn> z`@sLeAO6fQ3@^&>eke9;pL8~IafAQMai+!>6W&ko8C!b?;{5d#nmcb5$%EN(Bc=157@cga875Tydf;0#eZ(5Ui z{xDfzo$&l6AM&Jon!3qGUYUd*0usVNg`@(mhMy=i^H2EE3STynI~6GKzc!EDnALl4l`R8KP?hAM-SE(Pmfd zO$^1v~tsD)rneSop!DmMa*eH!1 zHx@B^t-%3GU7`3AEM@@R4mE6OZ*TONbLN>)c$Puv5IumN4&|38nTF5=aOcj&Dl`%X z4jkxe*QulSPJ+^T#_6bw&w8xP0lyr!XHwPvWc${0}|!0NzR)2sC$M zcf=t9RM`FVMVoOSz?&U6VvzA^cSS*!*vPX#@SM|yEAs4U2p+$!Jfk@te_L$&=!CaJ zhCXmVIehp~Krl*%s=N*O)lg4=Zk&1(FF~8bOo{2|!S^XQqq07A`MO}%JFxjAZ1Q*= zEAp2>k!Q1rpL8=brabor%wed6lm2P^8Lu@?{`;p)nTpCb{v;~9aXl3IjE#JA_r>(- z!e0U}W9{3Y?do8|EkAKR1sk~B{wFil(?1u#Xt0+42mBmYwz-@RgyJ1on+HO{Mm`B> zXnU>5n>eDK=ovi4vqB{l@$_bANNC*CT^S91bpcV2Dy>!_k!Dd|rlqWWtvz&?zn=}U@4U>kZ`p6=Yjt5SZImSh>n$@03xAA2_&h%d{@llo&^;9zx{;(1 z;%8EBR{QQNC<-hN{KvozA~#<$%*SW_2e7KfmE@gxx+;`u;#ly*JhxT(t+V@&oPeMH z`v>7bsnvLqA^jy&fq;s{DHmKLmxn@7z|V3hfdg;3TrU{v1p1lS$O8jU36>&{bmKM; z#M7oBeAqWR^}I~~QHsHihF33r9!P^*blJ$nLIu!2Z6nXqVR>>f4>u#xA>Hq!#t!=_8Frgy|fW-j6|@2!`IyEmR5nDjrG z`p~pXihK_KH2vi!miuEoiKxiWlp>EL6y#!3OB~d;V>QgJHOQE0+w+*moX>1;=zmer zqyxSll)^!2=`iMn!P>hZ&Pa{_kuA=_=JR7vl$$D7iI(Ke-2M|5zXI_oNz`LSIxVQWolMnIrMD_Y~vrlzqd{=24MA{{b>!emYuLMUhs1& zhPJW16`4Rlt9!cy0^#S;2{XdPbQd+JO_`f02G)srzykLM?a#kju@c>ltSM& zD_KIT$;38tlpSzHn9Qg57@jB=HYCsvil1BPL;(4IYyjh6T#pHH4UjAP7F|p|+V%5ckOosj*deT{L zPkfFh6+gqtj&i+-`n(tG5D9L53OrtH?hZV@9X2{1sH5dA-QZEl9Sx5ev!>Ccjur7s zAcTcmfz841!6Oqw1!Xn99)C%kgTRA`cG>GddD9UHl%9F{ZA^e?li_aUGXs(3X@(1nv4ZO0x6uJd;br7Vyr@-i%U3Led$r_@@>ylnkr0y$%0Dw{s-Ezqq*bHrl>gSiKFf)`tIpW83JkHn!Og zergMN#;4Yvt<*=(=Un`#lF+|+v6Akr9(O`1uB86=zY!b8vFTcUP{DkB)!R~sGu7GS zAblO?s575qKz<($k2JIB4$@OOFG1(8Bx!FF>LlXLd7pZ`gN__qnf{lDHLP7lx{SH@OGv;b$1u6C#FKlW_bAFhq2+b9byRt#ahAz<`A^SLY{l&)t8~a z9PaO-gJJ=Zx*PjK`Zt2D`g`wiAq)wgs5GaoUcI`TK4XTI+&+D7g2H{A%)O`2oauV> z=mFQ~g9P)xSlK@hw)zYNL61GXy=BH;d+iM?hI4QL*8bYR#Pf)50IbC@yD0_FH(#N@ zTj8sZ(5CPUSXP6OLZ(1T{xbNcB9^HbV&STSSwEH&@?}~Bp5?}C!6Ad=zr5Cf{|3X1 zsB)Fc%93ZB)~#Fdi!=ro11$zkLB~BmL)gi#-j+RJLNVHxpMLu3RsreU>HEP4@8cdA zQ8%;M6{aZI?Ap1Dz3m7)$~4}mvAgY#Tk&wRllfBp_At%=C*30^XnmwP?6i66Y7&wH zmNIk3OzS9aZL=#rwei$dwhhp@%1&Ezu5>0z1Y*^D4~3hkwrUG8D!Y3v?{@BeML-3~}X2kmrN9z{$(xw6OAIF0vM!KSZSnpr6% zeQ?+ye4REa6Mu6a&h_PV73qYpCT%%sJ?L|=k>>pmK1}di28wcKrs%&u{U@T(a-Qs( zj~6yT(>{zZWFv3zN&mB(>;ll3RBXFZRkcf$+!F2p%W5Ti@vlEaVaIAEplZz2Hr?u; z$ZQ-v1kg>S5LU#AoZt~?1I~?Yva5u^=jt&-E`IelMgMFCkhEbXT>N!i@mqOTLQq`; zlEzQLyH%Nr7;AjTz{8qtT#3)YMEFPhpNpS%w_BC1pvKU&=4ZIaTm|!kjn)6`4W}?~ z8QvG&ply38R~!#1+EUX@h8t&z(9WB6utCNX;#EI(VUu$jLt%v)3dCFOe{VzNOupHdR z(nONxOngzx(2JE31S4DiQ9ko)4!)z{3)MpJM?N2lJf3$z_#oU*rk}sn52qh_gU9(g z|D%Bd?InoXNIbk}``8Ay|5I#s6@Haw9{zl@l%)HR=&#wA0_vbZXiESWKnN&Sm;Gi} zB2)Jh#GxRs4A|qd(+=+Gr=O9@i5iEj4jtis5_$l=%5Z;UcAHTkg`XRGdGagup*4fy z?{(d^u6uXBYzk$j814rQ?C;vOJ6&m;K_>nL%Z=h#OihX+QngxDHv>vKLl4-sUyD;* z8Pa8jK!0k7eTO`4I0V|0Yu@>i2f}M1-AtB&BM=1upy_s-Hf`Obk3Jfzse7DT)^WU` z<7?4#IIrl?Lk~p-WTg79;Y=EmfUrnzdV%*L(Os4Q%AWyF3t;Z3s1`$fF^T zL6P5S$DOe1iyO2vv)OebR_0KF&pIk9p(1w%m%+~8x88airY`4MzGsl0cz z*8uUQn0;dVJboT>z!wSnB_%%Q&;?Sze*N9MSYaOsfAI_h9~lUDM|UKBv$D<|bkKov zX<4&YO;iP7kN@YNe})ax5S%_c*e%3KrcCBbIaljn!Adq)>tDiW81)A@`7cZ(Ah)uJviBxg!oe+~wiHT^xdqrcHhO-j4!#?vifZ@R{ml^uoy- z(M)`n3M~uxuekgQnJe*)_}Jr*cXM!38~eP|PCL5);$+-%7z6H}=SgZjD1$U+T?Nt) zH=%FLvk{s)rH(Mx+0ZT9Lm|lz;DyXZ7VH(m(Pa z=VF|Hlu@L^KU@B_zBuE4^5)K`Xd}CgtvV-}Ou@=Ar-WUf`QKfeU2R34&xpjd#xqZK znAY$)D!Gp@zCXfijcA&qe*mkVXRGlHGOsnZvL9Ndrlo%L9MDfH(Vh;L5STZejd-XM zt~uE-PsTw_*tnV<_+-ZGAzmzLht3VL_3G7!+sL!zW-?P2PC?E)A1B-2gGI`U?py3r z%nbo=xYD(FcdUB9jxYag9O=MNxpHN=le`Ff0WZu2p?jA4ZqB!E+wHd1{V~TJ4Hw~op;}n&c}3`ymI60X{WVuS9b3X zebMwJ=BYk@I95I3{DY_5hfT4}F9KSfaN>!|quF){0dKg5^z=28mx-nG8CT94*{Ot_ z@!th!1r>!NPo0B{OhNytU9%Qo<-xROkZ4ZWD#OjLyrGS}jfqVbv+rOI0!jd$CXCrg zunLJp?gk>Mrxc@?0H+fq7trQl(4D27M_uWvp1f!p+4h5k221 zp#0kKlUXZa6Dn67*@#)QXM-nvy>-M-7ey7|k!I-be++|CbZSeT7 zEB-Q2I%( z89H6PVugy1i@K8W$RjC#Gi(I>1|Mtec&N;rK2u(As#mXpualdt6aJO?E8Uz~bKLho zd@mQ3JWx}n+60{2-`xA8b8BX-nUWOFL{RAW=)X&SDDwDXCnX&{3j9|K3%XSJco-`; zyNcGx#EN`N;)ZDr4~>Bu;(3^9w&9HmyNWQ$MhUkBzz4I3gc4v(BoHxSFpu+24Ny7W zFdayR#N!eWW=^yzo&cA=M{0TPloY*&aX$ey| zn2RZxQ2itF_lkR;mtJ!HxlJX19(eH+K@R$dPIlF&eS_Ha^PoveWJgi}ESgGS$#APHf0#u53lTIk zWGllmBYw~&8Nr0n5(;HUQUtVUD#6oJz?96Nb*Fz_uE=AN8lP26*HCTP$k)b-JYF!l z*Faut)L4wER^)+W(v+|wZ#B|Q(%S<#Oiaeb?xd$4y1y{lnj6wHDHh|e0nmLwLcnyv zlByiC2r1&n^o#Tt|CO!Z7J57w94y1#_+KOkLK$104fm6QkHoLINrV@f2~y1NjoE^k zjHDZq$d{ZsoWlkPDX1d_oq(0jv`ju#i-^xtC3GOjT;=#l4M7nI`2dq{9pk${~MbqE$aLKp@ytS-X@Y7b$l<@5E0791CrtVVF^} z9_bo1Xb2m5D)C?ptYrrMueMg?y^TB%ghE{w$=JwaCo=>2$*UgXFC{V^6{PeaHIz|l zOoBXYVzgA0@r^R08I}|-U&r7pjmTLR!BU(GuQI;Wylv%8eoP8hQ2J3GnGabOz|?J{CZ@mxS?{c zdBAH(jrb$|!3#!&$bZSpah!iq{`__F*A9e&Y0bGXt-;)sA9+mE@ZGlt-A{4@Ay(zJ z+4aK#6zB!&Ka};f;$6LZg?sv$XI+&lRrI{DM+F}QfP)ZX;hIiUfi-^8C4FWANy<@r zDl>^Dk%%9J;z(pDk{$OHPF>O^eR2Uw%29eMGl?dV1aBtgxAA}T&1|{)rsD{i)w$}%>)!fOrv(~?i*eiAG4RKAUSGJfQF zpQKBPzGDrc2WaW^ zT9bbMa0)YMOz;ce4>}a;rknb>Bab-JGP8cIq$Q=}p%5?uzhncF3NE2uXXS-*;ToW= zc}U2;h`ZNRk!MT|e*G^uuE`H|;}JQ)Xnd2~NX4w(o)@?sZ_ zwSIIPSc{$m_25B73W}dkWCt=gktm`cvA)mn={(_$%2+6d2>4eDJQI@~74W49*w={o zNeR;M-%bBA4Ne2!(rp=qC|2a9$b&YXW$3lWc*f|Zmm)Gd^TlO}daZlC@gl}=O8g&k zRb2eXdP2uv{4s~)VC8Y6TM7e3by01U1T;1*;*azfOHj%M4kNeVjzSo*$bUAU0(C?# zn^Gi{rkf^OoPVT-5ky3?+bDmGStx3~uvsVhOZSsxsi1$D*BUNB5)c*n{v9QWMzZHTVw2qKhobssaS-30TvlI{7LISrnhiGiLM@< zd-3^~+%}cAvE-oE6r}&?iRF1!(AWGS3ul!;LR#?->qD$*Wm7dmRM!IZuZTaFx+G&( z2qa7rk+3P~KNr6Ik2>lo?7LX4gUqBmgAoG%)t0MtJK2?{HGFYpUZTizMV@N90xw`r zE$cJRUgZR`2MVI8=wHWw5r3qAXy? zv5M*t7RwuC-lRwr@kjdS@L$;mnH6~uM+ZU$URpo?^E|mB{1e`q65>~ur98mG>7y6_@*)>&tI z<%W3{yqF8Ox+<-JjC7LrUU`Kwot^~vgAcje@3&6_vRZ+1neChI^b>q1(o_D>muD0Lgq zrECXE;L3oid^5nisaMKb6Nyy*pnU zEfkmHQrz7M?(Qxv?vxgHcPQ>I#UWS%Avt-z_x#tnNwW8zS+i!%n&=EGqDag^$JEK= z3QWdE_QVr3l_AV;a`Z5Gz*FXC?K^U184gOwvZGoaP9yEU(*_QHjJ+=`7VMDm=_wSQ z$YPv4yKfwmC4G-4Jc@$T$ik(BD*qr{?sRcR00j!z z0$ik~DEk+MxPR;xkK!qL+$-#s>kNkzx=92&Qjp4(j$*`+1BbqViwWPn5(`gvnRvut z$GqH5b|Sex+m}AS;Z=!$Z!icc^pHUaRU1wz5J#IjF$&mAd|+7-o*c8UolQmw zP6zxe3N_4}&YzMi3VSt@GZV!xv^{MeBtjVFG;9KQLdoz@W;w{L?2`>@DXwO==Qc7NTFF zKp=*tl<^UAN#F8)`V%4e3y#J69y&C5%&2leL{h&{3AOdi?_S@m7 zw45qay+yUQx*wyd+?KY&ELLgQQUh(!puSPjzZF)o`2do7*>Ii=bCWc=B-I6Mq&lgg zdY!CEe02T_dlyD7}xA%Pr6}oB*VD`p$0LCzpoIO z0yS*2F3{(?u&_R0b8wBcdFgn z{a#n>djcW6V;oPjf0?jduTK;2dH*>(F9UD~%l~J5SBmMuy*I;t{?c|g0+zx@BiX#W z?-!~M`m9KsHKH25d|s3!Is1=$5BX9YzF1?Zzc1hqc3!S?5GB#t)UMV0p+F5CRY{p; zuwPOPq75!IYDLc*Z|44iwKFp3zWa`?ik5cUk;o!?m6?8#Nx@rNp5bMW36b=fINu(> zl{o;>2-2rl=+2OE>MWA5Hiv^bOhx5!f{LyqTNEW6OX7G*q6N}P8hPTXLdbp>&=ToX zPu@y#1^5hQPb*a6q@pAcpl!%PiS^#2*pOA2w`4Ge$UU7&Q9@MQ7^eYWv6A{ZV!j+X zzX@*}SYQSYa?&NilZ7xbTOl7I9lk?!E$8K&n{!tZ;jiB?7>Ue0{R7DEGUsB z7|x2MPEmGgmN+L@ee%h0$vPqU2gCD5@i8Vnv-k@Mp30yOXPLi3<(P zZ4tB}Z0!jSz%LwpP)acwv%CRl_)tA(i^3$t1WcvW?U`l z6Q>aCpKR94K1R}@f3lI=QnD)KI*8ZJkk z;QN1W)LQrQ**^b#6=cooMa=kUE$$Jol%~d%-QqR-f8XB7Hi+{FG#Vr*0o=P4{K7KE zOf5lrpI8BWsXr}5WUoFA{9MCGTsMdH=Bd^~`_*6kq(hB#h^`c1Ewth@ATOZ5u?;q27 z$*}}IoaWn9zr_ggR&+(O{Xj+0`ox_zsV}~SxtR4I1S%(|8yqO7<4$V)dH|Rvo2J1J zFVhXyZYSk}BoQCe4C|z>--+$}lnQ}x-UmI5^BdO4oojOhNU&M~DEniq@5IO>`)!T5Lq{bVm4Ags&qG0stzU ziWWj(&uCDl3ZBH|8@~ zXBo6(zqYUfc)E!4u+AVynXk+t)BBQJ)K}Xg_&2F2L*xhBd}*{eWxF$>pG|)uCWL;| zdc^5s)}}=L<(MmUkAjjN8~^u5NRtj8tN6nZM?ge-YV4~RiZ*oJQvzH9T{0mQ_W~eG zz0ey3Sh<(G)k7|g67}mM9MfG9>RPV45jZ9$qz~Um0k0Fr0Yd5AXGO;BpM= z_Af_(aRjRBp@Tyz(apY-ru<)-qXToTJLn@028LtpM+Ara>vBR8Pu1Xt zc*LrWlv2!KSJt2%?^ZS9+e56V@ia$9Zkm+YARZwD3c>F6?bQ#!qg>5!wEqFB`TH5B z&Lq+UA|Ara`z=LFEF_yf;AVMRNRrblEkS=R!Wr&uNy~ZTihJ{m=>L0ELlCmV8DkEh zji726ti^?uF;eD-_$UzQl(LTPMS6YWdE>e%+cO3v9n&-td_`02AAScn z(K8nSIW$lh%XZos*#-IRl|+O2N$;Vjxzgbg^8Bhpe^Mg8x90JMV{}*$G^zj(Owg9( zzmX`&V7ahr)C_=3F#(;}`q`=}vn%6>8x8k@;6Z=e-m|v&k{FZU8r~2x7-r8MZf4{< zoy3r`p{X#0i+hzr67wvn*e*%-UG@m1vv8$&{9y8BUoAAq`=!5i#h0B0TcHlhfGo}v zJYK_j8TJI*QKU;wteXIGv~u(>@lJPD(bflm{i5sY@@PkYERK@oO#}t3k_7O{z1oR! z=tyMKYmZ?wuxG0&8W2U7PjclKhaU!yPZ*ES$!dQvf72EZKGkB|tSaqk4Zme7C0`I)oE>7&t!ER`V9n^eI|*Vc8BKG>2$U({D~`THC!s^|-DHOzM9QC2=4DA!MR;>|K`_z4$Jmm(4(E zJALz46x$yk(-z-8cfrHb*rG{JFS!`v2Wm-_kEb+6k_!M>;FC`6nr`yQ0)_zSK-;&D z`!RJiOU#D{RB5|UlLY{ql$)8$*y>cVsGnNweC5h>R{G9hls!!Il?J7IckPk0gL7w@ zk*eGV5%PLq&=xZYR^wLl`uV@{%uPKPb9oS}Spj@B{opX^lLW&ui}F|;ZOG?6#X`#Q z8&*;>B{nvT^v9~Jbq;DP#o+(aWZ(PX+y744>KE;3Ix~>jEkCzKj_C2rn0WDLNtkrN ztXY)e#(SSqE!8qBffY5d`s`k!be0b4&zrRafX?w3C!^8Y9Koe2igkVC*cjqcNNMJ- zE)^D7SmJ?>@Bok``!b@kFA1Fe5EqI@dkP$mI3SrD2LI)?TCxbZGp-#Cow3n#^X-_I za(!EXE?1z+q>M1J2QcQ~<|z8po2r#vv+|FW2e*AIkdy8epI9C5a1wno(IcRXvkRf9 zQB~HlL?D?z7$hfnJWvUEAL~}FLEBt9v(emQ#rV5TGlxbB$Wn+)oj^r@w^ED<&uBt% zoc5^(lu4(G_OHyCfiAXFAN!1AP;a&=+bluPh0WM(2KQfdl!;ixB|w+hsfoVjLTBhFA8LVP`7d7W| z{^nX2tV4y8biypXGCbM2oG&iaw?2N1H^K7-IxRr1bi|zC3o+s*Iitvm0u<*L!joac zro-;Za3P>jv``z@-+d@=AMMbKF)Ofpd(ZiC-qJ6W#`TahEP5FLR>Av_0#BK%W{_CZ= zegD^BdU?nDb@@FB+V;^O$ITdRKGg#zdIADhAB^@Els8poiXVtUI#}O{cNWyfN;s|v z_^{m;e7?KX@0GYVPQkUR_dlmG2dnQ$&M7Hb{7wa< z!CmAEBufC^fxIxH$7uLn#93RGHzv{_e@WqM52WWx)lo2l3Hg5kA=_%XeRR8iqTnop zgw-igRFgm3ZX!tYT^jy(!7FkxtSXAM1z$~di7y0W-}PL^fxMgPaippX(w502w;TO8 z7npq{`-A})(Qi_VROF;-f%hPqmE>1NiiZ4e9u@X0n^)I+p5;<{Z9N%`L(s?EqY1;P=$Q6Fg%lyx;pZJC z@uQ@GXZ7qO3yOC)l<^8<-x27eJ{_L%(U=joUb^H6&ijnUz==A|C!bnN&xrcy`4*0v z@TdASq{UI&YSdX!G6hC~-zyj9gYyvktW=7~1yOhCPDv{4vM^-+|FFxk;K3EyVS684 zQC4OpF)jWFMt52ucjgiuP#9tOs&Tnf{|Z8c(kf1(8}>(amW?ue!?oqRrrY92qu;GN z3`H0z%7{CUxbJ9BP8T_U20LUT-y}aPo{EJ#lS&W&C)VI67~RR^xBk7kt~EKS_#^Z4 zFJbJYj>-*PwXekBR$3J7xCU!S6Q31Qvi>>7@-%66(qzPa2fl+ik!v4R7A+&@u1uE@|{14pC4FZ6Sx#G`Z} z$w4LbC*=R@Co%(-BvnhkZnj5e)}|3Dd_z!q7Zk(S>IGYiUvBxqAZ~=H{GC2!41p=F4`gT?;qoF$k5x8Cm`{(%yE!tLH83hlveXi77dLkVoBcwupj6 zhu8{sC-l-TGqIB1OoWY<@iQa)%05xXqHIMYQXP$t6cqmuZ;b1Y0p6JIJmzZ$Zwro) z1I$2ZEmQvCm#8!9|*l;&W6qN*tIxR*mP$ck@-`Z!5 zMWo;Vbb)hYH2Eh?>jmDD2s{VMi&*=@CywC8+%S#i?$N%hsMREw`0iZD4 zDD2B{wGnIlBH>#F!JFyV&6YXjrgx;h83{u%VywiRMtNGP?J;CQ#^J7kuCw%lsb&aL z&waCgwHL)~)J%E-xI_gHQ>|2pSfk}ed6(=f^vh#x|P$>(zvRSyVi z2Bc8cQ%XadzPF%};#p`?q`t>%now68b~Cx|7#|O28PKwMWkw{JI&=3bw`Lkd+h9NF zw1CMMb2sBQk->osKkR7tpdhb~8jJIb1CD+=!vA}Dkr|i4{1}5yQV3OWoc8x*=t>U5 zLYw&K^bdu82wt3*{yiH=4MS)(+NgDmX+@Nv_9=egVkCGK48A@Bcz|zFqEy$PBLfI?-H4Oi2Kg!T+yzZGBMcQAO6U7nA9?KZs z#Ys)jDYN8E=b^!dF=u{mSq-8>G#~>VleTrrKo@58JKG4hb0dATUUna}$yVQaidd2) zJQOUsB8URF%yqhSY`ye_qh5uQ|E(p@*}yKy$h-82tqe$dUdP3&;?EW-gIK=WC^9^4LsD7oi=0S8wL zpiPEcWte(IZrE=9Wd)EqUYc)gaAt_{ zi7Jn%eDd8_Ib9E`5qwE~3f?(eg_pptc8r_dD`S?bn>q-zHjo_6wuN7rsAD;Pd7l~W zhms!JtJuq9Z^;Oj$BrmXfuW>H?ocUe5jwF3U*DEuWO8W8N_bMt?wY zPp*4=ph^gO!?Y3K<%h2Nc^w~0!s@VZ z>&LYA5&;K)W;z`Fw$~to=W%;Hqw_|wOX39&9PgvhqPc={x2*}zluj!8;z7DGm zR_Hw=Qo+Ak6>DaMqplTo#>7m>@+Ke)Z=}S>tc6XGm*`V|$<8cA^a^iXJ9{(KlR4B6 za8r6~rGTXr)H1|nf9fbhjPKBCQKsQ&?1<+xS31#pspBHI)izT!I{$k(XUO;~4PedJ*YWvGV2tN3%80b)S3>b-& zX+?<7ph$Jx-8Vn8seyjKAUOLzFqqoQ?aV3AA#15&Kx{QV*U;lFv0foF$RJszeZoiv z^U(INwPmBUl`If4by__jna)>x0M+Z{(=iKtZ>ku61g3|8^l-xwhj9omuV| za~8EP$9@3OwVGAMOI7BB)6#4D2-sD-egF_GTkveQ#`m}nEv$jdzxl5;^s;GjbR6TU z5pt4*Bf})OaRT`#n1~u^H&*`dBssVNpnESwE~cRV`-=-&tqt!v{halDVGy9j+TkIr zQ|gMCkUIO7Kxxra_}G=pXA`6U$ChE-bdtG|pn2O-wB;~Ee@9{lv4n4`*67dY0ZH3r z4aisq)cB{o2iWB!j_$-}+zGG2pHnTvzdz)-#@1TthGHt-;YnO`u^x6DGT$D?Wm5n( zQE>+TENbhwQ&?M(*X-P8U87?2n1Hb;DtXboSMujU%ptX9(k_uRfz{oZV@ z6LEIxhFIEO-Ems^f|vGcn>XfOwx{_GPJWgCEt9@n7v641kbZ5(f!~6jXGza`5J92c zzyS_*s~H6#OvmE2srHdMpFksEo7Z!C%{wHeY_7}8jp0%y*R;RUlMoOFZ+M@RW4;2SiVj|R%#s*5F z712y;=!7*DKaG_6!fKV_G2kX*W$}0RFgJVt1u!NkxG}yO2>W*TzdnA&=zp^Zry-DJ zr_~to*|9oCu!f=ZQ)}ZsG&+09At%kft)i*xjt;;@s3<*mqesP&XXJB2(!xLuqm3(} z+YT>dt~dxC{b!Tyb8WR>Uj43%3nMcf>sIA9>z=?H6CC}+F~7YI9t4{Vsix*V+x_{H zxKpo-#0KJwAn}<+dSR)6pVFn1@Qs3$D;4&8_7g-6_5&W%*@X`mG>RsH!Ap$+l;v>xohYXKNR9IyUPuA{Ym>%zLSfD#{p)2-yVaaH4&p z-8dx6E%tAMk+_qy;N1zTdqg3*YgYwfj%D4%5Hu_ge>30NU!&!*+s}01vuO$M2nfS% zI#2krulr^5HS_uyGHOG$mQ)YNzxNj4i<_(Vi4eBy13q+A^H3C2P z#f2`pb5CD=SL>az2ly|b*4b)|z3Wi1>Fa_+uHM1>h1ik-%qqga?3LW&12F$_qkp7!l~=ex?7_7hQPX}`;hamwa0DXRhd%$p2Um$Yb@P>dbynh4CRfzJ=^8F z3(v}qSOHQPct5d1a5doW?}$S68QLB2tsFx7Iq1A{>?XJ6n=x$MXok9M{V2ZCyza<0 zs`+&b2uBt?UKj39MSMg@;-jBT6Ap7PixgCpx8=HRQaKmpR*n&*lGZdTITHl>>GQ4Nuz@Srt-AnUi>~e!**cIYrg`(Y6|jvP zU@x>csV1{(=rt>a-fEuU&`;}m)&0`#cY0%GaFtspM~hFzfD<_q?(5HTu>|4q2$xF` zX^$2KCn1L!s|s_X6lU4S9R~`|ygb(V6|-}i{s2{LSJwnK67-uYt5e%`-8?Zr?%i@W zHi&{o2mWS;u2NLvQT@Polz{_>WBa?j9z6hvoAmkb@1yZ*PAy?1snrG~!^fZx%&++D zmh72tAB8VZMz24*G`hf~4La2bZ9y@JY5kI(kf#~l9boJi*W!dta@9@Hhz_n#w^u6z zA+V2skBlJh*9T>UlnMYBgNjb}jFobhO1=WFg*E%wH$9y;r*F7Cr}f7lit}Dtd0o)E zl?<{)kL6r43WBLlgVVwLlt15u^TGh3WI=Y~S6%krbi{8Inm363Mti zoLuY*o$_Za*}`eOE8)5UKje`1b__JgClt?Ux~mRaQGVKfg36n!>?i=alK%eitX(r} zre&F@0I3>PFR30Bn! zevAc)Nj&bG z8{rDlAN8uf{?)JC9pKnG#@M}Keb5%;{nvXpoGd(E07T~1_LlnV(s>V7QDc}j7K;9K z7!GW`kq!ZFKI*k{_Uy!nh1QR*;lGZg@^9!k0R$u!97{Yyd6&zS^#Ld8`8z#FId(T8 z9% z3%K22ZiwG$tDl6!ta>Pyf^{)g&9-DB&93HkPbxa3SKc5v5~1G_e@@?M3CE zG%Z=-8B!8(qe0<^QqNog~9&Je}=~<0`Py+v8?#sLm?!%p|aBTrdcH zyAhs0<0X38kU!VTBmOeM`A)Ev6e(ZC(7$_!(5mnZ9!h%9k&c)F!Zv!llGjCf6+bn( z;?&>#WA&=*ncB#J$<`y+nD7B$0%#dJ;<}JpJ>f0y@VoMT#o2paIG-7jaO|PFd2Qlr zw;_5t!P(P0{zUhP$U3H}cV;ciMt-fSdVI)O&HZh765vr~Fj%AQyr%IM-X{KN|F3z) zHWmdwORG|4U&-SRh~`B9M_+wpUOq)hFK;RX{V>A$KHhVC>Yev&u(zb?o6y0uyAY2EDW9B0dVx9ua! zm*oG*&k>*D%AalKw{k2>k-CuT=(uo&4r}(dNe5x+ZMRqxxVT zCr}~_F*bAT8>2-_eP)<;#>q@v=^i}(dszDIi`7Up+G0lHK#bURI{P;Rn+#Jh@C?#a zJOdm9+{fQNR>SP`o_6$4o@s8(gCwz`tRKYU8lPMK7MAg!!yAf^J{7QmaCDQMlv(@S zUm4lSvIBh5&P?#|D1B;1X9~pEwfd_#FWi98UGy48>QJo>lBxll!6_sh_r%+O^ogtz zw`bWP?*U5vWJGjrT)he>Gx)dGAD%!6Jyt$q8K-*%Wk~zUJ1sWBI^lKPu@9w!5)hd z6X=jm&TK$Rn+48=*x1XeO0Fc91WWjiT(oq&Jgnn+P3O<;5er^?$#KL7A_|TS_7kIb zH&toU8H>yK{vUf61f`3=a(AL2R?T75Y{}fne4YQ97|tU&a&3YJEgJ~q{!F0X={;Xv(6>MGB#8X@*4$Y_O{suy zRU9k{JZyZ!{ITPvX}xe4be$kA6?it~)h^uBqn3ZmAvB^@srSu}5HMfMrr(^` z0cFivESmQBc5koCyy$D#I&5R}GdTU687eMvJ!co6yf9DFD(ZD{&>Z?sVqkrLyH6s4 zNsBQKh!~~X5I)WiFKfD;-kdx3%=(oTz}&;{J@7?UGc&hGrMI6e#Tc4X5`&bF@Apk z)uO)0&!RwBC9&|z7Z;qNCJq!X&>6@(Y7H1?j=wJ?7o_y+_dZB120 z699Q{l@$VC50<^3(d)P5&hUZ#%07FBTJd}~c9@8SH1VNkG`6_HhZ$7|MdRR0Ya#$Ph!Ddrzw-p|?$v+c>{{r0 zShLd(_sEL*l2~2mK0*TwIokI5WbVeSWQSwhZMW3f*0INvMP5XG1Gt^! zy_^!<9{harx@FxFw@926^GUB8RDOx649XkUcE@q9i*x? z9}hYvW`_yrjWlR})8WY#T%_A5TQw!C*!zo<`&%Tdl}L%CKLr)`I!4+|1;(3t;aHdV z|F)#iKEVC~Fg2XNrC`2RmfKrzNCAvK*IQ8^Lj#{L&aB<=OfEz5wj)6foXj1jaxkeL zI?htOR)zgcBVRQvzn^FwrZONKP@S|ckwo^+v@MX-eyk0YcaKG6u0ke0)&Mrdy@7c^8XG%rMY<{_%OZHjA5s%pqoS4zb9 zQOIq=(DFU%9u8(Jr`?RE!TQlFJufe1&m}xDtx`P;FKQ*bm5M`EO+Q1AQ2SMj=NP^{ ztJTyTnnV&%*TQtMOhlX4Hn)nh-{qeUQF69593@H-lq3G`O>}CUrkt`*#ev;^7mLfA z6P)p*{oBY!#-G!}mnqyU+0OTL=n02=tpE03`mW58q17PjYma4PM8`H5K@_0viNV6k z8cv)hD=2xuE|1ho*-MQuJ(&_*Bp-%RBD?6;u$d_q(-Vowl!-s zEl!pUFdOEZE;Q%AY?G+t*eHNcU@FX+LW}*c8BmtSZiiQ*U<)%ssqd)BsVp=W< z1HEUbNjN(pHJp!V5uXyiq<>wfj`MkI^H}5uH9JbsS&%h0)&?a$=U}d**;v;kPpYnyYOHVn+>)Pu3Dw7F{9i9(mD^-uXfJ4xS+D6l#O3b8uUXUwB&fH8*1sSdA1Ew2kq*72f@fa(z1^@#tjM z`3Gv}`@knh!?IkojhL3M85anmgTEq%%+|P=kF>{{RE2T+1WnK!Kdkw2dLf`H67JS% zlB|l1M%m8UD?Ksm*`IZmQ#n;|t`m2>f8Q51AAiJhpyY$K|83~+kA!|D(`|Awt~U~Y zl-%yF!{{cRbw3WF+WmGy zFfcDUwFbud36K1FM>ZoCJ&zc`;q2S|Yd?!c4;PNAL_85~@U_jWI8QS;84ka%syZErx~@0_7FcU>ECUqM0bz66Xr+=lQx*&v67^z-yN| zuFxLW!~6|YvI4e>R3;S{yp8Rdou3Mha5V*~3z-^X!H)g_#=e02`{m1o^~=6;uZ{O% zH^+e)UpV+-@PmEK%Vbz+(>1#8dHPy-dojEx(l|>JI7OmKH*Em5$KDOMGIE3?Rx?G> z{-6GDv*#9?s~KY~I8;Y0x}!Jc$TF?;o-ERtKMk@^k%`%&=V2ASuU>tx7foYw#76~} zrGABTr3A6u=$~t3s-CpeO>W^EfcwqR95sb}ZZZoQ-2pWh9m9uxVRBG7$J266_Kn_r zouQ~08Fu#V8%IxuCgnV#F}I)MRLB5zsh@OkZf(06v3(r&6TDCd7k(EOJoX&nfHJEs zG5a;KRgaF6To4)__4#GQo*?Nsy=gpxry|KU>^mQdA#*eTl$bunFK9vScK5eo+o%x@ zR+nj_MfRDa@Ic%Y8Ae6Ckdo_gB%erL`UVP~PvoMv!9sN*=faOwmh$|!r0?j0PVHQz zUeFukf3+pJQ!9iJRO{i~f9m+5I&EF$tTbjPYJA3QHvgR$h;9o z*|E_>tQP5^10d{$5ku=rTlP?+D=C2?G>L&5yVuu|ccPAw@nYI@Mp@?#76b`1BW54-MQf z^b5EVx-z==>|uG)J+L&qH%2Ky%8xalgrtY~k#!FZFO@0hj%RSr3*4c;kc+#ft>bXu z*%-P0D=s1m84u#HFtswOGZ21UjH~DBb|%U`1)_CanHSO5kyFv)MH`t|g5r4AP-6bo zsUF0)Dt+n=vLxOTd!#)dK*tTSQM9JIDIQ?&+W#urip_C~LG0tWw!nGw8Z*bVXq2c{ zMRf6Et*Cok0>dKfv&-tQ>!vMa<5bn7GxDYlM8wJuW{a9O^dV8Q1IuA@akOQPB+BGf zkNG&(2R!r^gc6~+-Boa3u9g>)ad7=vtMx>BXy)oV*(fDCJsS9PsO0r%_`ov~n18Hn zH=IBfnYntr)Z6p8<=1M-gp7WjNX|bke#Qf?0!)0^zL?K^Hdvwnv_V0V_2?4}({yftrzolC|fkUVv3Tek&ocHqHC<*WMrY zCQ4*buzrSBT5yb*(`U1$YT0N8{lD*lAz5k#=yWJbqpy@@3Dj3ZLJHuocY&I`y<`2m z#q!YiC>3dPxj?2DyQPSDF)(9mVac?#sE@s8c)ZwK3qoxK>ZC9awvCFTdCb^Q3`sLWlq6FF`ij0I#UdR;Ied-6S`M(uZ>@!U;{qU1f+>|YmOoiG^f~7 zVUQ7qSm2j|cpXQcGqcIDNEV#VCB7{kX_+#9-DErmG7M#-jKh4lV~_KSuGew%ah9(p z?Xm`Nn|P|Fpof`8o}2aO9PPjIv=33$R+B=k-$6D)Y> zS7CTLDK9Evixk3vJL*n3EuwMuDfk>;JpQqzxjud{=2r(-j-t=!%oLB_N;0EuOaCcn zH*hj?VYtw%992o8oAE@0(yF0r(ve32tOEiqJ}N%FyBDjT&o${j z#4bF4$X^NeGxv&|3)$%h;4!_TSH`!hQ?gUd^P5Y{zlGvzp%4OsMLJcav-~?*@mm?H zrfA7O(k&s3TkTO3Wiz9UKY#!Nj4V7a`X^2TAcTs2KIe#u-wQj?FzVLe$l||2g3um9 z2>P6hmOZ<~$^2lD0LZGqN90XjFl2Iq)6T-5*5Ll#>VZNnilF@pzNKJ*+^N@gz(~HI zM~~ zJ|>ID;TqAV!eeTyD1uftNM!Rvfl1~RYNDjX zfHHy=kMoxa7;&yf#gAh7YNhBLhD43y5Y`c-Rjq;c{C^klYKF=gy9z7+9b9;A?LMrx zaJx<+Y7P>gUggI^XKS(hhx2q_$k+Q6yn)%+`C57*yj_fHl2FTZV0O&u* z&H~PSYD|gn%=Wv@xJYVHM zB9Vz8LC<5u#adMM#W$jvTn@L|mQ8wKJkyptr|OGU4at!-KAx_VZ)=)ml4pjD6WES) zOopi7-7sI$Npl!BGfKB5TyB~;Fi^3`20!)RCkbXy4YhX5xC1UYbDfpqQ5$8RVvpVw zGnMu?+eR%mOmzvoa(S2AU;pVBMM2v158z;-a;7uc3H%vKV@(3#eMZ=K~B zT~=BZF2y|Va-Es{o4Y5>n(Hi{(d4V$<$H^%k=78hi82oNSYG}7yq^f5Z6cdWqk?u^ z>d^?Czo4XIu{ta!_Md|PEI+(PxPcF*sbd8jb7?JaIP@d)S7167164`I$f--7Mil_jwH)k{?vKd zyyA)t**eY))v9K0wm+vZndefri-J!HHm<&J%b!L0?oiZ4OI>p5>i7G+h8yngtD{?$TsWxI-eP12e3WwsDfbIvf(r03IS7-r!Q zc~vRM(2wZ-BW;13v5~?M*MLr2m8|P*O7Qi2X~8rW=-t5A@T?f8#7ptrCSc_sa)vRh z<@?J@+|>ya^@l=Ar*NA&|EZSmv{8x!+|V+Jo?#N@pHGDzH|W>ch|BM^Pkh43NHC&cndCFbz4yM9jRmxAECOeAT>S>-#t~>kel{z+ZR`~; zh$1ABLc+irU#?zrD5vctWxLVr#0(NqChVV(WR$Y-raXYoxqMVbzhpAiqu({#X)5T^ zF6d$3yK&mN-A;N1SgUXT)kb*@T}K$J?6pAPDnEhBx}Ukk(BukYJN&Ca-^!7T-y-Iy zUO(u5XL6uh30582VA4%4r&M6P6@t*pL{y=i8G5TRJrdfFw6P>nQnsz8Nem0KgxDxM z0~9N?sx>2A;}T?I}KG!w*4fd-19rTSU^)gb0iJadheq9mMDP|`g@f` z@h3u1Eyq=h88$iO54a#S1)KDblFVRd~3Sq-}6rXRyO-2FtjuqR}p@;RGWKx zH$aZ$we41)W)e7N0@HU2{(20FuQ?4$-jHxYA`7>qW5+EEBDwx~S|J^-e7{02kV%fw z2F?7g9uw(`?=7o{Z%OYI(@BC6_{#pUonE_6CKl1}XJY>CWww8+&X-BI(I$fjSb68O zK+taWLRo)?oG5XdY!0c>uT+=IyNjvk#Ii?M|635qC0u+Gy~&%3j|;ch>^M2lDZ};7 z;zTcG(fJvRE+3g-HBeGs^?o{{W1{1-lZ*{otvWmm$EPCpSm6FtMM8YnRd@N3=FXnv zWgFpM?m7}jftRSFda$Z$I(pY5Zlnskg4;637H(g*UN{Db?<2>)01lFn!$a`~jRRq* zTW(LSi`Bb-&^X`UwGT+j!FB^*{>nW}RrEikaq4UtuH|z^`1szCM@by*wG=dbBE_J+ z1Iz+!niIodlx4FY??li6Cn~!Al{fDba9+p}+O3ZW*%n+#(B2bjc)UG0C8|%!ojGri zJD3Vd49sGlJ3q#g$dOjn6*s!buKAEImm?#W#}y3H#v>U$sMr6rS%>& zCy5RN%ebEY*L;A--$P@;!W>jIfYCjyH=2FHrLOmd-wl$<3MhcfH;7)rx~O-O&H-Xj zO>DD@0bzc^zbCtbR(-?#;phQ{9FNko-4*&40ciT=B~VzG5K&sikOng|0-f4D>DOhb z7fHL=hwV&eqyE3YBb=3!Bi~W!^+og9-Z3lXOqE3CZ|r@b`UKE9Tv6C#YOKtfI}a|# zo^+K@@YfibOr#b?%jJ%hVfjXv`y%YtErqH1-hPKQ2DcWOntaDrv!2$#Ai1Y_CZ58{ z5obqUD^Oa$!6aC;?xgW@Sq?i$V@}c}s^_&5!s|dTtU7z+ToJ8^4zW{v(C5A8@iduF zXK*wQfN&8=eb?LlOS#&|g1eb0OmLpU-cxR^@}s^<(^+Os`cBIT=X1&Dr%l`HkDF0C zMjmpF1oymDTYTnEnYte(v@P`VHl)j2^u!mW7dJim=PCMkm=J7Mh8yUhvNimbxu0r; zL;GWQ$0mXj!xWnK==e=3hTBEE9EI?94xTc>%=2yUJcVFjH5>_$sX7X5yW^~Y4`yLT6k2{fR|)mC28hajMiSOJsn+g`5mQ!8 z8h+sN+6^(@9Gqc_Eg)fayr}98RWdR)n{eBgAK;zTn5pRAsX*gC>WFPPaeuzJxHJC$ zXgUkOD8DD(6H+2wl1qo6NH;4bok|N!NC`*@(n|?QcSwVDr*unqhjceAwZO*R@9*CG z4?M5eInOyWbLKPio)}!BPpx-f7)byA5`J$7JCn#hLX8xUX_ve@D!vrHxkZhCbb}Cf z@AbKj6V|!$#-xI(`e8S-lV3+z$7J66|9wt@BKFu)Wow1N2JD@JE^N=`K27MBTBCS4 zK0BEnfWj?+l^;Wqs2RFFy!fbzT1B7A3h;Cn^0_ z%Kw(5l|0RMKuDXH=e<&0a_QA8VcD%}g-c<@d?;R@~V+ew~0dG$X5FOH_?Dodb`BD!08*j%Lj@`%g2MHXEh|CqCRiP-^WCr~cYB zPcf!Vd?3P}P_qYx@&j%;u|B3IN~l~o1%}MKmv_bIxRcR3UPE3tMsmVwW8+mX zny3E}V;*<`Q5|fE3IPmI-yoFalFwhRgt^Hykm7GatR1tyZ%9ux3x6fL+AE03(f2G^ z%+#L?c$InI`-s_FHV~;N#I}3m?#AOBqgCK0PdTyRU;9PgDds}AB7=*&*<XFGK zo2FppZddhpO6--}IU(~f;%1BuT5bp_Mz?00%Sx<(FA79kWjT25+%M@YfQESEe$xD8 zelbZEPf-&rjV3*NB+iE_?|X@w^!zcLIYcGctlq7AQUe>Ko=2Z7k-ro9{hv$Qci>Ng zf=i=oEXQ4H*TYFxJ()UYk=vHx_QT8du;CBv676wpsCvoZCuLpiII#`i9p?mIQ!`5T zx0Ah8(S-Dq^HM$Z-%o4LX_JDF@x`T*@@ExSxO4er#hQxWXgHp&-plB};6Top`Xb>( zXjv&`NMB`%Hu$pm@-4-Z2k?RX6KlTCOXP1&h7^uSwBiS%Wo2Ull0G6h5rg+wLy?RL zC&Ym~?y|SNY#x;x5&p5gp$Qxh>$=ey;$ZJP{|g*B*!jG6bdK<;%|Rwt z?{pdiB;i$;HXkdgtyAZS`fWKP0E@m~CbMD74Btk_4`5o~v#9!((Y;3P^KL@4yz=y4 z+UPr)!mBRide|isP)!Tt)vGP~EkI7k%jnct8x#enx9GAS`Rrrmd)-WK=OSQoD%MK_ zfmwd<3h!^5W}d?Q_ql~uMsDdUKgO#TH0=TQBMn7QY+n{b@U(UT44)Vg0e@TnO7v+4 z(r)W%Q`_Hc9M?WO%ItB{k`-5}EJTuYUFA&k$z3ZUU%X~UF00LDLdYgSf{N-!u$F75 zZAt_3Yoairy{>VKL}COL!Hcu);n{?eis}R2Jz!3Yp57!#nYdf~*Dz?9yxV*Qa#dXB zEV{Sa+=q6VdUJ`Z8*`sj8xk+MN(KvX($z(!IKdbn-o9vfUJn|If-0M{Sjd*MTI^go z{RY3j@K1EV)e}eFyDDg;=m_rR}maE7hgjFaMW$u5Jjv7bk z2S2F7AKNepn8X$@w!+rHNS{=7jfPlWwEKVGiS!O4Il=!Gv!ULk1Fu|}{s_RbXYak( zyS=1OFM}6pwyWzS2Tk%rBBoDwQMtN6o#v(FzMom@zv#A}`z%$rKSs2{Am^NaR&>h@ z@QQ>c?TfA=Wod=obfSJ(mBk*1S-pSzPchOFi|DOeb3aa4a zgLxlK zkIR=N&{KbKD%(oFD2sG}5G?#0*Z3x2?pz4g~W%on8ixGAVj z$P+F#@HI+kKPO)qO(yDfih+)x2XR5|pzWy?xx`ERs9FpU%izAR`6M5o6*AC$M_xRw zkDVi8$Ye-ltVKO+Y}Uszyq-EloDM`J%EiOWCr^OaR!8l_!uSF^FEVb5SODWWUz!@b zqSH_MYE$Sk8TvemeGgZCE;<};-<1CpnLW9jUk2|4TY>@Zf5yAv`isl>jXxB6h_Jm2 zn<^bI<36HIlEFNu2&9iAVoC)^UkPr7_Dro5xR8OMZJbolQ!so(C5SrQu5#Q zd+4;CHr2bIphVh=sJiSx|L8s{5CA#|yb63X0duZ_P*pDUB5oCt`?&Kxv7#3SeT z1^NL|-e6x&W$`nsIA?dTuK82tM?`$ry`GM16y>94yVij8l9$i&;Q)m9E(fNIy?dJT zPH#U(cxo4*SEo~#LGWChZ|7g&TrG>ZL1NUbJgzO7Iv!ln|)lfy)dc& zp>0Ot=2c;mkytRCq0b1w63H8G-;&Z<+KaKoo}dUSJHKT$VkMD{Q-d1=JF!Z`if^Aw zVi*^dsz-zr<$iH`N}xuPf}aD-vEsXYjnPth+w^pGy%zbfpJ=>3&BSvA2-Wbf)<#i# zy{WYODFi_-&^*NPu!)}5u+*^aF>)vj^n}qp8{8Droxn^jiprbCjZm!F@cgG}8WFfh zZoD&EL}jsA#Y7SDblyg^;BL7kc?tkJ~Zqyc` z?Ev(`H!YFk{^u|y=k;gR+;}I|hI`~62?{kcfhVkg%_B;RN#B3WN=3BF`q zhiaRvBe}gkanN^|6>}dMbctZx8ROkGEfj)48A%zES}UfpgleVmLv3>WC@&~rDlwI= zm|bU0s|ynn_r8AL0*;uLm7QfCt}XRYlRs|DIKS3sJY7YK_*p?xGf)l{K0D>U*a}l zi`k3QmZ;+0CRiv_r)B@FP=|Y@#hxj6?gGnGF50g<-Ht-4eu1gN^m!hxt@LF@#39S@ zgBhCrLo0nV?cmp3)PqMLB57MXSWdUv>6Rj0mHgz_Zq)$$L+<=f-w_MOPAi%<)rI82 z&u}w`!Mj^wm*^#w9+Oh{77~4AcwY0Ptw*Ddn2PbLZx;q%NMX(zX100$3*~J#^?IP# zvi~)NcNPHyBO*J6ZILpM={Z)^%^nY}CsorP@gf#@gniK>|IDb&F5F$;KsM~2=xYRq zWY`3tyU1aV*_urA5CU4UUhA#vyLdj}vu`LOls)C+`cqBT&d(ob)YF{DuC>VBafWz~ zZ=E4XKy5X=s?97FK)$IL3)2nAa^eBs2|*OG^2KhGJ~%j47}`9d@Z8pebip}n8Jjt; zE7A9yZ){GN25YVMYYki>uTuq}^II~Tolh5iFRrnm(D?@Thl}Q&7>~_BoRp!T1nJl| ziNLE%9}Iy*Evm;!-bm9R-hGdL%t3quZ2Ugrhwt3C1=uym~) zXTAGqEn<21fKuD_q|*~J!v#H_%n;$$7#Iz6drWD=M9tiF-TMoN}qo?9$ByZHsCXfd?Fs_@PV8u=>P;ylZ+Lv*8(b>)ZVI;J=C^oGF+*YRev!-BvXW98h)b7I z+DBaFs>%D^;YQ{h>l5eg|?7+8A8F4%>DpirU4zY&O5u{W@k$j!@rnqD<*?rnF96H^%jPqcDff zS5ZG9ZlHP&(C{X2Yr8Dex*CRl(DNE?$5>!YGP585@^s#0yu4Bp1~=Qp+&7e)#1d{9 z=Xk>HIIR(MJT5hY?#-@ubIQym@ zxh?qjT}UwlCfUt4<#wSqE{5{JQO{Paaof+ry7ioYiRT5pCYt>tF4x}hvXRkNoa$d7 z`%`=-y`loeej@eHz;AdV7cKsp4;w7+AFG6V@!fmrA8OubU*;vgV^wu|$wlzYK<)K_ zVO}||o1vx#CzfT0!DydjYKcKhJ&5t6*k^v=)xD35HbpPdl}z~u4QU0gi|xWW0P=8f zNvx1?5q|_^zuPAb=?jZ(xSjxB&~ku5EU_l`&F{0D@ic_;8VH-dC$YrtDqdmDNhoe4 z-{#D?GVaj5kRwZx;eQr%u&VmoTJ|Jolp>=Y`PnaRb1fB8N41Gg6DGR0*V zCv44}!wDNuWbtox_sFna8%YMnV;u&(RfeiQ{^428(O>C~FD^}l7uJTX$!99hS%li+ z!AGhs8LAt|Xhsw3v}){-n9rEqdsjFfhxcrwtx{Dq5jdaIRR3_LQ_1tBb0A|s)J8^h zi7uNYF>m(*`yZ&G_U#R!mXgZ&F_J*x#0}B!_(E{gGRC~D_F6Gs%_u|DCIhO2^;#_E zgk0H1Tmhkj5#G8 z`)7zrHrhSPB1aoO+@h_!RzxDvK$U(vfc2SJWep1A7Y|Q>V|TSd5dF{(82!Bi)f-~U zAHZaz*E44XX9|*6->Fr2BAEBV$_PAg3@#^)JKxa;#2h^ENBkM^Mx@xPAN;59*H8fc zf=QC$-#`^W&=WxEl6i!=5ww5Q?tu)p;h2OIc~TQxJKT|BYIlC>`dI(C3)YeyM&=`R zApuG-5-nU%7;anQTgBuO!j1jKh^i=~*kG zq-{|pTIKi+lotJMOx}7utT=^kc*Bbgp+4N@o{?O0GQtu^O+dHjbhjx-WRaRi)-VaT z83x5jHigOaRayb{Rqa*et0VUe5k^MAR&P4Wd@S3vU_2@%Ue(`{>ab9>RWJXK?6ooE zg)v#crD~+aqY)<;#qnV7%xfpUE5tWU|c4tvI#)e_)8on`SHa3=0~g!l}E)4 zxnx6^v&oa!Ps^G7zIYiyoI^ni=`l&{vSy#{Y=k}#^~u)#tA59zMtIisYV+>Xi+}H= zy)T-RbeRe4(qrZ7Te*aHjtX>}gi;q6Uq1ljc%Lyn$SFNMeaicA!UW?I&cj@@@Y5 zCVd?v$$Ln$ZtMCNKJKv+zUOHzeDc9)8%47M4!Iv$r5;<&9SK-3MJ_$RXhA11T)?XM5!myPT_{>+QJQzCrT&HG8w_b`7q!Dmus z!a-pLJJD(6hM~ZXA|BSdh=gt(&*uZBfhqxv`XEfcll@-<{lF37Vc?eLqs#&KQ4Y5F z@%!ob6^C~=I@gYlDhd}^r%pE#`9khQ{qR^@aDhH)miMA?(ojegIj%t=OgVsE0>4t4 z-hOrfRCZQjyt(7mDlm);e3^h${3jNLu^&Pc3#Lan@%k@QFlEFaVAT*Pb^=$H1EiV; zPfiLP8ovT6UPx#WZi@8(I8#c5geZcjnQv!wIV6LJ-yzaO>`udZXnE zsiNFvAmPfYLJt5aD_rc+bS1|yf8w}H*n)=bj9J_Za2{xd5r<*O5|^6EpK=*PE%6RC zqY~sH@tZ@7jPWG{YmqXG+1(js z29!VLx75OLpPlpb^|l%WVR_1a0)GE1IKzl%B!~Do+7AkH_ENTPFKYi35y4ct%fOZT zjJpcf(jNNM@ZH$nacra-Tl<|3M)IgejQDOJwbw`tN6Sff$&ThB8A8;p?^8s2PqpL( zw-nuH78xes5Xg!Q$%+gCaw^UM|1b%P4kLWC;eK7D^^j!R_mCf1xs_V8~1EKjYt_9)()m?#&}zu2l(dA&w;Cs-AFSrAaL${;P|@3 zBIR4qV~8l1)aPRoTv8+}cu+}xAb-<|!Q-s-FJ-FyTI)vHfNIqzdqd}eKzq>WL{Abl zGW@o_=B2-T#M0O_fj28@R~5jEy7;fV=}=E}NPmtiBo{qp0NYg=qKR@b2g0l)Ov?Zz zw7Wm(s}xtyjXVtjoXc2 zXRhwt@#*Fvujb6HlC9ILuX?`tN`E`?tSFPx3Knha@n!wTE>d9`FR|G`2jNtGoZZX; zt2=pHURDM<4!rmmKS<-XCe)kx3BNu@4W}~c)l;;}vzv2mJcxUrc1oLl@Ej(~m@S=B zEY|xyPZf4ta(*LzS)=WoQNnE{NSJ7NJe1as1vbC1uw7od6iOy&vRF~4D^XeH8Ip=0 z=gJBVlF|Qx9@q;Y6#uXZWufQh909msf1hVT=qi058I9uQn5WWPW+`lZHF>kRe?qM9Rp2DXdR& zw5s*WPZBux`^72xt2Ytn!qpW7dMb}N55Gmw#hF7dZ)A9u|NTR1F`P9(3_0lTLXY(L zu5~l6*MW`f+3mk9DLWM+_iHwXkxB?Ec(ktr=o2#d2GocgJ>zJh^xswbzPm;e&$6Mp zB4BQj8IRLSg6k^sHe60Q|2qD)0GaMl;0;e|nB-qe$uBDNCOJbPGqc>?Uc+gBllpZ*zTg^3?q921hns@`4a|wiw)oZbcuheRfaL;G+@=`awr_ffkgR4 zuEL@cVd%Lbz%NEKKm#lU+tYR9(#CVOE4QHHIzu)m+5i3#n_pC)hl@NY1vY8=E1t-H zmybo0wl{3v*KYioC4~UhjPI}oPxf8N-SSEWei_tDh&=RlJ@&4Chsfnme#4T zxUn;m&swhi>I4*b#Afdt<@7sk=Fs z2X5j$2oAXFU04&k$2NCeSHo!XQgTsU9Vx)DLJbl2!WGiRi5;W-f+oW(e!JwNR0$_u zcv_3Wic;mZ6|&rB@=IfU+rqc%GM(!sYcYr*BU$bHZ1rnui~)-S*r>C&*R$c;zTcM` z*JB1XBRt{VzUM{s!u~o&aiN0+jLt=5zf+wIV>F0qLpIwXp6#w6q1n5e^jles z=ZrZ=7L{~BMs@pacM8STIEookaJo9x^O*l)O=V;PT=_0FSd#`k1I0J)mLUlo+y%2D zBgOMJ1EB98b2@4oLIYw!9))C+ah350QIcWGntw?&ninWiAD6RZS=QYAwO>s_G`DnKSe^co@O@l=0tcC@U?ZIFP~k z<F&HH`t1X+2M4=*cvFSlCU3J>bZ?XBKC|B+iBw<~nbe5Hxn=)@GQt_0MDA!D#j zgsbnM7K!Qzfbm8}zNFQcMR}KPsCli$B$S^g&t@j1LlcIKWDIVkkXFRf7caj&`#_I=yQ9@X5+MUxwX_- zB*ESy>PNUy4MO@8WZq^Npu9b{cZg(tftIi68tgguPTqz$={8nE4ZLkK*XJj(&4U)f zM{Jjr&luY<%jL8Q!za;`Ar*_Ben7s`|NLU=DG`BnT^F6gOWzqqX664Dzn04QhBy%v z4~rd(9)a;Z=nf-|*5R6OD)Zp9e7+&NS62Gea+!~}C%k{WWeGegr_Ht|3XIk0OPQQI zQB4fne4d+IA`Hg_AdOo|IvG_~7qmOSCih_ntYX%Fr7+~ToBFUH{lKqXTvu8a#M=Mm z&ce9DFfIQ(W9STK_I3)1@aR@_^WRrtvj1-%6pY_t9_^#Yzci7CYl=Y*$1X`&Dvg&K zLsd6MAMZ{$>P656Ag(1l7!?+iTzJoeILIM}p2Pi-HEMwaZc3^+{7A^(;mw$l1O-&3pX(1jynrR5{Urqt zy&7W9E)YzFana%y_an3o@W3-~!+Qrq!qc3`Z+?U0x@8AI&utZIftea1gjaj-IuEHq zq^$B0cd5Cnzxxr%%w=es|NS0w+ZW!LuU{#s)&Ubxz2*bHdNHRY3AoIMq5k3(xosvZScBd=n^EX?SM-yL2 zruNZhi7sWY0CP; z@z2~NR|?82v$kNo3@nUUTWgmHgkm4eOHW^7&D!_v`nuniM~3*3mEnFwgliOm^C5MR z-Q6oHFziZYYHSzTUBrQ`4(|#teeoGIg*=;g^XqQt*{1P&c#URixX}scT(7ak}e?A0t9?QuJfkoRL^T{>|GcPWBr(-C?l(8x{m%*blQ44o{&7MxLo;sDZ z-KnrF@z);AA@%qSAZwpi>p{M3Cu&EXi3{3YnRr&@wcC|Gednn-fBWfijO0L!KgxK$ zOk^Q4Y&Djh!V6T<&LmV(3+?9e8I+v$Vo?I%d43g-Q`rd8+0tUClr!B1WT}mWGU0Lz z1Ie%`I>WmjLm%`+stK|4{*|>>S~{*)@U*nBy4^3cDTm?Nb=zaN$^H)SlB90vk+IAbL1?5_eqd;5f!5L8ScXOb z71`vdM+M5r$W5p9_LiUt8k1JTLBh13I8Yk?1M#%)X5)!;;5znqpQsmO(7pS8`*Wf^ zLcy#yE@ncXei*qVTdt!=KmPAU0Mw_o3;)P`1W*sA6}`9S%=irEe|j=sm2xmT8K>t- zAO8_YT>S4)QP2t;{L$Pk=r!S8YNycA0{|aK1s0otNs$@@v zgagSDW7pEaq%urWOw6j=&q6R+o4L`ui$t$RPgEU2CbJMJ4;Bh-Swo{FopcC59#)J{ z7tIp!tA)SD$!h`}v%E|nUyG~L{5gC<;lFR;{=)aygslPf>}3w!OIHp5cqqpM?<>T< zC1YmTZS3oeKXvYlJq+b&%vb$`viUp!jP2xNpO`^6R?_lUDdzkZALMYiqMsdfKfa9p zDH@BX-pRCH0bs4hbZd)*^+@`1haljlR_aFl7TKgpKh8dDGlWV z6j&=##x5s4nBQi6p6M*{{|PKi7@ghgd5YV;a3%402Iwb6DKeFtcL#mKZ7XvN#=~j* z2%NK5H*7CSyxAG%~k<-B+gb~WUZx$H0cTp`i;LZj6#37mn3K&3FDfAzn(#j{N}KPqXV8~sldX7jg$CfoH_^HM6iY)^2c9bu z9>T~li?Q?U{U$1@$dJjQvRZGn^?r1Mp4bP;U$nc?`cbPcs7(SlX|{&-dd#ZhR8 z(vfm|0}XiLu*9K4*+idteFiyqPhX2UMc3(^j?F$-*)2*QcZ?K#*~IV zZ9`dT^qJN)vood{wIh<*a%+Mz*e+BAwM|8y@?C&)+}YtA2x2;`U^0bSrOPd0&#&+6 zgGRcVnkXTh^|wBb6*&rA+~ONAad306B4d9`_(yVmK{qGVd?`Z;4G(L=YY9Szyacc+!ogDAT79 zmSO9M24MAfQNoj@E&1zg{>@PPafwIruL8_9oUSi%gC*spQZRnhjabs_{LTfJ&FQKt z>f(^TnNGK_2Q2HSeOb>}fB-)^B!sC7vLQ3z;9A%%L>Y4ODTxWac&1QjEkJjILAGh75qp~+Xx<(&HfK=1fmikd{IRE!kxJTq zYn+6OYQ`5&w)WXhE``j_H(`QXmRMCKh@$AaMc43u3KfY^e_!vV$oKP4W~F1oM&QH% zV-bq4eI4uo+_dw?J*ee#f)}%vtRui;l&ob$r)IGx_LU55S08em5X}!M&qoLpGMbRE zKH!vEdA?TS+#>2#emw9l;3J#%J^E>!x2KK#BA(5fn%B=TBraHotfD)YBZm`t#n*nD zMmkgRFEXuA-HSvZ1~U~AnyjzivH%RiCD>YPt~gMs(?X&q8Pgp<5#AH69wqT8B^4PNfBpGZnqQmyos7qSyGOV4pKX|aOFqy}v5aV6 zIbwZ|xgDn!5N(z!MWXndMcpGw05dK_t-^RQ^Erw)Ew%&#bLr{4_u8E%R6OpEpcwZ$ zRb>7$-Ts3t=~6$?;8oJSxFy9pLu>we)0AlP^iYpK#VHs9&S$!5J0yMI{uW_vF!UEh zZm;aC5D(rd6zUhuGjiS?l=~EIUV8L>D6tIZ^e9C_Q)bmoFH&fsBUL`V%9-vqNLf_a|~ z?BW@|nLLfAIhO8r#!H1BeGZL=Es^yQ&tOLs8St(ibQ)|iq_|!|EkJzWoa>Nrcf@hG zPwTXW%n;s6rpeIPUt|TxD4yt5+e@6*RJxzFrZfhTRg0rP5&@w~dr1F-4n5Uk2D>>R zBvpg+Ytw3Wn^Dd>t-+v1xiw5|NA$Xo8>BuH37yH65fu=b<%2Avr_-G`a9qC zlU_4vJkE*a3=l1Ls#GHw-iu%yc6_emHU5wNH1^=xidO6_$|rdxo4-d4Zt6O{Hf~Tu zIqf7EaPxUHQ)f6(L;m`72k`7h=%MRZJkI&f3{8G|z_`t>eT_W$LZTXSPAWW#q5wcK)7hAA}*Th(~6})qX~WaB#SUP?d9Ljv|H5$82}D zUC%NGv|pPbLNngbzt!Iuy5v|1;Lss6aO#4C+Agx6bbZ6J{``^J8X|M5XfE2<1t>bN zHJzy}*fnZEshh-3v-BYYCr&H6`k(Bt_s2(}PBs3GXW&f)I*xVXR*hyR^jAuMMbqQ) zLfd4hd2tr@_&LaRm}0w7BB)Q(|B+?Y+y1;#jZSDuU=p#4c)9~uxS!}P_$00`6pkKK zBS*FF+#6&9DMApbga%CX96w(j0$fHmF^e-i0gPO7)|&z1>6+>9jkY9iKxHR}hRAvu zieyezoIf}We@UWHX=IjRU;yX#24CjoALhf2enJkxKrmxKXh}TWWp%4*2(0d&(KvMl zm>>E#{bq#b@3XG#>%-W;IjZ&%h{^mYWJOvdPZH3K=TlgH)f}UyRmLw!7b#e2pZgM- zqW@sONORbSN{v-~IeT@84bzPog+JZzRC7Y>?A43jcK^a5e<|@bb0J#W3%8{NQzH&= zFkacGc5`tyjcS8t<}CE(VQTEdbMe0|$F=!iH>wwJr(C!*gc)`X1S-WP zJN^Q9A)c0N6#)hNYT2;BvzEo|g#aS_Qmra&?Pr6B- z1JpK4XMffR#Yu(7LCG~6pby9{E|~qnSbT_2byrIyt-h-gRvEtp_t?;6Cf}-tm{(0J zJ%RYuK7{Jgx!jrle3?*UI)1~XxgFl5Exdo_(t72p;W7!O=+1g~4PWyhA)s|xa~D|| z@SQo_le?ZvG80S5GU}Wr6%b^{`BWQSuN_jmq(gZStZNb>5?m15{y)1gwVuFKp~( z-4&HE+X|KLUW~w5W0b$3mMfZQXaQ-_-&Q5X`*u8Jus~u(+g)u=u3GTvm(Mx#u^|<$ zJPF!ftPMnsTfbl~gqA;X<2J~Vf(Wn_B4>kzE>!@j8Y7weo43v5M1$AVU^I0hYH+j) z&D2*Z%9CEegJ&VSkyvt&nZaiHmxzQ;sED1}!xV?ad)>f~(rw!yfgV(OMNj)Lf zLGrx15m7vT6~-m)H5JzA2ViNWQZCI?Jo^mN3!iy#rd+MsfKtJ*RkwVhX~Q30A@>yH zaWD0KRh74M=`)fl8CjI!@TKj%wOqY7;n!B*`=w76YcZ8}5y-_gq|=P{9fQ{RNdlBv%bdn*cl!Ge5A z=XDg8YDF@GIOOnIU0C&3yix>ETNWt7O&IN$>skn;;h)O``X<>#inAo=_U9*z1Ca5G z^k3J?Tl3U{P29!_gb`Mq-@Yz*MZdXQ-JoYfICb5onD|ax;a_f;^_dFe65*dBk?TGi z6TpK>{F@wiyGp?E;xC=4RtL&;FSBdSPvnb#lT_SeKx(#TcbrK<9E7->rxv;K-gnGWI= zj(5ZSE!lIn1{?%6R5tTXSH=Q?Y^Dl`Noc zt=OmrYRGf<50MJr}$9k?v*^R907cb*vd zxOUkA9I0OCHrb-3OPq>o5Ut4D~XIFz|fL)jUDHO`oJrJuNccmZ zj~m(gSc0k}zkM&3U~c(PUm=SM{Bkb;1fT>)H=^7k_|>%5J61FmVLKZS+)SvPe@z)g zF*e;=ZgXt*3>dPSP z;}7E|heE*?sxh-}!~IAK4kTu8uGv3iB*fd%~zV^dlOcRQiy?ksUZj4oQ z0rI=m&3J2$VJZnnV_mm}%ZCZ;eN0*1H~xy=%J1`3`R5GPF9Hrkd^hZ<*S$ZsI@xCW zadHR(!`(BWEAme$ck~VLsENqbHF1j;a+FTdkbJoLkhWP^5Kf0XvKNAr3UOWiJUf1B zY+vA%CB8CdSbqvfcG&FH_}UHJq~l*Yg)aXa|Ffjy9(TVxoajg}Hg~uDnXJ5b@>RP( zd|c#oW72Yw6t)AOIR$MlrvaUePJy(=K9vKtIB$>x_g#U&;M{Yb!+}KUyGXH~z$Y5$ zBwpLX%>a#;6W&U1Oc;C44eX7*;?|33r3?_|C78^hSzn7VC zSJ*G14UNaXQg6**7=;!OVx{mYvY#0#o#EgB$A-y5X^>J^2kt{GE$4bnE`H z3-eQC{v`*&QGE$pe$1Q7M{V8fYfqGG}827?n3qKIl+_`zM`&1=P&Z;H})8#cRnxMb`D)Lu9)a|Gi< zJB6Kg&Lv_jIj;rW>P|fo4+pC#C_(kf24Y|2ZmN%HmEBpix*eiY7~Q%s)5B(KNc-WPE1lPVijzkqPl(aEb|N{WbMHN~etT~C$j{g{eHc82QKgA6_u zaF`8nRwq=pE`L%A*-npq+Hn{e1nd~a&grccgb~mRe{zpza4#mH7Ef~^M>%Il6wNHc z51eWunK$*zyP}{AQdWLiR$krfzwTLso416Nx?jRnbzo-+le$##3D^*`$tj@zXTYu} zVdGJ0;y-WY(dRCb^x5&(aCOhu&Id|k_r`gL!L#oMxkyc^+oUY-VC+(B7q2VqJQ^{kw0{ze$eW~bwyUc82#~U`65PQVCDNEBg6NY&Q%7# z5Q#<4+4gS}8#d#H1sjG<+%udjB}hJfdG$-24Fac#U^~<)p zu@vfpPC)8m8*Ox-#UxP?z0}i(d*j(GuP`Dyx3RH}%?V#l=EVL!uBS@vZD?%J4v@UT zDB+rV2Nl}^wY|oNMm;D_ZF>-}?RP+@5|c^#ERmUoo(!?79x#O;hDxw5?}z)DC|?7X zN#3^YP`7{MsS9TNzj{#hFF~j~SrMNjOd$Z-cw*p88?>C_^c1mrcW)Q0Ij9pLrF*5- zekyhxv+7`3LEH5_#y59tVz;m1q`HGJ1_mMr`8DdEJ)St#%x_kBziA=IcdfxMEf9E*j?HMuI4FB3J6+l8b|A(!+YJW5#4+eUi7SZq}^u=Rr{Q0vE~ z_rl`CC*&a&WZ9@7g}>>GT#Aj9_ag`Yo$C}o6?U6kyDz9c^}{s%yhgIBiLJ;W0Qt6C z%J+u$f(#@owZFttIa@{_k4o|a{8FVLQ~U)!cLF@Ys70_0O)~XZ+9ad4_>z?2L+OWObwn)<_Ct z*Ph5?;Wuey3S!gA)Gnb)wEK zDMMkszlQeMucH@ZjK7ZVOCvr+XH^GggBHs)s@j*G;@RD zO{>nL*p%BYI%!AF-fiHIx*ngESjIHDSPq2a z*z8twO5)Z|rkXK_S3SIW@_sWhsE@?_=76hgrIuN$!?L%bVF=(I^=4i!#PY`J?1RHZ})yi8ZsBi z;V_m~Yl>sbwfe^1qn7^rzloKq_PPxT(dmOMpLoBwc6FkP2nPbplc%2brBu^YF|hws zg#9Mc1qrOAOv{J)xxV?e|2CIAyVYZ3=c(!XO;zM=Gnc-pzmT-`?XSnZuLX;f2-Fua z`SiSrvSJ)(5_oiYF~AEHRweai=7Y?G{ash^)c*g(l@j}Bm_hcUqzMQ^^|p-=t< zrawwaxKQ{WY&BfZR%WuYgO~Y|~#DkC4QRIKkfR>>=8~ zW2Pz?XmCZOJ)XlBg^**=?dIY9;jp(5bnZH=)ZA_mEqwfs->)&5pE3{cg-)qhd5tau+Hw-1GlbgWT!r{?yk9#4t( z^w~}NsJlVf>2AiM5$;HX!GaPt%VKRms4RFvQQ z{Y?l6NQrbe0y1<+3?VI{h_tkTbP5Oz-Q6KA-3^k`J%kd{-8Iw<3^UL8{(PU`zbw|8 zd!2LN=j`j8yj}{JkWb?&7MGqu^!!ac~5dXs|MJPx(U?W z0B653W(jK_e+KxA3Yk>L?F!q0D!I0ih==5pzAM13-+>OlcbN|zCn)W4>TBz71SApZ z&>GhlyiUp~yM^_@0Q(l4bq2JR)5ValEq#g5CkRC}YW-IXrC-M@R=#|=^S-r)bM;{P zu8_~Qv`e{+7rwU2w;`>c2qSzD85F+KHtQSXBQ?nqp&YmETbr*~J|=jzvoU&P^PME` z3My?HW^9L&mk!wi`0(_Z%lKC*ySVDNmBpe6c7&QPLh9R1@8jNp_~)I&kt+mdzdCQV z+1ysV@b^gjQIAfvF=^oP0zL+GGOP7B$36-XuJo|RyRL+*H0yV_YR8cqK;`OX<0pcm zEVCFh75g4fiaO-^mGL~)dQshei!ZU05PWLN^UaJ>z%pW0{C+B?=eY}I-MYUQ|Ao!P zFIQY15>)+Tc=P-W1-*(Q;@A1hF(=(3bqhuR?!nV+s(rw;Wfs)wAEE!inmL9_ExHgi7*S zYu7D2@IJ3LQS=q=-v=e2K>$+o4pvSFf-|J}PYN&mWlJ|ox?`BEtT?LiA7)edV#)&h zhQ3>tGx%^`{-xnn6>_&MS>Rp19v-Ch9iez`&l4Cpj78Hk+kSvZ4o3ULp$W>;=eNV1 zEj&Cv=4O9D%oA$Pxp}U#GMq!?({xO$)u!lQ%WzWj^Cg4k?I&nV;&7>H`@;HpC&lqL z%(R^mlzKHg6*1h~L-Q#Cu^Sj%j+xw4P@UGv4QLYMpChv=`!WOyjM*%%Qn2vyps z>rur?%JK0}vxx#~SYj-EBQYNO4AIb-VJdHAKiuM6YGb=k(td4~@wcQaq90DmrdhwD z3)PPc8&juHr`=L6h_x~>)sh(Uqv5S)a=8XM7X&bm??8u#>hVn$pxID>G3;(jqNHsT zWfo1J*J6v9oIrV1efJzmHk{=S596u!mo*__WWkj#cov>M~@rv^&&(7FteM1AXW|0AlOozi4~ z#o!PV=UH9)aL;=ia(sz+6jGRu;qH+4XU^sO%cl5J>bsJjEiRn&6W zRx}Lafcb26S<h>)N_jH_q-{>)F$qKe{`KlF0D6-s&T0#F7rz3Q3 z)+y;=3J1Vtt(`m~$>O^_L%PF2g$CyeX5=DhUST0;-wR2>4oZ53ix9M)N~0B7Bkq`2 ze!Zhf^-kt92u`{%A{x#D?AI8ShOCLX+>!@qd>gNvu}|K2c^K`w9l|Lvy;fTF=eUsK zwX3_=bWDo;&5`07l8CIJlQ2`t{-nx)RE_=Af-V2#1>SGT)?xBWj-0J;oZu|saSdq~ z=X`XwBPL`;bH_4MI9-{4-PwK~O!U+5ZvC4VIPE3(VEA{A!dNJ&vigllN+9r3$;GD; z6CquSe24gV$sgAq(qlv}TabhpMK~+2sgv2~TFO=NO}Px75^Xo=2q{9hzfX+neeO>; zEVuf)wY%_Z!9~3@D(Iu|2FC8Yj0FYQ+`N*6j>MjIgAP3Q}7fgD1#7pdr^tLqu^q^ zCdR~l#zYmex22Z4&EF5l>&7!gIRK;j@_zUdNU1sfRAH#v9EvWQ#`AxFyRs1h!IJ7W z;v~|A zWn2O+4yf4ROZehk?guJwFas%JSEnWpz(`97Ll70rROR<@^Us=4W3StWbFhs7;c zi*BH4MDWRA`lo^0oP_n%1O|+CEHMfqE=P0tS)GcZV|(!cU`j<-sGKkjMF8IZyCmj8+y{$X|DuLrApj4Khuf(Xpao3^}(@ujFoCIZz)T=RMQ3vYp!>2)aIbl5+0aT5!EJ&^P- zeY7t%)ikPATa#Q95Nc>@|GuTpV+>)pi81cVPkN3}S`>^WC26{>dLHNjYfv7bP05XG zqxIHm8lqUY-CjY2-q!gB_Oz;z`Dp9oKZN<9E(&KUk(@N41EA0ZP;GA4la@r$)sRWG zTcVz6%x{ku4cvLJIp&=ea|0tbsBky~KY0$;2+x(=OGCy;>QZCrM1mU1-_&?jv7i0y zgGP{?MKl;lziYDyd`er@be}b)9)Dvzmckjl zHqeh3>sda1uj-$Gdm7YsspP5O2yo-aP)g5E%>bx3dZT9E3V~bxn&Uqln*bCrXkj!p z!vfjSosR`Hncx2aUTiRo3&^!k7sNK6Rj|GI;Lqoq#{`0yEIulEPVIAV0dX1^t9&eY zstc%Y2z?f862Jr0-mj`N`L!8B#w>>s7c!kaGYW?Q z_=A;X%%l2Us)y&nG>Ksp{TJ?5JA7;_uU`*j_Dipy0g}KAH+|5nc^{`mu_nSvD{YEq z7MNUP`D!b(HHM(#?}7DKWwF=~1>cHcR6$HO_`jK8NOLM`!# z$ds9d7CtZm`n`U)NoQ|b=}wKXpggbX^9k7uU6xd|7wFDW@o&sCjeVAOIs0=A+%t0w zh3pE54*3Gd4L`dq%L;97f}~P>)#g3?|eKA_H_ew?sBRj^7iihIwpipyZ$6V(-6Ch_*Gdqr*G)KcGpf(b-=rLPu(3MbgR@3U?mVPJ(? zW4k__<|v3;(GFz&xrzVdv5B88!NQ2ifH36OmIBUO0{4=NR(=q=l-vBOSEQDx_T<<- z_?j6+CGkVp1DTGE)HgD~M9?@jwHxdm$l>unBZ1Wj>lw%^A$~CBk78a@3>Kn4p&&s@ zoOQ~3Ej7#jJDAH<^MSA!lOP9a-vuBMIA_*0j&;(!;x?vqob)2={>MNA|M~W-FN=>h z7WSHd6$!TR7eVc)_M>3XxXDaSfRte0o zspVJiz{wPoNPX&Ulct{k1o;EBwQ1*^bQCJX+2X>$=NzDA6uFCK<@a!d>5GaPO^o2f z-;o5(Tmm{!b9MQgJ)H@voo}(?p>eeD8I>B^OGJGctX1^|I^iqxE?R&r;-zlOYP`#;>`{%r*4J+yxIn^3okG)dk+FuN-H-Fje}5(c zLWxYWAL_5T8~ek}Zb{*=hGSwe=n3*>!Ni)zzOI0l z{fr6RwZF=LIj0Z0Pd+UqvdBc)ngmbN44pzJZ7-<2?S)lL6yqds6FHEm+gg7t`nNmo zOjjr%O)Cg-#o$=h>y+wAcOk?7Y?-8Hik;&2=kRUL5>7$g>huMP&v1S{ytOw*a$zL- zk^q`Np9k0)uc2@0ZOBF}+T{Z5KEAl&%FL2!?8=hfejcUc+Xl$y~l$R$u; zq?PUOvBX|AHXbH~AU_M(0M<}h`}Ien$8O;S_?{HCEq)fPQ$#c+Uo;4}JygYSgKZPD zZWyZj*{pZko3M@RBrasjh8a-d<*;eTa~wCRlB(&=ksh|sU+-@o+HX#DPVe$y8V3$f zSZ1db8ESgYoNiH!%6$KQrJhx74{`cZFo90MdQ;-sGp(S#d^{V#5~f)KR=eHu-hpPo(Cqoo9X`UVNwCaTJ5 zaEVaJ5o&zfBzq%fQ{83b{M56G-lNg?w5?m9vw9h2bGl?Pd*;2053$y%XbJ(UHBehI zJycR7#XhnjxZ~iJ-)=BUsACvdR~for>4d8NNw};?@7i_Cj`ME!JW~I5W95E zO)D^ki5I#dj!5IVg@d<_mwfv26BL!S`xXWKC_iakFkYzXKKf#a`_1h?T;%OyQT)<} zrdIF}&GOJ9Mq^bFm;Ivb9^gKU2X!u6!+dKW6Ft2R1!>(Q(Rnr^b+v zXB~B{TUb0j8S$PcUzY6~VA-c$7mHkVr zPtQLWpcC;M+A8L%C@6I|G|ZlL+Y>EDkWWOK*;`U{*5tgzwqaM3^(K(C6H-m&#R6AaaJy_n$Osx)rkX#Q&hQ`m)4n1E#0m-w`o=bBeh7@T36uD>&>KT^4caAzDC8GVgiD{!m=jSC-(pIk+_E)oFde|8>a(mJ^b?2Zd zOZw2K=YE@70*SB^l{nYpDSFY8*rd$5*VaWPPXhXZ>_l)N{=I=U60?A#7tMl)43@+( z_$MRW4!(A_RJ>7Y-jdp%BvrmRf;g2a${|#xf<+X^7WCk!-&YDq8F<-lPhLs0`mBC- z8@ko`L}QP2q7xn^*EN0wrP%#3KQ!gF(1jH9Pu+IfI%hO6fYc5>5yI5{>KrTi80mW7 z`q&o(hhCBj6mM)ygz?AFNPLWw>@oc**nN_Ilp|nrgyRul>VlDl4U_ol#paO9(bEU2ijt= z7fCbcyWnncu_dtH!66!Iz!&>QzLZjO2me%lW(Sbh*xl<%+)FxdNT z!&cj@l}qv?c&IW0^G#XsAhDhRufwMJdDFt;3*^{dssWKEe4;meu0;L8y&OW(=;hWO zhMH(JjU?nk2?!T7&Wrfu=$9hWn?RT5R~)&wxOpqu{hjM3@lw&}2R?g_r}4VL=-cL) z??+k=h2!G|UgUcy&Vl!ZmG1tgeuXL~y7|lmH#c8oth4h=jtVhACy~i2G#!`eGQEk- zZPj|H+Z$4QwtH?fo5&z!lfXCb0r6TQv)e)cjhm4euQ&25I{vpL6oBgQmZRRs!=GNv z$Jg^;x{2QxF&X@Xmut3mg34n0WXl)*UxiS8`l0{Vgc~tRi9BR8+p0{{qcyh ztu0!i_`!ZF*;~k*bKhY?aVYV!a}hxo9p|Vks!OwkC+gE*`lD+SoBm|!OBl982?%B2 zhy;eZ63*8QBU<}DHS|bBl$X8tbE;Wd;$i|BvVF+x7GZ4jr8D%x082tCi?Y-1A&s&t3*$n0Cu zGuwEr6q9-+jYCu7C4SShba`o`p`wQXyPC;$9EO;R{&Xcxi5sDB2bY%a(ANwkJ@xam8CDHM^t*frq6hmFmBXw zoTztepMSBo;Vo-B7`L0%l4JhCfAP9DdZFIBz+jgHkq~B?y#t_0zu=HcjC-O`Fvj-i zmc8ZqoQs8}ace2>(!u}L16Q(acGM_gYo{3YGS3VeUzTMeTggeGvFB1E`+ULM)SDM^ z12{5))DaLDH%TV#K_Gmw<+O z@yomHD}pr8iv8|qo@y#F)BF?bP+C}B6G|gIk?E%{Pz(JP#{SbY272-Z+f|e2*K>fc zNb>X`D+gjf-+EG+jr6m!exER}1^I%|*C7phU#8a1PKqTr`JnQ z9Y3^)scYI3yPh_9^rMOQuFi#kgJIQNZPQo8Rov?_9jYSzNZu< zqXkGui=2;w(!?T-k|F3UF3BGtop%`E^E8!F$f5}Sk|GZv$!Kx0{Ah(#(q zNOVuWlNI_vJ2o0mZPzb++PQ2(Q9wi0D%wXC^|NS-gy^si%m%giJgxS5n<((MXJSu! zM)SIwZ*%XE6W8Cw6aH8++%v6r!%E*>g-pHIRN>9O>(K?@MCx_G_Ocx|dIMpoaQ4IP za0tm~l#nTj_?iF0m-p6KpN_qqUQCv1ZHuc$mW9l=fy0DMD@r%t!vFN4b1O?F5`_4T z>8*uQAo#=URa8a>!4$!$WYSJ?-p5G(A1_A)tyk22l%K4u81YM$luPE`yAxhZzj%X} z+x01R3;jmBp7MW)k$g=Rq$R2EXU3&GaAQwx(@>2^h=%lW$vBFh6kDuioF&$`FKJM~@LhSq7H$kh1;KlYkC7k40Uc`?N}ud@ znjcx4LhN>Mjl>}sum3PJW-7euIDR?=p-0LfC!bSUnhg?3l z(u#>m;2-LUg$ED1P3^tqv`rw=Un}aX)CWqMOnmdk`(jH*6dP^odf+H3kr@H;oklrW zc?NOTR|1uO)?}(>4dlsNB!S+%?=zVN{>5%3q&7}|)bB-T*m!gDSi3FF2U$)1NQ15Y zB63!vIS~L{A9FH8}MJKNqqAH9e-G_mZdV5wf&TRL$Ge$)Bfv)VQbc+{VF#1jK- z@EAqny2fK{wIB8xCc%Lb$N2X>D7+eq|1MCKSX&k_j-FTazmzEY@c*9@msVlPV#XSLR$1x&BiX~&Md*)s)Yq-%7rYF6*IN3nYO+rso`-3znaHrM1wAV0oH>zZ^ioyhUO|^%=I;xZ#{mOH`+?;NQ#8Si z44LE>_0@0jZ2z4N-a1nxVA!z2u?SS}(@!*69v%#$5g!fbKaC|VgZ}ChU2{rEGC!?-Jdf5KMDE-}s(K;QZzO#`nJKuiH<@Q=yinGF zf`y^ir(4piLAbJXdmz{5Db zFj%&UH3m>Gud7V9G`~#k66`;|7O(c-=cz{yr0!dHgke3XeLe$;Q9NcBEJ#mNVD`D~BcZL{_tB@QjWY3n@%dH? z?%c=nSI0KB&CqsVf+f(BHl^}nSm3)C*S z+7ub7nTt7K0xOd*vZkYZ6X;}kN|KSz{N$|||L#$1dfTx)U`WHK29Nk-)5tBpX}|K_QrF8@_ZrNPB4 zxbjBMSRZMoV6$yXj4@)lGJW51FABj39cIkLLI$Tp0gqRC&#M1ib2$7RPdEMCIPuqX z**2X&62m1X?sN#usdE;#gF1Ytf(T` z03AuVnTbg*)(mlNc!&Owl^E>E&h!Q_>qt1s$VLGD z-N%%AjepTm7ll-i)aPAmi+IpfMs0^ibM=a+m2yO(tIBS_d5WRig?OOE(Wb1dAoVhjOpApw3hd!7Hk)rO|@6JLJjKZD_);C7PG(IpC zu-i%9(LG!bGC7az>vQvzm-o(6UtXK{Q5OAT6n_0~(eY3rIGFEflxr7fKNHT7{_zdR zU<$kkZ6-kU7mZ6+^HUGS=-y~GJKtD1nps*?w;chRxPPA)ZC999e1&N&B3CqDIpwMx zzejyE%)G={5L4Ey6HP7nnqtrGK24dl4NuHz^zW5mOG*U9|T!s!5Z3-&(3aOIppb8`%)gOMM00- zr9Gmv`^Onj+o&aHAjlSN)qvWeIAo|+oUf(#nxmQrlT`#|X#mwP zQB+vp2_7-XHd8!p8!x$fk9=+oV-T+jr+UcB!oSF=!qEqfsd&}U1uQHt9KSNwfdsLpZ9~d04ywq zsP{yl6?JbLUBojrig-$xBnSkwzY@z(ed#Sj57oz!$ugw5*Ni8HDFIpDf0sk zD_u*Y(HzJqYg=SagKBd=S3}W^@=~I{B>vkgslaTG zAX~dJGw}E&I^FB@R4BuQfEmS5)((;wM0?vk0j)%TQB;HBAprw5IPfN5#YYDNCr9K< zj0^-(sBMUbfKslGW?AM&TpaX3W6K{(&AuTi#%^{&Fcs_Toird5Dw7ND3~$jk>?~e- zeC6FCnTC}^`^uODfuONT0`>-@DqDnU!gc)K0Dqt?RqyoIos^OtOHdMuaYfW5!xBz2 z^$!<@iOEUFv#1PR7)3%x+r{h;-d;LOMp77HcAvmZx}a9NA!01xQH?c2)JebYl^yE- zsHb}@w+u!_%rc=QA6#jT(eT2KEV zzr3UvBW4Aj49_%w7D~Cqv$GmhSb~DiSRamCSpQMm9E5U4HrZedqey1L1Fu&&h*Ffh zLC3_Cj&FMGf}ish=9FhOReZWJEG1fFaEWjFp>9Ol^*vhtT^@sEc+P(BA@IYSFIno5 z3RYHc^mF_0Jd$TIpL5ED@m(lpWfTdH|L+({1!*I1DdZPg6h@Nz2YWy)aUVT*8^zcwj=QW04InM5=F#Dtjnygs% zY3H4a2CqSrCE-?l0VEdzXwtJdUoS?7|A?HU(yX}+#nkMOk(_AR^nix_e>{sgQc#kZ}|T=Tsn2o3NN8DAPmq7 zMLf4{qmX(aVrQMMo~k zr`Y`j-u!YKJhT+c1_YQW#$qIZN8fekpbHT(#*=xY^{HsXqa*_p%@z71I+fy0cgeht zqcRlrj|(>hMnyL*zIGa5-=73>3J&2(jc?`m2PQ zx#!ZJ!*&^<5t-faJj?$74MO%4#Kb(AcVZ%o5?8{$b9!W}Febz+LkL%fehOO>O$i>o z`NgkbeE*nE7jb>0i2e+ND#Lh!qF`z{d z6IkQg=0n77$uzPLN;!bk?#P-J%BUO=&YJ{_A`ItS9D%6XJ=><@R3;u4Jl=eRC9ILNWpyU1>wp6d+t7Hk-TF& zEl9AwFH=9zf(+sQtQB2ygDCFPjtcr!UfzC+Ejm&GvJHFx7TW6KDeo*MulQbxKHFF- zOr_GGUye1!>P1}H7j^pd3vsebpz8(~E?sTrPxHd0bE%7@2x6>rye^@dPaCDL!M6dG zyZuVzOw&Yg(SRCex%PmFA7krT!tD*Fzu=6Z4b1}(+UR`)_UFZb7eje`h_L_0kq|E) z_+@?~DT<5>3tnV&)#!8&6WsvzSBtEsOu>^0{!mXjwJ_(gka6{z6I>>W0!PVU$}3cT zsTx%>M=LhF>P=i{Z7l5`0t~}e*~+$o545RjH0eU?}pcTFkn$+mFd&Z5x7YgAWMb_ve*3HuZMyb4i-Jc%-1M z+p>ur>_}%qaLop^np|vU^2(LN(DDNJu1CG~r+n)&sPWTffEPWCu8T=*UQk0upjSFqW z?ddk#MtlGeuaALd0cl^unpX0itOe<&kZY^h&8PbF>p3CHpq7Ciau6kXl~si!r(bjV zmX2(}f>MrTpSTK0Jz?nMj0Ge@@<;`5s9b2u)VlDn#)mwfeN?N+AJHP&USzc{Z%S*k zA|zJ?jcaygG%6BYxAGwNYr?)v%C#~R1#+rBWDCjrU$Zk=u$$d}r~ddIHK2A21aKk< zj_lJ%FxbTmUci%rfM%!aXmjaQ?GP)D&V8L`+sG53Mv4d{tVO5h{rQbVQQr!|al#B% zwlQp8+)4n|C(xNq+ned>@Wz<7p10HH(|0A66caseBQsChetx5H1hyTaP5QckkrNtM z!vwP3CN~`V@6F^0h^{e__-Q>xRU?4!nsnY|lt!W-5g9~4w&G-<;6X9dv(jhVtzNr8 z)N4wFN?n^!g64CPYS!?{u?k)?ZrSEAIm!#oUeN9{k@*f?e>0c3`^|D=1J}DXAC9 z2(l7HgdYh)nPF5ed|&+(FW_?OsC;cjBM#@JN?)vW+rWeRtIj`oHgF8{^4O>hNnQ zl;PjljU7U>bsEDAs&P?EA;A6p{umL$J|(BR%_>C@sA8j$YJM>CRWaIX;}AOA&j#(K}b%sgy|PKic$K?m7<0PrpQcT3`p*YQCYIO)~VL`adbusBKO**M8bO;_PupMQP8>AZ!0 zvsR5{|NhC~Ai4FNJvylH0m{|k1KHx>#pMS$_j74dJimVCEXBRS1X9hBC_KoMLOh~OXfy%EN7Ai z1-{K!g?x#{@D6wCJa~UciEYJ9tiU(b8m&CiwJ~Iw+4@!~`|8$)+sOhBme_I4i6tEDp;6HsOK47vGd-vq+RC}uxR@-lj7ds0t92M0m{ohwrJFAr9eh#2B_EvV zqgA3aj1zjj#sbqP%9QhoeSW~P=7aiSZcw|}lE-Fes25C|N!um*$9S|M|F}sC5Nm!E z^iTNUGB}RJCQ&1U;IJjue;SAq!8vs)TKO8)&)I)=09Eh?AjilM4m%p2bl8k0sHk(d zqj>{5q9ji8oNUAmBj^bY&t0-03CY>xd+{#xw#b~fT|)x1e>^@9wB1MN327v=P=82! zMLKjddO(KSj)t-N)=Lw6bhN7V+e`~aNqWwpZTH!V(2eFx2#%RfV}%tP5+#g)iZ)MO zmtuNk5kYW=Ziud6CALV%M^{0tePSGq*$JT498&8?K+l*noZAqg-5ka&e(jC3W2;2J z$Wf$bZSRT2mnr6gry8+@p6H%x#1v0zj;|7^^i1%bh}VF1Hv=)(uW*owIDz-f!#l7pOadhpf0^}%sdV|=5Xf)q>+X!+kFK1khT~w^*NlHVcoEsyv1Arb zXIYx#nMBvJ{^>I~mz$JBGl6VemlG|xCU(kvqF+L4B4Qjll0-^toWgNNnsQ?1cb%DQ z`|Q5x7}}ZXO*I4DKXLnEw_gJ}ZZg?9s;o`a2O%fTE0&)h)jAlC!NEl^6*O3b4M?PF z)Ol&d)z*Bw7?-A<&#^4uRhKGe!Tgv{@Zf_vaR7C_h=dhhFZ zqsCnZZl>MJG5Ahzp{bi~7puZ-lrwlG;8uWCWIU|?BCXhJY~nDKZ+ywSlz2L6qY%@C ze|FvHxY}wWgWoH;m2?xEfSNTPh-gc7!7l?YnUKSEB_LNhu+hSn9`ZHgq#aF7jffqf zK7udgUBcmtX5z6^`=ao4d@L{!`TA4)rRa!>FP!KR5CdcrM9i1HljqXek<5(&B|R{3 z(1p4r1v+e*rfhFoA<+HYe?Sifn(RrM=B|wrZu}I zwTx~&U#j_@s)JesEIM!6r_H~vUnZB!&{n5vAXs{~QhHGO3pkP&8-JoZrQjC}uIW+m zFF+oC*jlV}xk7Dhc(kA|mVZ*BB#Fla|E|9%g*$wPS_kj1YayApO?V4QXJ)?;1p0=6 zl2J>b-lzrkd;cz;b(di}*G02%7cZSw_tE#>+ts3$dwGIZ;dGuU>>ryI2#b~Zpe9D zS7U8ScpA|5h>3g&kVRCrUP$mfJVT1u!>t+Qx&Cifg%R_;=CnF?oYpDtatyx@up#wq zUf__-fq`ZS>%4X7CDHemxO4xOw+N2P`QMlyhmVByEw~|60xq*}_(sHbp;-LnwsR%T zD{kk0Px*)ZB>X##W{=JF8!T~sO}{@c%c~GA&Go4;@;qeYTeXzv&hy`+YDaeCBw)_X zz>Js{v%(PmxH;|Fy@K8G<)cC&s=~xrg}&^8qiWwsRi-sdg4?K>a+7s@}dl-R3v)@{$x2w9DdNY8CIzr);x^DPa3 z?mYgwDGQ03rt$U0&ttq2Nck)g2fY3V5342DAIMWtm>18iMKYW5cWF`j-wK_=l?x;L zCjNmZBzSulgC)C&Ne@0K*2B}!Q*YWm*Hp5E?YM5Jf2a4* zFK`5yUa0B>$9;6VItJ+re=(a1i}M|W+gZGYWuf+62Y*OyPeKH8IhLYK_l6|A*{id|6RnsIxwS*0=k^(#q~^3NOUeklFy5t`Vw_Z=Do+78-?G zJVAwXzfw$sPUT0{{kaZCDbdxX>t4R@5WdF{Txd|>GmiqCO;mIH)+W2)kJD{A~rIsKm!$FB9L(_7!(+h8x54 z)O!H#3)jouU?LSeG&m-aq)Zo)#EDjBGzD*3#;MDTTVbK8skFN_KizJ`9!+NnC9qm! zC*o_a+O2(kOm`OE4YvVLlNxy~P|8osHCo?&R-Kl2NEbiNdHbMNpP*8BI#IMea_%l^ z;AaBf!iVcW#VJP&2OO3XsMXWt-}bj8G_+pDL;JLt8}eOq><+Fwf8~@xyS|r_eo{rPX!nkSUsrtDx(%V5KC6Ug(MbY_pnfro`8p zAINPam@07$5hqQ>~0LN9)Y#bEHN7F8`{pYZB`szJa$4}`I3-b zlrKXo#j_h_w|A?rr>P$f^$!0u>JES^dW8n@$NGcmBTs+e;BrTej~;wOFnB`E1f14< zLv}||+_6`pcc-OR2Ep1|@k97SCpM(6g^Y@M!fzcxI==#IBcmR<4n3NBE@NW)XMs6G zgMyJSHlkF46f*w5Fn;UcD^N4a%=*0azE_P;mto-ff2FI)>18XsNJj-sP8*MBRkJ>voN&b zbbk4%$8mW{pBHE%gO{c5Ct~&M`Y}F+Zu)YoQ|;kGe$hK!)aoQ{6_&qgLSAKGwUKc* zY^r^hUEwb$oWpC}q5ScNn$xm7h5?!L{o9my5`IACVZG~_?)4wc>tWxEz)sj6&0?nn z#TRMyM8vbvAa5>mM>zC65n(?%*?9eReUc33Nj+b`&0oiYOrKx>WRBf$qF-Zzhb;w9 zs2IGcl(jD7qWgAIubM>R_#Ckrj?0+HfdVfTJqz#B${JvvM5SER6*ugD4MdV&R}R+? z)3UrWXo;DH?A6KFGXp{0HEeZVufIwg&n5YTbES6Y>hmz%sN$2J7kZVS5*c%<$L{9kz4tasG}+OyYbrC)FofX@S8b5XYz@9O5j`f8Lo9i!N=^gj zZ;9aHmRKe&Mzakn>0x>^L=6!OdP4^ojg(UNVH>Nf;WFt|vW4G6Zl8B~f zX!OS&SU@NGPDf$-{ z<{Dz^bpC*`2!@@6>+C8qd^36!|8YIUgi$RvY8%db%$kXm@H*G^1}pruPir8gR68W9QYi29Lcg^4@7j6K>b#Dn7CA7uTn_#<8Xy`@?c zp!e=>dg?2V@6JI+>kmG=NZ-y2ck%6KnOJHvO*de<4`DTM#;SQy+n=?2!wqvkr+v%j zr%xNfYh%f(Vy^d2Wl!ipE0+zff4Hg{Y>OdzrRPcL-H(|j?0<)^fT|6Fr(mCCB-|<& z{)+H^<(LmdA0P=G(S0_X-w20gdzJSih$|gmoJmn;A6`nyeGSN_MjLGCy%!A0!9f`& zK<}TTN@<->LGX42)&2V4ZGhsZXPn1$pnj;f`8I&!h)cWqv+*`aJQFEa`@#p|vaR7V z)f4D{cx6&OpFb0B9@vV~%QiZmSf+_J#vNMG*fRbNsZrJ2Mo7eCF{(-N{qNTENeHSk zVI-utaPc)#m4?XIMQ?=UNLPSrz5rm5@!B3$jfByoam>k06=$P0C zUz$GSF=(c7)OJ0rbtFZGpmK^S7Ij?erQ8sWP%EpA;-yz?VjQ_d%`+wUQW zqxR@=6|=x#uDg2eYl!Ojx}cd4sV&Q@pCzB)5Q8giulolYXgz`>=eJ5t*Ddvpzw1ac zY4bwi4{G#uI+wAtW2a6Mn45w>J&}qlywMK2bB2lK;%2ipvXud;JUb=eJTX$5m|ifVo&jo;xpv zc6|}r?YHTSNW%eWkqmvpTTRbuT_WykuY6ji3SrXvB{JoaSI@aW9lOn`L!8R|h{vQL zBPJRI6GB?QQhvg^nfm&o1;hh?Mnp<>FXbN{bP3)@tJt|x})A0ujVQ+aZi&k z#P;+I<`0Y3SQMY7BOY-M{EQy0$5KZ1s?VSQQATGTLJK-Wfo)>0(F02V^|NstvwCc2 zcFHyutXsOV(XN)I?TJjY^4_m*-Tk|^VB7un{CNGRI!!y#;V9C&F7gi5TzIa5=$n7p z3Hk$kXIcgAy64w>_1B0V#LzL?wZs?y|BtQr41{a_!oHOt2ofb~l;}hdy^iQ5j4pZ* zAxiWb3{j((DA7iVh#oC!jNV(+5Yc-#h?y~Fo}F|4=Xu@_?=eEcIT|Ds)p}-i6mAo!{JBYbd6$pv}`5$1=!9f^-CA7f> zaMBq%^XU4HiBe?_8Ut4PWgQnemyRfi?fReEbE6d1MdVt47e?N6hPSoI)56~o}@J`HW zSX#fF0P?r^{K)8T|JDgojCjW#IXMu_sPNZIoNapG>-rKoWq&`4XYpeei~6$HiQJU2 zK=0-VWE^DOfm+vUj8|e9O(?DW)6wClyz->>VEOF{CzvdNfs`5vzcj(70X}FGe3$x4 zSIuA`*`ef$?DJGnm@w+?Q48&(ZP}`2Yv38##i|{EA0WRSE%M&jeJ#5X%u0WGL)Cii zAIBj5?3lqnb576-n=$DHL99iKO*rHS82@!F%(?}fz)rek9WwgeX7^c5akZrP5Lh4) zacEh$xKQRo_sg6DnOfckOs=>33ph8#(}H8YAokCdcF&3=R-)m??!(cytlgdM>Qn+! z+9FBu0vZlT9~df?+G2&l4+eewbO1D2yi7kGkq#mMPBfF&lGYg^^-k+A=bvRkmu_vt zBGA7KEd$S42)~kbLU@az_kF!OS?s&8{bYsg6o=BDn>=Ew^)H8P>D)dlv95fq!kf+J z?R3jZ6b8A+7RWRjaJTF;dWXcX>lX5Ga#9rQR{Ek>VxxQ9aSj$D8X zTzM*&Sqgr01^VKXIq*9cea#Y|!rv@B|HFh5r9cBj-&nt|3%z~0D|kFh^-&tz72f!a zCH`q(R@SIOw;}3UEc&thbWg&~Ce@zLSzmKFa;S|+v(4A_vV+}wGoYJaLsjs$Kb?3` z)*Xf4v-=eJ!8hjLaFSU>)ep`u>SfgrQ)qSso(vTV%kmE-53(S9XU4wy}d({*VDL>l!hVUcy^;>cS6;qQd>`fLCutk?`#s0>Nh|T&ih9YCjCsOzOO_Sy_yYnX*ibB^+Ph>H)=R#5ZF^Z zIR~6v3tkg)V#8@)DX_2_SmNQJNL}^dianzl`%~7UK9ZZi(YH8|=Ix>MKZ3B%E=_j} zxp^ZK<>G8c*Xj9c14bg5s;;4W*O#2+d9gJp?AZW|C$I^dol{GhdgsI+PXKViw3^-A z{=l-XDY+}*x~$Ax$_!$fW{JLdrcGN4F)KAV8o7mrDg?q(tBsw3;k}wjQtQJcriTv# z5Wiw&jRFwI@j<4dOLWOs{BS^_OLmx~0h|JiPNqa}rXIjA42Z=*5!3eJ^Ap`uNq(Qr zHjFnDVhQQF{R!|5b`KaM--KPYBp2P+mRr9}nz=)zS1eO}Pi~!S&4BVE&3iNL3-ZYa z)*I!y{O6V5mih0M{XBpUE>e+RbP*4`c5`M?Bf!haq+0)-@lo34;=fGUBua= zwZDp=u6Vp~&EN<+Oq}-_vF=U`LPD`cQG)%bYgvg%27DLGXy;FVymD)7sk5&b+<7FHRTBZ72LUVSY+#`3lUv%?m#F z4+qn@&+rSIr~Owgb*sYS=Am31rjMoH!QY;=nIIt?qpr*Af2cOA&8h@7RuZxIZ8*{d z8=dE$q8)$|}R;=imggM_^Xa0`+8JgRkg}d=a%sSKb z_2fikBKjRtx5@)vYEKY6=c=@)WUD8#QRFecX?^|)w(TrtZF2Kb}BGXQ}}HWl7Ly-4z)=Hijarpg_xKuX9)e}bW^e=|>`-jlBk*0{}v zVX~2K09L@;>>FG9UMO<^v?nRq+zmEHoHbY~*{~Zs@;Ed7YuFn_HD89#eN2>j3 zmJpsyc4i9tqSxxvDbayLlWU*3nWME8?eh(nIQBX)E+Bl&k%Wrx*Fef7l_YFSC)cvETqD7?GqEa`pit zCYjc63it0{w|WR0W8gK$-q)?7SDs~PE`_A*(!wl8|UtoI5UTX`; z7)pV?sB*}0ZR(Q`6R!h;sGYO8IwQH;JnQ*Knl6paN2GRgBLz9$4w<`s=Pb4k{Gt0< z^^aIz!~p^fo%zEpA&ceZ?r^0n+f%bP-Udr3t8a@vX2P<~@p8SYQ+YD*} z%*UT*!>WpQq;#tx$n;EmLIszj#m16*%ar8u&EbY$I(=wbgQTbm`~Vx9S+P2piS0+Z zsQT`3NgAO!Z0LizTbog0=$;y-d0ra>qdx52LO;EJi9XHi+4A(6a!Z4r?eVz6;Aj< zCI3ugU>;g7MzNO6!zEN9b_*U8iS;>{n-#n(=3KBWU-(;ycBqeZmgLKQSNutEsWkk2 zd-XBCHCSiDU`gtRROxBiMSxmIho7p}r?!SoWjQe`&Wf?3wrFQ{I-hV_CG^ zGwntpQ=}9)a7>B>qZXtw|4@_F`ZkJ#u;^1w1w^F~9BDFBr4Hza|4O!i#^))AnQHgx z)4T=GtvM~GG?(9j)#XF*gNLKvzlSSs^$96*fS6CIsW7i|wfUnup))Bb`5QviK@ZQ< zgl8G&Jqx%Tx-{kXi^XC3V?8euGLCE@`F`E1{7gq6C5HQ%QShC^_l zC1udu)Kx^rHwPz5DNJ<5!Fu&liAXJRQ>0O#{i(UEE6fieC5IqCej6G+m0DkEqaEaSU|6 zx;6$p<&wl#1md0gWcFyk4P0hi{*MF}Oflz(MGt6Vn%3j*%9zULW>4(+XB*Y6g4!ex4RS`Lj{$NpBeLdoy3H7x<<+h>?%auV-7vb$(#)uRi9v znWDB&xb#<5;bQ#=$rpW->*_Nwoy?v8frzN^_)XX1^oqmD_pPDQC>|_JA8$>#jU0D6 z_R%e)Sl=!ZxuYeW>)`Cs?h!mcAqx5@w@RDWo$;A`HpkE}KQGdk=s>C#8#mj|R)&y2 zV@7Ek3{lT$cVOZgn%5#aOTr=Tm@}aYlcF2??Hu!@58yITPn;r%A}23lVwCNeCuc@K z{&xE98=u+#7!@dV1)vv5H^8=-q$EtPcC0i;l}OPQ?|h|o5tIlU%$dr7DZpNyQq+n* z2nzxrC2A*oMJldht)0go0_7IcU+@v@cR>~O2%@f@du++i$@%_L^6igp>L9~;=4G(U zGF`e+HS4Kc)8d#;Lge!Zd`tHCj74q;JPt>^WNKLk zzEdi`Uv5%nZ1fo`tYlpIJU>JJ+;;k`sd*o!)+4t$Ngn-h<<3XqylYx3rYhOZ{m>ea zM0*EN90MPlE>-VvFro~|8A^uL7AhI_ON@h%?X*SAD7(%IOdWVu=hdM6aOOcb$zblZ zXEA}F(_naeZNA&IWkvI#$$4S6{|45$L?q!LZwznOS*Edbvo2xEOo(az4lE%Da;L)m zv4g#YBSwh<$jy_-5}<}rWSbz9&4pL&)^sK1TcVGYW<7o##(vBFm!wklsXDw%s>HKS z{&v3c=?VHK?rv2$ya4Jzv2fX~Er3L))6vnX+lgI4nt69M+pBvCbV!6iDVJUEv z8KA|U8wOAi8%DR-Cl~@G|004B1^LyRu+JV{&vcTC zMnR9u#?I^~vG~FB!}g+BHy&(#;aV3Z#r#?Lm%Z04sI>ItOZt5u4Tf;TUa>K4)>(6GbeGP9_HnAP8kN6`QZ#y+-!BLk3N_y3Y$Q(q%f zsH}0abckMMa(XqGmqc(!@E1g%s7{v7(q}hwY!wQ5NZrj*kcbEod6N-2%aEb%`g?j= zMx@w|roqJ+K2-^+GHcvyUH-nc`sb*TXYX-6CZ99Y%9nT1y^f~k4x*;@s-MqQ?HfGm z6iW2Y)$tnO$y|LmpEmJajLE(_0|DgeV5zZBR%K|aRDv#z1>Vf41aR8OA^!NJMV*KZBmnae1i0OZQeiLfbV{xiZux} zR<9Uy6kC(j7!9AtHIHl(vUUF@H>wb4#}*$agj!1~cud6{WJs~uLrdr|n)I=gu~6j` z*z!!|ZEo2f2>G%yo>AgW`0jj}fm-0^MNA3SP|GP(`S)-+2DsD=F6}?*gl)%s=0;^j z=EypV>4Qy?W%9H2DwK8JsphS z(81LnHTP)McX%nS^~uBnFL)zHc#4h^0ky(hDDeEpsnL}0)(OK)t*z#Gv8=@r|25|= zXRHjtE*7D6hN1#*2GIa`Rau@1M`)d=Egfdl{rn4@vddwFR>MBlM4U!XfX1<^36#dp zhK+<3XRjLjs|LZjlDw&w!D;T{DcX~SJf^G4SxQ3p{5>bsX}`ZeWyU&^ki?!oPv7B- z7?J#AC%U>;-z(_0_vq2&m78Q3)=K;d2Q5*QK<7~PcJApa+dsS-CnqGaHO(-Xt!irbdc@h&bdY}bIw@L#iK&a#^ymZ8N)s_R| z(Zx>yKx?DP+35CfIHMPT z8TG!5kjj1FrqkYT8>*^$r{65-j(2NHtTJ6eUf(@?^5Z$bJb)sBSi&=3hNQwbEdB+q z|3T9ukbX*MYTm5Wq1v4+xR6Rx2;%Bqk__5SV?F7x|Lwa+;%$jcWxksM8B~_4JL8tM zJc%>`<#W9y%*CuQE)Nl>?$b0|SFJ%acd-XDt?=aK&L%?Bn|2~KgY#O2?^Dwc-Mcx1 z|K-=>zlUKDl3^CPhpq0*itRz=$tv)YDI-O=+QV#6jSlATlc$bo54wdE$@t>96jumot6KtxlD%z_Vf zXiztrW7jV%QC7GSFW`c}Ak0SaG_U9PbxB|#zB`r7!VYHy|6PYr@W9FifV0u@jbtRg z4tf$8R$Ut@yl3&EfTi}~DNALj|K@g8%GQXxzMB~`jyRn-b(pe5YvSRryGR3@E6-SK zv5HoCB*g`lU5p#f!S66Eqa<)-DLDXm_UCRP%!f@x+pO_yf!CIy{EnjyS>?9;sNd&Ue&@K z5#xDFq8DxF1>9l@_%PY(e8-g+IIGl9G16Y#8+6d)0rq!<*Fkh$(jQ83D0W1JkAs;o zC~x)10R_nU4z}E=;>U3^`2)ww8zb8}Cx|-HhOVMfNgO;n>+agSEs7s`T7yuSos}?f zEnS11khex*j<=9n^pLL9NvxqzHzHiva0d&Qz!+ESORch881#7LZb+^yq=ShwpI|*Y zd#pPliPkUFG*-a`@YM#Bs-cKk20UlW6z;wMOTSsv+3fnyuB%A=ahYV)9*DmiCWwQW zi)MVsp##u(s1`%@LP9tWoK1ze*Naa@?=CiC1Um?w3Lq<>>lAm z#MqRdT_-s%3K!~7{qWHH?`5j8@viP+z^Vt>294|X_i|XS0~2UF{$A6))6A;-GP18h zxyrdDkKPmuO62fhHeZ>{4gN}*=#3KTThYj|B(5_x?cbCO@pm?nm4aN?PV$h;N@XPL zJx=-j^0CER3FAe%;?tz|nZA(BnyKF40=GMwU#En5_68J-C2y1d!;iC|$gedv|7S>v zKxXzRcCs*$#?T{ZPv<{JgT6Z@><#+$IJMV*efd9XO0Ri$V{iM!pvBc z{PVkC({TVRDHSkb9cK^UUoR>S+XN0s4RF7uAwipr_hwOT zQqGv_3MpZ+d zWz`nj@wmwMAC|vN5|a8L@#_*|sNH?NuXEG>%3FND=)9rehLborU-cG@qU_&Wk=Vx!W@@&Z5f z{y-`R@JMlmf$9V5_;amOc0vqnoht|ZRQx6e!gQ%FFN~J0iv)p!ym094{o`D=f2#5HDP55W&f= zfk~E^p2xZ3gT!cl>@T*=L+kyad&i#0HnC*pav_#+KSFeui0}3e=twrcRBeC=^)K7L z5wq5xDTHxJ-4G1ZQ>k>n{|d7vYb&UQi>(yHID@{@62f^`L5#FhYZU7DR#nzcYs{*` zCKKqS^aFDj!ZmDKJ_9DXqs2%wQ5<-hHs6rX2~Wwbd3QbBB1V>>8$~$1(bL2tW@rZS z>Q52>{fcg$OyXi3u@rfNr*9)uraAt&Fh5qQWWCSXN{BLN!g_gm*>}Ao&@ukMC^@Pn zGL2-1T>hNs$_9hWE(<=@x*v&@Acnl0{cAq5nM!)R$}%5aNvr2csgO(|OnYilwkk*f zcf2jP&}i;A)M=v|7~}Y|@zq1|Hrv(nxXm|`4Ze0zB0`7Kd3OoAyFp&-t001PgjgoJ zcXyHdZso@78B9KjtVJpDMS@zs2#OH_HO9;?-8bECtlBsg$J2@DZBEz3?AuNMW>TwGXjp+Mdq z1GY0a-{St{u{&Sp?#!oIir@~5r0D?VS9>AjfX}R;kmtqxto*mc$K~}n@e1A)g?+^| z@1dC*{Zh+9th2)M)UqOlt906adKODx^}|Bm`MpqE1^s+V|9;5&%`u@rWBz=gJN>iLp(|DO#N5jdl$|8=N2gH-YP%Ntbb4e zC=B%Gt|REq%jDpPw=dGQ`_nohgX&%ICTaaDQ*+D(LEW7#!O7^`M3~G41ns8aUmh(> z>%o0tp}G*X9)7u4dahH#akSY0n(8zOPvZNoqE07-GZFHS`ej%(G&H0aNS~}%&z?l= zH$@G5nB*JA-mc1`n@KTQRgvGj()w^8Pd|hXx<~o=KSSjEyWiEe9|<&4tQ!F00aOQ* z>b=Rv&1J_0^3_KCd{EQJ0eN4^{?7R==U=5q0V+^}KFw?AxdC-8c0vZi8ckDAYh|p$ z(LfP<1HM+uKI0}W7n@1BdU|Mv3%P|(A%`^ENT4+00eR`~N4{A?73>isM%OJ~UyeOT1&>?p@^S|zL_TA5#Hio_35%UfR?fo=(dzYp}; zqWppL7>n8xq?Lw~ZYwo%aY3lX`y!j)znmJ%%v4Hmp+=q`U_K~mwS=Jfs(9fn{}|$m zUn&0O`)W;pC$(XkWRqZ@*(r=7Lm`ssDvwt|r!)yt-V>*&|V7u zVHz)uyKM_z03{UfL4bI!MD4)+hBuC{W@rBJJj`P0O8osxF!)G2d~bKx%^YKk;UZ&T zo(S6)=g}$vcbpw*mUNv-r&AiE>gQ>jDCM6iCO_;+clD*RH8@S>3`6sx5MPIDj))@DsYq%5|IDC}NHqxR7fV-C;q4%C(9rqJ4B3hPr9Zh=*U2nbpnVJBiz{=<#(VNcQw)o1V zN9a_e)J%z3hi61LtMN|i0A~=1IF_BLdTRPCTO+yp1t%QGUoGaH-{&|fMA>*spl~mI zt+?;6k>@7*H4SBO)3bD#nDcGGm-6IaK$Cc*Cdb}I;r2N=3nD=WnfdepNs^9->8fNH zaES;kJ9q1ZM44+*L0&2~MukTIRc$Jm300O3!(>f1*_+4i_p+`EO!m5eE9A3i`KH17 z7RJXyu$y2^=J$T$0U!109D5_XTfc~hO-NYBfpi!)G8NV?_LUv9^5AwlN9YGmr#;no zuk?*$|DVlr!mUK^uSo5<0rqA!>;&Or@2%8rP=5a0T-OatVC+G2FomJZO7r>RXazD% z>m6_h{i2-6NJi2uO(p*`TT*{L&#yR*>1h%%PR=G|9uz5A5&H6l*iKSF!c@{A<&GU& ziWh3L{gH!wFRR@7U%|mLS~DF~J@}RtJhUK$^m<}S4LG!ZKL^V5#!@28L2~oczwVSA z4RI$7@r~dGsOL!pM+f$Y6B$K>{s+px-tMhC5KonS#M|9do4S$5ZM!2Gnt~>qQkgn) z_=PUUmx|iDN3ZCk2kHMhD+~Vros|^SBIv)?N> zFJYLBKCH$5prAKD{BBH91586?Q_YR?$t%iN6J<7iypXUC-`{^rqq`B4STqwbJs4Kp zOM;o2{ya3RO;w`f1g|{J4229kqJPaGpuZ)}ulvrGqBQ~Q%-2Cucd})B@2*!S)q&uT zqci~~0#DKlZ>`Z;1d#Bb2r#h?dtbgo`jJaz*u+aatS8l-QXQuB)8AP4s@L}_0{89jmhd!Kj+Qc%T z1TE(*{_tVlH-5@f&@?av)PPgK5Lmvg@&xH><@P=MZyHwalsb$jXekm5`u z!?f8QZ=XnRiZElSC!bsGSKvblgt>}up$Mts9{`hA*pgVqSfXr}elj{Cz%72J(kRYF zM5nX^)30B&%SLUGaA9 zk}3X_pbS#*&dsZ_?4bmS-#pqmqO;OSQlsqSHmp?ukVbf^($0#Zox%Do*$@Q)v4`y6 z{LjCM^Cc^ZOY2Up+?+tR_W5hN^E}1_yZe(sGDpp6cj(+R4}oP}i)- zTa#4}%ABXoePjc3&S3E{HZp61tDx%J`|Y*z=n4g9uaQZco)W7Yl(d{X^U&^{S%x|< zz@w2~NS^EUP*d^=BN&Z~YBEwMwP*Z!yrs+p)e)AR1_q1r{|pv8R9I_8UiCg(9UG;i zzsg~|32d85`}L-ctX11*>pge$;Q54ny~-@C`4^PCjZ*h>8c6f5fR%QhtV+3Y8A-~M z81+4og(viJ*CA)fovFPU9Y#(F{_(zGY3_O``Um-7=`hGXH?QW_*2=uK22DyxT!z3_ zivu*BZ`FDR+qk(|*utiip9+|ldhvP88bx0OUtBFXSO*-PqZn^O;3`fKH-u_>Es*Hjj04WdBy71?cv5n7@o@|IWszoDez~ZMu{B* zziY>6j)b+2{N76>G84L26(Akw%)t8^{Y!b#nt$DSBfI451{Q^Y@eV@1z;ehqxL%=eH3@iZok~lfnUyJ?Tj^ah`bU0 zZsL!)-JESy?V|`!=`%E;%t+Q@iUZ#X(Pd6bA&5W@_h!-QyqozDtFIbY zo3R0!dFE_|$?Lrjr0@5?K;f@M%0UN~xkoibih64KK}ySeivTaotKZ^eM8+cHTRAnK zsm4HP+#_Iz$nyK1LP&SS!x3w}U1t@&M<=m- z*x8EtwN4bQ5@2Pd8+El13jq9;$Lswyme zhwRNxU|5WKq7;hmIg+DNdC1h;niCDr81EKOLectN?`$hJ5^}d?dyV!ewcl{#51Qa7 zLnYkuMPJV~Ahss-t`g=Uoj6jL9}H%i*o`+Kp-yCMHt&ZZB*F5)LVMfBwF|-To0KgV z`uH^?nSyEI6nm^94(PG~FDQx%d?$tc0lD5j4D%<4$@7%$C+$~ND5(bgdZq2_DeA+@BR2PcA(pSpS@J*Y+a&@Rzbx$fdor9m2|Y!5-edOUo|B&xe%@ zF42E>DR_)2y~CC^Ll5#dzl+7cq0GX@Bb$M|j({aPq*k5TMoVio#^J%zz|5TbXX}tt zwf7e=U5JD!^$a$)>E@qtx&C&T?E7>297}+Ve3}xmTL*mHmll3MLx{9Wg_Sb|%%f&2M*0RC>BpwNj*Yd<5s_8%WiUXNPp zQVU=*j`YnziE1y$QQNxmy9Q0)4(KgQ?t6a&d=F$uOxww~FMt18UVmNTYy=^??{C`= z&XlFG!HjB)WUaGBU2ThsMxG=msAF;`dm*`KqiPEyU~I>ez#tx>VI@sUqO?T(`2kPp#UkwukP0ZkM$Zc2RwKE?5*Gx7~l3YLF_2=H{1hJ>Z=6(X$`syNUbJ36rG!(+U9Q9ukDWDBv} zeG6SidQ06jOy`Nlj^@dzb={eMAs{cl=W*K`P$!Vh4xe?DSo%ROwkkqhx_rT8FBh}w z6TIT9RG$K#lCaz)$+D`X0qtBY;J0rK@qOQfP10$6C2zc$BktO-xdAEx?5S=)iloFr zp z0(R71|N537dwuAYoh1#3T#fx#G}~T%G$z9;w3eUN{QkhG_~gSW)7oh?kB}h%TTOz4 z!5j#_4p1QNCElP|FrY@pox>!5tw=x6G#bQA{GqsDudbQ}*lDbaRJt8TjYzTZTj5PO zI85N6<@UrkYw{8_%^3JeBH2H&u6DP&xGi{u| zIuE9s^l@_ixU`nY77k`TPH^z3X{9>N%Z{$L#;8I(=zQyZ0(Q4t`FUv+g|yh7NZWk@ za~@sOD2kgs-szB!Tq{9W38Q%; zCa=#GQPm+R-4=b$1v2QgUk~5Igx!hAj+2)z?*JmVtj>EcHm0KnTa8^ba{TcQbpQo8 zIG6fa zq_$@>a-MM_vwc^c^=0-j5lpF$pwR@8iQiSgtUQZk6zdbe`o2i&{+%WwEKV?r+A1F?o zY94^PzY98tVM|Itgc;7hzb~N-HNq}8kIfTlsiaeVdHo&6hkLHQ(Mi+bBSp?3f0ouG~%~f14W!4|q#%*^a%3MXY7m z#Bc1d1PiecN|g#loUg(p!C>_3pW~Mw3^Y5@oGU}pA1nGF0Hn^uS0hUReqZUaW-%p3%e!&_D z0&O|t1Fv++n!Gs@u|9s)KWz6-4FmV9}q4C1#JH4s)L;0tdc2sA?-bXRW)hM zI(;oF6c#HhKVJ&IQV7^9$+YWWAb542J*bU{7*BZZtw-BUHb(pB=xu#FZfFVaZHF78 zfNcMcU*u=w29&=1FWChHr3A z(WT=8o|%ZDt{gNmdVoZdY?5k0SF5#i#J?y zJCh%}){#Zt{>!q2Psd*KLvT8M>&?}QwUz%Gf&KYIm&xycvJWolzB72>g>nR-o+O|(W*E>i*#MGH|xt$iOAQ(nO1dzlt3WAwbc1&eR4lym85Zw zLWvi`L}j+#muDQ^Y}!GLDwBfBTfYKG%5fQN&?zFP;lf}5>U{(?!vU`(fn0xm>xo6z zGq?gT(3%#8efeQfKXv3&H958Ps@r(+J zO|UY0IGar#v+>}eG|vflQdOb!=~^@ktDAYBEK8b@&(?&@6_eCXJJvDN|8jP|AZL-t zKa?ML6fY9z=FRUwb-~b}RuiiLF}#)}S=8;&ATJXJvP4Ts0bwfiIwVPC!tlG7>I;}N zp;4?_t^Suznln0s2gMstpCCUNzt_)-)UMmf z_sa(k0&m2N@8sZfL78nmJXb)-!2W?;6;B=@0$O1%C%8|*|5#F;gc_)O zu`}aR&X;!|HQiB=_R4-9c-3Jwu7b6-DEIR|nbQuJh{DoZx!<#*tCUZGfXDYdA||+dgs05@qRI9L-wym|NWDJ6@M(nh24{qqq`c< zD{+C~A_i>SBz8Clt=ITjF;bI`44u`0dqYw7D36RyWZD*LBq!Wnn82X8%niM<bOB37d*SvScCJ`(my|h ztcMKlTKAKPo^SOA(HT>==%rBaBDCJlE__WHiuRh2q5b@b`0kULO-nDFfRhf?KEKfg zmL_L|!PfRWlXRuew~FwrcN1qD?gbN(gZK|fm?=PLA>$h}r+kZ#!9X*Ksh{ykt?9#@$ZUNHEX;8$vJGqMiRhGHg_@W`i5{K z1dZ{D?^4JXlrX(rBGv_$mz@o1b=+7Jv=EYW&no2YQBJe({*QXyR62R)!n< zlh_ahyQMTn+ATPy3Di^E{3D@6co$%muFRfdCso43RAE^Ld^m@?y@1Fr`7a zkFx4{2h&Y#1wo`UmKYQy_^b2bo!_=WPg{5L!xS$+@O|-$%eWo^_A!^|bkmRDkO@#o z;=BkfalVZ*O9Kk+$dZ$sIsrrHeB9)n^V79A_592~F^x`hb2TqM^-Y^oNKW=s%1@gz zb74(hz-~#L)*Zd+GrbpXQ$F36nv69>g`j^=qC}wYHYc2R58lTZ+i>a8)&SF5`Jxp9 z3~e=bK&4D}$of7oseR`(hU$l8v{dfE6rgq{3u{Srs}MVfkSTnmx*C9@GZlheUclCt`|GWbImr1s0&_?d?W>4QX29Q6m>lDQDo~Ugau!EPCcCE!7bhIp!MyFH>iN;30Ad~ zRISA;OfS%SU%Ds9qR5dc1!JuD5B|u!$;_54bDiCv_4Hclc#`w2{G;Gw8JDK-?c|qq z)vZ-w)jrpMjtb{nEe<5;tZ}o$xqg>!$fw7>wS4Fd_x#~eC`fMUusC;BEjM?=(zC8ur`hsZQO$Q9gk?~e zE&!Rl*;nl{7n4d`YFSpCH!>djj!QrW zmTcMC#U%FajSn~ERB@^1>&^z%&u`en^NCHZ=Rs@zv&z_!55V3Ya|{raTKU^On8ZnJ z@r4Wb*sNx9TAH0iGqd|@7=Q)Mo+W9>CaZCFrGmxut^YKus3P- z@iK<4n>F*-Y#F})hBQ!dO+;f+vy$a>BMh!Z9~LNLrg>uype7My1Ei$Z7D3kz zwiM$tylI6Z=JbO8BQ`Bx^ujLOz8@LyEtXW=-_vi8i0?P4D}t{~&%;|W&_t}J3;V%I znzNP0&B`VSm;}fJky*n5ZMQJaSUxS}&PSaTW^-ZZK2UU?8$@I6lR&b1YY%v%f13u_(;4-l`GT={}2xAO90`Mz_wNVbn7YZzs4 zM3n?$qo82ed}WEN4i0syeohrsI<<4}xDn^4lvdNN-C+WEuw=jL(~q&(r(hGEg7I@! zLmG0IfCAMcV0MI#HJ~@mRBt3a6ATwPqIg}Gh!^L8S;FxlWRC6OGmyXr%%rityL7m< z?E~z2ckD<~)OI+vv&b9K6^SZ03okXXYBcic--(+7LSf>;fgj67fN!689x;R-Eh&ZlI>x3}eSM#s zGt;t{Gd?{}^inhDkk(q!=hGc}|9MMKEItK~z;?49d?2<3)zmkl*6lRr-m<@6k>1X& zOqJ7iLY~gheDl?@AP%<~T!I~)2JhM8Z2XN!c@l#5U+ylcftA*HKUz1toVR`WE^#RF z%mY)(2-!fIehce3fpjdSLl2T6TRC>#9BE-#zTuoU^ej$d0aD{g?NA!?S@IW@KcJH( zC3nGc5d(w+iF7>Qbfek^(K7O8IH`4n;OAJarnN87SlB+0($EVT5`FVb+o-Dcgx&qfQstCV<(`Z&??+}0_V};M{P^5duCCH zjieQJl*Ny$axrc^S#Y?tBHztU;r3*flazba3x~G^E?(Rg^4@t(24W1=MJ6l0<_m#2 zuC;Zo0;8{e-Q$H$$I|FcnZap3GV7;3M)prhUq;j_CQ+HSLs=qaaLL>MJ`HqGwEQkdauESIk%&^4#sw9Gent z5Urr7s7JZ|qtFoBdj2u6_)`mhkr7MXn*f|eq7#^)5B7{=gwf+Q+fL0#NV$%@geS{P ztYB!y{Mz#rD8M_~F+N^g#Smua)2{qsFwMG_H|!$w@~NnT|=;AxN(RsxxKLGW* zWGu~aLiS*k1C@{cG@J`PFVXO$p{xjgOa!{hN4T1RJma0Y%$^truct-gS2NpN;#HfQOYJ zMONTG0Jv%0@#1ej0{LjWLcLtPUy$=W2&n`n0!7P>iLIDJ<0Ci>GBqaLG*p%>v>$X80wEth z^2zEEkq5C9@^RC*)}~7z$s9GLp;;ce3SClsy-*MflMDvfg4ygpN8}lOTC{e?&pu;& zsla?$YXe(36|5XUWyU z*S>X*!mf^Xe6Qqc_d`6uc*r*l^P%X>zzIaXC`t&3f;2b+5{l9(F)9K|ib%(Rlypf73`ip(ARQy! z-8po3BQbO%4Ku^Ux8HNl@B01$uFdRcKWnXf-J$1j8;e|%B;UC;LfjPtGak3>W!HvLwb2(EdCjXhntJ^3I)uSBT<9?gCcjn3` zaKNnV3C|G?_f*bvl*NoSLG$vR5(G@YZiwwUcM0LY4<@MOwH$KIjyL8hy zDN=W;VYk?zrElLq7KP4^8%5v$Y+$i6t6{|+7`Le1%z zAcg`FP=9B6=WA&^(p4 z@zpvuq%+MwPL;0_`DtYRYSAH%PQEqi9e!dyT2&~9|^NwJvg=kL$o(f-=Q%IC*TLfq-(RK{gktb>GBo%cxJ#VA1VS~IJWrA2I3P_ zSm)RgK1lWKmYI`6iVCUFrA?$0W9 z<#rTdIP2Zo^WQ2idwYOC!(sne5b8tP-H%CVh$@*->xt%_+6{{%rA(tOmV+_ExVEms z7EPy5dJpS|r zOX(AetR|+kv4kCg2-GU!8Uw|*{En(&K<7@R_hPweuik3vZ|iutH;^T<)lIEBE%kzB zQiv=9>~phJ!godP&B${uw$kszPOA(@O2fk0@E?jq#1}4$crx`icB}c)CifmO-?K>g z0Z6@zzpE$k8Y-SW^vOt^U*A!7g zq53Myq68aP$n|%rC+ZUsTJ8Y46^Z}P6H!h1E(`QOPOJ6`8m?WsmZbrxJulLj+AJDr zJa41IxLvZ=O7tymheFo@72NHw%wN>Se+bqwUAOE6(HZ@z%VB6Ck6X?p;4F45pn^5J zhKDevvmA^MSVQYGuoom+?i%B5)bW6DW!qD|<` z_uHO&rKf+qABx6Lo&{3%MiD{h z-fJku=jv@)g_R*l7A`DB8DO27C&3K-0$Q%@CO>bD*F)c2;>Vz)i8f`edW$e(0v6(U zUIY$vs3`4`LZf-*UjT{W!r}J)gzM$Yml#mmOE@Z_bsG%}<%|?O)@uQ8Mo4*dSjUO;fRcDIt12TWH86BUOB> z81+vn-fOk9tLCxyD(fMWfbPg%20O@`Uw&_*isFX+08W3y_*Fglv#zBY?0P#X!^RrD zU!#S!^6WqC-{AoPP7tF*fB;d~i+g2urtTA-nBQWlC-Iy3B*HFwc?-arC$KLsObs)> zuZ)JT3X!$&qG?046QtCw)!<*T;Eeycv5*Q3G3uPT0+$%nsy-cqXu$PVa<3%XEiEo> zBVMvVf6HN+EKTJeXYKn-Un|s zT~iA}5C%-cbY9kUJMK>$V$PVWRShOX5)i`ZY^msHT%R&>BWVwpy+ybO2XFiAUK?ey zJW++YZEh5a{3Lk)X%GBihVe0XfVW2)=3deRDg5nuvz-ssfAFuAC05=~MG0t=CUBFG z7r4DaryJkgBtm>Yp_|y?AZ4Sh-8{YOr$hu5Y?M5rdF*_D2=!4qK|*={et8n~souw( zvTahYNPrzaSeeb~JkIUOBdyID^M2wAIDAA@zZgw)q>ev3AM2mwG`UY!qwDK)%w*P= z?jGwo|78An*4O&EI*-ci^C{#IxhJdb52ZD@CnrP~n%$Gh*@%_+6XScEfURsh>|M?f z8Fa|cm)Ec5aazrNw1zP{jyQBAfV3-5C6!ZqNji5H%#^-0BDNwZ$~LOwt;u*Z&?in` zn!+5BxX@~%E~i_vcYYak*$4CeBtuSb%+)OTUd;{b`(qFvud@Z;Wh*=TsmGEvxG-0N zeA|2}^*WSqVDLZ$RR#UIElB*F(KgQK^v5ga*R70xufopW8QWVJR@sJS{1}|l^!&n) z!UL?bH6D5^og4Rdx*9GReO5F$)7x8h=hMMhUWsmXf?h&V@YKhFmAwi@7hoG;D(m{V zM4N?E0klziwfVDfhTPTpNfHUq8YXlsN7Vn7S&$_sfa?l{U8smH2PWc#W}+`3A-Q>! zm7QuRQ>>ae?oLU>f;B#oU*fxBeq3c^26N|6>d%-A=Ci;&Ea`XZSIj?v^dVe<-Z>Cm zZsjZj4jU;;iI)uUe^lODR*xa(bQ*Y{5TEA^#{q6IsDW~kTAP_qXbfZL=?|@{x2bIbg2+Epo8ZtKZpE+kEr2hL*U=!9=kl( zwDB@NX6)-D>5pUiA8`m%oPq5sSOP1c3UN;{#G1*p8Dm+ezvj`b77oWIbXC@w#U{ytw%o2QN&oR6SaI$_j zV?cG`85MKe(-xU+LvfjIPH-ksK~_$D;x~N#F`&Q0l7xfQTb~p*YTpjTZ0($kmds+Z zE-r<2bqb6zS4XX{q9@D>`Pv#1(+YFH*ZYPvq3Q(#th_@&C1+M+xvzGWHJnMI=fPxVe_S%!%y1j3dWF+ow6;mpx$!IF=y12&>GvU}PG7EMgg^Rj`kV7hhRdi#MUG4BB=jK{ z!&KGpR|<$Z*cS&mH!;BO=I7|QP1w}U!wPl>Et?V?GLQuCtrK5F9^bHYsC+Lnxd1*y zw54A}>^~uc3m%_4^td%ZbpS00c-XFHvIz?c8-V4mgaQ*eF|ODz5sE6#<; zDDOZFhqfNGX9kC+D?p6Vy{rFm8y(NwAs8Kd zjjYs1ku!agh^YjY-8a0PO!s$DpK`557MyO5ay!88JzU;pX_cfV%Y7MS9$$EzUErQ*c3i8T(32#2_QTm!yjB80zpKtc(R(NdKC3h)x>{AmM zO0B1R0%-+#Ozt5nS8H7aDe=#zsR&34QUD#-3FZ)#AN_t_+_4ISIVe|z@+-@W_BkC6 zA3yLrlN&F*rsx=`(=Imx9)8qJJTm_E%C`Vrg{A;?)tMND``*tyyibhlAT?}b&OE=4 z=jP}mJ(|1LdrS+6DX&bBwzh&85}tB0($}Z4lQ|62Zv{)|qzG8{^Zy79d(aRhtau-& zNK1um`P>}U=ZI-N*t@q5o_Yp(Bykc}LBztZa%1`S0g1uJrP>zwAqnEC5B4QKHXm+5 z0XT_P^P=%LZc_KJ3ZG3`Dr^6g-&F6oB!;pgHU;fMH2^yJVr|8%BdUQKgm0WL7QL2R z7x4J1NAt^%j=hS2QS^0>=Ur)M=k^irl1Csma2FNTo(VMkqx41!U}_zuz{0U;SI^=- zqHff<^L(}^%Dy?E3d#`fiwZ4D28Z0Fp=cu5p;l$IixV--p;Ca!SSk$MmNZ3lCoRgh8zI&-!@S8 zdK;pLi-YjfhPnE@4qxLUp%ID9-id^R^-Ery{Tz$vSmq zeoVfXwc3PG6OWg?63IveJ(}&#-xvBLbRnVeri-6gt~!sGzTqa3zu-jZ;|o+x8a@gf z2anQGXrnQ-As`Faip2v#QWP4j`IToOb>vnbTvn|`u{Q4xrogH&O!y1^a8$Xb%{V@@ zD>F(uXeA^r)i$#Mt;7fvesSZOxw4P(ZkpAB!wdx-v8e4%d6N8UVWaBYPe7+Z7F z%jD2o*Y2sg8m8u-r}{zsfxURXvBK;$t#%WNdN!;q8b`tZqY?9Tz#h)xy=l^~#O9d( zR+OERN_cZmPj*n-$dyK`1QkXz<&&S<_c8P)nG~`cPSzVZ8SU?DVI9<0E?TJoRjaJ@{t0dM%=Qlw?X+pK`-7*=^0KWG&qS+<31a zNb4y3K1Wf=$@E$>2Cr+AqyGos!j?vs$g_4fKM#D1cg^plI!<>Vo%)eKmMH&EJngR< z1~Pt8J4|iSWofKC6?V~Ar|{+5l+)uIwYm7K`Gp$uV%oLVYjFFW2gs)xxkOH*9$yWf z>b89@nRewU$rCb!tNQq>j6T}W1T+7WOtq_fmQvU;#J}X)hGCYQ66$D-8H3*yeXr}# zdMQguf8Nc4{Vzr1HNsLd`K%pDK;+E|vW4PrQaos{6Bs}MB(8Cawd;LXJsn#7g7=%+*Ff}dncQe&H}J9%cz8+V z9W3?C_9x}!SMu>b_rKSO4Wkm*PnoOMrd+mnL`=6MDwgX`)YHQZwI*I!Rz!o>?x3E# z2Ocxs^a7F9Mu4BVj{iCr^6P2hm+qDM?k(!W&;h|ji054k-TDdQhQ&C}geS`I8Uef* z|8L5~FmBU4*d3=<%JSODobg`5s!P97tgGhltTzX8!91!Q z#klWL$>9{jt+RQtz$KGo_GE)4LLPo*dJrZ9dsB(G721)Xn%L>piCa4pk@WHHvwO}m z?J22?XBazHY1N47#{W)q_SM1DQJ@VTTA;R?`^T4D>z4&F`7%_py%cF|xKBodx{q$~ z4&Hvll;&2ir+fxrP(zfy@QK19H^Ju*_!lZ?J6H%N18m3Q8v%x*%RPJHLDINa05jDB z1Woye(!O|3EUL7sC-kn4Yc3}L`n7$3mx9r5ZeH$e(1gXDTY^BJ`eUMW-8(N{AKVv< zTqgp^BEOW$Blxpb=NS*0AWkzh+}c^uuTYtV&kJp*#v!gVn&$H^TCS6pun6@@LCn41lBZ)ns}@STTHF7kl^4h_1|yS0`f(-U^bP zk3j2c9MNMCfiqu2V)q;+3`!E0+h>R~#E+EYj!DFZX)&~gq(?9m2>PepH2^#eWGs{Q z;*gnF$Cwkiw;{5>YxS|Ht#}(DOToa*r&XF@Z&23lGLa1vcvndea+o#c4elf@m}Bcv zQkYVDQc35;uQxug5Y>^wgxPci@zsi@m1lj@j(mHwt*Lsg1U3UMcj-!%#>p@CK4oia zx07GSixt_=t36U#hUUXpkz=k?BefQ87*c~kgKoACwGvS}Je??(LTrh9i4x{{T ziFRE}48*y(uiX4=h6pM_U7K)~rrxaiZHj11lXJx0M&*m1vz;;MsW5N zPlW%6@IhGM@Cg4ljoORoynC!_YcDXq$bv{hCw<$Tu9Ig5x>vX!nM`Zil^2(!-;Cmq zUc5)UaYf2dT{+Ml^D!)9TCC1I2#YB5pkwD2XDE$BlV>$seKvWadDv@B~hZOfkO;tzKsS-bVMS_***Y;(TgOYv|B%8%z+V+WfH z8ra-Jnwom?$v*9R+CLnL-*eAL`QTI$sAJ>?C7u^r1}#u=X@xLb9_-`gv_e=If{`+kvoG9otC z*wt45@@5g45GKClDX8s>Nh9c^k`Sk_7PAu!^G96`#OI!yDwjQxg=SnMpx_8b2|;h< zgu?Ry*mp-h6-C%}4wnK&*01aKJ-!UgPG4(Wr8Px-StggvUk}Z0n6old-&-;?s=8Hb zx(Du@CPSla7JCl18o!$jWrj9hjL6^`Fo1C&bh`-ps$HSYoWXl@pv>d(d%)@_bb07HXF{N zqL`7%(K6vpcG@1k$NL?yM+KS6)!=UC@KI{q#m{NVZuk^kLYT!4d~N1*=WGY9V4dl- z0TtRsM6pz0o{C0XoYDnk_iEIGX*q6+TfP|T1^vJ1%T~IdRRhO zp#QyW13)T8t)QzBDucrj++^swK(kM@BMoJ_)S{T)-umGABw;tV`Q&8hA{}TC6XC*bBGV@7lB%_@XObv?Ex zpY(}9_Y{MrZ1%ZaKD~Km$6aP%g(~UM(hX0h{YDd4Vs0evvjg?cWpCT|1CRD|$!7E? zi(%xHaSt%0x#vHj`FtPTaY`JcaX1&u!h1o;M)VMD2(}<7{2>+pUHfwy9w4<##znd)*_@lMQ!i6xv{(kdA3w^_SZ!lb__ZFeK6RN@|s2W zrF^Q~DD&I2;n{(+FKI%^8KmW9a?mGXoYi=7#Mb9H{&d3luUz-!VuIC;Z<$RGm{Qn` zRV@fpr{N}h@QUbuwMd~e@Ck}cuUm~JLWB;N);0h?hrJrUNVodXd(_11kTjb1dxU!f z{0=XHi->gfJUoMcTaT7(ueY5LRMeRof;=)*3Fjq8xuex1d2>#oMkt875vw^kXH~CD zR1v*ysy*rz1?pq)B#I)D+IY>*96;v2X0~)DVXPxCI+!Skc)m)%A%IX7xAyze9C{8; z)wl^A?^mlXd#d!jq`*7>7z6ktY0k`;psx{fm?~~PPByuvVnG00hr18NbC#cj zKIwy5`>O<^PmllS)!&OA0{wo++-rb;oh&MhKv>>gsQA-wODBXx;&Q0PH${5j22xLwO(B*T#PgvcEjt0x%p6YV%&lwn}*kJZ%cp$aG zLuv_ML|3#*!-UREdm}{bLcUKatF}OzJ21iLQd>`@f04<2Sx|nk@rel;*%%BKaiNtF zH4vZq@?**!NBE%oPJl_*Y2u2bA&yEkkbDk2ollj}PifGEyQ{P=zOg|$;s$~);~Z~n zK2(6eYql#qekJn~JaG5^E?0E4;Tn5%;YDRqj9BrrjEmHl*7MvkRY`w5oDvkmq;lSH zJXeqZEuSl9fOGNCdLX>}Jj*HcaziI4ZYsJ&?BBcDUrM9UllD`(HP(0C0Yi{H*$DPu zjB{Y)paaN%zxv2%AAp+gGv&E%HdgbtP&gx(}xI*5naoJv@Q)rdoSx!Bw2x6V-2n`Y@010qQ-Iez4S|yEK8xTaAOi(C;-a zm4iD}78jS80Nsta(AScDp{&b0nhF99!}`**A7+v^fh$1nbxy+uUm|rPcn(|!^}$Sd z44hIMua*mc(B6+2^7;`C=Z7r)_YMVW!N7=q`uLUES9;;EAGv|+_G;?c-Xigk+83*S z0n=XDL)Ps(5)quOr%J3JFXFR#40rv0B++sHAhE*slA%Mt`I_h3eM%$R}!U zLbQn9&Ej6%z$iYD{dV`Mf|C+eZ^KS6YBZsWa0bH8m!d}B}r0F<98yQnbo6-xV`Nj+?pyW zn(1@Z;>TU<>wMNB*9`|;-}zh3X@dfect$C!38zn1kM21pGmmh!q@iyT7qc! zevvh%s2kD$AP^oZptRC(*?P1?)9@XwBRZ4I4y)9hQ!*@1{9hf{pFTtwwR&~Ta!;JaUBvA0m6@9x z_{PedkqB>EwNtm;6Z}H|Gv+VX!dXq`c+~z^ebZF78uUWU(E#f+F}JTK>e&L_)Y(9L z3g!bY>}o;f^jGF9rwS40k+CMc$RYj#=gub8^AIRde%vu1Fb22?3Mq@GU^;b#nN=Ay zMGo(_JO8Za>jc{dlvei_#{H)62oCy+KQq&>eeyoZq#M-(HQz4jrjK-a9vfg79H8e8#Bsz|g5vIdS#zw8;`Lp^yex7b5rPoc4+9}_GoGm&cKIToWYC>xKo0Z94 z{NP{_zvtki4fO!5*yF#&o0!#T$^gIGE979${1WQ;f>eJ8o*5+M+f9z&R>42CF9^ct z5zYVD7;AWrgSHKx5-qV z8MdX%WUl#V{!=*Nd{Gy1Odc>A5qV51aV(?&_o!GPTsOYjR^G@VpqpA&8cF?!s)+VB z&|d7_-y_+9Rv&=-U>kf>F}zThSa=kRWMD#4j9PZsBFG*pYzAGW-nXcS4t+PBVVp;F z1yjzP#npHyuHDlbstx{A`^@vNtiWh&&}%;z17q%9iW6^|P$D2V*rLhk_Jg0IzV?T7 z_g?wzfSulwh}?!&KVQ=eVDKc{(;PqL8h8;JKLn~+LlZM1F3Y3AA4@;SESTxPQAZ+~ zUb*cu>(z9}`bN@2!_!=D`uZTMZ^UY``g65;cMkepQd*7sqx~I~I?d@8OxGejMQp!6 zQg*nQQ2mn0e2|b-HOa0iEB@eQX*v)__II08_~pF$cFsqb&VS8e7qU1-lYz5KVZ~AD z9mA|O`v;2>-5tfvw_`-E|3JMfJ$`H;LBU|O0EqBXz||HSRhS7v)}Qsml0 z8?X}<*h>CG#M!;!{;u{9GH1U}Khd8HZtm$z+b@Z4E)FuEg@@TZpdSC+gd%uH_}KdG z{R~^8cykF>70W#J$Z^}Zp+Y9k#x=heqRx);nj@!(cTS>9zTp*saaKpaFlBUDUO-FzO+KZlIDRJ(Zavx$O*gP1VCp0%zS-(C<{Bo|Bzp)|MVNV+ z0Iewkr6#wE)ED142HZ$lgxL{P0mKwEAn1-Tt_e&0Pq-SZ-;r+ka56#)QU`qX@@=r; z9Vg^`d^1#i`wryimvxprb>|mmED2G;61;XE=bSJ^l@c)s=*pE?pM3SkvY;bhpC9WV|Eb&SX_p)yMw@ zpaatl`vyV>N*w)kv|dHp$e-rx(5gH8wiAj;O3{X<*QVGK)HWU%O{6KPw_U`_K<-2s zt>jE`nWo|2y?dw0O7)gQ#w|To#?2Mz+0=uc<}~2KngU{FZ#e@Oe{?jwM!4!tV3!Tc zO*VT1Z2Hw%EW>|wN2lhHkKwF=eo^K`6KN%-&PEw_FFRBcINsgw-v*CTtd=Vt^sAK- zLorc{%CFv(`hmYZ#IuW117=5Gj0TgbvBCe>o)1R-7=no68gaTcmkwMFv%h<`xTeH2 zNflKGJ9pi0LQM0W^43m)F)=W`3GLu}ga|jV}8!_8Al(wf3#50*W?HkjQ5f!a=IOnW(~V|#Lj4%^QhE1m}hsEaO{ zo2R$cw4H;ZtgS=OgTRpf`dIL1b`EINguv zd{eH&L%r}VE+0(#0Wx)CEL;?URI+i|s{s$q1w~7d|NH)rfl5o?PGmuj!=o!O+8gA2 z_dn1K{FeC-sF008@tfg0Vd_(bu5hl|&uazmfBoT0kLi_SL<+Wlj-pe}UyCLYJnB}q zSWvs83b7T;5Xbw+1fTW+ruP4#V|ydctm|WYgq@@KpgcS?6qs+p5xEnz0;XpFu0q`0 zdLpC(7Qr<5HvpS}c@4>v_5B?xYnUVhUUSXm| zX~Fy1`qddVa1r)$wrZAnE}r{bIaiE4$fi+OfPz<8kZ`*9mn-dK`)LqJ}(gR{nyps%Vgb9bK6 zZa8&Zz#o6Pd@6y>d6NVpS$Qus8qP>3@_Owf0?8xXjMW~(eYFMviI&8Y@Mf!i_XEzf$rodyfsB+O z&*yrJJ@Eq4oGh33Be@1ra~^j5`dyWtZ0+!OFam(PAgs&d4}NaG9A74C4&i%%hNDI@ zGaV2Coy4Fv(5Mu4nE$j|?G4|GtI#-P9t%m?)$XXZ5Y#HooD{jpSy|NVga88guGNl! zF8k)A@Y@f`Lhag6v?lA_q|VpYFK~79-sg@x>F&W=rTUhi&uJrZcJ<4&qY~E!Ey^e6 zC55dBN?v@?Eio@`ob>!h}g1}f$L>_rp=sYlI~A@szakE(sunFhlQFvCASPmt)q>iElEG!_ZzbKie7g8+X50iIx z$|2p1y?ACA9O#z*n|gtQua0*Mw0D?@4VJoaWx;~QZcL-lbHHE(rS${g&`MroEh=$# z1O)(Z3vO0#arJGNiM(@kPGFEmH;(72*a*2%)v4=NpWkk(`k2j$x$I5tgxDj0a*V&7 zyL~EbWIOu#vmC=S{uf`TlWKHc7}D$&lPG1r7y0}7W%uPoiIF%}BV&Vd%{@vWmi7;S zmT|*w=g-c5^=nZkpOv`V&DFr+s+rgc{;07!`sI*lbpHenLHMq9l8yFNuP%N2#iYX3 z_Pa=)l>GQ*XFja+SM>XL@3t5_5l&p`^s2Ac?$DA=<*85t(Wlb;rt+XUZ}*nHJpE2U zVOp>)gWT<^=(>^&+}~H+6r;o)T)`W+#`c;qO5z6iMjS-B53XvU&{kq@fhQRg7w*M5 zVt3Hxg_n@K1~y+E=Oet`8BTMOQT;CRGD2cVDdWJYNe~v}lIaT6Zjx+;=!DET?=5X3 ztF3XnNPCZf+VU3N?b$_pia88wEYxX50N*GVO)vK6rCqg|wfA>q?J)K>Ry-!uxt4G! zaUC~V;yt+Ru@TudDtR1{JUiBD=#GCmoFt&3J0gepC(F)|rK?byI5LjU4fA!?q=`6x zHoK*#kTROP7S6*<37>a|Y_*(V&esAr)JW!VTj+`bc_uQqv^Qe18CB~1^-S#up)sjQ zO68`)*5b0Z=LSxwt=f~hYRj>c{_N=X0v;UE?mY!M3EKIyyTLDzqqqQ(0B`3Y$zizX0Y2!52gd^DRoeT^;^@IU?d~S>UYWk?!}> zT3I={yGG1Sfze*DHC4S;KdM?fVMp=u_n1g8#OoBY5LLtDccu4LR#mpr zwkT=)%gFq5(&HAaebNQWvR~Bq4^rbjhFrhzcp7)HU%~>!CaB zxqDH3I`$9dMQ}-m)n>Mu`TEmA-C9OHOjxJb9v|}r{VbZs)AAL6H0za>RTks}2vz7L z-U(d{_xY zu4mjMzC4ET;jfo$#!f6TY&|K^)>EPecPwic)MpH0wy ztRe8YYZJgk=ws^;?+tYx?zpm2{b>5@0vvK&4@rSQ03zMf{J*a5cTY_`=$bxkWi^~=D6>7IXwA9Om17a zs!423pNP(`bMbv9m|Vyzk()P9G<+*}9DdT@MH!+E`L5!LmJ0|HcRbVnc~Kl-Pj0nY ze&&L1q1)u1?t@xXyo-@Zoe41)=VZ>M9bsUCa<2NL6{UOSujlB66@M8=Pvf8f584mFy$yIv0JP~t_I(u;lzB{%k+_L^S{3F)GgRCe`T zr4S$~k9e}yn`%!YhS`#T=Yj?!_IB1Np~wJ>PTi6=erlPMn@&ov&UX#(pN#W)0d^#s z-u2RMaph8^EbfO+CIG;tSRUPm{_$hDc|(-(Oxsedp);H(SkeK`UQ)4suLy!LncRVe z;F5B&-jh#HW77ZHjV*WH-7);Lu6eSv0(3u}bP#V=#$g4um=G|-#UBGoye5O(;f8Uu z%Zj-n?tH_HsIK2_UIy}7M=dO2WJ$;QG#{cXGBg{b0ApN)GA<0%S`1xO+OQ*`!xMdh z8fwy-nC6RErPU(B>^49AJ1B?oRxVwmmj(&Q`{K>K{o)bhCj2F3*v=FQTICdzF2gYZ z>{J7sr-K87iB(s)KSm0E3{n*1Vre$MC?7&(aY0$c*+Gf0nvR;i40tsDQAJQbluiK$ z$gK-puchY~M9TDJ=U{-1;?3Y8(@zG^XBS`7p1O>Ydm{TcvO8G>*vBXh?PxO?M~lAg zj>{~4KW@!*$shIN1@a6_j0omW$|R4dc$;?FriHqkcE>Z<_VQutEtd242X>vl#LT}2 zh8Qf<{G;!JloMb^At0Bm-PR8}LZ$37R8j$4}A)`jp$--~R=$$|VkW!WEQyd-Cj z@+1lQc=K67mYeR2(u`;C0kN8jPNtGzcTD@JtjW|v3#H-V45#hnug}_#QO%CyZ?J2E z=l*3VpZ1qx2&!2_ItbZDolN`v4EJTH5QFct`;A)k{Eh?(xAX|jmH62Kv=00Snj~VA zu%F>Q_$+hGpLoA*9@?h-3#Q0DZs~Ao(Q_;3K7Qr_DZ5g>7jjI*{^gXa7P00>72WD@ zV8~CW4JPO#_cFVSOH7Cl1e&Uko>O^hWUZm8O_=B{Y8<01)CdNZ#m=3(qdjCAF_QR! z`;I2J_Cb9REV^$iT8EvERt0r1Ul^cO8Z+5zsQ1VANpD;J=X{WjtR3$DtK*)@Rx#(aYEE)&fG`e69`MgIv89>_68pIV&~%OR<@F{x^Yt(Q6TF zBCesD(f274FjXp9&p`oXrY|Z;K|_byEn2}}60`gzwX=F_N!ii&?G|g}oV@#v>JL_U z;Z(sG(Z!R>ltW@9W+}x5{`zn*TL1*#x+MT(QcB(LN4PdbbZ={>Iz4 z|Fi_pF-x&ia`UcCy6hvhmZ58{qvY4cs^B-lfXh}(;Sz=DJb~{dl)_wRX2)fzPv+*3 z8cFsxM&>xU=393rFFyJ890*ro=ag;R*YYDHf$`b4KHv5b5B?{ssz|%%*0b!z|5CAIUeJDXO>^pSOVva zaz_a{=A3Fj5J)27|0H``Jt3Q9ijE80wZ(<1v?vOH-sf%(QtL8;TPAZUG10+NmYY`JT%L*gUg~w3d+EpoYw2j6 zC+OQJARaC@Zj4HHYHAy47ULD#U+#**QITjU&T}W^2Iq^ny`W@t{FNYTR^5X;ZvviE zHgS~aYyLuVe6{t2rA)v-sY}8gHau{%IR%cK?;nE~H5lsp#Bn0iP@YQUWceL%zOkh? zSq$h)MfE|x(DSEzvgRyC+IciKC}3Pqn(Q>pTgHt!oj`)V_=*9-E(JEtUH>at19#yl$1<{ z0|YT^-1=`-c^!yyFn_noO|Z=c2Qak4oBFTQSf$L0Q3x|v)xK*ElD8g^c*4MCNjt9d-&=N zpO;+k+!z#{=Tt2a%w_WDI?=yu2t$DW-o6UI`fC&Kc0M8&a^A>Fli~OhLyG*9ERAoR zisqQ+-8jDl>Y2xXD;5sy!5LSsu9++|KE({98+fV4_3tp80VRwR`+%Q%9lr-Hyc9p_ z*3dXZ?Aa)fULw}Jpyx3mtdw&pBC|TOVvj8WluSNYiu_@#imoX6Ov(wsAE$_sch34=$_azL`Wy7|J;fGP+`nbcy({C!+@cUE4>x%-@sC#&? zHAw%ni3qSE{q9eD)`Xrg5FL)J7h`u4t@cefIQ0VP1w}*ScQ+!vm@azKabQ-kV^^qO3pfp)>mVBfghoQRQ%)mpF3O ze;-_Zs{guUcW{qKwln1>?nt>*=bXI=gAzI4o;syWdacOh52kiagbuSD*3%0uAQccd&p}mb0Ff}G;QpG*@259&uRJ1zJ_NMYdoIwKYYI7-B zGYW>c%uin!*x0godLK5i%eX~S$qJoXAX^-e*%a}e3`aod=UXR=R|P{`pfNDsJLlQm zf07Ql1(9om&q!Mx#cc#YOwykKv#^wbVRu`6G~%kw(#D=sB}$8WKN zz;0WkWz?MBoIx-1MuPAdBQwoI|JT-uNq)!+J!d6r}hN7p+?m7I|h3Ls(^Z| zq39;5D^sV`)%w5iuX95qGhvJsf5m%uV7qgJ4p}>DjH>WDj08R`REh)?1vU-UF{RQ7 zk!?!T#*Qh@GDh*@YcCwu%aP}Vr*oCk7MrIkOv%bOU44-MrFs3|V!i2TQY78D=Pa(S zx|rPaX)+l>uHU*}x?wHr?^5r2!Yn|ha^3ug^cL|%| zv!Q$)A%%%1Gbu;!;)bNpOy|C=GKpD!Xq2FlbefQZwl^WzHQX(;!N^#puCbZpSboRx z_7lL1xPR3GZYJG9TZkRRHM(y4DEC>A#6#*VR1K`AqnQUp=l=Ib;p*H^;2KqPFl;?cPO1n zqrw978RucESR3IRe^Qw8IG?Q5-R!_%6cqQQ%v_F+7Fb%Ig9wTJS{yI_XM+8PJeaN| zlyS2obyZOrA!rd8g^YapO>Iv`iEm)Zhfklu%0#-?##v>%{=5lz`$3oCk${od%~i)) z96xz)1cQU09P0=bN&ozVeF6bwd{3=EJiJAi$f==m+I1Fgfud z9J2bNajor6FjiL04roo3l<2u428>*Nv$o_iHlovl1+YMXzNt31i9bU+q8cp~ zY;vD+;;!MKpyIM7M(58#H0i#Nkt*`{EKRbj;LE6b&;`cTrBH3pGi$9;1}}dB7eE5} z6W?ORGtL3VM!TTJphe>-zHtz?LD`V}6X%Z7_c-edYpW8(WuXZ-LA42ar|zQI9jwFI z4xj8}c1lB=2EJG~w;+>32G6%{%wRY3_Pf38Y2^J@vfRF(lScaW1={G>hZaOC6-PeO zujp`6X%a!4o#l&ACCjt#ug zrEq`v_Yp$dJ3_u?Q9e@_J78$dh!Z~st*wr?pTjDz#y#I7U7GGIHXe@FKcPIFR0=TZ zwH%oI6_z<%Kj)6aN*!NRiQOJr_-&~?HIqI!qJRC4g@a1&uzjfK zT2?EP6aH~zKqHiH+;(Xm5!>RD>tV*|Fj*|GRn^XPRJUW?=VJ;m;n7C{>+<}k?Ffcth%u6*E!ziaSQ&keG6XvniD_Tl5| zJE@JOGoGs$J^{PeRz0qEsePoHXT z`|_ugZd3v(lEto$y%nI%s*uFR{MLNof!wj(4E0g%UH@d_a1o7O_x%_zF z49MW0DTuWYgG-2WwfnltllI2-)hDdi`wLpt`|PEazFzk`dR>8j4*_@3Qo|gLG2{u4 z7WnjR9P0};JGwu}jXnwngnADNp*i3pBk`tv5QdH9+~_xw;D8@4Xt)KUb}YCmW$ARzlIkVSfvoL;m^lu9Xo-&5v=d`kTN91Hg-SL{`}9pE+_m5xM8Z^ zsvx`lu<1xAteY1rAt|;0$yw^Mg9y)CHf%)e`xE}TpIu|+10S%a=Ws>FO?36?mO+PY z6X;w5xb>?(4h_4M>1wtIPIdQ1#;zm0X7V=TE%33Wjqly~DL65FG7w>(<0{1*ES53y zh=u~SM-*dP^^W5`X5^!O#Xw5;4D-^0j?`tsyK2Mq`|Cgpp*{050%|c1W1m6BSxwhm z>*3i{>vk^3nhS-7h(QtN^q=-E4d-1rU4C~fQ#{6z0GMwncyFjyTu)P6} zZ|NeNDw2j)Fw$C+NgsdiBA0Op$O?i_*zQYis@PNIqcoL?+P7D> z=64d)8P0P$ByWQWc%E~nH#+u|$4c$)yPp3JazoQg8D|bL@mSDt(_MdgQMcqohA(;j zZpjx*v^cDrgxvBD7pNFCKGJaHU9C7?AB_8NhBJp|>VNwI^3IbsdnkbT8%hoF=P&UE z)L3&joo81bl{W{oK}XNNEg&kGz^o67H166W+k~sMXHo^U+)2rkjao;LiZ-z$s-qg0 zc4Lv2;#vI2K=@Agya*S(L z8_gNNRjU!5*S;Oo9=+9m2LZI4STGZGu(r>4`_W2Bs=SgxjwZM-iaIE#2vb~EourU& ze>%|UZ#u*Cs%^SxKVRA6BEQd)uPQ0@|0A@1^onnmoF`RUj?^4Xt@~g3LEo0+ktls# zcdXSP*4DMR)6@m%fL5ba9YJ%1gQZ62Im2d+q|(~JA3@*4&tH;ge!%5E=K92!_x^Ke!4>l5aTXLGRBAMbGI6s?HeI&5c08E&dH*45j2f58UuF!KzC?XK!DFd zj!3!VN`a4_y&qjdCH_+x?a(-D_nU5AS*toWY8wBU*IUj$12V!HkOMO}3xC?lrEP%S z)?S}~6=@HELbk_4)AM$ib7ETn)?oZ5_Q=e&qn|b=M+$+BZHelbps+|wE&ioX?9$p9 z*#Nh#V>#EtMXO?qAde7;{U7p0>-OnCC6Y6|GI&R7|ZxSuagnnF8+JhKg ztqFPBeT0!R4jvE>hyKTW2^dWsf>YXk89d?q+z(;SJ=gJf7b(vVZ~HTjV?!*E$$I7r zH0bJ=@dY;{E*ehv${(Vr`gIKLK5o(0NE-%a1k@>hw53q?X_k*7lVFKYa%IKqLT&v< zxS3x0vVqgOF2}d|z__1jgmr?{XpMYub2c3p_Z0}`^PdDKZ zEA%>WgNh<@QlJv!Epg}Md0Qy%NlvZ+V@bwn3q3c*EfZ^+Ul!x9kxeDRq`m7w1N1$x z)w$j%#eGGB1KvFPZr+>>o$v-@6FJY_S61-C5&zLVL!eUJt zA6d@~W5=_3W-TZg`)_j{b1Y(p?Mk&cmr@!Ty1s;FUyR?=CB@pB41W5fO=13z2ql1) z6n~>blHz)=+m%C6M}A~ELshZlVJP<Z&|GB|&p{?BiEQI0tI z?!ImGw-((i@8f?&=)dO%aQtGSmbP>Mla7amWR=_f{SM@VeyR7T>Bu%Sh1gYIy}x~1 zro=`vF5R}UQT0JLWezhD_f*VKU@yKmMQUTj>P|sr^vYtY6y1yJU7FUlA%% zc3t9l>5_r`@O{fM>sH4mhu&z*p6N+AcDmg1Klef5Io}uOCtw{zMgLEd&@InpruW|{ zkBCjHH08KIKmP}SeH{ARi1P5p8xHmS$DGqzD|Q0IlAxnfWPeTFwTv}#SR>y}mYCt+ zXspMaZzpXXviJ;KYSS6|{~=NEwlDhyu=U){Oa?0r_RV$G9jYH3-v46ziT3@}4z?8$ ziZ|O}f%D=AL+;TRg&Rtd`4m96q&ofc=ljIrC*c2eMWO5g7?6P|F;#?G?-L6ftE-Jz zVfhXwn7H3i)m3}rHRQi(LPHC1q<&RaJFq50{-cr_=V<}&MQ4;3kyL?0>)k>`KqVxB z`;zA!w_aWcaNGB|@6jJkP4Gz3&3lci6C8?dLPiv=Z8KT987`X=J2mDsLMb;-A=Mg0B;r?v71c}8QnWGt>Vp@yWlIv#QL-iCGqv~UvNUS2<$`Pv zWJ-9Ri{l>PLy+MpKr%!@x+l!Uct=CL*-|01^S^^VV6p&4BUxs-;4)#vtp}Fl8zV&; zJ}D31pYdd&jA4I>+ze17jjJ=229|Nl^vkSjcya8#BYM526a7!fPz=uupE6fO82#~l zmho@Sl!EE#=!My@fgyoRibiiY#rYC7d!Fw_O{v6N8uSr#n5}BR*0qyl6!inJ18(B# zZ@~>PM$~k+!JBc3SfpAmc&XNjSuZ5m;Nz!y&@!TM9HErsD>jZlDA8*vF$RGfWl`J* zP~2PaZ7Vk?_WAD8gIewPKGQc z+q^8+x*`#L$F$Ca1hX+@-=@oMl!5y>#ygaP9>__AwCXpAQaTxI=evNh;ZEQB{|*rU zW{ek#@TDdl_Cg<8FqJssl5zT^eYYAsJp6S~VYgTi6$9_tnTj06ZFn4=#@1H}-);N- zd9Ly(Myyza%mbB-i?+}8EQyKA9DuM-&A zw*cxty%)Bda#*!<*qJ5(%R_R_al-=qyxeK+Afp9~bo_iuB&!+(g&y68Et%!>les@u zH^jg?0#Pd|NMKu{^y53stW#E%F)ga72AF?U<3@}!xCDQ?E4~Ruz07X4N0%?P+?ChP z%xAWI&Se)xLFyJQ7V^Z>GCAiz>IttvLp?6+-?(ynBJoa@kcBCNKbt)TbRiR^%+VQg z1SwR*B8Oq97$X%-8OEScdeOmc=F8rGvj|*wV6u=xJjZYwiSAbU9NoS{dh|`e)1x-l zL@kUyPil^G)f!=ia+)ZNgQka9#*Kl7|I{w9`Wjav`BU7765Qkf?C*BxAO7ipzA=eC zFCD^CbjN@xAx|iTaN{YV+kYNW;H&fY|0ygpQ0b%2|6C9B$7;YSo<=S&B+)j1H+GZIle^CNfN~VO)0=i|4&{i%OQ6^xWzDa>Ifg1;A0;?spwrraB%luCIE^61|B8D zQpf+CKXM%46Z8e9?f=bH5KC$L8_+3Y8))ZBOxO_mE1u#~`HPp390{}eAPJsBJnm;r zR&)S;aY!RWOUOL=hrSBf{l7L#klWvXgK`;RczCQaHlT&XqvDmoB?rVkqq-FUti6{| zr;<(6F%c-X_L{nlQI{U!w=ybN=2FxdTi4ry%-FtoZV#LKt?1TERH9(S_+?Bx`xo^6 zyWDtFwhSq9tuW#3)C*z>D;m`2|6wlwxh+$PM2SGw69B2giB8a_aGk-NZMU}a-XT$~ zbZB#wKf#QCKv}}JwLf+%N&=NrHq-6*^WeXuCi)chzLZhAxXSv78JmeRg{XdC@r}Zi zCv}k`$ZFBrSj-TP(WYvm0&bdv_5a5PbAv}(bpEqetY0<2Eka4_2b0ci3!j zh6y#4nRctPe-!0VK_$_H=KL${nzEssOqfSUidGRZ(9azN5v)fh$rrl^SmIdps|)1nRJV zvaB_Rp94L|pEvL`c^+r7C=9D7ioe5er^M zwM=6Ju@wA5)zpFsWHW)R-;FJ{z5h;(Br3Kjv^p}yxmBsfoTrj*a`-wnxbjgdDXDFa zumtUh!mmTOU>fsE=HaiD*E<>MbhV<#r5w(K#1;MdqA!}DRx%$b`ZzL86|9cFys8%3 zq{_d5kR}DGI?xQ*D8qUk@|Q?IQFefE0MP-&OI#HWB=;M@Pz**XgW%}%3burq-?*_v0TH^RmH?RM4^1r<4! zKg;&J+ow+qWF}^k8;SyxhdqFMUeCYe5L}yAj-2D9UU9;a5t>PVoCzuvKI)| z9m;yms3AEN)?3i)`M1+tPV=*Ouc~xCBB0~tq;3zE#aHdWkss-Y00_obtoueIVc(2g znzOQ1vh=q|*S96@lYn39)5!NXV!0fW&+`QY5Yub}IQ8Hj{d(kx0%5?9EMo^4ar$lq z=qkRyzudKqVgejs`Y`liKyi+e;MAmayI zQ%jItdko6tD{ij}g@Wk2x5HiD@~0$q3I?J}%4v_z=KdS6Bk6F9SJ@n>5b~`^s7mUP z^ermD^qk^TRRg$&b*GI7o*%W7Yag}guGHmQ-Y6!k5x6uJn1^yWywN@Zi|<#QjY zs2wCY9G`Un6q>eYqW>iHu`WG5Jrt9?tfRLjAv-B&mRn<}bbi>qCTi?2*mM2}iUcRvxiL{ttV|=$b(VP#CAB{8b6*+XB{Uf-Fmh2Qn;kCc( zBIQrl0`HN}D*p7Mu8~__%63mvXjH6k_unYa;k{MlY#{Py?UWuSf#sQ#@6NO=wzqDM zm3LbV3@g$!D6qrkXFNQT=wfp}wD3$lb$Q}O$-?`#V@jQOg#Xt)2L$`v7;MgZf|fT_ zrCuB=Q2>cjy$q$}>%WIoWpg2l9g1alO5b8?*zdO~#8xizo%9NCeQ=hdWVYvdh37SD zuJ@hfB854}QDDL#5*1}9@Z>1}%dbow!!Xv`o@%rL4Yf^R+LvMa2PR8%z7VK|+)F+s=IkEpoTK$jy$7W5lgg@lM zaPflB@vs>);={;fmDi*Iid97}hJt_fNu;cg3Sk`mcTVO(*%tm?Qd6@e6{5{D%G1Wb z#2B3UI#;C}L;=R@hHAfY_vhA``KKq<#Na)Od+j_H19pVqYMW8*DxIcsqGSN&^`cxz zbECc?(6?6(1t}9Ra%P>8|5h9+BKl=1k)yP3XgH)iC&4^I^;>iHzYFQ-%$PCC_|E_~ zhXaLPWdE}Ovx_nE!^7Rxjt7p`q7xh3ew~+w*LkWL(Cm#Max3oX+1TWIgFRsY4-Wb>|{0YHshR65S^lW8JnQP;2vTYXOjst4x*5nCj zUC{EA%tAZgcIUyiVNPsamcAgph8uZ*tO*6Czf`cQ(xqaOBtN^tBSzkc-H#u;Q|z#j z-d%F2kXrX!YiLVaC}`4pU6kpl>mUJ7RFBWpnrJKS3KYzS8(h7PoX^);s`MZzt-_9r z|GbT-nM8ncFf_^8zuBkOr^)n3(hUR(x`)L+%lWO+qxE`>?f7~SeXo6D8=4UOZjpt% zahqg|3%nf@lBa_lajcSs7EZ;RKQ%EEkyrW}*d(}t9xuDjEe2DaIVN}{oo15nRDl&J z49Vje92M9;KlMT4qAL`Ly^FBqg=N6DksxvZcG97-zgs`Mysf~>$vOZ9evDF*(0U@ z6vCmkC`uS8WFAA#s2ifRnMfw`T9Zf5+ZZRZ^mWJU=dFzVu#)_MF*Xj3Qg-J7t{TOU zy#2t7fhGFtTjzzMO#_Vc8b}fMqP=y$|=FrGZI;BsK1K3AW0kXox$ACbXPJ*f4YpTxy+`FrHiutn?$LLRyl;@tj-4jZoO*Ax z1!U^(Xqu&43|Uv5OhWgroe||O|X_~VtiSRuX~XHnYMf$`adp$t#x{JRO=IVY^>v0I)|RW zGwAG@Pz6L=Or+iLk+pao`_J#hFV)w)-snzS`1<1Z*@+v*=fxxWZAPAe-pZau;N4WS zc+1=I6FPa+Yy{Y|${)mgcS&sO;xKAs>OIMRMq+!BJhX)akKj&aE^3Ii>Kx5FgUJ(m2o&h~k1|-c@(;Ap!}9(jc(}+Tf94uyaAhZvNeYO}4xdN8~7mTam$7X=>yH2|1}A z)z9LQXRI2w^>k5%s$@TWvDZj`0oDl)t!#K!=-lV7e^to9ydRY^XVLuE*lL#6+sx`5 zR(kReTCy;rI>Z`#(kffa&+(im}h5&M$Wj*Qpt zU>dZQjQmA0A@BcP-2J*eaKQ0n}r6Bc# z(47ip8R7Qm^0)H;8o$8xzj$fg;>RBphY5z|g3rp(&0aUoQaSess4q|{{c1qT4ALP9 zl9izldO3a1FP+E>&$rek43~cj3~T}pLj35SHPw*;e#eQg#3f|Wil}T<5vcUPd&?S$ z5h4=g$Dc;R8IBYmY%1x^WLU;FnXp;}D624kYzdYStpU*-Z#@209|!wB-Sf}LNWJ-0 zEpP;>Z-%L}O&dU%Q^3>TqP`Aii@TEFNF((gIIft{dS4|u8 z?S9sTLmPiTfviSqr7gyO*W=i@W<4K6hF%E!G2wZS3{;Q_6K3VxL?qT_;j*DBz0OXh zkzMjQk=+~nC07tbZEl<#%9>CRT+#Nx=6iQ?SEP;Z%xZy>{?rN@%YdC7@buOUwSRe^ zf6b=QU9sg&C0g58qyL))03G*Oq;qVB&u&zYGlht|G>fbL;*)QR4$IHx{<#DfQ2lu= z->`Mt+!0%ZQ--`?6@?cXr&EE)kSqyf;6jEeQK4($Ux=(*s&<%x$os#Bzj-sK6q`;N zg%PUN{eKGkW8ahN;~@)?dGF)xG_;E<^G-63%u0md_9Fv1zh+&_EWYOvU0nc%)cSC6 z$1BRL7j)QBg!ON7Ek-N5*>O19e*aIG&guLxK)kyQ>Q9Ee=>s~9$GHrPD5Gyup0}{B zW>d)WF_vn5a@~v0<~tL>iaCW~{M5kw@MQJP4VxotHT6w$)C2HZjc-_*o(o&`ec5mc zy7A{IWh7@KQc9?lhm|#-idh^vfhm2GD(G>4yckw~M&x(*%d@*$Ly~OO=cr8NGd+Jq z**ZkpSy7e-s>jxmjwOIWC2LZ)RHd|5xS%O-*&ObFG+WjcLI$TPVGf`y23>d1Z^Vbu zI=p{yTCBm|GsFDJc|M7oO#r`&KL<2R&O}5)VJ22;AA)LbP%+Y}X(8X@uGNFIT`{H! z1lyaL({%2<06SlM(>77P{J9C>`=TK59`Z+I`^7Nw#V2)xP=D1+4eJO2IkXUbc)eo+ z$h0Kp&ljbKxlhw1hx+3A1)MRVI5q1)baR_h5x$dQeOkdq*j+y|ZPWH@g81JJts(=5 z-J>4=Wrw7Rg8;n-49GmX{txjdN_d*>OL(lvx^toG&=;M-eSResi>qoEO7UVJKJSn@#T}qM9#MLeX>Ul zn|Z+o9Rm1?=e+KCe4fJU1fiR)cFTzI@TLMaHH+~$D=Y_XQhhrV(bd!-YCz;6xQUNNpm@gf6azZRw<*d znlybWVu-yWq!}Tk?LN)({DfJ7K3ime_D#3`gi}CUYeTl?Hu5%uUv5vGngXvN<4fq| zcS-cRg&f7Z$_oXm(80L+0*H^0_E_l~^h+{Azl)F>C7vO(@ywNO1u7(qub}luuU+Md z3_*7MxNSNdVNSRuk{Ia}P_71Urmn{UIyRM@`fn?dUye%&kHab|ke{>L7i>WIk7q~~ zZFVeztvO)edZlLYa|A?QJ;3&o5ZOIQv_S9~_+>q2HVYV7Y$q>a{hm@Pd^$#m!PjhU zix2%@^rp{7FcMrdMiH;fM)_niNh9YR=@`$wr-KhP&e$Kml(8;?K+}kcEzU{d2qmH& z!|S4GZ-SczGj-aGdz9O?A45R5095aW$IP#?cY@8tVy<2&p^rC5!@s^*48#sU3srq5 z9Rfsv0xhuBq1y?Sccx!+xHj~erGI}IVX1gFg>VS|dPZF&knDRnSkMJ@b^lH-&(%f2 z{4Z{xmlca+m|ZErK&nLZJBui2ItxHok3yE5+tu#rD{*Ff$;_U%e$oDOvyQ1MH)EOZyf2ME+j0HyBlwAg`7zPVd%vV zOA6jeHWGU2G5@>r@M8%!e-E@wc!j2~MNs#IJ)Q-PNf1XGlm3H-$Y$&l&dGW{mMGr` zKg8)W|4zEbjNf56&vv}-XcNcxx}Bv#A+PATTH1=;6e+4_$~Cc=2g-;S1P@DR(sKS} z7vSmdkKOyWkn#H|CB(P`(t3av$}UOJBr1tJmviq8jLS6$sOo1IY#4F{{+o!b7l+sr z{0z{bqe5m8B>1W%z5kkqh)@Wdi9`9uJBv%Cl{sP|K8dW3Z8t_ucN+IX=RRVYQ{cax z%{Z*$d{+1Sza)q6ceX#9U0twR2Rt9GJP^W3@Dz}^6i+Z?Fp_T@CGV^GfTvLoRhid@ z5OUrfNMJy-6CB_t@at!d@;?=YM%Rv&$BN(T=SpgfDWzf>Blxvm@!{z)L%zJF!oYVjQ`Sv-X=)ezt-k@=RdNC@b_7>i>nVQK>-sfb%1b-jX`r)$!s<1l_0VERyca{&Zn)I2eP)+$^EzK?L;3zuXd-v(}Y4f6W}qGCTGnviu%* z&I7N7H@1*)f^lEV!qU>x8Klmfz90w29H&&V=c+%mwhnJ7p>1pFDB-N3?a9Pbadh?m z;&;(mE>7zM=(`t4a9l|1RxIv!ZblByO})BDf1cSfSVLV0|!9a=Mn*?EdD*MS^sLs`@LAT(O< zX(yoKYAImFXfWi^^_k|FpcysmIjJV(`rH;8dq6>IT=L}6(K@4T-sa^ege|mWUk5)k zv7>gk97q=ML?%^ijGS!Vy+NEH~semakQGW2L+eUnUYbIv;6R{4MvO|oaHzM zYIiLgW__U(7DWA#&ph~3IN?%Oz` z3J5sFYMqu6DG!~G4Tx>6jP5h)BBz1%_^hUdBJ=9&7GzEPtcE>};p@+Q*Q0z*Vs z;&Ot1V*}%YmpwK45KrubbfG?No98Q^jr&|t(HiF|O78YOTfaEoQq4a7-kgW9#}DGR znYYb64!U`hQ6+T8t!J4V%tlT{oVmn)_?a_2G?1@|5ZATu?Fk3al&iH&)U&xoHHmHh zW#@jU?o(jv{&*cv3qQ6a`LoB+*Yox5bs48njO|3RH6VbRbj>coo()!mkU*PaB`DOC zTUPL)&?D@mzPg$Cu@v7W`2+-;u-eGN*s#(1-3~k_Y$L=69Ey%!shM@OgZn)ssluk- zGk%)VvP?;z;G7syRXA<5?P8T&Smo+N>tE7q$~Oagw-9bUxicD?eBbDq;Ax(-K<}BX zqvDd_KvaTBPcl{O`)M%cMehA(yXmCgpVOiONB~(!(hba|Z9@qD+^9uQcUNTv=*GtM z;L9uVSb8;K)y!Pi83$|$%~Je5=!*kFHS6b*!9aMu+Pil`w`+%4K8LAOW}8q@IkEpQ z?N{mnm{Ca0mEdlR9@@4&?4efhb)u_Ol=rtls4%RzScFI=7>i)<`i$~+HG+GZZW)Sf z!vxfbW~g3+Tx2migb%HeV8{pg-K6&=*EiiPNMCS(mBqZutnH8<*$D%euRh z`me4M^9MkbSAR^~J(2OohkiY-wS~eP+f&S0Xh|}CvH33xGJ?h2)_X_a@hIh}q>y&) zrn?a7n%d`0Jg;d>z4j(Y-a^lJkMxtiRdhbD^}-;^5X)MLSS7V+T|ElnN*#nF9qeOVyQ_E}Pd45` z$Yw8V6Uixr(;f=uzX!P=Zo8sHY3C11zY2hcsb+e~;L?j{_#9$o9kH>q-_U}=uOx1I zriMf>S;#MsuHSvtNlkJIB#^%kxbfBb=|tgIj*!*EHBG6tBSJUxk+S(xT8c|<$uL=v z<+(W6P8ux}hW{j!t`w6j)05X`qFn(;rn*Mp!Ar(RRjW`Aym&YDG+J})&?wCdydI8D zKTYj1-U{gZNT=xERY3#?v_2IWsLwrJrM#{MZAR^(rV>!!#4Mmd5PXya-aX*izCdv~ zM5>F!IgVR~$KSbmJQN`bBw;MRW3+>3S}!YEpK3l6y|>=_rlCOFY&k zbrWpfccp4*doEh{S5A$A_T?-VZBT<{x zJY2NPTL8JHHDK^2*>|TC8dXDb;v!;?yh0_x5^KTWuhWbh#+2J2-k=P>Y+i8ZFK8R< zrraxSi*8W#dG(_^-s8mgSg%H8yzIo-(nA{@Ru^~IXpb%gUXhGXUFsk3dITk z@`$R$!)g1wau`?aY2*RdW-n9AJ5-(TTLQa_Xj63YU9{c~hdS_Mwg_b*$#X+>2?fSy}Us*B1Rpze2F;Wu*zon4y0x^JlK*NE_fB&K|~H59&? z+Vfk8?$l^?SKPt{{ky_6wJ~22uhM{|{z*8%9ZMA&r%zdok^2aF9af5ohr#@(NKWip zjt9sbBF6xV!`1Q*t9!%nGZx@p4GbJWd@lz#<{nSNC~978e|S-Bp~xyb}1M&tTSqkcpi0yj^1r z-rWaU)o&b^x-xDcP@#Sf{uzI^fI5zw(T8-|Fub^)%ISV|W#4c-j1^uHvHPVbXiV(< zEf(IudtRlupiV6|^c#f8&5>KGJNYsm#Ql(P36(iYtSC6QG1s(7D!sC)hHXxB9Ncb7?@p;`_XfO92epw~E#c{HfKD-}9gGt+hl+i~= zAlsKcT^by|XhK$lBL|!sgjbkOg9X@U&+?jJq~>hKVgt{c!fQHIFb_()Rng!;WSi@J zd>9MlfV=J7l3eI|Rn^|9zP5~&;6s2h4c2p%0c!$&2C(F#5fmB>`%Wyr(wU}tz(nLU0HBY9ewsvEs<-QO*~ zwKmHpb7ams2|-*lXX&ITzxJGpt`EJiTAPi;UD5uIN_mir>}of~)sVQ%-3J42lxz6U zBBA>%*Y+TMZ84X$REjIztoEgIIfv#F@v{c@mfJ(BLoh*u6t(v<(XDDAsO_Z`bUduP z*;TUmjV~2PQqzG=dR%WYct&R~6zP9%_IHDusUPC1txuFdyS?+F#W(&mjMkS#wXoi4PR@O?zgpMhjQTvscJe zE|RoPM-avIf*EFsr?tzj|FLI=s7>l`=3QkF^KXXUOB{jVW!YapaxDO(Gd5%%OZ^S1wIJ39Qy{nzj*)?LM9?rKmWJVB_^=?md__t&)qH-*# z9Aj#LtExlM7Q`aehPvaAp0F>^Y&5=hk)@3n78mmwgICPD!_NGILvfLL=V>6=cR4u? zXxY_4E7sP{Z2FnD_%*>^8T>SZU$#J<`WmJo`B&GU-2k6Fkm3TsBmQZLrB-jDz zYqIW%ify;uqZKnXjJEU@Sb_wpG9Jc5Z z-KxC9Q7v~;kMlYs!#4!Q*+gFvzy|-6s$;>|G`16GWlBSmUiRSr?D5F23&{H`hcVNB zWrM9D;ru!6th!~QPiUL@u81r#PX#<;KN|aqTqIg`u`P0?i+p`8@x*Fb#QOTKqXKiu z#pSxcx?Onr^~0{=XRm2AK`7dV3zYOyEpF%$Jk1++6y!8XE9LmX3x!)PcRZyv5XuPr(|;pm+gIj%vbR(5O|PR z4zk9tiVtphl0c!&R%OyG-2BGsBmD+suE@9wS*qoWU<-RxfpKp_{Mc_H)@jL$?hieE zKl+hw)ap6~lUMa-Pw7vC1An-V1uUXJz;l?xu3Rv7i@+$a$3ipx2yHgOi>xRL)pvU< z7vYA4!CS)x##^la1-c%MP$iT6B zrEa4Gei!Ram*L%)wn2BD(DrMXv)ZSEwwMxf7mz3o(7ec&z$`DN}PG92++4nrZQNkKsn6!2jf zIIGgc1Zyh7)S=J>*L!QQLh;}2t-a%mMty=z`?VnEJ=m>;bqU-HbR5OhwGww(2X22&@TxA_gp)Wx-JR)f(}~ectjqX zpmn6w-mWSJ`BeoGqE=GfEUvhf;k?MQB>eX4RfMIdIhlv&=ibw2$G1S7|C79S-pSG-&$e*J2T?V?M7vfx zF3(q;_H3c|+$JFEM0LW!a*Oe69uCvnCj;Fu28ag1;e`?R<)Fc;<@W(J*eqAjIya}I zrhJG_%l+=sX{dv_rQQy;N6(yVvrOC`6#T4MKpX9{wzH&HFFDkC-5x$;N+T_0{bVsx z>g|PA-r;s2+?*(VBy#e$qWW~vxxmDO1kq31 zt~457#9 zLp~DT7V)V!R70*UmEQRlFeQFs%zA2JhVf9`n7#^+m~uPHdQ>wS3H|hP|-2l*}BJp2KQSJH%Aq|h>4G*MW`BD z&H8Y{`7W2{Qfii#-NTY)c{{Q&_l`B-Dy1G+2&-A}cO;K|uJ?;H#4-rSB+ncHfAEO? z?0O2L(a5<`$4>B8@baF1R?j_a(R@It26B+0#nQq4@(J2{mNq1$>fi|`!>bOoA7KAB zi^^PhH8J8sA_KF261(n7y~}P9UDIR9imd*Hhc>H>#0Zs2R*ouVgehA{Y6_Rq&iZVB z;J}IJqPl=T)wh*e!eb@msl@RRYJTCyVM)3BS=%VDmAc>7hMZ4oOGF@kp>yt3j=Q z6WQCk82=ge^z%wBEE{thYD8Sj*$n@zrxqS7!`Sb!ofc->L#y-7=T0;@aXKfpi2y>o zKoe!4y=H;Z^((uY0JO0z$KJoDzwD*w1t>vGg|T3<0C8G=)E+J1)8 zF(+MH6lsK>Y`xDVa%OpbPgZ4YazX&Tmvys5ZvB>;e!^zBWddqZQp>b7)As57sD(2T zq~W?)WZQIRH;NH+%}fs%ncbynLlB||447j@!MmA@lsdB{V)s6f$`lwX0X6)NBDeag zI_!ePOXfA!XQH?c#GN;)gZUgBL-n)Zd^oq-5rI3U%aWnHQETB-Buh<+7zh~)w$XbX zdQ+9(Ws&wqMqM839FmEJBFRX@Skayv%VJ$qIVKl@UNV0Eo*mt#+ z^r6XP-32&Tc0Rofcb)5gvgB6}ZtBju8VjatkiE^u;r1Ap636{Zso%_dmEc>WFwqFM zndhJHEk0`tGC2dH(a>LTGx6fY-KxM8UwVj0Ihak>Fw7f{J z!0<|64$lKOtm_p1lIKYA$}QbFLe8=9Gh0KQo#~5Las87!ClHhKsbch{_QQx4!lAanLh-+P zMozVjONnBazxS40WL||GeltdcbEYM3x3oRdG&j!BbyQUpt@)8ni;ySBNX*y-tWRr= zJw1CrK<9Cb<}KJTTtgNrWW772ZWAwoQ zMqsSS;xoUG(vm@u*{5w+{#Mkn?V4NYA{j#|8;1M6;Lwy)l@Q4I=mXVlk-|%!y7ZLc>x}MF)w*=`~ zC@Gt^YM+#V7#)rv?$)P!7E*7L|3dhEMg3>1H<(DavkZfLMnB@Ac?YQ;aR2-iqI6!- zG8EyQV&eHp%{4WeAZc@tzskQ!c+Gd%1Uwq|9pYOl?ztU*$Kp{)+exRlIcFTyQ#-M0 z;<*wa^so9#7lmCo&b6p>hI z;D8`6L-P?N?JA4havN}0&z)_uiV-8ii0(&qVMP|4Xk6qx^{qYL*9~}f_#KT3c#oMFvd9po+Kl8!7^EQRcSBzh z>=~$9DM=J)e`4*^SqBAc%zq7MX;{KFZ-3VQNKv?Kj7&;9wi5kR5kFR4ve*8sGuYOX zn5lS{P^6+g`=?zOEv^AK@>NGW2RJ})@N?gLB;7M7vt>p(2@Y-zr?jW( zE(*=Il{p%B660=P^OBL(ot*4Z^U2OQKBtEytMGr$4F10hW{slD2Lg!C;F~cyh}K+8|x-GTQ?Su(s28y+Ur>c zZq0Axo7~4}nq^x1G+r9HwANDLu}c=wOSnu=W+&OO93Nr3xssBF+v+o_Bd24%!AFUa{$$*kelST5&qKO`~LrM_tp(1ywfiP> zS1a0t-4$6{sjHG_Qt%ooJH#Sf8iaxbdPw0&o~No&7WxuJNcGZqt~x$nUJs4v zI@5URz!HHodq4Q$gz1yZWmxr-8YNOl$jdExpFXUg*gmCuxUW>Y0(F>zaiy-rM(-M% zmyRA!!8TePP3FwD&C305H9@C9k{BvYANxCBGjMnl=`^XQc zPUs6VxwTN%x1z%`f>u;`hf7D>+Ic7+N(f}cP=N67p6DN}4c9!Kg}rB2a5Hc&`C_(f4d%&WNZjv=#F?W@eq@SzGI(&;l;5TLE=Iimdi|zB5Lt zc~ucYGj|?x7vwi5Vo^<;Q+eZ4vlj_3=OV%xVBnYUQ>KaHN?o?d>-p%;mfpg+zr+Ji z7UNcpKm%kwV_#YH_?lKmFm4c631}!(R39!7892`xBSwc@7WM&WvGZ%a{a;^Y`980# zhLv(xoO$P@?uzN$?&evyb?bQEn*iU%MT4Odk(zb!`)v*W?uc#%a4xbhrSIRv=(ow| zxO4#v->-j4_1GkpcgaZ4;)`{+HV0`X>^@YlAQER{u_fj)g~7|NPh03#>=||iV(YKa z_=z0^tGb);##QuPpFxf12+DKX-Cs~}b>)@<2l%XW=VxLd)O(m8S+Ueq?NPTo$&*+D zys7vHm;ZaHR%@j#6`I7@aH4GvDJW8j)kx+2+5`93=x(ZOwZ+vL-`Ry^j0AT(XqlBd z++!z1A5hneQnyqU^QG#~B`&$7s+ak7Fcm>%$io&p64o3Lzr=AfYa=R!Ch z5EbjHQocPJdUsg)0z5O6*z3=>;39PuEWd@~w^A~OgnBib$@jFii)?{>fN{fswU}E( zwq%W2;+OVDY>;W5SO30aBLqPb_{5wLuyTHXdu@!5%a>*1DDU_1D<)3)d_ZVwQ={>< zYp7*A)5){Y;~H(hwA|Nv#*$S+0XoWc`zOR33j8(|IIyl34`xDa7^3cu-S<=LBO&!- z8)1@vKlOcCf@Ricy2}J9(qWxVi*bw->SjL}TG)sLlMTDl)s#Mesv%&eI(@kzw~n>v zoFsmvGqPbA)QIwtGlYyh%7_;y;DcKG==u2d?nzhi3??-)XB7dj3$hI+A|hm-29-?FO;&W+O|gaNPrnmTEA^w*TSy-l&(40SP@>|`5AeP z2|n*$zV|j~;BU1sn26Ih&vb-ET|&iR1C(fA{gx!iEMDOg61O`icX$+vDKnPLd7p6bVV;$SB>9 zS5e(`Z_!7JdSUhu8~apHC(8`PZ+yDKr1T5(m8c(B0!#Twfg^f+*z^VEXd=XG=D1Z- z-?_U#5bqk10Cc=hGP{7#GO`zVr}pgs4f4+RMv!6|<_r-?4az_k()@c|P{^oH#%<(Q zqI5p#|08`Q^IJXd-0}7MA&2#w);2H|1+LCr4{qqj!Dab#jgYZezjx{QJ$j3&)h$&l zBz&wOvd&aKw9Et(Oe#p z?(|-wu8KA3(4Z)J6O#bDl?7%|1tfT^L!k>O<>O=tpOCuIG1768-)hIel{BmA!9GBsXGN9sx=WpyPZWXudT{*^*GDTcEt zx9Om_dlG8O+uBBPhkV~eqc_sgDpIgWmUZdb6!srGNfOv>NWjkzzu^N=)=9!pzV^ru z38pnmD`zhFz6d`HwSKyJI~q-fZo>yjZE|` zkdW3rTGKB2F8(iv7@@0CxjEpxeAA`Tvwud#{v|Sq3p^DC_?@zm7-RN*k@{%-J#xAJ zF&yKz*wcGdjneHFT^I@FNZVgm?Fl%~E9&uH0YCm!j9cklz15EU3YpYZkA@08zs`_l z+>O?(3>YPH=Yf>gcb9@F*NfKwim@#y-16pQXG2qSB>h@OLJ- zTKk?>)zS;6xsW+H(n^1&XhCaB4;VR3H6m#})S8edN_qV}hz;4So;2^(B)BXn+@0?--A@- z%03CNq+o7>5R9m|Ve7sIYu$F-Sj&D@4Px#AdK4!&$Mq_l=rG_(Z@d%+KYsprWAD`m zzh5I1UFF3saw0U{Viy_3@3VeZ{Z&rG2Si@_S>B_Xb-1ph`AJP_=Pu?_XJ5W2rg*(x zakF1EG$}trKk~lgEiUvU2%~RGwaM2jAC*#Gl;`8a1cn-7=h0Axh+qpMTfv%nP&HKm z!yO^|=eS&eog(_@F~t{b^@hzyeBuuirx5oT-BIJ2O(Q+ zy4sq-g=!|@9o){{{+$#1)?fs)UQZi?gbcEUdbs{_7eWjg!O8YCkYs=%_?BPc^ah(Z4m|YVK zuz$cT>b$zrlKT2DW)#SWJN?v$@qz^P9idwn$sN?~95Q2`qyX)fZ4@u4{ zz|#rtR{%qti<#{f`nnY~7YRCAbT@2h_BrBDr*kOb;2k<_Uz2@oUt@Kqt^1&n7#mKa z-QS3`%?Z!y0PJ9Xl7`RxxM7eGxYxm_)LEmx1cLM{DY_?hvSQEDgVbG%!ZJ;w(&?+KEi3n~D z7~66kes39u?k$kGoDUBobqi>(?srcP7zhgqM>Z3TB7GQ2AYrq-`#fXs!9dD3ykf;c z!;T9GAWGgn*mQs{9>Q@E0%Tt^3{8iyULIlOGGmvmdz}Nr4Gl5!-$=GCTnf3iLt#DV zBk{ctj?)CS@=`FO;tlW+5pW{!{Q&yyS5kfD`*c3ZUFQ-SFQQfxI<~R=zMhGUAO@nA zY5Pg?q#Nb-%BhE*Lg5oULi6<;>&~_Y4;z%EX>w{ZZm_NX^U6-8`=j(po|l5P;`=to zl12)A8nFeIkbB_9j=(Xd!fY>X-Tp^48W<$i9N5$Z)QKHzckXbPKpO} zGV({^*1ball)0bx*MFRb?~gBdE;o>vzCU|Ta>h^?U!p6o8wl$hgLDG#M>YVHu$Rf<-4K$hm zjkM(ENEPGL*tKNhI+I9OUMNWE@_=AUZO7uA-y!r@GfGYSQlZ0e!5ub?DeE?c6N+^A zfXWgXIMIiVE|1k^Q?Ck@%m;L?cujb{+Qdhd6KgM$NWL60rS1 zr+hJEDKGwup=-R9{x+G$m9x3*^&EYynZDUwq~}Hg%;0g;%G2RHGC;qwp%YImvTr8d zH(OCY%_alWeK}k|iw@r{94?R>@w!)-uqP04=Jh$M6q>E()`-Y3-fT(OwMW;8{18Nw zJPBMyl7X}9)OQ^~H3pO&&BEbLLxCFp8HvF~HpvdBV)A|89~}DCW>aDosZIPO0xUt} zT7A@89iGFk8q{lkgC060)K9e+X%&+L`Ma_Y-8x0yFY^ndtd#N3s~Y=8M+_b$2OGIk z;9n~7XG4tR9;!`R@%hqfqKlHlU?p&V#C>aO$VE!uSMR@W49lImHsMOplIOMNU^X97nO%BZpN}abwez*-Vw}9cqR>2Kv@U@iL9lXL2X$UqE(~B7Zf)gckriL z-7{iSuu=xgVB0_{yRvYKL6SPq8AoxHpDV9UCTQ zsd{7pdU?JJKq+TPDcM_xVz_V!5&s<^HX2Ra(tc48p5iJj^}O+=Mr1oV)%SPcIL6oZ zO_DWqhgxxietFgiq3P>pRjMmSi<@%i#-qy}h(DL(=oLxf`S=g-hM5Z@0dZ#KPOcCG zG9i4n^sDufmVoD(s~H9+c)Lqd zKg4yEyu-JoNGN--hhWb~+1TD28vCQGvV&CQ_l9X4fGS_8R7n4!Mr3$kA}>>dZ)j_! zLR?h?`p2x4jQZ^L+h;qagR$#35s3yL`1&qTWN0rNqa)f8@+gAu0;oCQ&gqEi8j7~b zj*n}0W^3PkTMTm^x=wxnoT%4|xDRiH1YjLS@Xs@u%i{a}HkX4~j6S0T%}sk)Ef}mq z*r6;*vNgiU3*FA4q6lJOMS*#Hmk{t)%(~zhU2PcGhvt&IaUQNQ$IF44i`M7B7xG3- z&)YF?>{qy2(6kJOXQt$UVGpkYRCWG-j4m>8J}`)4E!4RjW{0pIVv{(;< zZLEJ?v9qr^P}^u(uWD3LgG0-++&xG&gyNr(i}uP1q+`W;_^NxuI=5{nqFNN8@W7(l`bG@l@jU!_a4FnXPl&Hc_x ze@>aZ1Cr#Z!K!?5L*$<(RYL^}K*qYRghh%*WcH%XBgJI92$yCBpR&(KcGGoz7%?pD zljM-b6%Rw^trR70u{@VPY*(3bbjI^!F>(uC$B;}eh3=eX2XxBCq`=8T5Z}SK`JYnn zHkYjnnZM!u3D`G1=zGYI+4yj4XJwc@Vw$igb}Q2Vv!?fEd$OD9o7hj~g_svV`9%== zc|EIS7yOZ^y);{FYZ;b<#MWqgVHn~MkDWm$M1M5;V+?{MD%<_?T$%-4+eZu&E~GEI zCZ`)Yq-Q0OEf|L$vhW}KR_f#+?C!tkhX`ev)yq!ZKJ%l*+(hEw(?^S+?harnFF=L5 z7MJpd{yrbQWx#O^poGNFfRc4&f4u)g{7E;Uf2J?b>!*FghT2%I!d_9(9eOq)h|3AM z2}BX3&ChLM__lOyoIs%y2G1V(R`D>8li|il1tfz}Koz9*eN=$lGbY`VGY5ux!t=Jn+U)0O9SWQOxp)2h3%SSNUBe9GC@zvxckXU zWJ_QgD-!MPc-PuJ##D>X$WOl%1jmmFW)EM16b)ktg7O+*`td4vU~{6AMO(h`mav== zNJ-EWy|1#0kl*5-6l!C4noUFgnuCiLac7+O zm-{j!c^F1E_tB1Bq2N_OuTVvF}s(WN)0}k{yuQC0prhd2T@X-@oQtwQO7>GD`&XzWDkAD*8^0 zWe8AV38PL}+W}6Q3C-0b!kq#r!<^UXC9gJX8OPT&6E?Jh6jbOF)X6gXFS;SpQlA55(a|`LaGlC<8`- zP{rAXAWT@jz!1plNJ)W{em8egWryHz_w%ZjzQ_Z3pzi<816Lq5F4h!PgEK@Cyn5tOSOhe7)uYYhnJ7 zPaCnYY{T{mVk?)a;|ZkXrGd64n2;ks8eRa!W;2Gie_@PS!4yXAr-0|%0wRN)$e+b} zMs%p0yO`VaS&GKD1-G`E1mDwl8F=){-pKQ%+M7TfB(Gk8t4-#=mS;*Fj{6; zb=A@V;t8|Z(x8gCuy|%%zbuy+4D6Gf2Lw}icBE&pu)XMgPrTN>2objMDuz|A-DGviZx|C1UOpQ? z)PHb}81|R(4>oBR-kG^6qKCJSHT>*8u4k#0_~Oo(GZeulr!58Xrt|zln~}g${<$p& zuG%CG>&FCcLWP49&9@alncbw-h`%9`P;}o^DL|Fex%M8nk%S$3pqbogPe5lRWQ}p^ z4En#eeCGFl8fCt zG*k+K-AqWjAAcNMa@~|VT>3~#7 zQh+xX$v#lyQv2k%jhdN9UDLlWZB)CpLj0y7fQ%k`SPw9UOLtwV7{xO z2%rLCC*{Dv_>wcUJ{Jr=;0Yk&3Ptj#z|~3kr&L~XHNB9xBr9~nZL+P*`bNxfvOy`Y zINQiDAU-We8J+LyYv4hciY=k@Q%#5yn5kY!U$sp<_BNDe|-HCYH4VDyx~v)}iCQ=TMn)JMa& zM;Etla|FPi@3FAty5fDyIbV5rWUE`~3!)Bk?eHj2ZZ_7BpxQy1Z7P79EJEXLkn=1H zAbQaopVKTe>=+d%O!^B;St+DAj{FY;NfSX>?z5LJsR2-5oTb)+;7HV_3+jZUHavK3 z>I1U$+wl!gYsW}om?r$eQ@82e9w0GW8g>PZkZAb`doI2#bXCCD?2VI2l~}Yl(5e<4 zQ4afzoeU7)k8Tr(f50*51`~qkp={2(jnx%_wEpSJdLS?au{vuIp6z3rBp6^qnDff+>wD{8 z2Cf4TE0Y%qlvm>8!{95$1XFv|M0Y$I5whm2H>~JK6tcvS7_^}m>e0hG96RjOD`I6j z6#yl}IVPQ<1MBo=_dUSdG1@VTv7CG2q8&Wo`1sDu0lH@>c^JoqQX6OwDqOP_cm}85 z3|HH@L{z=G*BwZui5MkVU*klKlj)J3I?%BJS2(!W+dG6tm1uLC8pQUD@8^329KLw@ zexVVYxJpvZxeN7)j~%fU4>X3c&?Uf!!{O+VjsmfIdb9v~Z+&eKl|njO1LtI{=D!Yy zOhW_seV=|u;c$%V4id@jOJ!cV58$+>*M-i$)}iqlwu<2B)x-@Y%(#pYQWn<5Mr&Q) z)`Wk6ilK{61*mpm`|hmqw}$)Adu5((lO|x1D-t|t?7YEqKuwto`~Hzx_aZi)!Wa#k zr?{Mz_J+K*R5ZfS%|o-vm0UM*OARA@*A|)svt_sRam@yV7M3o4e@7YPmx3BWH1Hjj&X|(VAd+Ewk3k0r zruX%+Mf>gYlGq(d&kx6`t}mpvx#zT_>GtcKKQbLqWha62X~4C)LFR*Z*+s8OEjlz+ z=wDYb&LF=go}}W(ym~T^r9b}qAu-qoS0(NHS7XS-VYs|DW63&h;U*WL+kXFnO zyZ;oeZZ){39oXmk^_PE(|ESx|_{~3)P*lVM-i5@Z9aUi*cb!w$#SS2=bcgcDvk4$4 zPo#>zs(s7-7LuKn!(Z}VqEGtxSN`cPVeQ-=@X7WdH!*tOT5Cm0xSLkft$EH-wk3gQALjxKQr&mPq+TTig@09Xdad>B16WCe|^m(Hpqy$z;a=hm*eg-E`sSb zbHk4#FE`@@T@PQbb$9LON`1@6Q-tlH$T{@x0A=cjud!6H>{u|A!F;Hnu9=euK9Dwt z)Mg6|%YF3DOhcgX9S{mBGkcNJaMo*PHI(p3h+j?ZxvWcDJ&hru7z+!3S)QDwa7~d3 zZ;}_!<{Tu;>kDuA)Xl=Ew2XdQ>-0z;+w#Y4hqL_-t#CQn36UuEH=lBw>^->I-r-SW zByj{{J>-}RJp5`kYEl24v^8~=q|05G$f}nKmyQn%iI+a#LFa~``Ipy-@b=gKkwJ5L zIUd0jlSG+l!MzzDU=JppO5(Cq@|4)Hy>w9*!dt|D5sQhgA6|~k9okm$0>h>@p_{nr zu5;^6-X&7?(TkY};pfURAPJjh|1&Dd!28=j#PRL!xI9fwm)sC+JUh(naWhVM%Dni1 zfD9T6JnEMv^pFXSr$8fv^T@t$(MB($o=FzneAHsLl@ea2T>@zgnmL4_kz#3s!R&tK^?S@vOg{S*b|Cur2?{I@06 zDr_t~0g0V|7Po`R+yY!zD9OFH@qTtkOkSyEg6!!bwd&w|n6kFTbEgix%PiY*x&QrrIGU#Zb(xmPZisM)oFHU_CkU(z6r&+q|F@IGg71 z!1Cv`k(@U)Vm{Z1pg7ODA;2_E`IWPQMYk>D+Jo_P3T`3Ea+g8T=5N(6E?HUcXa5HgG6r$4ZclH9j(D^gpZF(I#ncQE#n#$*aBX-72CU8>UnZqV|nM6KBd_|8teq~|h?bgz>Lg|jl@>gvyc66{BR-48jAcJX#fE}n9_6JcC zHm%Tp(+hG%P&~Y?D$vznMK*c#@K0-~qsjp~TLSZFq;F|1&WI#uG|A-?TD39(El@$P35!#AZZTt#gK@os(E~C0 z!k#D=mBiP53DQn1f-C*3iZmK|{J#5NxzO1GNa?Y!AeyDBGfJ3RO;a}ZK9%e0y*&RW z%38ez7tN#E5=sDLseF?oJ@F=52jFw{t`CRGnK|Ml)&iY05{ce#pg(qa22enMFLjjp z!Olpwx4)%VS*eO5N1a7;`11>rErl+G!&juq^lki~;bJ4xd=UT}$esm5VrD?J&aV~? z8$&)}pW`$3KRM;t-aEfB2dV-2g5VR6lZ*V5Kwu)YmM&n5LcU#DI)Q-O7pyY)0e{)h zJ4{kR5-FJFRw4N_Pa))GUp9|+@}@W=l7rNFZdRSPOW81tSHg_`$M$C-g!!EU(!R)5 z+uFU%+AorZdulaN*nAF%-Mv-Gb7o99(*>u?XDx5=0Bs@8nVEpKYkR}aCYBj0-6y01 z$)?o=ec^;n{)^Tq)1%_ zo&8oj8fH;RcvYS6>Emw2PUpeL^&>6}3H(Wwo-HOQ)F{(KT%AqdTe|De@y;@(Q3%7; zPA4;CP^nF@7`)Anh&kAC(X}R^`(s-4z+Be<_rsPP?lQOW9KHWgHY}4Cv}8bhh+eof zy}nq_H%GNkf)mqGAEw=Deamf=2Bt-32z{=p_BDtiUG;$SD4r|+I9hil^K)&E)W*qY zbdxN^M{H$Xq3DAA{^;4sEJeHj4!0>_0MtrhbIJx30GqgWGY4KhI=5i$qj?ceS|`ja z{ach|wv(V1=kG}mty_t58uuEhk?1#A`CMD?S zGY%+A)KCm2E=vg|md)gc7K^gQ?WT`191kAEsqsy(Bij-{#dPB-hlHNVDrKcgg-j7E z>|*#j!$R=?Gt0Tj@y_b17qS&e{~7(q+KHmm%&0_Zw8A{sZew3CCx!ae;X*Y_IrG6d z2opjzt)QHZ6;Nwe4{%qZlmJNluu(%^`$@TF=Y-~zKEi3KhD1ioq8Zcql9iqUSsy=J z{UJC*Q%^eM06kV6Pgx0(9FM}wIV^(kCjq9E{*4CzG8XK<*D~clO&BZ-oj_1Jf-<3M z*J%q;Ac#E)_zi3{O27LznG4mWIif(<6l-ixP1&CEi~WA>)+*7<*X1m|DKEifoaEb$ z`o;cogK?!28r(E3l#_@PMEHOD%P=(&0OKWRd(XAqST86TCzoQ)1lzlf@g|ZPWX}8F zwO#w#A73$|bz+JY_-#3{cR9uFcWP4Q^tD2}JsMg7+cwD8kI334eCG^Rci9Z`v}DHVtV zYc+)jVWqxA2K{Hs*^colV*fMJxr^YLw^CQUFJB%vaZhV|F3}K3T@jN=pnj;OjV~=z zetD4A;Je=Z#58~$VE&gpdoX-IspuV5+qSAg3x*e^$gUwqic=}vd z4?DD}|BOf~p#XiSLBB^f_;!p$VPX8Zu~Q*o>o9efFqu`|*u!;&H-+y(vJ z@Z3VVC1S7AXUF|9cnxZ(-yUro)q;nCGKNAePFA0Z(AScD#S*3=jT|0IT9=VH%ot@` zoX$EoSIYB@WFR?1KERKg-AauxWx&t%VZ8t8Fk*(CU&5P~uDB)Wfd2}Zl6ZjAk5>*s zRZH&uYts$&f|%*pSASBHwnYz-nWT;SXEg_?SUgB)j!QC|xKnMJs;bfZHmKxOS(RHF z(i0~lg}1X}pQwQUJFDU#cfQS6x_kd$3uZ@B`qdcr)%tfG2Vz1ouhZPw#yj2`)rb^G zDzsz7zjX86-5%FZ!zOOLRMZp86`ivEcV!&cE<_R4 zt2f1NJe~0GxB=3_FH4gp-ex3%M!g@~i#>U$pq1-_hi=V5_GeN7S5UV$w)ShNw#hpSxP3Ytk*tOUjZcgJ!#9y`>mwFRYKaO0de|(g zle2khyuQPCmHjjO3mV2|a=&`zc2S0dAnRL>uX6m9lQ}pEBDqoT=4y1Gu#~bBHaI@s zAW6vehl`d~{L%x*y`H*h1OrIo5`6R3KL}iNW8_Bn3XCL=f@LJNts-A6=hdcu3Q4Jb zcbnwVuxq5?|10DCcyS%8j*O8CP=&v0|Cna6DFlaHW{g<~4uy(cK^3EuTOnP=iQ4aE zua9zp5>jg?1hMe1P@=tuSU@A-=tnOMB4I&0=`*_ryiyw&W9wj)cSgNl?Qoun?eQ3Y zv0S6iJ@JPYRiwauC8LM_{0lfynl2;?6+k*TaMO#PWHqVsjmv;|3gmTAg%t8>Wj`;* zqJMJ?hxx;Gs&-}aj2Xqd54R+VGXqe>sSfwMDDuc#88_nJ0<@5qS_5XBe_bo8LRI9h z7-XQy@S$+(2fY_lI}DcSQ=YLb(;xH_4)Ay}39qvI(O=pcg^V^zL=GoUB4!R#8R5RA zDHQ@k=ApH}m$g~)-6RgJ&lU+P_AIpeSALE~Z!_=#DMj;EKXYdQp`~C*k*!tHLqm#~ za|7^OzOE=6ZuH1BHJ?YQZR}wooArwZVEGlHzB5+fd~%*C_iBovM8D5b8@u{aHxY|f zyYgT+85`*H_}`A9Y9xIj1{~tfB1?$sx0%EEV)~^C3ZzWjf{_+)gZS?Z+-Y>w7UV)9 zzJ;_Csw%`aZbHvrXy>;9;FNuFq~W6N*H35~g{;_WhG_jxqw$(N`vCmpuZVz(hav8L zCE{goN8y^f1;axqvWbiZjr!E$!8j30HIFOTz)=ew9M&HDPk=Ho`uQYH<3XS zon?V9eUhn&YG*vue8@((?V~K<8C5*t%CI!XB$Mbr-mT8RYF^Gk_~L{R$b_Ll6-xyF zMO0&Qyz&;3l{Shos_4cqHKwd*JE;_OO^&fRW4bWTh2?om%LW{#Bm-HOV$fGLrBDJnupm4}r{`rj=V2Z_CHtTG@>mmln0pXU-<*Hq$ z%v3%FN*#LOp(<` z;9GXOK%BInjJ|^N)3Q**zaf`9Vx-BUys1FH2J{SHaWSROL>sDjA`2Ags0An=Al4TF;#|YHRK+g! zzv{33zkkSqClJ8xU{BE2nRSW%{NqCE$Ff>XtG|x}k~@GD-~uIIV!*z-lVqkPkVlqI7ADW|C8GQf6+u}v*OsuVLjVVOLm#LEEBA2xI0z{9O-(uJM})DX2aXh8=o^~t^7U>iOhY_Y8->P% z-2$()&4D#vq%|gY_+!cD3xN&Y#8CfVQ6bf+MXTNpM;#N7%im)_7kn7P@pYq+EQKvQ`Yg4(->_ry2^CX1Xh+ zFT9`v7Aq7paHbwIh(|08@JRZmm3?8;SJaBb_SGhGRoxQW+~ejB=CQso)vA z$_D(}hSJ`oEQ{P7c>Q)zpA@WNqSG2;fq4+$67!55jD>7U-;RM8iz)y*y>vaJZ)kq_ zvu6{)wqoon1#=+{u%W8=W^CpaTh9^8tT+VZ$u4* zVte!|##AvHoxU^cc$VhJ@yHiiQdQO;-ziwpQm>+?weH0V2Kc7$geltF@XHcOetMH` zp3P)6!D8pZ(8l%8U3a=JVtYSRfTu?RJ87K}QYP-G=+(fuZisYCQzfp$;Y!5t!|2?^ z+&p-%JWdc|{4I&COHH6ZlMP~|iWh>E*k>1cvYBP~2DItZx&i%NApG(%#SQ%O2nf$@RNC>{;9nx;vpQo1~LOa}`MBxgt@l-!NX?e#-1 z&L1yI?t;6C92?`Un5CUofI3l9$qTHIum)yZMjupzz4S`bOG=!uO+!mwV!Sff{vY{Q z>CE(sOfR4)W=i97M;LWeoZ6h*f}|(|jr}I9l>hZ<^zOLiumyah9KFJ~!!{gf;TB=- zssjJ@XD6rsIKE}j(-Fr9m*?q*4*BHDJ>;2y z{*d0oalO1=$bz>$RXTY?!8->LyDkzhO%GDwc|fUt_}qmvY=lBFcQ9`U%Ugt; zBWSgK31x1Qt6vrObio`;TN+^ZsTtyEfPiPFk~xnZc8*_-H|%m${s}MTn7+uapNPQ{ zB_Ap8u^v(~Gaf`a&2O#&`p&AU?_K*w&>)Yu@`3IjD-#6(*z?JV;ug%WVg9zZq(aJf zuiI$tUnUBY?8a;1?IsTf7H28aux;6g%$wzr3Pb4V>HA|30%gbc*V zlwbAF67CAYeuu#}xthloIQ9MgG)b4Q{Fi&KLhvq;#bs$G5c53>f z=jm)jsIW??AdQY z`+^k4?XoH52v z?-}U?Wj8f_M9qu?B6txV1cFHfaA({#b0n#xxCnccD2x{z_XAX?#u$c2dIU>e>7>%J z2zmvdUg5sSZcSnD zLm;~vog5cQ#5SxL>Xx-iD7Jf`Dd6IRn|%=w^mxSGv1stV>+watcNYP*pcw>H6K?n( z$d;TNnUybpG7=!#%? z@x<>>5}C{B_@eql;ItPqfk17qqXqP1@%w?XEe)Leb5&nPQ0Kj6RC)QQZ}_b8uintS z(C5Tr@7QBWW3=pYTXY3s8K)QcS*YoDrQ!b;{^-L@XI!uGF`IVbwt2uU*>aT?OSAPM z0)Rr&lNk;%suGLhP{V~S=-yrB z-$&3rzAt~tG-2|u$g4+%{?of};g}X4UF3;KDUU$qClDNlx!fDzRU)x};$_;X804I& zbs2q86b3Xnfy*@>J}K)Caz+*zhrug1&X5GJJAiX#YOd{wvP<-EeAe$edrUK)FKaYl z*e`^yAB7Rz9e%rHhMjrtxCdhlz!L=ER;ptY1-Dy7s%}@Zu03gja-io5QGq zA!Vy3k6NDE-bQLnq5W{7>^Ib3-cVh-d&H)Ei*MuM-e%lte*}TsJ+W)&VuhHlKcpxK z6~=y-gY>wpeT!XkROu7;2IuI3WIZLf>Wk#^rp(XL^0W6!MVWRJ72gh>w2_%Li3cQDc4et_~HZqVNat0Yh!ivDp|G~3%+hL!QpTh<@{ zDy=+0BXJcX6uKAVY_+;sN#@RK7waIw2B{DV(&A1z<148{+pkL(2E=(;!t(YCoc*dI|4?-bD3 zv9KB;GgUlsJyNE}5*zWqJLSJ^TbP|yo7?QdyZPM@ z5S3<8?DBoi%MHmx&wCwc0pNos?m}ZX?nv8}ACDHp!QBCwG*-z~Q7_qwv-AHtf9G(s z3cmo@1#IKq;;?TAtBpT=QD#+QO|Fa&P;zndRWG1 zZB{5Bi|vr>b%?%%I=!V%L=v>@nvee6-^Y*Z7!~J;ripkE?IO(gbhSH$Wy6a`E*TM7 zLMrwN4u1`qd2dDK9zASRHN+Y!ycHkGi6(i$(Duh^i1L7N19%SG^8?SwjG+Z&)V>ve zqdv5F;r8wK-r40CgP#U(d-~I61RANrj6h%SDuc!3bF z8$w$-+_69o*F!xUSM;3gB7bMPVAgW+Vf-gzW+M=AUEftFb}^6b*F=`Tl*m1L;MYYB zKDC*NVrzX5a~iHIO;)-mk`Kudr=C1=h`LjJigY}t2+b`;EfEl{$t73Re>X-v1fzgJ z9zkJ%l|&~+9uB4TQ2!>RL|6VlZPWewKmSl9hWtVcm`SR``AGTB+4cLW!E(UH&uwET zkCnvWFEp~&;+qLpGHZ{On}zIRw{Pyvdd>tS)U5Ji@K3)m@(#GK)bMNX+bCq;yAh3$ zhpfblJ+zT-{o8$>u$?6os%)sB)7vgahUf@~Enozwbw1GpkzjU1n6 z2m*NN;WXQFwo$6Q^~>3GED`Q)Xzo+)JSToIN>jPvvs};=I5++Lz~Xn}{KqBYyo&Mo?0dfN97gAr1qz<~{d+&aSuDePY)rUbpt)0FA zU%rfjtdgqhJHMJhM`!*NVZ~d{S8y@cBq&W`${)?1Lp*QH|JT>*YL1={=LwsUA}gV2 z2L~?0t3F}T0MaptvMFy>21i_BRP{c>Ey$sBDO$Dn$JpFBi!} z_*%y<68XGDU5My1xZpD8lA+32p$dfe>?C6eR@JKyP?x%zKZDc9$hGfn$1n78uB7n4 zbFS$CPOTKJKb?Qt+$#)nT}BA+E+Z)@cX9JNJ*Yefk$T^Q+~aw>v)_9+E9rwVtCpZ| zYD7fNPFjC@#L%yDSJ$*<76GYuWFIa*b*ya1Z)-&>5qJLBtjKfOD~y%8I@k7tdQe;= zV8Gs`Otw627bvcmFEa%+m~t*$&lr$6H~pn;f#TI&Ib!uWJ4x}GU-=>^z!Jio$u4<~(F3%m#j3+O=Tkzn~&7WjJm z{SC<4W2Z?}(Ct78iZ_sJigQW={IpAJ4r?Ygf20xlz?grhXQI)3 zvSb|-od1M~3>k&B7Qa;dDbX@>nm~cS>jy$cvRoGq5aKLRS-=+Hei5l&?J4%l}VlZyD9r7yW(W?(R-;FD^k!(ISQ7THM_w#akR&w1q%% zcb6c=io3fNibK)JA( z1GpI?5q=k1QT@{dl{ABloRt1)x15si-_+-)9YBYw9ExLLn{ zP?vn$I2rUB1&Ix^q<(ndpnF*VH7DrtB(}@Rfr2MDGW)v4<^+Ixf zK9Z*KuSy$`J8=pHCV69%h!(7p8JsMNFRO=R%ntnf$}9YB7K);)@gIO<6h{!X&aVY80qQ9KUK9cN{&5+gIe^;5n28;ZER1r+%S^iPOf zkaqk>h^BYh5AVjJsTV{s9>cgm_Z7>0E?mQfGdS4Mf>V*Jd4L+YW|t;&Xp4RQ?S`3;;+b(Ch>q10YD^IM z_1|+dV)5{TIxSXa3?I?y5C@&NEN=*p^OACzCc)Nsgn29XJR#xFCl2n{LzEBI=6|<> zZ{3!w3<)3RcjAzY%)ikC2016@`x%5UEgI z$5R7~AohTneoShMOkNw7%9VU!Po~q}zOaY$fZhul*N+hVTZe!Elq2drhnPagN>*(I zYdWcx>YuVI^@(;l+dIA0_l;lqaz;nbjjt+h?mf_0H&)FVpf=Da4Z)e{K;@0{lUe>K9JQS)^}z z6r=`N`UyiFqT$Z{z|gdVq)@$b6V%0LpNPyX2CX_ zc1Qfzkp$sbOl)dkanWq?%|o3kFno`$Vz8FBy{z1XwX~)S-qB}{rI0^`$ZN7mh~sq4 zzZzX`M0^7otxUx)3yQr;L`?qA;f$2H_bhiZZ=zYSVaFf$DT~}mxKwvD0V`{tZO%T+MmyR!iV`aEL5iE(; z*RqEPdK*F`R^rmT;0sLufGl${s>;w?g>z5VfS2;x1C3mB0As+>szX7YMBGn{6p#RZmW4mp+FDl?wm2-Px8ITIo#+-p}iQ9u$J6#u-4LX4^M#YXJ! z_@hDu!tUP%n5gGQ!oPB8dtY|rD_HRF&i4WLxTihT6R3MN$?H(q-d#z|6&ooJPdD&6 zt33bugzrgkcXQ2na*RsOWdEr+M#AIJ&vbu5alBbebiDgit{Lbf`R6I$pzf1obrZoRK*xc0p0&~G_m zg_yebqF4XD>vK4}zsSh4MCjIoY315%m}CDgR+ULVY}}F|cDd=zM{Lqfxs|jKSP5lw zLRlS2V1DU48iccL$#Rd7rbGoz`7N1`(1-5yyCd}Aesw*t*sh&O+zp>;*)pl=z8Lp7 zdjIT!XSJ>_mR;aBeD6Bm;ULQ353Xj7Kahc6@Cj1sC^0rCd3f_+VJH<3!38p)rg2Y*c`Vf9FgS@o}n^_ zK3)Z61PoJ`*-S7pjL+r6*Ieq(dq0`riTyNB$?w^?f6SG*4829ZjGFleUJFA!gOOl# z>A9}+Mi)ART!-g7@+|@#%2Thae)q)>hCX%R>&`Bu+hZRc){jTjQG(NSTaJs98e&NCW^TSl_1IuQz(tapb2h@@G5Jw=zs zpx+BO7cVp2Q|MiA=nI)%6A}#T9Np&)dyz<5IaATNP#!;%!;hfIhj87Tl2i2m6rc{z zQ7Vrt(y}ntU*EVaQb&?yc27oikFq~D5aEP>;tU;EyS0(PK=Dm4C>VOu$hxMbuf}wE z_pG;);XH{!bPy+3VF5CLGtQA3xbT;^vI1x#Ap>IKk@~U?fq4NJ9V`W2SCY2CXO4z**rT3C}Men!Ld8SWF;efN_}wg%Lvhm#?N zW62QaxVxEw&K;E7uBzIOB~iDv*O?%skYNqp&i0r4E{|*D5W9!xzjc2*6v6HU^iLzj ztlJoI=74G{;I_PqAMu}4y%E2s3#PvF11F0^@gdnzc|36Sp1FYnJ*nVJoFf-_^JRQ*#`Xl1Xhob_8uy%15sS2T3KDa%eBxtT9cg>SBM3{K;v z6rZqYQ48Pv&yAt{O=*;%^M3BrzK5;~C!@6u6A}kDj>!+OZ(%6HSYcaKTd&qa7*aQA7Zdum$Ac%KH8fA5ggENiB^yHU5w=ly%E)l7F1dE;C|^Wvf7d2~w0jB>y9CFFj{flFkcUlwKP9Wd zeCc82OAb)@=^B3*FQQ(_d)>V)sq1pTeG3#(cZe*t$~U%Fo)Br@apIp4S&9<9aF`IGU$Vf=q>j@F zb3V%!dM1&tjzV~Q^iQlY4w?wF>xYgMK4-^kg=u@jQOnNlK_K#X5AL&<+X)PPmjOde z?6WOr{DVtVkFH6-zf12Pn_0of`FZ8aMbp1peQIT1u2G~}krjwlchHb@8TfX(;&VyH zg$c^E@$B|w8XtfBs*{z;5c1?1#AC&E(dEP-k-*Re4g+3hK(6xPhZ^jDp)IVIiA*E| zB6!HVWQ1R~1pZZiL+-=7W%RU#u(X7uR4B#oG0(n~Y+1&`$m7hF=GE6|Sys&VtPL+l z2jBH4hss`z7EN&SDkjKN=1nx&{$09|U9l^pmg$g)lM0}qm>z@->Ah~Dad$9x?>J<_ zA`Eb;ZJ?O=-D7q5_+5rD=0#*mLhnnd$vm#L;9tBMu!^>fxZg_ymG-d2{ITow@ppyP zlp~JqxTi5;7!awm4bj^UA2f^~@=X2k@NY?|8+-5N0!-QEF3kyLr<*XfnH0rx=XV0S zYT=<3AmM`XfOaxu{BD7&_$lNrQ;!OItj>9_ael=k&7Y%VUd11mom8h?=f)BWQw zA|lwjYdQ=J*dL1MI{U$z5?7%8(1%KBmA|TWrA_!kIrXoa5~PCk*zu760>O>*5$K5y zvQH947x{yi4f=&_J(q|Aqw~p4?*9F*2(HvKOPGX({bB#VzsX`u#3$T=_U~WZv>vuJ z9=mXo2n6dX00BZ=9Nff|*`WR4@^(*=b1?x_Py0JkU5P2wX)d{2)xPVBY~x8gvb|t6 zeU7SMnbcDJuV~p?7?HJa?kR`Ndu$GnS2wE8pXo<6(jvSHnXWlY8O)<>oZ)P;4=yOn zR2n7V-xZ@;BjbEb^E%(9Po2l74m2_-lRkZ4orjK{}TEq{4>S!pIJQlDcJqVGJ`T~mD-PiH@ZdtRbOs($lkA@wh3+(dp=@oKO#24 zE2cTpv$+soc#_qO?DX|DI3tc=bo z6l>FYwJCCt8R*r1ilD~B*O!wr#?8X8=bg8s9*f9y4){!GXl;^bqw}uj-%Hlo2^&}~ zw=|$H4{gc}^W=OVCuYe?H;GzT6IvGbJkPUchFbF2^=MBSDzg~#1s~aSf3T)gB#%U&;JcQce?(6h7Jcx!LlsFB~RGa6P%ePD&qR;4{t^$+F@@*i4*HF3mL(S zOQtGf5behlgL)YBA7>~wWEPqjw2KKj&Mtv?O*D+`9PZjCIeAMG*6F2sY?K$iP)B-w ziJ_go2=PBXaHkVSi@TemFuQQ!ZlhqAlz*qo?f!vJg^+;1HEr$uI>s#EXZk1Eh6?P(+y{gr$8`UU z#DE)01DE)GGxQmy_hghP8R=VX&s7qYbmpt!k5SZ2jvY!ggb53S4^^rcSPAk7^#K4C zVrYt6MZJ?T_7#EjY%@{vPkFB!UG0LB$XFaFrRJ!;*KtYiXB@UD5E1j$?+iN6?`~=&o|u4F$=b zgWZ;pRD)(}8Da;uRta~?{uTm9mEOtS+YXOA|EhglSo%{@EZrbPNH zWyJ1wfvzeHB3fc-_(^u0qru3k>Z0q{-M-JfDzdt{KVWdmbbv$9x~ zZa`;C%!-w1*KdwCXJY??n6Yn%%9@Ihovr%2V0E-H(JArNqw=ewFMkkP*ji%QM9xQi z-5HK-P%)D>KX3W*Zl6u2YfN)!P`$8Ye+iDry4qiCW#T-o5R~UwF4lU1jU3+ zqa>RxM2*nfeFyD*mIjt%KU)OC0+0j3B%TEbGcp4T=NjkKdB*k<3URPvD$(HiA*=jk zAgA946_0Gr8&YeG{)e_lUr~n4*|~ zPgZS~ZM8UvLVNfKV_3DhPv#+RUm?)}_0jyj^fg)liKadYAm;RKJxc?c=FDUqs=bWh z%L?fNoP%vb^4^uF(0k-MjBwo90OM$cY;uXA%Ma17 zvThwQyVByyViGNyrsZB1v|^un3FRXD!yo&NM~pN*2zLmusTC$=kzV!s1*H>(2;Ffo zSEKY}uvJUsHZ|WcCp0874sy+dBjJyGDhm0a{VfF0XZsZ8D>{J7GWXlBbSo6tOJrf~ z4oAzkSL_XJ*`MIOR^>d8YJ4SXYP{I5cEAEur#Rv7HJsj)XhdQ^UgE}fMuHaJBTfxX zEnSQqSdtDNV~ru{z)h;+rt!Yblur6RHthyHZ0gx$WSbm_byq|oH671<8% z3ZG?2Udx9glkC`iHW-hb%B>Ei_{ZRXr|4C}{;CFk9Doh4ZEOS2c21Dbl(8US?7Mj$ z?d>hqawzPmGtT-8MM4*|IKeEskgHxFaKme)U+k7^fM<~g*nl&RaDstI^iRS8h~wjO zvgmB|f-ZB#WGqbB;pwLT{|5Vi-Y5RYR#cq+FN(y`MX5vaf71c>t^eXqxh4y{w7xcK z@OKo`BHy+r73{q)Q)0CYfr>s1+zP0}em>JLLjGf3j?jlm@h*IEEQ5!WJx)+eSM}y+rA50M@kdLSqOAK z?kN^%g`n1}#m+bVel+*OgAFot-BB( zQ8Mq>=C1!9JC^b-Y1VcW_|KZiiaHcpEOiZi*k(LG(a6kVB2;>YK%@|vo!QUQBCSe# z3lo&wG^Gxt=gT&jk-52wg|Xi`(F9AV6)E1RfwCIAfNDIEMJG@59(M-cE;ql-veUqBtN8oE4pJ7`CmEALRYiz+MVPJ#N^}Q zl?MiD@0m+DauC@_V#Bl>(Ec_M4&6W)ixDd2UJn~#W$WQ_^fmazYIyA_axf}!npR!8 z$*S3dkE8gNqQ%-SyuFu$!b)L4r!OBI!N*n{d6_Ja2Oi*m6xB4<1&N3hbPD>5dD|=g z^x)kX%Jf)tyeWDwHiEAF@YtOw`kb_yuf+rK|^W%ugQkrMvijqR~Oe0O{-vsmCvHtNqpw zPN$9%B62b>X0zoMWGf__<1keF{iuLE%g(HYH{53#340ANW|SK9Z%-`>e*SxjFp6}j zj&rDzo1zFM-@LL=(nbS?jtdIt>@izzF~274y$>TGr*T)U?mXqbz|W2|x9UePF<_N_ zD8AC!uj~18B_%uW;!=0pW zsuS#eEfM_mwL6Y*X_a4e5|-Ka?%Vy=@L`0yn==38VZ$ejKZk|t*kG!mL7oKX5u0)V z0pisLyt2R|_r^Xk&tN;`lpRa#r~2F;{|yCL-n&re$HNQ$i58FxDi+j!s$=fWY;|fX z7UQp&Ye%5#6SRXg0w)n^cvB5Mc@Xfv;sxEq*FS+rH?~%&DZLSzd~ow+g(113#Ppx$ zF$cHB4VgyH>v)O0jN<9(nr7PH-x5K2Z`IK%?Vb~&`lQ)35apKH6|VuUSax}T%%Q(lLFJW z77w>M`#mq}AqPfu)mGWyuJ6>jaO4O_L`oy-Wfha!cObYM%tlsGxc&#Lp{+gQ1X=k$ zOUBHmu+M3iyTi+(%~4aEufkD|{0@2Nv5z?7HUe1{Tk19S>=WoKre#DlA1f#ZQ9_xq zC>1xu#0mO?65$o0@RcqkUm%}%gFPjcLw*|#>5Tmoi_9-m>+Yv=3Qwn|1mds>Sa|B#Mz?&lOhBNit36F*@IVsL#QKtNw+vMJYJbwyUcUkS=?AToYo zEmUHz*gS2*V;^>eAIgg(15<%aX@I|0wK*+`zbtdVO?HTVMd&92^+(_IMyohxBi%nMrbXDwcW|I$<#{d^`R%T_+$ zW!3m)gKWWQ0Zi^H8%||;>}Y=#%WtOw82h>LM&M7!pahNY!5BI_Pkl8*xP?|uvE002 zFANyww>8Xy)8`B^Q)jCVd3^uq{luI<8@PPBDiZ=IWwmtZa|b0oIhEmio4PlYyj+(t zZWN`+Vf4d{#2GSO|43{gxRM`}lX9cHPJR!;NF~rcO{yK|?YaI;tS@6`N*`t0Y*dux zRo(}Dk^>M9y47z5OLR+5ZH8US&gs&}U77hzqvMKMu1u!Cp-?F$94V%<`UhSen!~b% z36%53vP~)%sIt#2$JxBcgV?V%$@3=lO}qh$JgD?zS0${_0&dQ~j31`t1VMKbo`dSS zrjI@5+Ogdegg=%Jz2-t6F2{qtV?XcE2efv(C92`AR$ z=z7@0X2Gd9QFTePPhIbQuJrB1JoMcX61ppbo#1=m)#YrC*gNbI!5qC7j7W%~m&z7z zV9lnr+XI(aMhEU@kptmGl(ZrMX#G%!pxfVi5srLL8(ic=to)R{ub%4)%JpvX(0t{5 z;|N^T4S}-R9S0lUghx5^TJ*)VO~(V2aRLB1T_j9q4=hq{5~b7=%5PlbB!*dOrI|;# z{ol};F59|{Xd;}h5#l`>`HZ)aQ9iG}TY}`FEF(Ps>k&sA&L44Bj7vM`m6^*2H>3i7 zvgg<1)zdpc#(Vh}6da~L-)}g!x^i>Hw{FA`6WK(2$lDl2 z<1g2mz(x~Ec$!D)fH!{45`xKNTAc34?*B2>^+OVzYMSNOx6?f0~2q7r=twf%IQ*tKQsr2WAi(Btr1J*5`Shj3mcSC-^D1F zGSo8wYM7nT%$Kqftn0E}YrgrxP$zH1lNfz-Fh%CHb={w*d&V1es0$5fdDfce_v(B! zq)N1>LeVCsLcqhGC*xdCa(sCp(Nv^u?A7=77ti{n8PcnP4I<3iKPv|L#4CaONo0h7 zGA(E@Ew`i4X(_BgyKW(MJr7>udy%<+Qb(>60qr!K3~F?U>OdznYlf=N$ydC`q&JVK zd@@3b2G~Q2I^p?1?gK(Eiybx_&YYLQ1VMC;Qxd90mWMBvaPGg;WJW_$?IZmc!e+Jl zi-ljTA$&b|!u4w5RRrglD-T!bo=~{Q<;KT<+_lnjBLS|YQ4L7~4JjU<1a-f!RG6S! zkeHDCNX2ARFQNE=pQNu_!L2IKo!cyM7(qczR;|viAM}~0L?FzWl=Qodeo1$#x>HcO z`fm0x9f24I?aK<<%MTdPWpXX#9@aZ8#%wm77zwMI=dSfXI*WHroo&9fJxA_M&uAD(`;1(25DW-Y;=fPBwiM~f>(JHL zrqy#H;%NgVaxbt^?m}IGYmzrK60urpa>&6GW~M$naZt`)(V|qN0AKMG{^8>DdJtyXwpKyjb+CUkSa+x)q;v{6 z<>qUt?wxxPJc0LPGB$^$l*K1c1D9P*YG)s|7kkHkEOim@K4t>CRhY-t|^m9rB?Qk*Ay|=YdlF zQvpNW=q+n2WJ~?QX>stQ-R9$2+50bRbMX%@U2-nqlbsZhI*&k5R zK!=B%!+ve=v-97?aYl70M1!90_HkP_KlzVpKva)yj>mKsCPBuk*CxScx=p|EGnOo{ zYkP{F!SgYPdiKf%n)MNqgPBhw+B-eH{q}&8g7rq&oZlAic`U-X2Xol&y>vo@TCacL zai$pNthf0Esj}=*PS|E_K)(%xYHMn)j6Q*(FY_lgA|9(Yy9>u%m%Zx*ANL?QQ-hC3 z;@}Jk0T#2+64R=>#IIlDl#|oRC)(l9jN!6j$6qds-+dQbTcDrIJcJ(r^4}3cR#Bh1 zAB#1W+2?@0Vizk7N*tDOwN5=$ni824uV#^E_7oS)x+0a7FNzWQnJrSlYrYl{P-kri zA<7>F+5fU-{fC^W8;7as&j)HCbQ_g{vVp#`G}x|XbxMI}F&SJSYS``Ry(Z>C>8c

|}N?&o(b9FaPl`{QjWSQ#P_ zQy<3KLa~31F<5JbetxVj#VH!j|5ert(oT`uTOReHxlb*@YL&vHxeUw|U$@h5l09H># z9YITl>i1#sRj1s|`1NHU(<#_$iV{ZuC^iwzVwCeTehW2g!*KH(DOh3Njzcgiqm-e3&ISs{+&pkfc zugmWg>BE8OGWO=vBaKVTaR>~Jla51$n=_Kdavg%I&kH#cO1hfZLbSVEveGW!mndqh zn=2Q4U4+~;x@+`He#VKnM2)*2A?OZ%AOHC}MnT(YmB?mXev9SDpZpI$e&S6$O*EL6 z9~Pxl!YP1?pDif*zf7iiP;_Xl#5VO2y+7FHHFtUX z06*@x6lr-Y9Ktx53g5>FiztZ)ycR5O#Al8%s{!hFt-;J(U&gh#aPL~iwETxWKQ;>s zszCW%N&no}-2k>uzp^jNMw#iL}wGeJLUe~@hIvMv2 z`3kjBU?VFMJuBetCSSloH5pGqrsY2X39#>t$97_@9P~~kcesFcM})#!R4Wz5YcMIm ziu1KtY!9}^mb~Lwx30hyNaXEYg>j}Zi+QHdbmKw^SCQ?~&2b}a+}wp>eqg2R;<*Sf zIUZbkk*)jTFG!U2y?R><*ZL1(vu2@8ovbw&nYvSyCiTl%tvOB0;sVV#7htE^D2zE_ z4Q?A#m>3Uk_5Yc2wq*^q0l^Sly_2E$foO@1V8fG7eEujO82||xlJyA!s;yP{A3_!@ zKQvA3cf(?*onkJV4Mep|eV*Sq6Ugr$yzJ?Bu__C#+?w)OWqqEs^VY3GlF`Q#oXh#d z!7d)MVJ2JhXg{OTXF&lu>_;1;QlS{GI_#3FggLaVd^9k{7UZ7hcmd9Bb(prg3YFV^ zlDxo^-_(#7+OfNdX|Pf-^cCL?`j{spo!0*b)fKBH^xKRa&cc!m6U%V`eV#u{)CtB4 zV*xBI8DjEM6T-M&MgQu;o6~dMXXjyk{t4+HqQfu7Q9F!~F%7sdJaku8J7G_<9N{+EiT%|Vu71e+VXn0s zZ4$l+=Y&K^QaA#2Uo4w476ctua4?9?kc7n*U(FNy3e~cX*7Tt(=H0#VYzbXfS_DV* z3(g^{8>Dm-9~kVQ6lw*tw8sOMm4-n7FCP#R!!F>%;&X(+E2ZziiC{Ufq`QuI2=YtW zJ>bt@$i*NG613%?l1J+XoX7I!@r=;nkR{#-x%k`J?8BxoIUA_!aCmWTh0tv zQZV|U@Q)oky~7;bX?2-ga$C38*4E}o3`*Xx=yeSWhCb4sl)pWpK6xz>wd88O&Kcsv z@l!k?CqlqNghbm2{yMh$Q>jAzJ1q#&_@IDA?0{rKYocc5p&^v8A7rhuB2fl7N{5aU z!i-HXp1wQxRYf?wWg((HHR)Hl*MZ>2;}j`<7TsyIS8%yjTZ{i5|K=d}rjre7K zuYC@TW$x9edK8z4U}=gLSb&QxdOuuj4s2+elyiwz8P?1ZSY}bnfr3xDq+0?Iovq!w zam#Hq(e>6}x0=Vw+^!=?u=vv9v^ER~)^*{Q%v0(4*WX1#^2ZKu+Ue}34_ZZzPIA85 z=D&Xd2=$L?CZm76A19o&E!k17aU$cZ3ETK5fEQtDA>{#q zBsop9hL{l+Qp9QhY6Ow9*M&NrVHxk^+c{z+RZ2Ut%>4oC-$njTi=#cBKOFC&qz_NYD?)F> zignHUW}|=yh^F2C@0%l2vyrA~O0ECvbjE?C%S5ZVZJT4qe!fWNb@oFeqCvvl;efaG zgO?3H%{vtRspsk5yq|aDJolwpS9Tv(BuzXMtVaFSBoPoDrS|JQ;HT4ET`0;8DdOu0CxT&hq48fE20#(}FsiE2GU-mxD zhf=d&hy~10XA$styXo>sH>4KAOV+WUyzNu4LcV~;#N2%SiY(%aZ}Tc4A(4Xa;Ev0! z8(WzPva3wHsQea}!29`|*j6+PQ8*v47{g4th+K5^L~kvSG+zFZRfF&@`Tuf?2}~8g zJ^n3^bLxEA|NN>-rUeLG<=FfaLo)2Q>ss3-QhE zB7!us_-FmE>i*uXD>;SU9yJGrZU}K(t@+Y~=C!{|5JWgT_KUcaKT3z9pXsG*-|elA zLUKeL<{Flh$Fcz~9>`Q1D_e4NTKGEAvwk9(10KLY>PQy{1>-&5gd}Ah0@O`QTw&&$ zL_@pr|7GuHEbFYG@qo4TxXinbFju=Nd6nz791TvK#`{UkOzJ z1++AaRUXdNFbNo%&!O2iu5o%e$4pSlMu0P%bu#1p?FEyQTrhvJS@#$6_X>j}VcqMX zFJF|LQm#gQbB7YL9JL_7n-9|Mq?lhwsIN!na0R*@n`~=4ox2kK&p)yb=Jo%6qMD_; zF}afB`R?F~s6A-}Zm^9+9|68kftLQraHxen&x4&Y=9-TtK@wgAffFiXT|GIkNq?6K zKcW_FiQHY@PT5*j=p<@~2vV8<3VJ*0=%Rj)-Zr3C_A8$dMG~N0;Ub|?tvb4+B31If zITH|f2a=|t%>ay`7*RNJUWywSny2iBjp8Q>&N;`L-as74)v#^{W?Qlvep6ec)v`Ux29cwVFD$O3@ zobcxjU`Z4vK`EnbP#;F{Q9}zTk!8&mLv*_Zs`~Amzb z$^?sx--vMn2J)t>S5Ae|6|=`+3HbRLqZJ5BbZ&5vbGl6tRIr9A!P zCzskJ6k!vZDrL4I+8oW1qSt2py^Xw^ny^Lq*yE#Yx-(5l#Zh=ic-H)tev=I1f^0+me5sQ4f%1JJ8d#UF0Qreop zaMAbfV%f5K;mH@5c;%!cay`%*Ve|r$OaWU%0R=)65zJxeCgo{&Lya}YGo6~!%KY<@ z28nk*4$~zLz?vgMpG>20xDLIM~_qM`t2kn+V z|7OHB){cH@&(JEL$g64Gh)Vrl@yfg&SJf$!ucc3ZELGDyfFCNA?Zgl=Vs!@zOe-{MV1 zTrZQ*dSBc3g*zH(lauxPZ3wzzlXi{JK?jRSBQnb;dpQE1gC`rIv(6Tm`Ak03xlTWq zj5P2n&5qi!(!uXeUl~Q9!cGL2hGhN^bZw%5)qYyW!?*WiB7 zz=+9f%It;L_u|^6*fr~cCo>vO;|6D^eqwpUc)&2(0b8bt9jPF%xhhaVLHBDKfVOZq zPqN#3t+@-0AWy0ar5hS4vXHe8^vM*^7E`ZTV_i!~yI5z|A#7ghB>ta^ppAY%+z^;X z-l@9&$EwQn)*S)wzJ}Na9Pa(P$H~|)F4Q@|dj8OmDq=E@PaN*MU|eL^VFKG&0`z&& z(m=+HSqTl$sCq(a(tZ2I`}V##{FT7d&)<4`dMl_sY7+jc=uS(Yf`W;X3**^HNtpl_%T z^tzGWSf z5ufqP7x#tfhx<%xDXeMSZuikA@&e#*c|g&djIQ6NYV>Sx^}L@lhn1m0hjWdth>#w5 zudd>?h3?Jb8t&p(6SyKk8C8jhl6L^T%L(rM5@kQpvwSY_qypbhQHGk8!LKnFWQ3L7 zIzbFSWvA}v4)z0TDF#S_g2CK1C~udBp3{E9{W{HQC9Aa6cZ0YbO6m?n;89u$aaR#1 z;2^OLg1|lO7n=iuNxytxipoNGf?9fJ+}X+ADHZz>UV^ypNWHT3VV0zd#Dm!zzKD1| ziTK-4u5Co8cPu0CE+dKKbP=z*8kJEE6t8habxC~arLBFpg_H(Ton`M)E9FhT4A!IM kHm~i2z3T-sL+>~o@FEFneFWxrA|YOC%35z~6|Exw54mv?FaQ7m literal 0 HcmV?d00001 diff --git a/docs/install_desktop.md b/docs/install_desktop.md index 923568d..2b81976 100755 --- a/docs/install_desktop.md +++ b/docs/install_desktop.md @@ -1,23 +1,32 @@ ### Install DeepLabCut-live on a desktop (Windows/Ubuntu) -We recommend that you install DeepLabCut-live in a conda environment (It is a standard python package though, and other distributions will also likely work). In this case, please install Anaconda: +We recommend that you install DeepLabCut-live in a conda environment (It is a standard +python package though, and other distributions will also likely work). In this case, +please install [Miniconda](https://docs.anaconda.com/miniconda/miniconda-install/) +(recommended) or Anaconda. + +If you have an Nvidia GPU and want to use its capabilities, you'll need to [install CUDA +](https://developer.nvidia.com/cuda-downloads) first (check that CUDA is installed - +checkout the installation guide for [linux]( +https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html) or [Windows]( +https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html). + +Create a conda environment with python 3.10 or 3.11, and install +[`pytables`](https://www.pytables.org/usersguide/installation.html), `torch` and +`torchvision`. Make sure you [install the correct `torch` and `torchvision` versions +for your compute platform](https://pytorch.org/get-started/locally/)! -- [Windows](https://docs.anaconda.com/anaconda/install/windows/) -- [Linux](https://docs.anaconda.com/anaconda/install/linux/) - -Create a conda environment with python 3.7 and tensorflow: - -New version: ``` -conda create -n dlc-live python=3.8 +conda create -n dlc-live python=3.11 conda activate dlc-live conda install -c conda-forge pytables==3.8.0 -pip install "tensorflow-macos<2.13.0" "tensorflow-metal" "tensorpack>=0.11" "tf_slim>=1.1.0" -pip install deeplabcut-live -dlc-live-test + +# Installs PyTorch on Linux with CUDA 12.4 +pip install torch torchvision --index-url https://download.pytorch.org/whl/cu124 ``` -Activate the conda environment, install the DeepLabCut-live package, then test the installation: +Activate the conda environment, install the DeepLabCut-live package, then test the +installation: ``` conda activate dlc-live @@ -29,6 +38,10 @@ Note, you can also just run the test: `dlc-live-test` -If installed properly, this script will i) create a temporary folder ii) download the full_dog model from the [DeepLabCut Model Zoo](http://www.mousemotorlab.org/dlc-modelzoo), iii) download a short video clip of a dog, and iv) run inference while displaying keypoints. v) remove the temporary folder. +If installed properly, this script will i) create a temporary folder ii) download the +full_dog model from the [DeepLabCut Model Zoo]( +http://www.mousemotorlab.org/dlc-modelzoo), iii) download a short video clip of +a dog, and iv) run inference while displaying keypoints. v) remove the temporary folder. -Please note, you also should have curl installed on your computer (typically this is already installed on your system), but just in case, just run `sudo apt install curl` +Please note, you also should have curl installed on your computer (typically this is +already installed on your system), but just in case, just run `sudo apt install curl` diff --git a/docs/install_jetson.md b/docs/install_jetson.md index 33f6ee3..2db456e 100755 --- a/docs/install_jetson.md +++ b/docs/install_jetson.md @@ -1,17 +1,23 @@ ### Install DeepLabCut-live on a NVIDIA Jetson Development Kit -First, please follow NVIDIA's specific instructions to setup your Jetson Development Kit (see [Jetson Development Kit User Guides](https://developer.nvidia.com/embedded/learn/getting-started-jetson)). Once you have installed the NVIDIA Jetpack on your Jetson Development Kit, make sure all system libraries are up-to-date. In a terminal, run: +First, please follow NVIDIA's specific instructions to setup your Jetson Development Kit +(see [Jetson Development Kit User Guides](https://developer.nvidia.com/embedded/learn/getting-started-jetson)). Once you have installed the NVIDIA +Jetpack on your Jetson Development Kit, make sure all system libraries are up-to-date. +In a terminal, run: ``` sudo apt-get update sudo apt-get upgrade ``` -Lastly, please test that CUDA is installed properly by running: `nvcc --version`. The output should say the version of CUDA installed on your Jetson. +Lastly, please test that CUDA is installed properly by running: `nvcc --version`. The +output should say the version of CUDA installed on your Jetson. #### Install python, virtualenv, and tensorflow -We highly recommend installing DeepLabCut-live in a virtual environment. Please run the following command to install system dependencies needed to run python, to create virtual environments, and to run tensorflow: +We highly recommend installing DeepLabCut-live in a virtual environment. Please run the +following command to install system dependencies needed to run python, to create virtual +environments, and to run tensorflow: ``` sudo apt-get update @@ -32,7 +38,8 @@ sudo apt-get install libhdf5-serial-dev \ #### Create a virtual environment -Next, create a virtual environment called `dlc-live`, activate the `dlc-live` environment, and update it's package manger: +Next, create a virtual environment called `dlc-live`, activate the `dlc-live` +environment, and update it's package manager: ``` python3 -m venv dlc-live @@ -42,7 +49,10 @@ pip install -U pip testresources setuptools #### Install DeepLabCut-live dependencies -First, install python dependencies to run tensorflow (from [NVIDIA instructions to install tensorflow on Jetson platforms](https://docs.nvidia.com/deeplearning/frameworks/install-tf-jetson-platform/index.html)). _This may take ~15-30 minutes._ +First, install `python` dependencies to run `PyTorch` (from [NVIDIA instructions to +install PyTorch for Jetson Platform]( +https://docs.nvidia.com/deeplearning/frameworks/install-pytorch-jetson-platform/index.html)). +_This may take ~15-30 minutes._ ``` pip3 install numpy==1.16.1 \ @@ -57,25 +67,35 @@ pip3 install numpy==1.16.1 \ pybind11 ``` -Next, install tensorflow 1.x. This command will depend on the version of Jetpack you are using. If you are uncertain, please refer to [NVIDIA's instructions](https://docs.nvidia.com/deeplearning/frameworks/install-tf-jetson-platform/index.html#install). To install tensorflow 1.x on the latest version of NVIDIA Jetpack (version 4.4 as of 8/2/2020), please the command below. _This step will also take 15-30 mins_. +Next, install PyTorch >= 2.0. This command will depend on the version of Jetpack you are +using. If you are uncertain, please refer to [NVIDIA's instructions]( +https://docs.nvidia.com/deeplearning/frameworks/install-pytorch-jetson-platform/index.html). +To install PyTorch >= 2.0 ``` -pip3 install --pre --extra-index-url https://developer.download.nvidia.com/compute/redist/jp/v44 'tensorflow<2' +pip3 install --no-cache https://developer.download.nvidia.com/compute/redist/jp/v51/pytorch/ ``` +Currently, the only available PyTorch version that can be used is +`torch-2.0.0a0+8aa34602.nv23.03-cp38-cp38-linux_aarch64.whl`. + + Lastly, copy the opencv-python bindings into your virtual environment: ``` -cp -r /usr/lib/python3.6/dist-packages ~/dlc-live/lib/python3.6/dist-packages +cp -r /usr/lib/python3.12/dist-packages ~/dlc-live/lib/python3.12/dist-packages ``` #### Install the DeepLabCut-live package -Finally, please install DeepLabCut-live from PyPi (_this will take 3-5 mins_), then test the installation: +Finally, please install DeepLabCut-live from PyPi (_this will take 3-5 mins_), then +test the installation: ``` pip install deeplabcut-live dlc-live-test ``` -If installed properly, this script will i) download the full_dog model from the DeepLabCut Model Zoo, ii) download a short video clip of a dog, and iii) run inference while displaying keypoints. +If installed properly, this script will i) download the full_dog model from the +DeepLabCut Model Zoo, ii) download a short video clip of a dog, and iii) run inference +while displaying keypoints. diff --git a/pyproject.toml b/pyproject.toml index 0566071..0978593 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "deeplabcut-live" -version = "1.0.4" +version = "3.0.0a0" description = "Class to load exported DeepLabCut networks and perform pose estimation on single frames (from a camera feed)" authors = ["A. & M. Mathis Labs "] license = "AGPL-3.0-or-later" @@ -9,10 +9,8 @@ homepage = "https://github.com/DeepLabCut/DeepLabCut-live" repository = "https://github.com/DeepLabCut/DeepLabCut-live" classifiers = [ "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", "Operating System :: OS Independent" ] @@ -26,18 +24,36 @@ dlc-live-test = "dlclive.check_install.check_install:main" dlc-live-benchmark = "dlclive.benchmark:main" [tool.poetry.dependencies] -python = ">=3.7.1,<3.11" -numpy = "^1.20" +python = ">=3.10,<3.12" +numpy = ">=1.20,<2" "ruamel.yaml" = "^0.17.20" colorcet = "^3.0.0" +einops = ">=0.6.1" Pillow = ">=8.0.0" py-cpuinfo = ">=5.0.0" tqdm = "^4.62.3" -tensorflow = "^2.7.0,<=2.12" -pandas = "^1.3" -tables = "^3.6" +pandas = ">=1.0.1,!=1.5.0" +tables = "^3.8" opencv-python-headless = "^4.5" -dlclibrary = ">=0.0.2" +dlclibrary = ">=0.0.6" +# PyTorch models +scipy = ">=1.9" +timm = { version = ">=1.0.7", optional = true } +torch = { version = ">=2.0.0", optional = true } +torchvision = { version = ">=0.15", optional = true } +# TensorFlow models +tensorflow = [ + { version = ">=2.0,<=2.10", optional = true, platform = "win32" }, + { version = ">=2.0,<=2.12", optional = true, platform = "linux" }, +] +tensorflow-macos = { version = ">=2.0,<=2.12", optional = true, markers = "sys_platform == 'darwin'" } +tensorflow-metal = { version = "<1.3.0", optional = true, markers = "sys_platform == 'darwin'" } +tensorpack = {version = ">=0.11", optional = true } +tf_slim = {version = ">=1.1.0", optional = true } + +[tool.poetry.extras] +tf = [ "tensorflow", "tensorflow-macos", "tensorflow-metal", "tensorpack", "tf_slim"] +pytorch = ["scipy", "timm", "torch", "torchvision"] [tool.poetry.dev-dependencies] diff --git a/scripts/export.py b/scripts/export.py new file mode 100644 index 0000000..320ada0 --- /dev/null +++ b/scripts/export.py @@ -0,0 +1,81 @@ +"""Exports DeepLabCut models for DeepLabCut-Live""" +import warnings +from pathlib import Path + +import torch +from ruamel.yaml import YAML + + +def read_config_as_dict(config_path: str | Path) -> dict: + """ + Args: + config_path: the path to the configuration file to load + + Returns: + The configuration file with pure Python classes + """ + with open(config_path, "r") as f: + cfg = YAML(typ='safe', pure=True).load(f) + + return cfg + + +def export_dlc3_model( + export_path: Path, + model_config_path: Path, + pose_snapshot: Path, + detector_snapshot: Path | None = None, +) -> None: + """Exports a DLC3 model + + Args: + export_path: + model_config_path: + pose_snapshot: + detector_snapshot: + """ + model_cfg = read_config_as_dict(model_config_path) + + load_kwargs = dict(map_location="cpu", weights_only=True) + pose_weights = torch.load(pose_snapshot, **load_kwargs)["model"] + detector_weights = None + if detector_snapshot is None: + if model_cfg["method"].lower() == "td": + warnings.warn( + "The model is a top-down model but no detector snapshot was given." + "The configuration will be changed to run the model in bottom-up mode." + ) + model_cfg["method"] = "bu" + + else: + if model_cfg["method"].lower() == "bu": + raise ValueError(f"Cannot use a detector with a bottom-up model!") + detector_weights = torch.load(detector_snapshot, **load_kwargs)["model"] + + torch.save( + dict(config=model_cfg, detector=detector_weights, pose=pose_weights), + export_path, + ) + + +if __name__ == "__main__": + root = Path("/Users/john/Documents") + project_dir = root / "2024-10-14-my-model" + + # Exporting a top-down model + model_dir = project_dir / "top-down-resnet-50" / "model" + export_dlc3_model( + export_path=model_dir / "dlclive-export-fasterrcnnMobilenet-resnet50.pt", + model_config_path=model_dir / "pytorch_config.yaml", + pose_snapshot=model_dir / "snapshot-50.pt", + detector_snapshot=model_dir / "snapshot-detector-100.pt", + ) + + # Exporting a bottom-up model + model_dir = project_dir / "resnet-50" / "model" + export_dlc3_model( + export_path=model_dir / "dlclive-export-bu-resnet50.pt", + model_config_path=model_dir / "pytorch_config.yaml", + pose_snapshot=model_dir / "snapshot-50.pt", + detector_snapshot=None, + ) diff --git a/scripts/fix_deeplabcut_imports.py b/scripts/fix_deeplabcut_imports.py new file mode 100644 index 0000000..e73b4b0 --- /dev/null +++ b/scripts/fix_deeplabcut_imports.py @@ -0,0 +1,82 @@ +"""Script to update DeepLabCut imports when copying predictors""" +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class RecursiveImportFixer: + """Recursively fixes imports in python files""" + + import_prefix: str + new_import_prefix: str + dry_run: bool = False + + def fix_imports(self, target: Path) -> None: + if target.is_dir(): + self._walk_folder(target) + elif target.suffix == ".py": + self._fix_imports(target) + else: + raise ValueError(f"Oops! You can only fix `.py` files (not {target})") + + def _walk_folder(self, folder: Path) -> None: + if not folder.is_dir(): + raise ValueError(f"Oops! Something went wrong (not a folder): {folder}") + + for file in folder.iterdir(): + if file.suffix == ".py": + self._fix_imports(file) + elif file.is_dir(): + self._walk_folder(file) + + def _fix_imports(self, file: Path) -> None: + if not file.suffix == ".py": + raise ValueError(f"Oops! Something went wrong: {file}") + + print(f"Fixing file {file}") + with open(file, "r") as f: + file_content = f.readlines() + + fixed_lines = [] + for index, line in enumerate(file_content): + parsed = line + if self.import_prefix in line: + parsed = line.replace(self.import_prefix, self.new_import_prefix) + print(f" Found import on line {index}") + print(f" original: ```{line}```") + print(f" fixed: ```{parsed}```") + + fixed_lines.append(parsed) + + if not self.dry_run: + with open(file, "w") as f: + f.writelines(fixed_lines) + + +def main( + target: Path, + import_prefix: str, + new_import_prefix: str, + dry_run: bool, +) -> None: + print( + f"Replacing all imports of {import_prefix}.* in {target} with an import of " + f"{new_import_prefix}.*" + ) + fixer = RecursiveImportFixer(import_prefix, new_import_prefix, dry_run=dry_run) + fixer.fix_imports(target) + + +if __name__ == "__main__": + main( + target=Path("../dlclive/models").resolve(), + import_prefix="deeplabcut.pose_estimation_pytorch.models", + new_import_prefix="dlclive.models", + dry_run=True, + ) + main( + target=Path("../dlclive/models").resolve(), + import_prefix="deeplabcut.pose_estimation_pytorch.registry", + new_import_prefix="dlclive.models.registry", + dry_run=True, + ) From 4c07e045121a17a28d7d022c019c3e85934f2f0b Mon Sep 17 00:00:00 2001 From: Niels Poulsen Date: Thu, 27 Feb 2025 13:58:12 +0100 Subject: [PATCH 20/24] working on README --- README.md | 214 +++++++++++++++++++++++++++++++++++++------------ pyproject.toml | 10 +-- 2 files changed, 165 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 311d22c..09d5cbf 100644 --- a/README.md +++ b/README.md @@ -11,34 +11,66 @@ [![Gitter](https://badges.gitter.im/DeepLabCut/community.svg)](https://gitter.im/DeepLabCut/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Twitter Follow](https://img.shields.io/twitter/follow/DeepLabCut.svg?label=DeepLabCut&style=social)](https://twitter.com/DeepLabCut) -This package contains a [DeepLabCut](http://www.mousemotorlab.org/deeplabcut) inference pipeline for real-time applications that has minimal (software) dependencies. Thus, it is as easy to install as possible (in particular, on atypical systems like [NVIDIA Jetson boards](https://developer.nvidia.com/buy-jetson)). - -**Performance:** If you would like to see estimates on how your model should perform given different video sizes, neural network type, and hardware, please see: https://deeplabcut.github.io/DLC-inferencespeed-benchmark/ - -If you have different hardware, please consider submitting your results too! https://github.com/DeepLabCut/DLC-inferencespeed-benchmark - -**What this SDK provides:** This package provides a `DLCLive` class which enables pose estimation online to provide feedback. This object loads and prepares a DeepLabCut network for inference, and will return the predicted pose for single images. - -To perform processing on poses (such as predicting the future pose of an animal given it's current pose, or to trigger external hardware like send TTL pulses to a laser for optogenetic stimulation), this object takes in a `Processor` object. Processor objects must contain two methods: process and save. - -- The `process` method takes in a pose, performs some processing, and returns processed pose. +This package contains a [DeepLabCut](http://www.mousemotorlab.org/deeplabcut) inference +pipeline for real-time applications that has minimal (software) dependencies. Thus, it +is as easy to install as possible (in particular, on atypical systems like [ +NVIDIA Jetson boards](https://developer.nvidia.com/buy-jetson)). + +If you've used DeepLabCut-Live with TensorFlow models and want to try the PyTorch +version, take a look at [_Switching from TensorFlow to PyTorch_]( +#Switching-from-TensorFlow-to-PyTorch) + +**Performance of TensorFlow models:** If you would like to see estimates on how your +model should perform given different video sizes, neural network type, and hardware, +please see: [deeplabcut.github.io/DLC-inferencespeed-benchmark/ +](https://deeplabcut.github.io/DLC-inferencespeed-benchmark/). **We're working on +getting these benchmarks for PyTorch architectures as well.** + +If you have different hardware, please consider [submitting your results too]( +https://github.com/DeepLabCut/DLC-inferencespeed-benchmark)! + +**What this SDK provides:** This package provides a `DLCLive` class which enables pose +estimation online to provide feedback. This object loads and prepares a DeepLabCut +network for inference, and will return the predicted pose for single images. + +To perform processing on poses (such as predicting the future pose of an animal given +its current pose, or to trigger external hardware like send TTL pulses to a laser for +optogenetic stimulation), this object takes in a `Processor` object. Processor objects +must contain two methods: `process` and `save`. + +- The `process` method takes in a pose, performs some processing, and returns processed +pose. - The `save` method saves any valuable data created by or used by the processor For more details and examples, see documentation [here](dlclive/processor/README.md). -###### 🔥🔥🔥🔥🔥 Note :: alone, this object does not record video or capture images from a camera. This must be done separately, i.e. see our [DeepLabCut-live GUI](https://github.com/gkane26/DeepLabCut-live-GUI).🔥🔥🔥 - -### News! -- March 2022: DeepLabCut-Live! 1.0.2 supports poetry installation `poetry install deeplabcut-live`, thanks to PR #60. -- March 2021: DeepLabCut-Live! [**version 1.0** is released](https://pypi.org/project/deeplabcut-live/), with support for tensorflow 1 and tensorflow 2! -- Feb 2021: DeepLabCut-Live! was featured in **Nature Methods**: ["Real-time behavioral analysis"](https://www.nature.com/articles/s41592-021-01072-z) -- Jan 2021: full **eLife** paper is published: ["Real-time, low-latency closed-loop feedback using markerless posture tracking"](https://elifesciences.org/articles/61909) -- Dec 2020: we talked to **RTS Suisse Radio** about DLC-Live!: ["Capture animal movements in real time"](https://www.rts.ch/play/radio/cqfd/audio/capturer-les-mouvements-des-animaux-en-temps-reel?id=11782529) - - -### Installation: - -Please see our instruction manual to install on a [Windows or Linux machine](docs/install_desktop.md) or on a [NVIDIA Jetson Development Board](docs/install_jetson.md). Note, this code works with tensorflow (TF) 1 or TF 2 models, but TF requires that whatever version you exported your model with, you must import with the same version (i.e., export with TF1.13, then use TF1.13 with DlC-Live; export with TF2.3, then use TF2.3 with DLC-live). +**🔥🔥🔥🔥🔥 Note :: alone, this object does not record video or capture images from a +camera. This must be done separately, i.e. see our [DeepLabCut-live GUI]( +https://github.com/DeepLabCut/DeepLabCut-live-GUI).🔥🔥🔥🔥🔥** + +### News! + +- **WIP 2025**: DeepLabCut-Live is implemented for models trained with the PyTorch engine! +- March 2022: DeepLabCut-Live! 1.0.2 supports poetry installation `poetry install +deeplabcut-live`, thanks to PR #60. +- March 2021: DeepLabCut-Live! [**version 1.0** is released](https://pypi.org/project/deeplabcut-live/), with support for +tensorflow 1 and tensorflow 2! +- Feb 2021: DeepLabCut-Live! was featured in **Nature Methods**: [ +"Real-time behavioral analysis"](https://www.nature.com/articles/s41592-021-01072-z) +- Jan 2021: full **eLife** paper is published: ["Real-time, low-latency closed-loop +feedback using markerless posture tracking"](https://elifesciences.org/articles/61909) +- Dec 2020: we talked to **RTS Suisse Radio** about DLC-Live!: ["Capture animal +movements in real time"]( +https://www.rts.ch/play/radio/cqfd/audio/capturer-les-mouvements-des-animaux-en-temps-reel?id=11782529) + +### Installation + +Please see our instruction manual to install on a [Windows or Linux machine]( +docs/install_desktop.md) or on a [NVIDIA Jetson Development Board]( +docs/install_jetson.md). Note, this code works with PyTorch, TensorFlow 1 or TensorFlow +2 models, but whatever engine you exported your model with, you must import with the +same version (i.e., export a PyTorch model, then install PyTorch, export with TF1.13, +then use TF1.13 with DlC-Live; export with TF2.3, then use TF2.3 with DLC-live). - available on pypi as: `pip install deeplabcut-live` @@ -46,11 +78,25 @@ Note, you can then test your installation by running: `dlc-live-test` -If installed properly, this script will i) create a temporary folder ii) download the full_dog model from the [DeepLabCut Model Zoo](http://www.mousemotorlab.org/dlc-modelzoo), iii) download a short video clip of a dog, and iv) run inference while displaying keypoints. v) remove the temporary folder. +If installed properly, this script will i) create a temporary folder ii) download the +full_dog model from the [DeepLabCut Model Zoo]( +http://www.mousemotorlab.org/dlc-modelzoo), iii) download a short video clip of +a dog, and iv) run inference while displaying keypoints. v) remove the temporary folder. DLC LIVE TEST -### Quick Start: instructions for use: +PyTorch and TensorFlow can be installed as extras with `deeplabcut-live` - though be +careful with the versions you install! + +```bash +# Install deeplabcut-live and PyTorch +`pip install deeplabcut-live[pytorch]` + +# Install deeplabcut-live and TensorFlow +`pip install deeplabcut-live[tf]` +``` + +### Quick Start: instructions for use 1. Initialize `Processor` (if desired) 2. Initialize the `DLCLive` object @@ -85,62 +131,125 @@ dlc_live.get_pose() - `` = path to the folder that has the `.pb` files that you acquire after running `deeplabcut.export_model` - `` = is a numpy array of each frame +### Switching from TensorFlow to PyTorch + +This section is for users who **have already used DeepLabCut-Live** with +TensorFlow models (through DeepLabCut 1.X or 2.X) and want to switch to using the +PyTorch Engine. Some quick notes: + +- You may need to adapt your code slightly when creating the DLCLive instance. +- Processors that were created for TensorFlow models will function the same way with +PyTorch models. As multi-animal models can be used with PyTorch, the shape of the `pose` +array given to the processor may be `(num_individuals, num_keypoints, 3)`. Just call +`DLCLive(..., single_animal=True)` and it will work. ### Benchmarking/Analyzing your exported DeepLabCut models -DeepLabCut-live offers some analysis tools that allow users to peform the following operations on videos, from python or from the command line: +DeepLabCut-live offers some analysis tools that allow users to perform the following +operations on videos, from python or from the command line: + +#### Test inference speed across a range of image sizes + +Downsizing images can be done by specifying the `resize` or `pixels` parameter. Using +the `pixels` parameter will resize images to the desired number of `pixels`, without +changing the aspect ratio. Results will be saved (along with system info) to a pickle +file if you specify an output directory. + +Inside a **python** shell or script, you can run: -1. Test inference speed across a range of image sizes, downsizing images by specifying the `resize` or `pixels` parameter. Using the `pixels` parameter will resize images to the desired number of `pixels`, without changing the aspect ratio. Results will be saved (along with system info) to a pickle file if you specify an output directory. -##### python ```python -dlclive.benchmark_videos('/path/to/exported/model', ['/path/to/video1', '/path/to/video2'], output='/path/to/output', resize=[1.0, 0.75, '0.5']) -``` -##### command line +dlclive.benchmark_videos( + "/path/to/exported/model", + ["/path/to/video1", "/path/to/video2"], + output="/path/to/output", + resize=[1.0, 0.75, '0.5'], +) ``` + +From the **command line**, you can run: + +```bash dlc-live-benchmark /path/to/exported/model /path/to/video1 /path/to/video2 -o /path/to/output -r 1.0 0.75 0.5 ``` -2. Display keypoints to visually inspect the accuracy of exported models on different image sizes (note, this is slow and only for testing purposes): +#### Display keypoints to visually inspect the accuracy of exported models on different image sizes (note, this is slow and only for testing purposes): + +Inside a **python** shell or script, you can run: -##### python ```python -dlclive.benchmark_videos('/path/to/exported/model', '/path/to/video', resize=0.5, display=True, pcutoff=0.5, display_radius=4, cmap='bmy') -``` -##### command line +dlclive.benchmark_videos( + "/path/to/exported/model", + "/path/to/video", + resize=0.5, + display=True, + pcutoff=0.5, + display_radius=4, + cmap='bmy' +) ``` + +From the **command line**, you can run: + +```bash dlc-live-benchmark /path/to/exported/model /path/to/video -r 0.5 --display --pcutoff 0.5 --display-radius 4 --cmap bmy ``` -3. Analyze and create a labeled video using the exported model and desired resize parameters. This option functions similar to `deeplabcut.benchmark_videos` and `deeplabcut.create_labeled_video` (note, this is slow and only for testing purposes). +#### Analyze and create a labeled video using the exported model and desired resize parameters. + +This option functions similar to `deeplabcut.benchmark_videos` and +`deeplabcut.create_labeled_video` (note, this is slow and only for testing purposes). + +Inside a **python** shell or script, you can run: -##### python ```python -dlclive.benchmark_videos('/path/to/exported/model', '/path/to/video', resize=[1.0, 0.75, 0.5], pcutoff=0.5, display_radius=4, cmap='bmy', save_poses=True, save_video=True) +dlclive.benchmark_videos( + "/path/to/exported/model", + "/path/to/video", + resize=[1.0, 0.75, 0.5], + pcutoff=0.5, + display_radius=4, + cmap='bmy', + save_poses=True, + save_video=True, +) ``` -##### command line + +From the **command line**, you can run: + ``` dlc-live-benchmark /path/to/exported/model /path/to/video -r 0.5 --pcutoff 0.5 --display-radius 4 --cmap bmy --save-poses --save-video ``` ## License: -This project is licensed under the GNU AGPLv3. Note that the software is provided "as is", without warranty of any kind, express or implied. If you use the code or data, we ask that you please cite us! This software is available for licensing via the EPFL Technology Transfer Office (https://tto.epfl.ch/, info.tto@epfl.ch). +This project is licensed under the GNU AGPLv3. Note that the software is provided "as +is", without warranty of any kind, express or implied. If you use the code or data, we +ask that you please cite us! This software is available for licensing via the EPFL +Technology Transfer Office (https://tto.epfl.ch/, info.tto@epfl.ch). ## Community Support, Developers, & Help: -This is an actively developed package and we welcome community development and involvement. - -- If you want to contribute to the code, please read our guide [here](https://github.com/DeepLabCut/DeepLabCut/blob/master/CONTRIBUTING.md), which is provided at the main repository of DeepLabCut. - -- We are a community partner on the [![Image.sc forum](https://img.shields.io/badge/dynamic/json.svg?label=forum&url=https%3A%2F%2Fforum.image.sc%2Ftags%2Fdeeplabcut.json&query=%24.topic_list.tags.0.topic_count&colorB=brightgreen&&suffix=%20topics&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA4AAAAOCAYAAAAfSC3RAAABPklEQVR42m3SyyqFURTA8Y2BER0TDyExZ+aSPIKUlPIITFzKeQWXwhBlQrmFgUzMMFLKZeguBu5y+//17dP3nc5vuPdee6299gohUYYaDGOyyACq4JmQVoFujOMR77hNfOAGM+hBOQqB9TjHD36xhAa04RCuuXeKOvwHVWIKL9jCK2bRiV284QgL8MwEjAneeo9VNOEaBhzALGtoRy02cIcWhE34jj5YxgW+E5Z4iTPkMYpPLCNY3hdOYEfNbKYdmNngZ1jyEzw7h7AIb3fRTQ95OAZ6yQpGYHMMtOTgouktYwxuXsHgWLLl+4x++Kx1FJrjLTagA77bTPvYgw1rRqY56e+w7GNYsqX6JfPwi7aR+Y5SA+BXtKIRfkfJAYgj14tpOF6+I46c4/cAM3UhM3JxyKsxiOIhH0IO6SH/A1Kb1WBeUjbkAAAAAElFTkSuQmCC)](https://forum.image.sc/tags/deeplabcut). Please post help and support questions on the forum with the tag DeepLabCut. Check out their mission statement [Scientific Community Image Forum: A discussion forum for scientific image software](https://journals.plos.org/plosbiology/article?id=10.1371/journal.pbio.3000340). - -- If you encounter a previously unreported bug/code issue, please post here (we encourage you to search issues first): https://github.com/DeepLabCut/DeepLabCut-live/issues - -- For quick discussions here: [![Gitter](https://badges.gitter.im/DeepLabCut/community.svg)](https://gitter.im/DeepLabCut/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +This is an actively developed package, and we welcome community development and +involvement. + +- If you want to contribute to the code, please read our guide [here]( +https://github.com/DeepLabCut/DeepLabCut/blob/master/CONTRIBUTING.md), which is provided +at the main repository of DeepLabCut. +- We are a community partner on the [![Image.sc forum](https://img.shields.io/badge/dynamic/json.svg?label=forum&url=https%3A%2F%2Fforum.image.sc%2Ftags%2Fdeeplabcut.json&query=%24.topic_list.tags.0.topic_count&colorB=brightgreen&&suffix=%20topics&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA4AAAAOCAYAAAAfSC3RAAABPklEQVR42m3SyyqFURTA8Y2BER0TDyExZ+aSPIKUlPIITFzKeQWXwhBlQrmFgUzMMFLKZeguBu5y+//17dP3nc5vuPdee6299gohUYYaDGOyyACq4JmQVoFujOMR77hNfOAGM+hBOQqB9TjHD36xhAa04RCuuXeKOvwHVWIKL9jCK2bRiV284QgL8MwEjAneeo9VNOEaBhzALGtoRy02cIcWhE34jj5YxgW+E5Z4iTPkMYpPLCNY3hdOYEfNbKYdmNngZ1jyEzw7h7AIb3fRTQ95OAZ6yQpGYHMMtOTgouktYwxuXsHgWLLl+4x++Kx1FJrjLTagA77bTPvYgw1rRqY56e+w7GNYsqX6JfPwi7aR+Y5SA+BXtKIRfkfJAYgj14tpOF6+I46c4/cAM3UhM3JxyKsxiOIhH0IO6SH/A1Kb1WBeUjbkAAAAAElFTkSuQmCC)](https://forum.image.sc/tags/deeplabcut). Please post help and +support questions on the forum with the tag DeepLabCut. Check out their mission +statement [Scientific Community Image Forum: A discussion forum for scientific image +software](https://journals.plos.org/plosbiology/article?id=10.1371/journal.pbio.3000340). +- If you encounter a previously unreported bug/code issue, please post here (we +encourage you to search issues first): [github.com/DeepLabCut/DeepLabCut-live/issues]( +https://github.com/DeepLabCut/DeepLabCut-live/issues) +- For quick discussions here: [![Gitter]( +https://badges.gitter.im/DeepLabCut/community.svg)]( +https://gitter.im/DeepLabCut/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) ### Reference: -If you utilize our tool, please [cite Kane et al, eLife 2020](https://elifesciences.org/articles/61909). The preprint is available here: https://www.biorxiv.org/content/10.1101/2020.08.04.236422v2 +If you utilize our tool, please [cite Kane et al, eLife 2020](https://elifesciences.org/articles/61909). The preprint is +available here: https://www.biorxiv.org/content/10.1101/2020.08.04.236422v2 ``` @Article{Kane2020dlclive, @@ -150,4 +259,3 @@ If you utilize our tool, please [cite Kane et al, eLife 2020](https://elifescien year = {2020}, } ``` - diff --git a/pyproject.toml b/pyproject.toml index 0978593..200a5d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,16 +43,14 @@ torch = { version = ">=2.0.0", optional = true } torchvision = { version = ">=0.15", optional = true } # TensorFlow models tensorflow = [ - { version = ">=2.0,<=2.10", optional = true, platform = "win32" }, - { version = ">=2.0,<=2.12", optional = true, platform = "linux" }, + { version = "^2.7.0,<=2.10", optional = true, platform = "win32" }, + { version = "^2.7.0,<=2.12", optional = true, platform = "linux" }, ] -tensorflow-macos = { version = ">=2.0,<=2.12", optional = true, markers = "sys_platform == 'darwin'" } +tensorflow-macos = { version = "^2.7.0,<=2.12", optional = true, markers = "sys_platform == 'darwin'" } tensorflow-metal = { version = "<1.3.0", optional = true, markers = "sys_platform == 'darwin'" } -tensorpack = {version = ">=0.11", optional = true } -tf_slim = {version = ">=1.1.0", optional = true } [tool.poetry.extras] -tf = [ "tensorflow", "tensorflow-macos", "tensorflow-metal", "tensorpack", "tf_slim"] +tf = [ "tensorflow", "tensorflow-macos", "tensorflow-metal"] pytorch = ["scipy", "timm", "torch", "torchvision"] [tool.poetry.dev-dependencies] From 82daf43cdc082ac9387f79204124bfc11e9f7c85 Mon Sep 17 00:00:00 2001 From: Niels Poulsen Date: Thu, 27 Feb 2025 14:44:37 +0100 Subject: [PATCH 21/24] improved docs --- dlclive/benchmark.py | 3 - dlclive/benchmark_pytorch.py | 1 - dlclive/benchmark_tf.py | 7 -- dlclive/core/inferenceutils.py | 1 - dlclive/dlclive.py | 71 ++++++++++++------- dlclive/factory.py | 37 ++++++++-- dlclive/graph.py | 5 +- dlclive/live_inference.py | 2 - dlclive/pose_estimation_pytorch/data/image.py | 2 +- .../dynamic_cropping.py | 15 ++-- .../models/backbones/__init__.py | 5 +- .../models/backbones/cspnext.py | 9 +-- .../models/backbones/hrnet.py | 5 +- .../models/backbones/resnet.py | 5 +- .../models/detectors/fasterRCNN.py | 4 +- .../models/detectors/ssd.py | 4 +- .../models/heads/dekr.py | 6 +- .../models/heads/dlcrnet.py | 11 +-- .../models/modules/norm.py | 2 +- .../models/necks/transformer.py | 4 +- .../models/predictors/__init__.py | 17 +++-- .../models/predictors/dekr_predictor.py | 5 +- .../models/predictors/identity_predictor.py | 5 +- .../models/predictors/paf_predictor.py | 2 +- .../models/predictors/sim_cc.py | 2 +- .../models/predictors/single_predictor.py | 5 +- dlclive/pose_estimation_pytorch/runner.py | 31 ++++++-- dlclive/pose_estimation_tensorflow/graph.py | 5 +- dlclive/pose_estimation_tensorflow/runner.py | 13 +--- dlclive/predictor/single_predictor.py | 5 +- dlclive/processor/kalmanfilter.py | 11 --- dlclive/utils.py | 16 +---- 32 files changed, 192 insertions(+), 124 deletions(-) diff --git a/dlclive/benchmark.py b/dlclive/benchmark.py index d7db674..2f0f2af 100644 --- a/dlclive/benchmark.py +++ b/dlclive/benchmark.py @@ -549,7 +549,6 @@ def benchmark_videos( ) while True: - ret, frame = cap.read() if not ret: break @@ -656,8 +655,6 @@ def save_poses_to_files(video_path, save_dir, bodyparts, poses, timestamp): writer.writerow(row) - - import argparse import os diff --git a/dlclive/benchmark_pytorch.py b/dlclive/benchmark_pytorch.py index 8ab0cb1..bd5826f 100644 --- a/dlclive/benchmark_pytorch.py +++ b/dlclive/benchmark_pytorch.py @@ -214,7 +214,6 @@ def analyze_video( ) while True: - ret, frame = cap.read() if not ret: break diff --git a/dlclive/benchmark_tf.py b/dlclive/benchmark_tf.py index 32850b7..d955496 100644 --- a/dlclive/benchmark_tf.py +++ b/dlclive/benchmark_tf.py @@ -305,7 +305,6 @@ def benchmark( iterator = range(n_frames) if (print_rate) or (display) else tqdm(range(n_frames)) for i in iterator: - ret, frame = cap.read() if not ret: @@ -321,7 +320,6 @@ def benchmark( inf_times[i] = time.time() - start_pose if save_video: - if colors is None: all_colors = getattr(cc, cmap) colors = [ @@ -399,7 +397,6 @@ def benchmark( vwriter.release() if save_poses: - cfg_path = os.path.normpath(f"{model_path}/pose_cfg.yaml") ruamel_file = ruamel.yaml.YAML() dlc_cfg = ruamel_file.load(open(cfg_path, "r")) @@ -407,7 +404,6 @@ def benchmark( poses = np.array(poses) if use_pandas: - poses = poses.reshape((poses.shape[0], poses.shape[1] * poses.shape[2])) pdindex = pd.MultiIndex.from_product( [bodyparts, ["x", "y", "likelihood"]], names=["bodyparts", "coords"] @@ -426,7 +422,6 @@ def benchmark( pose_df.to_hdf(out_dlc_file, key="df_with_missing", mode="w") else: - out_vid_base = os.path.basename(video_path) out_dlc_file = os.path.normpath( f"{out_dir}/{os.path.splitext(out_vid_base)[0]}_DLCLIVE_POSES.npy" @@ -614,14 +609,12 @@ def benchmark_videos( # loop over videos for v in video_path: - # initialize full inference times inf_times = [] im_size_out = [] for i in range(len(resize)): - print(f"\nRun {i+1} / {len(resize)}\n") this_inf_times, this_im_size, TFGPUinference, meta = benchmark( diff --git a/dlclive/core/inferenceutils.py b/dlclive/core/inferenceutils.py index 64f94f5..c160f40 100644 --- a/dlclive/core/inferenceutils.py +++ b/dlclive/core/inferenceutils.py @@ -841,7 +841,6 @@ def assemble(self, chunk_size=1, n_processes=None): # work nicely with the GUI or interactive sessions. # In that case, we fall back to the serial assembly. if chunk_size == 0 or multiprocessing.get_start_method() == "spawn": - for i, data_dict in enumerate(tqdm(self)): assemblies, unique = self._assemble(data_dict, i) if assemblies: diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index 68b348b..db706b2 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -28,31 +28,41 @@ class DLCLive: ----------- model_path: Path - Full path to exported model file + Full path to exported model (created when `deeplabcut.export_model(...)` was + called). For PyTorch models, this is a single model file. For TensorFlow models, + this is a directory containing the model snapshots. model_type: string, optional - which model to use: 'pytorch' or 'onnx' for exported snapshot + Which model to use. For the PyTorch engine, options are [`pytorch`]. For the + TensorFlow engine, options are [`base`, `tensorrt`, `lite`]. + + precision: string, optional + Precision of model weights, for model_type "pytorch" and "tensorrt". Options + are, for different model_types: + "pytorch": {"FP32", "FP16"} + "tensorrt": {"FP32", "FP16", "INT8"} tf_config: + TensorFlow only. Optional ConfigProto for the TensorFlow session. + single_animal: bool, default=True + PyTorch only. - precision: string, optional - precision of model weights, for model_type='onnx' or 'pytorch'. Can be 'FP32' - (default) or 'FP16' + device: str, optional, default=None + PyTorch only. + + top_down_config: dict, optional, default=None + + top_down_dynamic: dict, optional, default=None cropping: list of int - cropping parameters in pixel number: [x1, x2, y1, y2] #A: Maybe this is the - dynamic cropping of each frame to speed of processing, so instead of analyzing - the whole frame, it analyzes only the part of the frame where the animal is - - dynamic: triple containing (state, detectiontreshold, margin) #A: margin adds some - space so the 'bbox' isn't too narrow around the animal'. First key points are - predicted, then dynamic cropping is performed to 'single out' the animal, and - then pose is estimated, we think. + Cropping parameters in pixel number: [x1, x2, y1, y2] + + dynamic: triple containing (state, detectiontreshold, margin) If the state is true, then dynamic cropping will be performed. That means that if an object is detected (i.e. any body part > detectiontreshold), then object boundaries are computed according to the smallest/largest x position and - smallest/largest y position of all body parts. This window is expanded by the + smallest/largest y position of all body parts. This window is expanded by the margin and from then on only the posture within this crop is analyzed (until the object is lost, i.e. BaseRunner: @@ -14,9 +16,18 @@ def build_runner( Parameters ---------- - model_type - model_path - kwargs + model_type: str, optional + Which model to use. For the PyTorch engine, options are [`pytorch`]. For the + TensorFlow engine, options are [`base`, `tensorrt`, `lite`]. + model_path: str, Path + Full path to exported model (created when `deeplabcut.export_model(...)` was + called). For PyTorch models, this is a single model file. For TensorFlow models, + this is a directory containing the model snapshots. + + kwargs: dict, optional + PyTorch Engine Kwargs: + + TensorFlow Engine Kwargs: Returns ------- @@ -24,10 +35,22 @@ def build_runner( """ if model_type.lower() == "pytorch": from dlclive.pose_estimation_pytorch.runner import PyTorchRunner - return PyTorchRunner(model_path, **kwargs) + + valid = {"device", "precision", "single_animal", "dynamic", "top_down_config"} + return PyTorchRunner(model_path, **filter_keys(valid, kwargs)) elif model_type.lower() in ("tensorflow", "base", "tensorrt", "lite"): from dlclive.pose_estimation_tensorflow.runner import TensorFlowRunner - return TensorFlowRunner(model_path, model_type, **kwargs) + + if model_type.lower() == "tensorflow": + model_type = "base" + + valid = {"tf_config", "precision"} + return TensorFlowRunner(model_path, model_type, **filter_keys(valid, kwargs)) raise ValueError(f"Unknown model type: {model_type}") + + +def filter_keys(valid: set[str], kwargs: dict) -> dict: + """Filters the keys in kwargs, only keeping those in valid.""" + return {k: v for k, v in kwargs.items() if k in valid} diff --git a/dlclive/graph.py b/dlclive/graph.py index 4cc3d40..72b3b76 100644 --- a/dlclive/graph.py +++ b/dlclive/graph.py @@ -106,12 +106,13 @@ def get_output_tensors(graph): def get_input_tensor(graph): - input_tensor = str(graph.get_operations()[0].name) + ":0" return input_tensor -def extract_graph(graph, tf_config=None) -> tuple[tf.Session, tf.Tensor, list[tf.Tensor]]: +def extract_graph( + graph, tf_config=None +) -> tuple[tf.Session, tf.Tensor, list[tf.Tensor]]: """ Initializes a tensorflow session with the specified graph and extracts the model's inputs and outputs diff --git a/dlclive/live_inference.py b/dlclive/live_inference.py index 3763fee..6db7597 100644 --- a/dlclive/live_inference.py +++ b/dlclive/live_inference.py @@ -197,7 +197,6 @@ def analyze_live_video( ] if save_video: - # Define output video path output_video_path = os.path.join( save_dir, f"{experiment_name}_DLCLIVE_LABELLED_{timestamp}.mp4" @@ -217,7 +216,6 @@ def analyze_live_video( ) while True: - ret, frame = cap.read() if not ret: break diff --git a/dlclive/pose_estimation_pytorch/data/image.py b/dlclive/pose_estimation_pytorch/data/image.py index a9d71dd..c6f1705 100644 --- a/dlclive/pose_estimation_pytorch/data/image.py +++ b/dlclive/pose_estimation_pytorch/data/image.py @@ -109,7 +109,7 @@ def top_down_crop( # crop the pixels we care about image_crop = np.zeros((crop_h, crop_w, c), dtype=image.dtype) - image_crop[pad_top:pad_top + h, pad_left:pad_left + w] = image[y1:y2, x1:x2] + image_crop[pad_top : pad_top + h, pad_left : pad_left + w] = image[y1:y2, x1:x2] # resize the cropped image image = cv2.resize(image_crop, (out_w, out_h), interpolation=cv2.INTER_LINEAR) diff --git a/dlclive/pose_estimation_pytorch/dynamic_cropping.py b/dlclive/pose_estimation_pytorch/dynamic_cropping.py index 2853aab..bc84533 100644 --- a/dlclive/pose_estimation_pytorch/dynamic_cropping.py +++ b/dlclive/pose_estimation_pytorch/dynamic_cropping.py @@ -39,9 +39,9 @@ class DynamicCropper: The margin used to expand an individuals bounding box before cropping it. Examples: - >>> import deeplabcut.pose_estimation_pytorch.models as models + >>> import torch.nn as nn >>> - >>> model: models.PoseModel + >>> model: nn.Module # pose estimation model >>> frames: torch.Tensor # shape (num_frames, 3, H, W) >>> >>> dynamic = DynamicCropper(threshold=0.6, margin=25) @@ -57,6 +57,7 @@ class DynamicCropper: >>> predictions.append(pose) >>> """ + threshold: float margin: int _crop: tuple[int, int, int, int] | None = field(default=None, repr=False) @@ -424,16 +425,18 @@ def _prepare_bounding_box( input_ratio = w / h if input_ratio > self._td_ratio: # h/w < h0/w0 => h' = w * h0/w0 - h = w / self._td_ratio + h = w / self._td_ratio elif input_ratio < self._td_ratio: # w/h < w0/h0 => w' = h * w0/h0 - w = h * self._td_ratio + w = h * self._td_ratio x1, y1 = int(round(cx - (w / 2))), int(round(cy - (h / 2))) w, h = max(int(w), self.min_bbox_size[0]), max(int(h), self.min_bbox_size[1]) return x1, y1, w, h def _crop_bounding_box( - self, image: torch.Tensor, bbox: tuple[int, int, int, int], + self, + image: torch.Tensor, + bbox: tuple[int, int, int, int], ) -> torch.Tensor: """Applies a top-down crop to an image given a bounding box. @@ -487,7 +490,7 @@ def _extract_best_patch(self, pose: torch.Tensor) -> torch.Tensor: # set the crop to the one used for the best patch self._crop = self._patches[best_patch] - return pose[best_patch:best_patch + 1] + return pose[best_patch : best_patch + 1] def generate_patches(self) -> list[tuple[int, int, int, int]]: """Generates patch coordinates for splitting an image. diff --git a/dlclive/pose_estimation_pytorch/models/backbones/__init__.py b/dlclive/pose_estimation_pytorch/models/backbones/__init__.py index 55bd515..0d32951 100644 --- a/dlclive/pose_estimation_pytorch/models/backbones/__init__.py +++ b/dlclive/pose_estimation_pytorch/models/backbones/__init__.py @@ -8,7 +8,10 @@ # # Licensed under GNU Lesser General Public License v3.0 # -from dlclive.pose_estimation_pytorch.models.backbones.base import BACKBONES, BaseBackbone +from dlclive.pose_estimation_pytorch.models.backbones.base import ( + BACKBONES, + BaseBackbone, +) from dlclive.pose_estimation_pytorch.models.backbones.cspnext import CSPNeXt from dlclive.pose_estimation_pytorch.models.backbones.hrnet import HRNet from dlclive.pose_estimation_pytorch.models.backbones.resnet import DLCRNet, ResNet diff --git a/dlclive/pose_estimation_pytorch/models/backbones/cspnext.py b/dlclive/pose_estimation_pytorch/models/backbones/cspnext.py index f0595cf..681f2ba 100644 --- a/dlclive/pose_estimation_pytorch/models/backbones/cspnext.py +++ b/dlclive/pose_estimation_pytorch/models/backbones/cspnext.py @@ -36,6 +36,7 @@ @dataclass(frozen=True) class CSPNeXtLayerConfig: """Configuration for a CSPNeXt layer""" + in_channels: int out_channels: int num_blocks: int @@ -79,7 +80,7 @@ class CSPNeXt(HuggingFaceWeightsMixin, BaseBackbone): CSPNeXtLayerConfig(256, 512, 6, True, False), CSPNeXtLayerConfig(512, 768, 3, True, False), CSPNeXtLayerConfig(768, 1024, 3, False, True), - ] + ], } def __init__( @@ -136,7 +137,7 @@ def __init__( stride=1, norm_layer=norm_layer, activation_fn=activation_fn, - ) + ), ) self.layers = ["stem"] @@ -177,8 +178,8 @@ def __init__( activation_fn=activation_fn, ) stage.append(csp_layer) - self.add_module(f'stage{i + 1}', nn.Sequential(*stage)) - self.layers.append(f'stage{i + 1}') + self.add_module(f"stage{i + 1}", nn.Sequential(*stage)) + self.layers.append(f"stage{i + 1}") self.single_output = isinstance(out_indices, int) if self.single_output: diff --git a/dlclive/pose_estimation_pytorch/models/backbones/hrnet.py b/dlclive/pose_estimation_pytorch/models/backbones/hrnet.py index 7d99b77..942399d 100644 --- a/dlclive/pose_estimation_pytorch/models/backbones/hrnet.py +++ b/dlclive/pose_estimation_pytorch/models/backbones/hrnet.py @@ -13,7 +13,10 @@ import torch.nn as nn import torch.nn.functional as F -from dlclive.pose_estimation_pytorch.models.backbones.base import BACKBONES, BaseBackbone +from dlclive.pose_estimation_pytorch.models.backbones.base import ( + BACKBONES, + BaseBackbone, +) @BACKBONES.register_module diff --git a/dlclive/pose_estimation_pytorch/models/backbones/resnet.py b/dlclive/pose_estimation_pytorch/models/backbones/resnet.py index a71276b..f661159 100644 --- a/dlclive/pose_estimation_pytorch/models/backbones/resnet.py +++ b/dlclive/pose_estimation_pytorch/models/backbones/resnet.py @@ -13,7 +13,10 @@ import torch.nn as nn from torchvision.transforms.functional import resize -from dlclive.pose_estimation_pytorch.models.backbones.base import BACKBONES, BaseBackbone +from dlclive.pose_estimation_pytorch.models.backbones.base import ( + BACKBONES, + BaseBackbone, +) @BACKBONES.register_module diff --git a/dlclive/pose_estimation_pytorch/models/detectors/fasterRCNN.py b/dlclive/pose_estimation_pytorch/models/detectors/fasterRCNN.py index 1656402..f250b9a 100644 --- a/dlclive/pose_estimation_pytorch/models/detectors/fasterRCNN.py +++ b/dlclive/pose_estimation_pytorch/models/detectors/fasterRCNN.py @@ -13,7 +13,9 @@ import torchvision.models.detection as detection from dlclive.pose_estimation_pytorch.models.detectors.base import DETECTORS -from dlclive.pose_estimation_pytorch.models.detectors.torchvision import TorchvisionDetectorAdaptor +from dlclive.pose_estimation_pytorch.models.detectors.torchvision import ( + TorchvisionDetectorAdaptor, +) @DETECTORS.register_module diff --git a/dlclive/pose_estimation_pytorch/models/detectors/ssd.py b/dlclive/pose_estimation_pytorch/models/detectors/ssd.py index e1c9da8..7140cd5 100644 --- a/dlclive/pose_estimation_pytorch/models/detectors/ssd.py +++ b/dlclive/pose_estimation_pytorch/models/detectors/ssd.py @@ -13,7 +13,9 @@ import torchvision.models.detection as detection from dlclive.pose_estimation_pytorch.models.detectors.base import DETECTORS -from dlclive.pose_estimation_pytorch.models.detectors.torchvision import TorchvisionDetectorAdaptor +from dlclive.pose_estimation_pytorch.models.detectors.torchvision import ( + TorchvisionDetectorAdaptor, +) @DETECTORS.register_module diff --git a/dlclive/pose_estimation_pytorch/models/heads/dekr.py b/dlclive/pose_estimation_pytorch/models/heads/dekr.py index d0e3d47..1ef1ec1 100644 --- a/dlclive/pose_estimation_pytorch/models/heads/dekr.py +++ b/dlclive/pose_estimation_pytorch/models/heads/dekr.py @@ -14,7 +14,11 @@ import torch.nn as nn from dlclive.pose_estimation_pytorch.models.heads.base import HEADS, BaseHead -from dlclive.pose_estimation_pytorch.models.modules.conv_block import AdaptBlock, BaseBlock, BasicBlock +from dlclive.pose_estimation_pytorch.models.modules.conv_block import ( + AdaptBlock, + BaseBlock, + BasicBlock, +) from dlclive.pose_estimation_pytorch.models.predictors import BasePredictor diff --git a/dlclive/pose_estimation_pytorch/models/heads/dlcrnet.py b/dlclive/pose_estimation_pytorch/models/heads/dlcrnet.py index 44e7664..79cc315 100644 --- a/dlclive/pose_estimation_pytorch/models/heads/dlcrnet.py +++ b/dlclive/pose_estimation_pytorch/models/heads/dlcrnet.py @@ -14,7 +14,10 @@ import torch.nn as nn from dlclive.pose_estimation_pytorch.models.heads.base import HEADS -from dlclive.pose_estimation_pytorch.models.heads.simple_head import DeconvModule, HeatmapHead +from dlclive.pose_estimation_pytorch.models.heads.simple_head import ( + DeconvModule, + HeatmapHead, +) from dlclive.pose_estimation_pytorch.models.predictors import BasePredictor @@ -38,9 +41,9 @@ def __init__( num_limbs = paf_config["channels"][-1] # Already has the 2x multiplier in_refined_channels = features_dim + num_keypoints + num_limbs if num_stages > 0: - heatmap_config["channels"][0] = paf_config["channels"][0] = ( - in_refined_channels - ) + heatmap_config["channels"][0] = paf_config["channels"][ + 0 + ] = in_refined_channels locref_config["channels"][0] = locref_config["channels"][-1] super().__init__(predictor, heatmap_config, locref_config) diff --git a/dlclive/pose_estimation_pytorch/models/modules/norm.py b/dlclive/pose_estimation_pytorch/models/modules/norm.py index 1cbc0f4..9bf839b 100644 --- a/dlclive/pose_estimation_pytorch/models/modules/norm.py +++ b/dlclive/pose_estimation_pytorch/models/modules/norm.py @@ -31,7 +31,7 @@ class ScaleNorm(nn.Module): def __init__(self, dim: int, eps: float = 1e-5): super().__init__() - self.scale = dim ** -0.5 + self.scale = dim**-0.5 self.eps = eps self.g = nn.Parameter(torch.ones(1)) diff --git a/dlclive/pose_estimation_pytorch/models/necks/transformer.py b/dlclive/pose_estimation_pytorch/models/necks/transformer.py index 939ddf7..eae5118 100644 --- a/dlclive/pose_estimation_pytorch/models/necks/transformer.py +++ b/dlclive/pose_estimation_pytorch/models/necks/transformer.py @@ -16,7 +16,9 @@ from dlclive.pose_estimation_pytorch.models.necks.base import NECKS, BaseNeck from dlclive.pose_estimation_pytorch.models.necks.layers import TransformerLayer -from dlclive.pose_estimation_pytorch.models.necks.utils import make_sine_position_embedding +from dlclive.pose_estimation_pytorch.models.necks.utils import ( + make_sine_position_embedding, +) MIN_NUM_PATCHES = 16 BN_MOMENTUM = 0.1 diff --git a/dlclive/pose_estimation_pytorch/models/predictors/__init__.py b/dlclive/pose_estimation_pytorch/models/predictors/__init__.py index 5220642..0662ffa 100644 --- a/dlclive/pose_estimation_pytorch/models/predictors/__init__.py +++ b/dlclive/pose_estimation_pytorch/models/predictors/__init__.py @@ -8,8 +8,17 @@ # # Licensed under GNU Lesser General Public License v3.0 # -from dlclive.pose_estimation_pytorch.models.predictors.base import PREDICTORS, BasePredictor -from dlclive.pose_estimation_pytorch.models.predictors.dekr_predictor import DEKRPredictor +from dlclive.pose_estimation_pytorch.models.predictors.base import ( + PREDICTORS, + BasePredictor, +) +from dlclive.pose_estimation_pytorch.models.predictors.dekr_predictor import ( + DEKRPredictor, +) from dlclive.pose_estimation_pytorch.models.predictors.sim_cc import SimCCPredictor -from dlclive.pose_estimation_pytorch.models.predictors.single_predictor import HeatmapPredictor -from dlclive.pose_estimation_pytorch.models.predictors.paf_predictor import PartAffinityFieldPredictor +from dlclive.pose_estimation_pytorch.models.predictors.single_predictor import ( + HeatmapPredictor, +) +from dlclive.pose_estimation_pytorch.models.predictors.paf_predictor import ( + PartAffinityFieldPredictor, +) diff --git a/dlclive/pose_estimation_pytorch/models/predictors/dekr_predictor.py b/dlclive/pose_estimation_pytorch/models/predictors/dekr_predictor.py index 6f8fd05..7261f29 100644 --- a/dlclive/pose_estimation_pytorch/models/predictors/dekr_predictor.py +++ b/dlclive/pose_estimation_pytorch/models/predictors/dekr_predictor.py @@ -14,7 +14,10 @@ import torch import torch.nn.functional as F -from dlclive.pose_estimation_pytorch.models.predictors.base import PREDICTORS, BasePredictor +from dlclive.pose_estimation_pytorch.models.predictors.base import ( + PREDICTORS, + BasePredictor, +) @PREDICTORS.register_module diff --git a/dlclive/pose_estimation_pytorch/models/predictors/identity_predictor.py b/dlclive/pose_estimation_pytorch/models/predictors/identity_predictor.py index e7e7d06..a4837c5 100644 --- a/dlclive/pose_estimation_pytorch/models/predictors/identity_predictor.py +++ b/dlclive/pose_estimation_pytorch/models/predictors/identity_predictor.py @@ -13,7 +13,10 @@ import torch.nn as nn import torchvision.transforms.functional as F -from dlclive.pose_estimation_pytorch.models.predictors.base import PREDICTORS, BasePredictor +from dlclive.pose_estimation_pytorch.models.predictors.base import ( + PREDICTORS, + BasePredictor, +) @PREDICTORS.register_module diff --git a/dlclive/pose_estimation_pytorch/models/predictors/paf_predictor.py b/dlclive/pose_estimation_pytorch/models/predictors/paf_predictor.py index b6ba68a..de83636 100644 --- a/dlclive/pose_estimation_pytorch/models/predictors/paf_predictor.py +++ b/dlclive/pose_estimation_pytorch/models/predictors/paf_predictor.py @@ -206,7 +206,7 @@ def find_local_peak_indices_maxpool_nms( @staticmethod def make_2d_gaussian_kernel(sigma: float, size: int) -> torch.Tensor: k = torch.arange(-size // 2 + 1, size // 2 + 1, dtype=torch.float32) ** 2 - k = F.softmax(-k / (2 * (sigma ** 2)), dim=0) + k = F.softmax(-k / (2 * (sigma**2)), dim=0) return torch.einsum("i,j->ij", k, k) @staticmethod diff --git a/dlclive/pose_estimation_pytorch/models/predictors/sim_cc.py b/dlclive/pose_estimation_pytorch/models/predictors/sim_cc.py index 1b4e007..e4ec134 100644 --- a/dlclive/pose_estimation_pytorch/models/predictors/sim_cc.py +++ b/dlclive/pose_estimation_pytorch/models/predictors/sim_cc.py @@ -157,7 +157,7 @@ def get_simcc_normalized(pred: torch.Tensor) -> torch.Tensor: mask = (pred.amax(dim=-1) > 1).reshape(b, k, 1) # Normalize the tensor using the maximum value - norm = (pred / pred.amax(dim=-1).reshape(b, k, 1)) + norm = pred / pred.amax(dim=-1).reshape(b, k, 1) # return the normalized tensor return torch.where(mask, norm, pred) diff --git a/dlclive/pose_estimation_pytorch/models/predictors/single_predictor.py b/dlclive/pose_estimation_pytorch/models/predictors/single_predictor.py index 96f31d9..c622cf9 100644 --- a/dlclive/pose_estimation_pytorch/models/predictors/single_predictor.py +++ b/dlclive/pose_estimation_pytorch/models/predictors/single_predictor.py @@ -14,7 +14,10 @@ import torch -from dlclive.pose_estimation_pytorch.models.predictors.base import BasePredictor, PREDICTORS +from dlclive.pose_estimation_pytorch.models.predictors.base import ( + BasePredictor, + PREDICTORS, +) @PREDICTORS.register_module diff --git a/dlclive/pose_estimation_pytorch/runner.py b/dlclive/pose_estimation_pytorch/runner.py index c7029d5..9e94b3d 100644 --- a/dlclive/pose_estimation_pytorch/runner.py +++ b/dlclive/pose_estimation_pytorch/runner.py @@ -33,6 +33,7 @@ class SkipFrames: detector is run, bounding boxes will be computed from the pose estimated in the previous frame. """ + skip: int margin: int _age: int = 0 @@ -80,6 +81,7 @@ class TopDownConfig: skip_frames: If defined, the detector will only be run every `skip_frames.skip` frames. """ + bbox_cutoff: float max_detections: int crop_size: tuple[int, int] = (256, 256) @@ -112,10 +114,8 @@ def __init__( device: str = "auto", precision: Literal["FP16", "FP32"] = "FP32", single_animal: bool = True, - bbox_cutoff: float = 0.6, # FIXME(niels) - max_detections: int | None = None, - dynamic: dynamic_cropping.DynamicCropper | None = None, - top_down_config: TopDownConfig | None = None, + dynamic: dict | dynamic_cropping.DynamicCropper | None = None, + top_down_config: dict | TopDownConfig | None = None, ) -> None: super().__init__(path) self.device = _parse_device(device) @@ -127,6 +127,24 @@ def __init__( self.model = None self.transform = None + # Parse Dynamic Cropping parameters + if isinstance(dynamic, dict): + dynamic_type = dynamic.get("type", "DynamicCropper") + if dynamic_type == "DynamicCropper": + cropper_cls = dynamic_cropping.DynamicCropper + else: + cropper_cls = dynamic_cropping.TopDownDynamicCropper + dynamic_params = dynamic.copy() + dynamic_params.pop("type") + dynamic = cropper_cls(**dynamic_params) + + # Parse Top-Down config + if isinstance(top_down_config, dict): + skip_frame_cfg = top_down_config.get("skip_frames") + if skip_frame_cfg is not None: + top_down_config["skip_frames"] = SkipFrames(**skip_frame_cfg) + top_down_config = TopDownConfig(**top_down_config) + self.dynamic = dynamic self.top_down_config = top_down_config @@ -306,7 +324,10 @@ def _postprocess_top_down( for pose, (offset, scale) in zip(batch_pose, offsets_and_scales): poses.append( torch.cat( - [pose[..., :2] * torch.tensor(scale) + torch.tensor(offset), pose[..., 2:3]], + [ + pose[..., :2] * torch.tensor(scale) + torch.tensor(offset), + pose[..., 2:3], + ], dim=-1, ) ) diff --git a/dlclive/pose_estimation_tensorflow/graph.py b/dlclive/pose_estimation_tensorflow/graph.py index 4cc3d40..72b3b76 100644 --- a/dlclive/pose_estimation_tensorflow/graph.py +++ b/dlclive/pose_estimation_tensorflow/graph.py @@ -106,12 +106,13 @@ def get_output_tensors(graph): def get_input_tensor(graph): - input_tensor = str(graph.get_operations()[0].name) + ":0" return input_tensor -def extract_graph(graph, tf_config=None) -> tuple[tf.Session, tf.Tensor, list[tf.Tensor]]: +def extract_graph( + graph, tf_config=None +) -> tuple[tf.Session, tf.Tensor, list[tf.Tensor]]: """ Initializes a tensorflow session with the specified graph and extracts the model's inputs and outputs diff --git a/dlclive/pose_estimation_tensorflow/runner.py b/dlclive/pose_estimation_tensorflow/runner.py index 3fa4eec..fa05f8e 100644 --- a/dlclive/pose_estimation_tensorflow/runner.py +++ b/dlclive/pose_estimation_tensorflow/runner.py @@ -35,26 +35,20 @@ class TensorFlowRunner(BaseRunner): - """TensorFlow runner for live pose estimation using DeepLabCut-Live. - - Args: - path: The path to the model to run inference with. - - Attributes: - path: The path to the model to run inference with. - """ + """TensorFlow runner for live pose estimation using DeepLabCut-Live.""" def __init__( self, path: str | Path, model_type: str = "base", tf_config: Any = None, + precision: str = "FP32", ) -> None: super().__init__(path) self.cfg = self.read_config() self.model_type = model_type self.tf_config = tf_config - self.precision = "FP32" + self.precision = precision self.sess = None self.inputs = None self.outputs = None @@ -172,7 +166,6 @@ def init_inference(self, frame: np.ndarray, **kwargs) -> np.ndarray: self.outputs = self.tflite_interpreter.get_output_details() elif self.model_type == "tensorrt": - graph_def = read_graph(model_file) graph = finalize_graph(graph_def) output_tensors = get_output_tensors(graph) diff --git a/dlclive/predictor/single_predictor.py b/dlclive/predictor/single_predictor.py index 96f31d9..c622cf9 100644 --- a/dlclive/predictor/single_predictor.py +++ b/dlclive/predictor/single_predictor.py @@ -14,7 +14,10 @@ import torch -from dlclive.pose_estimation_pytorch.models.predictors.base import BasePredictor, PREDICTORS +from dlclive.pose_estimation_pytorch.models.predictors.base import ( + BasePredictor, + PREDICTORS, +) @PREDICTORS.register_module diff --git a/dlclive/processor/kalmanfilter.py b/dlclive/processor/kalmanfilter.py index dbe05b3..ff46805 100644 --- a/dlclive/processor/kalmanfilter.py +++ b/dlclive/processor/kalmanfilter.py @@ -26,7 +26,6 @@ def __init__( lik_thresh=0, **kwargs, ): - super().__init__(**kwargs) self.adapt = adapt @@ -42,7 +41,6 @@ def __init__( self.last_pose_time = 0 def _get_forward_model(self, dt): - F = np.zeros((self.n_states, self.n_states)) for d in range(self.nderiv + 1): for i in range(self.n_states - (d * self.bp * 2)): @@ -51,7 +49,6 @@ def _get_forward_model(self, dt): return F def _init_kf(self, pose): - # get number of body parts self.bp = pose.shape[0] self.n_states = self.bp * 2 * (self.nderiv + 1) @@ -76,7 +73,6 @@ def _init_kf(self, pose): self.is_initialized = True def _predict(self): - F = self._get_forward_model(time.time() - self.last_pose_time) Pd = np.diag(self.P).reshape(self.P.shape[0], 1) @@ -86,7 +82,6 @@ def _predict(self): self.Pp = np.dot(np.dot(F, self.P), F.T) + self.Q def _get_residuals(self, pose): - z = np.zeros((self.n_states, 1)) z[: (self.bp * 2)] = pose[: self.bp, :2].reshape(self.bp * 2, 1) for i in range(self.bp * 2, self.n_states): @@ -94,7 +89,6 @@ def _get_residuals(self, pose): self.y = z - np.dot(self.H, self.Xp) def _update(self, liks): - S = np.dot(self.H, np.dot(self.Pp, self.H.T)) + self.R K = np.dot(np.dot(self.Pp, self.H.T), np.linalg.inv(S)) self.X = self.Xp + np.dot(K, self.y) @@ -102,7 +96,6 @@ def _update(self, liks): self.P = np.dot(self.I - np.dot(K, self.H), self.Pp) def _get_future_pose(self, dt): - Ff = self._get_forward_model(dt) Xf = np.dot(Ff, self.X) future_pose = Xf[: (self.bp * 2)].reshape(self.bp, 2) @@ -110,7 +103,6 @@ def _get_future_pose(self, dt): return future_pose def _get_state_likelihood(self, pose): - liks = pose[:, 2] liks_xy = np.repeat(liks, 2) liks_xy_deriv = np.tile(liks_xy, self.nderiv + 1) @@ -118,15 +110,12 @@ def _get_state_likelihood(self, pose): return liks_state def process(self, pose, **kwargs): - if not self.is_initialized: - self._init_kf(pose) self.last_pose_time = time.time() return pose else: - self._predict() self._get_residuals(pose) liks = self._get_state_likelihood(pose) diff --git a/dlclive/utils.py b/dlclive/utils.py index 615758a..94a3dba 100644 --- a/dlclive/utils.py +++ b/dlclive/utils.py @@ -13,12 +13,14 @@ try: import skimage + SK_IM = True except ImportError as e: SK_IM = False try: import cv2 + OPEN_CV = True except ImportError as e: from PIL import Image @@ -65,17 +67,14 @@ def resize_frame(frame: np.ndarray, resize=None) -> np.ndarray: new_y = int(frame.shape[1] * resize) if OPEN_CV: - return cv2.resize(frame, (new_y, new_x)) else: - img = Image.fromarray(frame) img = img.resize((new_y, new_x)) return np.asarray(img) else: - return frame @@ -89,15 +88,12 @@ def img_to_rgb(frame: np.ndarray) -> np.ndarray: """ if frame.ndim == 2: - return gray_to_rgb(frame) elif frame.ndim == 3: - return bgr_to_rgb(frame) else: - warnings.warn( f"Image has {frame.ndim} dimensions. Must be 2 or 3 dimensions to convert to RGB", DLCLiveWarning, @@ -115,11 +111,9 @@ def gray_to_rgb(frame: np.ndarray) -> np.ndarray: """ if OPEN_CV: - return cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB) else: - img = Image.fromarray(frame) img = img.convert("RGB") return np.asarray(img) @@ -135,11 +129,9 @@ def bgr_to_rgb(frame: np.ndarray) -> np.ndarray: """ if OPEN_CV: - return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) else: - img = Image.fromarray(frame) img = img.convert("RGB") return np.asarray(img) @@ -165,12 +157,10 @@ def _img_as_ubyte_np(frame: np.ndarray) -> np.ndarray: # check if already ubyte if np.issubdtype(im_type, np.uint8): - return frame # if floating elif np.issubdtype(im_type, np.floating): - if (np.min(frame) < -1) or (np.max(frame) > 1): raise ValueError("Images of type float must be between -1 and 1.") @@ -181,14 +171,12 @@ def _img_as_ubyte_np(frame: np.ndarray) -> np.ndarray: # if integer elif np.issubdtype(im_type, np.integer): - im_type_info = np.iinfo(im_type) frame *= 255 / im_type_info.max frame[frame < 0] = 0 return frame.astype(np.uint8) else: - raise TypeError( "image of type {} could not be converted to ubyte".format(im_type) ) From 40005ae13bc7cc9c5af9159ac55b8fbc60284173 Mon Sep 17 00:00:00 2001 From: Niels Poulsen Date: Thu, 27 Feb 2025 15:30:43 +0100 Subject: [PATCH 22/24] improved docs for PyTorch code --- dlclive/dlclive.py | 45 +++++++++++++- .../dynamic_cropping.py | 20 +++---- dlclive/pose_estimation_pytorch/runner.py | 59 +++++++++++++------ 3 files changed, 94 insertions(+), 30 deletions(-) diff --git a/dlclive/dlclive.py b/dlclive/dlclive.py index db706b2..74a1ffa 100644 --- a/dlclive/dlclive.py +++ b/dlclive/dlclive.py @@ -46,14 +46,55 @@ class DLCLive: TensorFlow only. Optional ConfigProto for the TensorFlow session. single_animal: bool, default=True - PyTorch only. + PyTorch only. If True, the predicted pose array returned by the runner will be + (num_bodyparts, 3). As multi-animal pose estimation can be run with the PyTorch + engine, setting this to False means the returned pose array will be of shape + (num_detections, num_bodyparts, 3). device: str, optional, default=None - PyTorch only. + PyTorch only. The device on which to run inference, e.g. "cpu", "cuda" or + "cuda:0". If set to None or "auto", the device will be automatically selected + based on CUDA availability. top_down_config: dict, optional, default=None + PyTorch only. Configuration settings for top-down pose estimation models. Must + be provided when running top-down models and `top_down_dynamic` is None. The + parameters in the dict will be given to the `TopDownConfig` class (in + `dlclive/pose_estimation_pytorch/runner.py`). The `crop_size` does not need to + be set, as it will be read from the model configuration file. + Example parameters: + >>> # Running a top-down model with basic parameters + >>> top_down_config = { + >>> "bbox_cutoff": 0.5, # min confidence score for a bbox to be used + >>> "max_detections": 3, # max number of detections to return in a frame + >>> } + >>> # Running a top-down model with skip-frames + >>> top_down_config = { + >>> "bbox_cutoff": 0.5, # min confidence score for a bbox to be used + >>> "max_detections": 3, # max number of detections to return in a frame + >>> "skip_frames": { # only run the detector every 5 frames + >>> "skip": 5, # number of frames to skip between detections + >>> "margin": 5, # margin (in pixels) to use when generating bboxes + >>> }, + >>> } top_down_dynamic: dict, optional, default=None + PyTorch only. Single animal only. Top-down models do not need a detector to be + used for single animal pose estimation. This is equivalent to dynamic cropping + in TensorFlow or for bottom-up models, but crops are resized to the input size + required by the model. Pose estimation is never run on the full image. If no + animal is detected, the image is split into N by M "patches", and we run pose + estimation on the batch of patches. Pose is kept from the patch with the + highest likelyhood. No need to provide the `top_down_crop_size` parameter, as it + set using the model configuration file. + The parameters (except "type") will be passed to the `TopDownDynamicCropper` + class (in `dlclive/pose_estimation_pytorch/dynamic_cropping.py` + + Example parameters: + >>> top_down_dynamic = { + >>> "type": "TopDownDynamicCropper", + >>> "min_bbox_size": (50, 50), + >>> } cropping: list of int Cropping parameters in pixel number: [x1, x2, y1, y2] diff --git a/dlclive/pose_estimation_pytorch/dynamic_cropping.py b/dlclive/pose_estimation_pytorch/dynamic_cropping.py index bc84533..b337e68 100644 --- a/dlclive/pose_estimation_pytorch/dynamic_cropping.py +++ b/dlclive/pose_estimation_pytorch/dynamic_cropping.py @@ -260,18 +260,19 @@ class TopDownDynamicCropper(DynamicCropper): def __init__( self, - top_down_crop_size: tuple[int, int], - patch_counts: tuple[int, int], - patch_overlap: int, - min_bbox_size: tuple[int, int], - threshold: float, - margin: int, + top_down_crop_size: tuple[int, int] = (256, 256), + patch_counts: tuple[int, int] = (4, 3), + patch_overlap: int = 50, + min_bbox_size: tuple[int, int] = (100, 100), + threshold: float = 0.6, + margin: int = 10, min_hq_keypoints: int = 2, bbox_from_hq: bool = False, store_crops: bool = False, **kwargs, ) -> None: super().__init__(threshold=threshold, margin=margin, **kwargs) + self.top_down_crop_size = top_down_crop_size self.min_bbox_size = min_bbox_size self.min_hq_keypoints = min_hq_keypoints self.bbox_from_hq = bbox_from_hq @@ -280,8 +281,7 @@ def __init__( self._patch_overlap = patch_overlap self._patches = [] self._patch_offsets = [] - self._td_crop_size = top_down_crop_size - self._td_ratio = self._td_crop_size[0] / self._td_crop_size[1] + self._td_ratio = self.top_down_crop_size[0] / self.top_down_crop_size[1] self.crop_history = [] self.store_crops = store_crops @@ -363,7 +363,7 @@ def update(self, pose: torch.Tensor) -> torch.Tensor: ) # offset and rescale the pose to the original image space - out_w, out_h = self._td_crop_size + out_w, out_h = self.top_down_crop_size offset_x, offset_y, w, h = self._crop scale_x, scale_y = w / out_w, h / out_h pose[..., 0] = (pose[..., 0] * scale_x) + offset_x @@ -448,7 +448,7 @@ def _crop_bounding_box( The cropped and resized image. """ x1, y1, w, h = bbox - out_w, out_h = self._td_crop_size + out_w, out_h = self.top_down_crop_size return F.resized_crop(image, y1, x1, h, w, [out_h, out_w]) def _crop_patches(self, image: torch.Tensor) -> torch.Tensor: diff --git a/dlclive/pose_estimation_pytorch/runner.py b/dlclive/pose_estimation_pytorch/runner.py index 9e94b3d..5e1d89e 100644 --- a/dlclive/pose_estimation_pytorch/runner.py +++ b/dlclive/pose_estimation_pytorch/runner.py @@ -32,6 +32,15 @@ class SkipFrames: then the detector will only be run every `skip` frames. Between frames where the detector is run, bounding boxes will be computed from the pose estimated in the previous frame. + + Every `N` frames, the detector will be run to detect bounding boxes for individuals. + In the "skipped" frames between the frames where the object detector is run, the + bounding boxes will be computed from the poses estimated in the previous frame (with + some margin added around the poses). + + Attributes: + skip: The number of frames to skip between each run of the detector. + margin: The margin (in pixels) to use when generating bboxes """ skip: int @@ -78,20 +87,28 @@ class TopDownConfig: """Configuration for top-down models. Attributes: + bbox_cutoff: The minimum score required for a bounding box to be considered. + max_detections: The maximum number of detections to keep in a frame. If None, + the `max_detections` will be set to the number of individuals in the model + configuration file when `read_config` is called. skip_frames: If defined, the detector will only be run every `skip_frames.skip` frames. """ - bbox_cutoff: float - max_detections: int + bbox_cutoff: float = 0.6 + max_detections: int | None = 30 crop_size: tuple[int, int] = (256, 256) skip_frames: SkipFrames | None = None - def read_config(self, detector_cfg: dict) -> None: - crop = detector_cfg.get("data", {}).get("inference", {}).get("top_down_crop") + def read_config(self, model_cfg: dict) -> None: + crop = model_cfg.get("data", {}).get("inference", {}).get("top_down_crop") if crop is not None: self.crop_size = (crop["width"], crop["height"]) + if self.max_detections is None: + individuals = model_cfg.get("metadata", {}).get("individuals", []) + self.max_detections = len(individuals) + class PyTorchRunner(BaseRunner): """PyTorch runner for live pose estimation using DeepLabCut-Live. @@ -242,7 +259,7 @@ def load_model(self) -> None: self.model = self.model.half() self.detector = None - if raw_data.get("detector") is not None: + if self.dynamic is None and raw_data.get("detector") is not None: self.detector = models.DETECTORS.build(self.cfg["detector"]["model"]) self.detector.to(self.device) self.detector.load_state_dict(raw_data["detector"]) @@ -251,18 +268,23 @@ def load_model(self) -> None: if self.precision == "FP16": self.detector = self.detector.half() - if self.cfg["method"] == "td" and self.detector is None: - crop_cfg = self.cfg["data"]["inference"]["top_down_crop"] - top_down_crop_size = crop_cfg["width"], crop_cfg["height"] - self.dynamic = dynamic_cropping.TopDownDynamicCropper( - top_down_crop_size, - patch_counts=(4, 3), - patch_overlap=50, - min_bbox_size=(250, 250), - threshold=0.6, - margin=25, - min_hq_keypoints=2, - bbox_from_hq=True, + if self.top_down_config is None: + self.top_down_config = TopDownConfig() + + self.top_down_config.read_config(self.cfg) + + if isinstance(self.dynamic, dynamic_cropping.TopDownDynamicCropper): + crop = self.cfg["data"]["inference"].get("top_down_crop", {}) + w, h = crop.get("width", 256), crop.get("height", 256) + self.dynamic.top_down_crop_size = w, h + + if ( + self.cfg["method"] == "td" + and self.detector is None + and self.dynamic is None + ): + raise ValueError( + "Top-down models must either use a detector or a TopDownDynamicCropper." ) self.transform = v2.Compose( @@ -283,9 +305,10 @@ def read_config(self) -> dict: def _prepare_top_down( self, frame: torch.Tensor, detections: dict[str, torch.Tensor] ): + """Prepares a frame for top-down pose estimation.""" bboxes, scores = detections["boxes"], detections["scores"] bboxes = bboxes[scores >= self.top_down_config.bbox_cutoff] - if len(bboxes) > 0: + if len(bboxes) > 0 and self.top_down_config.max_detections is not None: bboxes = bboxes[: self.top_down_config.max_detections] crops = [] From c6a1f6920f1958259c4c93a23f9ffbc704308eb6 Mon Sep 17 00:00:00 2001 From: Niels Poulsen Date: Thu, 27 Feb 2025 15:41:07 +0100 Subject: [PATCH 23/24] improved readme --- README.md | 47 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 09d5cbf..2536c93 100644 --- a/README.md +++ b/README.md @@ -112,24 +112,41 @@ dlc_live.get_pose() `DLCLive` **parameters:** - - `path` = string; full path to the exported DLC model directory - - `model_type` = string; the type of model to use for inference. Types include: - - `base` = the base DeepLabCut model - - `tensorrt` = apply [tensor-rt](https://developer.nvidia.com/tensorrt) optimizations to model - - `tflite` = use [tensorflow lite](https://www.tensorflow.org/lite) inference (in progress...) - - `cropping` = list of int, optional; cropping parameters in pixel number: [x1, x2, y1, y2] - - `dynamic` = tuple, optional; defines parameters for dynamic cropping of images - - `index 0` = use dynamic cropping, bool - - `index 1` = detection threshold, float - - `index 2` = margin (in pixels) around identified points, int - - `resize` = float, optional; factor by which to resize image (resize=0.5 downsizes both width and height of image by half). Can be used to downsize large images for faster inference - - `processor` = dlc pose processor object, optional - - `display` = bool, optional; display processed image with DeepLabCut points? Can be used to troubleshoot cropping and resizing parameters, but is very slow +- `path` = string; full path to the exported DLC model directory +- `model_type` = string; the type of model to use for inference. Types include: + - `pytorch` = the base PyTorch DeepLabCut model + - `base` = the base TensorFlow DeepLabCut model + - `tensorrt` = apply [tensor-rt](https://developer.nvidia.com/tensorrt) optimizations to model + - `tflite` = use [tensorflow lite](https://www.tensorflow.org/lite) inference (in progress...) +- `cropping` = list of int, optional; cropping parameters in pixel number: [x1, x2, y1, y2] +- `dynamic` = tuple, optional; defines parameters for dynamic cropping of images + - `index 0` = use dynamic cropping, bool + - `index 1` = detection threshold, float + - `index 2` = margin (in pixels) around identified points, int +- `resize` = float, optional; factor by which to resize image (resize=0.5 downsizes + both width and height of image by half). Can be used to downsize large images for + faster inference +- `processor` = dlc pose processor object, optional +- `display` = bool, optional; display processed image with DeepLabCut points? Can be + used to troubleshoot cropping and resizing parameters, but is very slow `DLCLive` **inputs:** - - `` = path to the folder that has the `.pb` files that you acquire after running `deeplabcut.export_model` - - `` = is a numpy array of each frame +- `` = + - For TensorFlow models: path to the folder that has the `.pb` files that you + acquire after running `deeplabcut.export_model` + - For PyTorch models: path to the `.pt` file that is generated after running + `deeplabcut.export_model` +- `` = is a numpy array of each frame + +#### DLCLive - PyTorch Specific Guide + +This guide is for users who trained a model with the PyTorch engine with +`DeepLabCut 3.0`. + +Once you've trained your model in [DeepLabCut](https://github.com/DeepLabCut/DeepLabCut) +and you are happy with its performance, you can export the model to be used for live +inference with DLCLive! ### Switching from TensorFlow to PyTorch From d738a2c8f1c576c9fda0567371a0368902c16da5 Mon Sep 17 00:00:00 2001 From: Niels Poulsen Date: Thu, 27 Feb 2025 17:20:20 +0100 Subject: [PATCH 24/24] fix default top down dynamic cropping parameters --- dlclive/pose_estimation_pytorch/dynamic_cropping.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dlclive/pose_estimation_pytorch/dynamic_cropping.py b/dlclive/pose_estimation_pytorch/dynamic_cropping.py index b337e68..ae5991f 100644 --- a/dlclive/pose_estimation_pytorch/dynamic_cropping.py +++ b/dlclive/pose_estimation_pytorch/dynamic_cropping.py @@ -261,13 +261,13 @@ class TopDownDynamicCropper(DynamicCropper): def __init__( self, top_down_crop_size: tuple[int, int] = (256, 256), - patch_counts: tuple[int, int] = (4, 3), + patch_counts: tuple[int, int] = (3, 2), patch_overlap: int = 50, - min_bbox_size: tuple[int, int] = (100, 100), - threshold: float = 0.6, + min_bbox_size: tuple[int, int] = (50, 50), + threshold: float = 0.25, margin: int = 10, min_hq_keypoints: int = 2, - bbox_from_hq: bool = False, + bbox_from_hq: bool = True, store_crops: bool = False, **kwargs, ) -> None: