From 200f02bd7709622e7bfe59f6372a6257c602984a Mon Sep 17 00:00:00 2001 From: Franklin King Date: Mon, 15 May 2023 16:57:07 -0400 Subject: [PATCH 1/4] Initial bronchoscopic depth map destimation model package structure and model files --- .../bronchoscopic_depthmap_estimation/LICENSE | 1 + .../configs/inference.json | 85 +++++++++++++++++++ .../configs/metadata.json | 74 ++++++++++++++++ .../docs/README.md | 58 +++++++++++++ .../docs/license.txt | 1 + .../large_files.yml | 9 ++ 6 files changed, 228 insertions(+) create mode 100644 models/bronchoscopic_depthmap_estimation/LICENSE create mode 100644 models/bronchoscopic_depthmap_estimation/configs/inference.json create mode 100644 models/bronchoscopic_depthmap_estimation/configs/metadata.json create mode 100644 models/bronchoscopic_depthmap_estimation/docs/README.md create mode 100644 models/bronchoscopic_depthmap_estimation/docs/license.txt create mode 100644 models/bronchoscopic_depthmap_estimation/large_files.yml diff --git a/models/bronchoscopic_depthmap_estimation/LICENSE b/models/bronchoscopic_depthmap_estimation/LICENSE new file mode 100644 index 00000000..13dfa24d --- /dev/null +++ b/models/bronchoscopic_depthmap_estimation/LICENSE @@ -0,0 +1 @@ +This work is licensed under a Creative Commons Attribution 4.0 International License: https://creativecommons.org/licenses/by/4.0/ \ No newline at end of file diff --git a/models/bronchoscopic_depthmap_estimation/configs/inference.json b/models/bronchoscopic_depthmap_estimation/configs/inference.json new file mode 100644 index 00000000..50317429 --- /dev/null +++ b/models/bronchoscopic_depthmap_estimation/configs/inference.json @@ -0,0 +1,85 @@ +{ + "imports": [ + "$import glob", + "$import os" + ], + "bundle_root": ".", + "ckpt_dir": "$@bundle_root + '/models'", + "output_dir": "$@bundle_root + '/eval'", + "data_list_file_path": "$@bundle_root + '/configs/datalist.json'", + "device": "$torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')", + "amp": true, + "network_def": { + "_target_": "ResNet", + "init_filters": 16, + "in_channels": 1, + "out_channels": 1 + }, + "network": "$@network_def.to(@device)", + "preprocessing": { + "_target_": "Compose", + "transforms": [ + { + "_target_": "LoadImaged", + "keys": "image", + "image_only": false + }, + ] + }, + "dataset": { + "_target_": "Dataset", + "data": "@test_datalist", + "transform": "@preprocessing" + }, + "dataloader": { + "_target_": "DataLoader", + "dataset": "@dataset", + "batch_size": 1, + "shuffle": true, + "num_workers": 4 + }, + "postprocessing": { + "_target_": "Compose", + "transforms": [ + { + "_target_": "SaveImaged", + "keys": "depth", + "meta_keys": "pred_meta_dict", + "output_dir": "@output_dir", + "output_postfix": "depth", + "output_dtype": "float32", + "resample": false, + "squeeze_end_dims": true + } + ] + }, + "handlers": [ + { + "_target_": "CheckpointLoader", + "load_path": "$@bundle_root + '/models/model.pt'", + "load_dict": { + "model": "@network" + } + }, + { + "_target_": "StatsHandler", + "iteration_log": false + } + ], + "evaluator": { + "_target_": "SupervisedEvaluator", + "device": "@device", + "val_data_loader": "@dataloader", + "network": "@network", + "inferer": "@inferer", + "postprocessing": "@postprocessing", + "val_handlers": "@handlers", + "amp": true + }, + "initialize": [ + "$setattr(torch.backends.cudnn, 'benchmark', True)" + ], + "run": [ + "$@evaluator.run()" + ] +} diff --git a/models/bronchoscopic_depthmap_estimation/configs/metadata.json b/models/bronchoscopic_depthmap_estimation/configs/metadata.json new file mode 100644 index 00000000..341a10fc --- /dev/null +++ b/models/bronchoscopic_depthmap_estimation/configs/metadata.json @@ -0,0 +1,74 @@ +{ + "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_20220324.json", + "version": "0.0.1", + "changelog": { + "0.0.1": "initializing the model package structure" + }, + "monai_version": "1.2.0rc5", + "pytorch_version": "1.13.1", + "numpy_version": "1.22.2", + "optional_packages_version": { + "nibabel": "4.0.1" + }, + "name": "Bronchoscopy Depth Map", + "task": "Depth map generation from bronchoscopic images", + "description": "A model for generation of depth map images from bronchoscopic images", + "authors": "Surgical Navigation and Robotics Lab", + "data_type": "nibabel", + "image_classes": "single channel data, intensity scaled to [0, 1]", + "label_classes": "single channel data, 1 is spleen, 0 is everything else", + "pred_classes": "2 channels OneHot data, channel 1 is spleen, channel 0 is background", + "eval_metrics": { + "mean_dice": 0.961 + }, + "intended_use": "This is an example, not to be used for diagnostic purposes", + "references": [ + + ], + "network_data_format": { + "inputs": { + "image": { + "type": "image", + "format": "hounsfield", + "modality": "CT", + "num_channels": 1, + "spatial_shape": [ + 96, + 96, + 96 + ], + "dtype": "float32", + "value_range": [ + 0, + 1 + ], + "is_patch_data": true, + "channel_def": { + "0": "image" + } + } + }, + "outputs": { + "pred": { + "type": "image", + "format": "segmentation", + "num_channels": 2, + "spatial_shape": [ + 96, + 96, + 96 + ], + "dtype": "float32", + "value_range": [ + 0, + 1 + ], + "is_patch_data": true, + "channel_def": { + "0": "background", + "1": "spleen" + } + } + } + } +} \ No newline at end of file diff --git a/models/bronchoscopic_depthmap_estimation/docs/README.md b/models/bronchoscopic_depthmap_estimation/docs/README.md new file mode 100644 index 00000000..5c1ba0d6 --- /dev/null +++ b/models/bronchoscopic_depthmap_estimation/docs/README.md @@ -0,0 +1,58 @@ +# Model Title + +### **Authors** +*Anyone who should be attributed as part of the model. If multiple people or companies, use a comma seperated list* + +Example: + +Firstname1 LastName1, Firstname2 Lastname2, Affiliation1 + +### **Tags** +*What tags describe the model and task performed? Use a comma seperated list* + +Example: + +Segmentation, MR, Spleen, Self-Supervised + +## **Model Description** +*This section should describe general information about the model and task that it performs. Any high-level reference of what the model is doing, general architecture of the model, and brief mention of data used. A developer should be able to understand the purpose of the model from this section alone.* + +Example: +This model is trained using the UNet architecture [1] and is used for volumetric 3D segmentation of the spleen from CT image. The segmentation of spleen region is formulated as the voxel-wise binary classification. Each voxel is predicted as either foreground (spleen) or background. And the model is optimized with gradient descent method minimizing soft dice loss between the predicted mask and ground truth segmentation. + + +## **Data** +*This section should talk about the data. Where is the dataset from? How many images? What is the split for training, testing, and validation? Can the users get the data and where? Also, if you had to prepare the data in any particular way to make it work with the model, what steps or preprocessing steps (not transform steps) did you need to take to make it work.* + +Example: + +The DETR model was trained on COCO 2017 panoptic, a dataset consisting of 118k/5k annotated images for training/validation respectively. + +#### **Preprocessing** +Images are resized/rescaled such that the shortest side is at least 800 pixels and the largest side at most 1333 pixels, and normalized across the RGB channels with the ImageNet mean (0.485, 0.456, 0.406) and standard deviation (0.229, 0.224, 0.225). + +#### **Additional section** +Any additional sections can be added with h4 (####) and bolded header to make sure it's easily navigable. + + +## **Performance** +*What sort of training or evaluation performance should people expect from this model? How long did it take you to train the model? If you have training/evaluation charts, you can include them in this section, but aren't required.* + +Example: + +This model achieves the following results on COCO 2017 validation: a box AP (average precision) of 38.8, a segmentation AP (average precision) of 31.1 and a PQ (panoptic quality) of 43.4. + +For more details regarding evaluation results, we refer to table 5 of the original paper. + + +## **Citation Info** + +Example: + +``` +@article{radford2019language, + title={Language Models are Unsupervised Multitask Learners}, + author={Radford, Alec and Wu, Jeff and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya}, + year={2019} +} +``` diff --git a/models/bronchoscopic_depthmap_estimation/docs/license.txt b/models/bronchoscopic_depthmap_estimation/docs/license.txt new file mode 100644 index 00000000..13dfa24d --- /dev/null +++ b/models/bronchoscopic_depthmap_estimation/docs/license.txt @@ -0,0 +1 @@ +This work is licensed under a Creative Commons Attribution 4.0 International License: https://creativecommons.org/licenses/by/4.0/ \ No newline at end of file diff --git a/models/bronchoscopic_depthmap_estimation/large_files.yml b/models/bronchoscopic_depthmap_estimation/large_files.yml new file mode 100644 index 00000000..80d4e653 --- /dev/null +++ b/models/bronchoscopic_depthmap_estimation/large_files.yml @@ -0,0 +1,9 @@ +large_files: + - path: "models/model.pt" + url: "https://drive.google.com/uc?export=download&id=1hBsBRrET40FbB9WJcNg5hIL67h2p2pRi" + hash_val: "5883296f3d06021b7068d024a00ac0c7" + hash_type: "md5" + - path: "models/model.onnx" + url: "https://drive.google.com/uc?export=download&id=184YGCmG7kHScBC8Cf5cxrt1qJCcfo9Zh" + hash_val: "636a469d5fbf72ab568dd232c8aa965c" + hash_type: "md5" From 4ea76dacf4a95d5a7e4aed89baa4e4d31a187a62 Mon Sep 17 00:00:00 2001 From: Franklin King Date: Mon, 15 May 2023 17:05:39 -0400 Subject: [PATCH 2/4] Updated metadata.json of bronchoscopic depth map estimation bundle --- .../configs/metadata.json | 48 +++++++------------ 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/models/bronchoscopic_depthmap_estimation/configs/metadata.json b/models/bronchoscopic_depthmap_estimation/configs/metadata.json index 341a10fc..aac74d4d 100644 --- a/models/bronchoscopic_depthmap_estimation/configs/metadata.json +++ b/models/bronchoscopic_depthmap_estimation/configs/metadata.json @@ -8,41 +8,31 @@ "pytorch_version": "1.13.1", "numpy_version": "1.22.2", "optional_packages_version": { - "nibabel": "4.0.1" - }, - "name": "Bronchoscopy Depth Map", + + }, + "name": "Bronchoscopy Depth Map Estimation", "task": "Depth map generation from bronchoscopic images", + "copyright": "CC BY 4.0", "description": "A model for generation of depth map images from bronchoscopic images", "authors": "Surgical Navigation and Robotics Lab", - "data_type": "nibabel", - "image_classes": "single channel data, intensity scaled to [0, 1]", - "label_classes": "single channel data, 1 is spleen, 0 is everything else", - "pred_classes": "2 channels OneHot data, channel 1 is spleen, channel 0 is background", - "eval_metrics": { - "mean_dice": 0.961 - }, - "intended_use": "This is an example, not to be used for diagnostic purposes", - "references": [ - - ], + "image_classes": "single channel data, intensity scaled to [0, 255]", "network_data_format": { "inputs": { "image": { "type": "image", - "format": "hounsfield", - "modality": "CT", + "format": "magnitude", + "modality": "regular", "num_channels": 1, "spatial_shape": [ - 96, - 96, - 96 + 200, + 200 ], - "dtype": "float32", + "dtype": "int", "value_range": [ 0, - 1 + 255 ], - "is_patch_data": true, + "is_patch_data": false, "channel_def": { "0": "image" } @@ -51,22 +41,20 @@ "outputs": { "pred": { "type": "image", - "format": "segmentation", - "num_channels": 2, + "format": "magnitude", + "num_channels": 1, "spatial_shape": [ - 96, - 96, - 96 + 200, + 200 ], "dtype": "float32", "value_range": [ 0, 1 ], - "is_patch_data": true, + "is_patch_data": false, "channel_def": { - "0": "background", - "1": "spleen" + "0": "depth" } } } From 4f1e6332dc96932c82eee00a10fd429f62a84e53 Mon Sep 17 00:00:00 2001 From: Franklin King Date: Mon, 5 Jun 2023 19:13:43 -0400 Subject: [PATCH 3/4] Updated README of bronchoscopic depth map estimation bundle --- .../docs/README.md | 58 +++++-------------- 1 file changed, 14 insertions(+), 44 deletions(-) diff --git a/models/bronchoscopic_depthmap_estimation/docs/README.md b/models/bronchoscopic_depthmap_estimation/docs/README.md index 5c1ba0d6..36151be7 100644 --- a/models/bronchoscopic_depthmap_estimation/docs/README.md +++ b/models/bronchoscopic_depthmap_estimation/docs/README.md @@ -1,58 +1,28 @@ # Model Title ### **Authors** -*Anyone who should be attributed as part of the model. If multiple people or companies, use a comma seperated list* - -Example: - -Firstname1 LastName1, Firstname2 Lastname2, Affiliation1 +Artur Banach, Franklin King, Fumitaro Masaki, Hisashi Tsukada, and Nobuhiko Hata +National Center for Image-guided Therapy, Brigham and Women’s Hospital and Harvard Medical School ### **Tags** -*What tags describe the model and task performed? Use a comma seperated list* - -Example: - -Segmentation, MR, Spleen, Self-Supervised +Bronchoscopy, Grayscale, Lung, Depth map ## **Model Description** -*This section should describe general information about the model and task that it performs. Any high-level reference of what the model is doing, general architecture of the model, and brief mention of data used. A developer should be able to understand the purpose of the model from this section alone.* - -Example: -This model is trained using the UNet architecture [1] and is used for volumetric 3D segmentation of the spleen from CT image. The segmentation of spleen region is formulated as the voxel-wise binary classification. Each voxel is predicted as either foreground (spleen) or background. And the model is optimized with gradient descent method minimizing soft dice loss between the predicted mask and ground truth segmentation. - +This model is in accordance with the following publication: Banach et al. "Visually Navigated Bronchoscopy using three cycle-Consistent generative adversarial network for depth estimation", Med. Image Anal. 73, 102164 (2021), https://www.ncbi.nlm.nih.gov/pmc/articles/PMC8453111/ ## **Data** -*This section should talk about the data. Where is the dataset from? How many images? What is the split for training, testing, and validation? Can the users get the data and where? Also, if you had to prepare the data in any particular way to make it work with the model, what steps or preprocessing steps (not transform steps) did you need to take to make it work.* - -Example: - -The DETR model was trained on COCO 2017 panoptic, a dataset consisting of 118k/5k annotated images for training/validation respectively. +https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/8MNI49 #### **Preprocessing** -Images are resized/rescaled such that the shortest side is at least 800 pixels and the largest side at most 1333 pixels, and normalized across the RGB channels with the ImageNet mean (0.485, 0.456, 0.406) and standard deviation (0.229, 0.224, 0.225). - -#### **Additional section** -Any additional sections can be added with h4 (####) and bolded header to make sure it's easily navigable. - - -## **Performance** -*What sort of training or evaluation performance should people expect from this model? How long did it take you to train the model? If you have training/evaluation charts, you can include them in this section, but aren't required.* - -Example: - -This model achieves the following results on COCO 2017 validation: a box AP (average precision) of 38.8, a segmentation AP (average precision) of 31.1 and a PQ (panoptic quality) of 43.4. - -For more details regarding evaluation results, we refer to table 5 of the original paper. - +Model takes as input grayscale 200x200 images scaled from 0 to 255. ## **Citation Info** - -Example: - -``` -@article{radford2019language, - title={Language Models are Unsupervised Multitask Learners}, - author={Radford, Alec and Wu, Jeff and Child, Rewon and Luan, David and Amodei, Dario and Sutskever, Ilya}, - year={2019} +@article{banach2021visually, + title={Visually navigated bronchoscopy using three cycle-consistent generative adversarial network for depth estimation}, + author={Banach, Artur and King, Franklin and Masaki, Fumitaro and Tsukada, Hisashi and Hata, Nobuhiko}, + journal={Medical image analysis}, + volume={73}, + pages={102164}, + year={2021}, + publisher={Elsevier} } -``` From b97c7257929b35553b6798bf2f182588b94e612b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Jun 2023 23:24:06 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- models/bronchoscopic_depthmap_estimation/LICENSE | 2 +- .../bronchoscopic_depthmap_estimation/configs/metadata.json | 6 +++--- models/bronchoscopic_depthmap_estimation/docs/license.txt | 2 +- models/bronchoscopic_depthmap_estimation/large_files.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/models/bronchoscopic_depthmap_estimation/LICENSE b/models/bronchoscopic_depthmap_estimation/LICENSE index 13dfa24d..b085295c 100644 --- a/models/bronchoscopic_depthmap_estimation/LICENSE +++ b/models/bronchoscopic_depthmap_estimation/LICENSE @@ -1 +1 @@ -This work is licensed under a Creative Commons Attribution 4.0 International License: https://creativecommons.org/licenses/by/4.0/ \ No newline at end of file +This work is licensed under a Creative Commons Attribution 4.0 International License: https://creativecommons.org/licenses/by/4.0/ diff --git a/models/bronchoscopic_depthmap_estimation/configs/metadata.json b/models/bronchoscopic_depthmap_estimation/configs/metadata.json index aac74d4d..bde2b71c 100644 --- a/models/bronchoscopic_depthmap_estimation/configs/metadata.json +++ b/models/bronchoscopic_depthmap_estimation/configs/metadata.json @@ -8,8 +8,8 @@ "pytorch_version": "1.13.1", "numpy_version": "1.22.2", "optional_packages_version": { - - }, + + }, "name": "Bronchoscopy Depth Map Estimation", "task": "Depth map generation from bronchoscopic images", "copyright": "CC BY 4.0", @@ -59,4 +59,4 @@ } } } -} \ No newline at end of file +} diff --git a/models/bronchoscopic_depthmap_estimation/docs/license.txt b/models/bronchoscopic_depthmap_estimation/docs/license.txt index 13dfa24d..b085295c 100644 --- a/models/bronchoscopic_depthmap_estimation/docs/license.txt +++ b/models/bronchoscopic_depthmap_estimation/docs/license.txt @@ -1 +1 @@ -This work is licensed under a Creative Commons Attribution 4.0 International License: https://creativecommons.org/licenses/by/4.0/ \ No newline at end of file +This work is licensed under a Creative Commons Attribution 4.0 International License: https://creativecommons.org/licenses/by/4.0/ diff --git a/models/bronchoscopic_depthmap_estimation/large_files.yml b/models/bronchoscopic_depthmap_estimation/large_files.yml index 80d4e653..d86eef7c 100644 --- a/models/bronchoscopic_depthmap_estimation/large_files.yml +++ b/models/bronchoscopic_depthmap_estimation/large_files.yml @@ -6,4 +6,4 @@ large_files: - path: "models/model.onnx" url: "https://drive.google.com/uc?export=download&id=184YGCmG7kHScBC8Cf5cxrt1qJCcfo9Zh" hash_val: "636a469d5fbf72ab568dd232c8aa965c" - hash_type: "md5" + hash_type: "md5"