DEGA-331-Xenium-pre-processing-bug and lower max_workers default to 1 (#173)

* adding variable to set max_workers

* Fix preprocessing path (#174)

* fix preprocess data dir

* tested on pancreas

* updated notebook

Author: cornhundred

notebooks/BNB_Streamlined_Pre_processing_Xenium_V1_Human_Pancreas_FFPE.ipynb

@@ -10,18 +10,24 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 1,
    "id": "b81ab32e",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "The autoreload extension is already loaded. To reload it, use:\n",
-      "  %reload_ext autoreload\n",
       "env: ANYWIDGET_HMR=1\n"
      ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/feni/Documents/celldega/dega/lib/python3.12/site-packages/h5py/__init__.py:36: UserWarning: h5py is running against HDF5 1.14.5 when it was built against 1.14.6, this may cause problems\n",
+      "  _warn((\"h5py is running against HDF5 {0} when it was built against {1}, \"\n"
+     ]
     }
    ],
    "source": [
@@ -42,7 +48,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "id": "13350680",
    "metadata": {},
    "outputs": [
@@ -51,6 +57,7 @@
      "output_type": "stream",
      "text": [
       "Starting preprocessing for sample: Xenium_V1_human_Pancreas_FFPE_outs\n",
+      "Created directory: data/landscape_files/Xenium_V1_human_Pancreas_FFPE_outs_test\n",
       "\n",
       "========Unzip and extract Xenium-related files========\n",
       "All files have been successfully extracted or skipped.\n",
@@ -59,7 +66,7 @@
       "Transformation matrix saved to 'data/landscape_files/Xenium_V1_human_Pancreas_FFPE_outs_test/micron_to_image_transform.csv'.\n",
       "\n",
       "========Check if all required files or directories exist========\n",
-      "All required files or directories for technology 'Xenium' are present in 'data/raw/Xenium_V1_human_Pancreas_FFPE_outs'.\n",
+      "All required files or directories for technology 'Xenium' are present in 'data/xenium_data/Xenium_V1_human_Pancreas_FFPE_outs'.\n",
       "\n",
       "========Make meta cells in pixel space========\n",
       "Done.\n",
@@ -71,7 +78,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/Users/whuan/dev/celldega/src/celldega/pre/__init__.py:213: PerformanceWarning: Concatenating sparse arrays with multiple fill values: '[True, False]'. Picking the first and converting the rest.\n",
+      "/Users/feni/Documents/celldega/src/celldega/pre/__init__.py:213: PerformanceWarning: Concatenating sparse arrays with multiple fill values: '[True, False]'. Picking the first and converting the rest.\n",
       "  df_sig = df_sig.dropna(axis=1, how=\"all\")\n"
      ]
     },
@@ -112,60 +119,13 @@
       "Cell clusters and meta cluster files created successfully.\n",
       "\n",
       "========Generating image tiles========\n",
-      "------ xenium\n",
-      "generating dapi image tiles ...\n",
-      "Image tiles created successfully.\n",
-      "\n",
-      "========Generating transcript tiles========\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Processing chunks: 100%|██████████| 81/81 [00:00<00:00, 1101.60it/s]\n",
-      "Processing coarse tiles: 84tile [00:08,  9.64tile/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "tile bounds: {'x_min': 0, 'x_max': 34126.65, 'y_min': 0, 'y_max': 13744.4}\n",
-      "\n",
-      "========Generating boundary tiles========\n",
-      "technology Xenium\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Processing coarse tiles: 100%|██████████| 14/14 [00:10<00:00,  1.35it/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Done.\n",
-      "\n",
-      "========Save landscape parameters========\n",
-      "Done.\n",
-      "Preprocessing completed successfully.\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
+      "------ xenium\n"
      ]
     }
    ],
    "source": [
     "sample = 'Xenium_V1_human_Pancreas_FFPE_outs'\n",
-    "data_dir = f'data/raw/{sample}'\n",
+    "data_dir = f'data/xenium_data/'\n",
     "path_landscape_files=f'data/landscape_files/{sample}_test'\n",
     "\n",
     "tile_size=250\n",
@@ -394,7 +354,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "celldega_env",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -408,7 +368,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.9"
+   "version": "3.12.7"
   },
   "toc": {
    "base_numbering": 1,

notebooks/Pre-processor_xenium_merscope.ipynb

@@ -289,7 +289,7 @@ def make_cell_boundary_tiles(
     tile_size=250,
     tile_bounds=None,
     image_scale=1,
-    max_workers=8,
+    max_workers=1,
 ):
     """
     Processes cell boundary data and divides it into spatial tiles based on the provided technology.
@@ -316,7 +316,7 @@ def make_cell_boundary_tiles(
         Dictionary containing the minimum and maximum bounds for x and y coordinates.
     image_scale : float, optional, default=1
         Scale factor to apply to the geometry data.
-    max_workers : int, optional, default=8
+    max_workers : int, optional, default=1
         Maximum number of parallel workers for processing tiles.
 
     Returns

src/celldega/pre/boundary_tile.py

@@ -109,14 +109,22 @@ def _setup_preprocessing_paths(technology, path_landscape_files, data_dir):
 
 
 def main(
-    sample, data_root_dir, tile_size, image_tile_layer, path_landscape_files, use_int_index=True
+    sample,
+    data_root_dir,
+    tile_size,
+    image_tile_layer,
+    path_landscape_files,
+    use_int_index=True,
+    max_workers=1,
 ):
     """
-    Main function to preprocess Xenium data and generate landscape files.
+    Main function to preprocess Xenium or MERSCOPE data and generate landscape files.
 
     Args:
         sample (str): Name of the sample (e.g., 'Xenium_V1_human_Pancreas_FFPE_outs').
-        data_root_dir (str): Root directory containing the sample data.
+        data_root_dir (str): Root directory containing all sample data. The
+            ``sample`` name will be appended to this path to locate the
+            specific dataset.
         tile_size (int): Size of the tiles for transcript and boundary tiles.
         image_tile_layer (str): Image layers to be tiled. 'dapi' or 'all'.
         path_landscape_files (str): Directory to save the landscape files.
@@ -136,7 +144,7 @@ def main(
     print(f"Starting preprocessing for sample: {sample}")
 
     # Construct data directory
-    data_dir = Path(data_root_dir)
+    data_dir = Path(data_root_dir) / sample
 
     # Create necessary directories if they don't exist
     _create_directories([data_dir, path_landscape_files])
@@ -212,7 +220,7 @@ def main(
         chunk_size=100000,
         verbose=False,
         image_scale=1,
-        max_workers=2,
+        max_workers=max_workers,
     )
     print(f"tile bounds: {tile_bounds}")
 
@@ -227,7 +235,7 @@ def main(
         coarse_tile_factor=10,
         tile_size=tile_size,
         tile_bounds=tile_bounds,
-        max_workers=2,
+        max_workers=max_workers,
     )
 
     # Force name to be str for MERSCOPE
@@ -268,7 +276,8 @@ def _setup_argument_parser():
     parser.add_argument(
         "--data_root_dir",
         required=True,
-        help="Root directory containing the data for this sample and other samples.",
+        help="Root directory containing all samples. The value will be joined with"
+        " the provided sample name to locate the dataset.",
     )
     parser.add_argument(
         "--tile_size",

src/celldega/pre/run_pre_processing.py

@@ -36,7 +36,6 @@ def get_local_server() -> int:
         int: The port number on which the server is running.
     """
     server = HTTPServer(("", 0), CORSHTTPRequestHandler)
-    print(f"Server running on port {server.server_address[1]}")
 
     service = thr.Thread(target=server.serve_forever)
     service.start()