Density map

.gitignore

@@ -13,9 +13,15 @@ data/
 *.pt
 *.pkl
 *.bin
-*.png
-*.jpg
+# *.png
+# *.jpg
 *.mp4
 *.gif
 
 debug*
+
+.env
+
+wandb/
+
+tmp*/

.vscode/launch.json

@@ -0,0 +1,46 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Debug PowerPaint (Accelerate)",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "accelerate.commands.launch",
+      "console": "integratedTerminal",
+      "justMyCode": false,
+      "env": {
+        "CUDA_VISIBLE_DEVICES": "2"
+      },
+      "args": [
+        "--config_file", "configs/acc.yaml",
+        "train_idm_pp1_v2.py",
+        "--config", "configs/idm_pp1.yaml"
+      ],
+      "python": "/mnt/disk1/aiotlab/envs/powerpaint/bin/python"
+    }
+  ]
+}
+
+// {
+//     "version": "0.2.0",
+//     "configurations": [
+
+//         {
+//             "name": "Debug PowerPaint (CUDA:2)",
+//             "type": "debugpy",
+//             "request": "launch",
+//             "program": "${workspaceFolder}/test_ppt.py",
+//             "console": "integratedTerminal",
+//             "justMyCode": false,
+//             "args": [
+//                 "--config",
+//                 "configs/test.yaml"
+//             ],
+//             "env": {
+//                 "CUDA_VISIBLE_DEVICES": "2",
+//                 "CUDA_LAUNCH_BLOCKING": "1"
+//             }
+//         }
+//     ]
+// }
+

configs/acc.yaml

@@ -1,12 +1,12 @@
 compute_environment: LOCAL_MACHINE
 deepspeed_config: {}
-distributed_type: MULTI_GPU
+distributed_type: NO
 fsdp_config: {}
 machine_rank: 0
 main_process_ip: null
 main_process_port: null
 main_training_function: main
 mixed_precision: fp16
 num_machines: 1
-num_processes: 8
+num_processes: 1
 use_cpu: false

configs/finetune_Ploc.yaml

@@ -0,0 +1,81 @@
+pretrained_model_name_or_path: "/workdir/radish/hachi/checkpoints/stable-diffusion-inpainting"
+ppt1_model_path: "/workdir/radish/hachi/checkpoints/ppt-v1"
+#tracker_project_name: "ppt1_finetune_bs64"
+#report_to: "wandb"
+output_dir: "/workdir/radish/hachi/New_ouput_3"
+
+# learnable task prompts
+task_prompt:
+  indomain_inpainting:
+    placeholder_tokens: "P_loc"
+    initializer_token: "P_ctxt"
+    num_vectors_per_token: 10
+
+# training data
+train_data:
+  resolution: 512
+  datasets:
+    name: "FSC147"
+    data_path: "/workdir/radish/hachi/OBJ_INS/phase2_V2/train"
+
+# training hyper-parameters
+num_train_epochs: 20
+lr_warmup_steps: 200
+learning_rate: 1e-5
+max_train_steps: 8e3            # max training steps
+train_batch_size: 64              # batch size per GPU
+dataloader_num_workers: 8
+
+gradient_checkpointing: true
+
+checkpointing_steps: 2e2
+validation_steps: 2e2
+checkpoints_total_limit: 10
+# resume_from_checkpoint: "latest"
+
+
+# validation data
+validation_data:
+  data_root: "/home/hachi/obj-ins/PowerPaint/validation/"
+  cases:
+    - image: "test/2_b1/ground_truth.png" # cần resize
+      mask: "test/2_b1/mask.png" # cần bbox mask
+      name: 2_b1
+      prompt:
+        - task: "indomain_inpainting" 
+          prompt: ""
+          negative_prompt: ""
+          promptA: ${task_prompt.indomain_inpainting.placeholder_tokens} A seashell
+          promptB: ${task_prompt.indomain_inpainting.placeholder_tokens} A seashell
+          tradeoff: 1.0
+          negative_promptA: ""
+          negative_promptB: ""
+        - task: "indomain_inpainting" 
+          prompt: ""
+          negative_prompt: ""
+          promptA: P_obj A seashell
+          promptB: P_obj A seashell
+          tradeoff: 1.0
+          negative_promptA: ""
+          negative_promptB: ""
+
+    - image: "val/560_b3/ground_truth.png" # cần resize
+      mask: "val/560_b3/mask.png" # cần bbox mask
+      name: 560_b3
+      prompt:
+        - task: "indomain_inpainting" 
+          prompt: ""
+          negative_prompt: ""
+          promptA: ${task_prompt.indomain_inpainting.placeholder_tokens} A flamingo
+          promptB: ${task_prompt.indomain_inpainting.placeholder_tokens} A flamingo
+          tradeoff: 1.0
+          negative_promptA: ""
+          negative_promptB: ""
+        - task: "indomain_inpainting" 
+          prompt: ""
+          negative_prompt: ""
+          promptA: P_obj A flamingo
+          promptB: P_obj A flamingo
+          tradeoff: 1.0
+          negative_promptA: ""
+          negative_promptB: ""

configs/idm_pp1.yaml

@@ -0,0 +1,218 @@
+base_model_path: "/mnt/disk1/aiotlab/hachi/checkpoints/stable-diffusion-inpainting" 
+ppt1_checkpoint: "/mnt/disk1/aiotlab/hachi/checkpoints/ppt-v1"
+tracker_project_name: "idm_pp1_density_map"
+report_to: "tensorboard"
+output_dir: "/mnt/disk1/aiotlab/hachi/Output/ConvIn+Down0_IDM_PP1_DensityMap_OriginLoss"
+
+# learnable task prompts
+task_prompt:
+  object_inpainting:
+    placeholder_tokens: "P_obj"
+    initializer_token: "a"
+    num_vectors_per_token: 10
+
+  indomain_inpainting:
+    placeholder_tokens: "P_loc"
+    initializer_token: "P_obj"
+    num_vectors_per_token: 10
+
+# training data
+train_root: "/mnt/disk1/aiotlab/hachi/data/OBJ_INS_FSC/train"
+density_root: "/mnt/disk1/aiotlab/hachi/data/FSC_147/gt_density_map_adaptive_384_VarV2"
+
+# training hyper-parameters
+train_mode: "conv_in+down0"
+num_train_epochs: 20
+lr_scheduler: "constant_with_warmup"
+lr_warmup_steps: 200
+learning_rate: 2.0e-4
+max_train_steps: 8e3            # max training steps
+train_batch_size: 8              # batch size per GPU
+by_bucket_sizes:
+  - size: [512, 512]
+    value: 8
+  - size: [512, 768]
+    value: 8
+  - size: [512, 1024]
+    value: 8
+dataloader_num_workers: 4
+gradient_accumulation_steps: 4
+gradient_checkpointing: true
+
+checkpointing_steps: 500
+validation_steps: 500
+
+
+# validation data
+val_root: "/mnt/disk1/aiotlab/hachi/data/OBJ_INS_FSC/val"
+# validation_data:
+#   data_root: "/mnt/disk1/aiotlab/hachi/PowerPaint/validation/"
+#   cases:
+#     - image: "test/2_b1/ground_truth.png" # cần resize
+#       mask: "test/2_b1/mask.png" # cần bbox mask
+#       name: 2_b1
+#       prompt:
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: ${task_prompt.indomain_inpainting.placeholder_tokens} A seashell
+#           promptB: ${task_prompt.indomain_inpainting.placeholder_tokens} A seashell
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: P_obj A seashell
+#           promptB: P_obj A seashell
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+
+#     - image: "test/2615_b1/ground_truth.png" # cần resize
+#       mask: "test/2615_b1/mask.png" # cần bbox mask
+#       name: 2615_b1
+#       prompt:
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: ${task_prompt.indomain_inpainting.placeholder_tokens} A deer
+#           promptB: ${task_prompt.indomain_inpainting.placeholder_tokens} A deer
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: P_obj A deer
+#           promptB: P_obj A deer
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+
+#     - image: "test/7420_b1/ground_truth.png" # cần resize
+#       mask: "test/7420_b1/mask.png" # cần bbox mask
+#       name: 7420_b1
+#       prompt:
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: ${task_prompt.indomain_inpainting.placeholder_tokens} A deer
+#           promptB: ${task_prompt.indomain_inpainting.placeholder_tokens} A deer
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: P_obj A deer
+#           promptB: P_obj A deer
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+
+
+#     - image: "val/560_b3/ground_truth.png" # cần resize
+#       mask: "val/560_b3/mask.png" # cần bbox mask
+#       name: 560_b3
+#       prompt:
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: ${task_prompt.indomain_inpainting.placeholder_tokens} A flamingo
+#           promptB: ${task_prompt.indomain_inpainting.placeholder_tokens} A flamingo
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: P_obj A flamingo
+#           promptB: P_obj A flamingo
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+
+#     - image: "val/568_b1/ground_truth.png" # cần resize
+#       mask: "val/568_b1/mask.png" # cần bbox mask
+#       name: 568_b1
+#       prompt:
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: ${task_prompt.indomain_inpainting.placeholder_tokens} A flamingo
+#           promptB: ${task_prompt.indomain_inpainting.placeholder_tokens} A flamingo
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: P_obj A flamingo
+#           promptB: P_obj A flamingo
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+
+#     - image: "val/573_b3/ground_truth.png" # cần resize
+#       mask: "val/573_b3/mask.png" # cần bbox mask
+#       name: 573_b3
+#       prompt:
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: ${task_prompt.indomain_inpainting.placeholder_tokens} A flamingo
+#           promptB: ${task_prompt.indomain_inpainting.placeholder_tokens} A flamingo
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: P_obj A flamingo
+#           promptB: P_obj A flamingo
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+
+#     - image: "val/578_b2/ground_truth.png" # cần resize
+#       mask: "val/578_b2/mask.png" # cần bbox mask
+#       name: 578_b2
+#       prompt:
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: ${task_prompt.indomain_inpainting.placeholder_tokens} A flamingo
+#           promptB: ${task_prompt.indomain_inpainting.placeholder_tokens} A flamingo
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: P_obj A flamingo
+#           promptB: P_obj A flamingo
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+
+#     - image: "val/3536_b3/ground_truth.png" # cần resize
+#       mask: "val/3536_b3/mask.png" # cần bbox mask
+#       name: 3536_b3
+#       prompt:
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: ${task_prompt.indomain_inpainting.placeholder_tokens} A chicken wing
+#           promptB: ${task_prompt.indomain_inpainting.placeholder_tokens} A chicken wing
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""
+#         - task: "indomain_inpainting" 
+#           prompt: ""
+#           negative_prompt: ""
+#           promptA: P_obj A chicken wing
+#           promptB: P_obj A chicken wing
+#           tradeoff: 1.0
+#           negative_promptA: ""
+#           negative_promptB: ""