Cleanup IO logging and fix file converter python script (#1072)

Refactors more of the logging output and fixes the python file converters to be able to handle json files that come in the merged layout, but did not actually merge any portals, yet. This allows to use the script on the current G200 ITk files

Author: niermann999

core/include/detray/builders/detector_builder.hpp

@@ -135,6 +135,9 @@ class detector_builder {
         // TODO: Add sorting, data deduplication etc. here later...
 
         DETRAY_INFO_HOST("Detector building complete: " << name());
+        DETRAY_VERBOSE_HOST("-> Built " << det.volumes().size() << " volumes");
+        DETRAY_VERBOSE_HOST("-> Built " << det.surfaces().size()
+                                        << " surfaces");
 
         return det;
     }
@@ -163,7 +166,8 @@ class detector_builder {
     /// Put the volumes into a search data structure
     template <typename... Args>
     DETRAY_HOST void set_volume_finder([[maybe_unused]] Args&&... args) {
-        DETRAY_VERBOSE_HOST("Setting volume finder for detector " << name());
+        DETRAY_VERBOSE_HOST("Setting volume finder for detector: '" << name()
+                                                                    << "'");
 
         using vol_finder_t = typename detector_type::volume_finder;
 

core/include/detray/builders/grid_builder.hpp

@@ -216,7 +216,7 @@ class grid_builder : public volume_decorator<detector_t> {
                           det.accelerator_store().template size<gid>() - 1);
 
         DETRAY_DEBUG_HOST("Accelerator link: " << vol_ptr->accel_link());
-        DETRAY_DEBUG_HOST("Finished grid building: " << m_grid);
+        // DETRAY_DEBUG_HOST("Finished grid building: " << m_grid);
 
         DETRAY_VERBOSE_HOST("Successfully built "
                             << gid << " for volume: " << this->name());

core/include/detray/builders/homogeneous_material_builder.hpp

@@ -59,7 +59,7 @@ class homogeneous_material_builder final : public volume_decorator<detector_t> {
             std::dynamic_pointer_cast<homogeneous_material_factory<detector_t>>(
                 sf_factory);
         if (mat_factory) {
-            DETRAY_VERBOSE_HOST("-> found " << DETRAY_TYPENAME(
+            DETRAY_VERBOSE_HOST("-> found decoration: " << DETRAY_TYPENAME(
                                     homogeneous_material_factory<detector_t>));
             (*mat_factory)(this->surfaces(), m_materials);
             return;
@@ -68,7 +68,7 @@ class homogeneous_material_builder final : public volume_decorator<detector_t> {
             homogeneous_material_generator<detector_t>>(sf_factory);
         if (mat_generator) {
             DETRAY_VERBOSE_HOST(
-                "-> found "
+                "-> found decoration: "
                 << DETRAY_TYPENAME(homogeneous_material_generator<detector_t>));
             (*mat_generator)(this->surfaces(), m_materials);
             return;

core/include/detray/builders/homogeneous_material_factory.hpp

@@ -128,20 +128,20 @@ class material_data {
     /// Output stream operator for material_data
     DETRAY_HOST friend std::ostream &operator<<(std::ostream &os,
                                                 const material_data &mat_data) {
-        os << "material_data{sf_index: ";
+        os << "material_data{\nsf_index: ";
         if (detail::is_invalid_value(mat_data.m_sf_index)) {
             os << "invalid";
         } else {
             os << mat_data.m_sf_index;
         }
-        os << ", materials: [";
+        os << ", materials: [\n";
         for (std::size_t i = 0; i < mat_data.m_mat.size(); ++i) {
             if (i > 0) {
-                os << ", ";
+                os << ",\n";
             }
             os << mat_data.m_mat[i];
         }
-        os << "], thickness: [";
+        os << "],\nthickness: [\n";
         for (std::size_t i = 0; i < mat_data.m_thickness.size(); ++i) {
             if (i > 0) {
                 os << ", ";

core/include/detray/builders/material_map_builder.hpp

@@ -85,15 +85,16 @@ class material_map_builder final : public volume_decorator<detector_t> {
             material_map_factory<detector_t, axis::multi_bin<DIM>>;
         auto mat_factory = std::dynamic_pointer_cast<sf_factory_t>(sf_factory);
         if (mat_factory) {
-            DETRAY_VERBOSE_HOST("-> found " << DETRAY_TYPENAME(sf_factory_t));
+            DETRAY_VERBOSE_HOST(
+                "-> found decoration: " << DETRAY_TYPENAME(sf_factory_t));
             (*mat_factory)(this->surfaces(), m_bin_data, m_n_bins,
                            m_axis_spans);
         }
         auto mat_generator =
             std::dynamic_pointer_cast<material_map_generator<detector_t>>(
                 sf_factory);
         if (mat_generator) {
-            DETRAY_VERBOSE_HOST("-> found " << DETRAY_TYPENAME(
+            DETRAY_VERBOSE_HOST("-> found decoration: " << DETRAY_TYPENAME(
                                     material_map_generator<detector_t>));
             (*mat_generator)(this->surfaces(), this->masks(), m_bin_data,
                              m_n_bins);
@@ -112,30 +113,34 @@ class material_map_builder final : public volume_decorator<detector_t> {
     auto build(detector_t& det,
                typename detector_t::geometry_context ctx = {}) ->
         typename detector_t::volume_type* override {
-        DETRAY_DEBUG_HOST("Build material maps...");
+        DETRAY_VERBOSE_HOST("Build material maps...");
 
         // Ensure the material links are correct BEFORE the surfaces are built
         // and potentially added to an acceleration data structure
         update_material_links(det);
 
-        DETRAY_DEBUG_HOST("-> defer to contained builders");
+        DETRAY_DEBUG_HOST(
+            "-> Let underlying builders construct the volume using correct "
+            "material links...");
         // Construct the surfaces
         typename detector_t::volume_type* vol_ptr =
             volume_decorator<detector_t>::build(det, ctx);
-        DETRAY_DEBUG_HOST("-> back in build");
+        DETRAY_DEBUG_HOST("-> Back in build");
 
         // Build the material grid for every surface that has material
         dindex sf_idx = 0u;
         auto vol = tracking_volume{det, this->vol_index()};
+        DETRAY_DEBUG_HOST("Build material maps for surfaces...");
         for (const auto& sf_desc : vol.surfaces()) {
 
-            DETRAY_DEBUG_HOST("-> sf_desc=" << sf_desc);
-
             if (!surface_has_map(sf_idx)) {
                 sf_idx++;
                 continue;
             }
 
+            DETRAY_DEBUG_HOST("-> surface #" << sf_idx
+                                             << " sf_desc = " << sf_desc);
+
             // Construct and append the material map for a given surface shape
             darray<std::vector<scalar_type>, DIM> axis_spans{};
             auto axis_spans_itr = m_axis_spans.find(sf_idx);
@@ -188,7 +193,7 @@ class material_map_builder final : public volume_decorator<detector_t> {
 
     /// Set the correct global surface material link for this detector
     void update_material_links(const detector_t& det) {
-        DETRAY_DEBUG_HOST("'update_material_links()' called");
+        DETRAY_VERBOSE_HOST("Precompute material links...");
 
         // The total number of surfaces that will be built by this builder
         const dindex n_surfaces{static_cast<dindex>(this->surfaces().size())};
@@ -202,18 +207,20 @@ class material_map_builder final : public volume_decorator<detector_t> {
             }
 
             auto& sf_desc = this->surfaces().at(sf_idx);
+            const auto id{sf_desc.material().id()};
 
-            DETRAY_DEBUG_HOST("- sf_desc=" << sf_desc);
+            DETRAY_DEBUG_HOST("-> surface #" << sf_idx << " (mat. id = " << id
+                                             << ")");
 
-            const auto id{sf_desc.material().id()};
             if (!mat_type_count.contains(id)) {
                 mat_type_count.emplace(id, 0u);
             } else {
                 mat_type_count.at(id)++;
             }
 
-            DETRAY_DEBUG_HOST(" -> set_index (id=" << id << ") to: "
-                                                   << mat_type_count.at(id));
+            DETRAY_DEBUG_HOST(
+                "--> set mat. index to: " << mat_type_count.at(id));
+
             sf_desc.material().set_index(mat_type_count.at(id));
         }
 
@@ -222,21 +229,19 @@ class material_map_builder final : public volume_decorator<detector_t> {
         det._materials.template apply<detail::material_coll_size>(
             size_map, std::make_index_sequence<materials_t::n_types>{});
 
-        DETRAY_DEBUG_HOST("Update the counts with the detector offset");
+        DETRAY_DEBUG_HOST("Shift material indices by detector offset...");
         for (dindex sf_idx = 0u; sf_idx < n_surfaces; ++sf_idx) {
             if (!surface_has_map(sf_idx)) {
                 continue;
             }
             auto& sf_desc = this->surfaces().at(sf_idx);
-            DETRAY_DEBUG_HOST("- sf_desc=" << sf_desc);
 
-            DETRAY_DEBUG_HOST("sf_desc.material() = "
-                              << sf_desc.material() << " ("
-                              << sf_desc.material().id() << ")");
             auto coll_idx{static_cast<std::size_t>(sf_desc.material().id())};
-            DETRAY_DEBUG_HOST("sf_desc.material() += size_map.at(coll_idx) --> "
-                              << size_map.at(coll_idx));
             sf_desc.material() += size_map.at(coll_idx);
+
+            DETRAY_DEBUG_HOST("(Shift = " << size_map.at(coll_idx)
+                                          << "): material link = "
+                                          << sf_desc.material());
         }
     }
 
@@ -321,6 +326,8 @@ struct add_sf_material_map {
 
             // Not every mask shape might be used for material maps
             if constexpr (materials_t::template is_defined<non_owning_t>()) {
+                DETRAY_VERBOSE_HOST("Filling material grid...");
+
                 // Add the material slabs to the grid
                 for (const auto& bin : bin_data) {
                     mat_grid.template populate<replace<>>(bin.local_bin_idx,
@@ -331,10 +338,8 @@ struct add_sf_material_map {
                 constexpr auto gid{
                     materials_t::template get_id<non_owning_t>()};
                 mat_store.template push_back<gid>(mat_grid);
-                DETRAY_DEBUG_HOST("Adding material grid with gid="
-                                  << gid << " mat_grid=" << mat_grid
-                                  << "(size is now "
-                                  << mat_store.template size<gid>() << ")");
+                DETRAY_VERBOSE_HOST("Built material grid:" << gid << ":\n"
+                                                           << mat_grid.axes());
 
                 // Return the index of the new material map
                 return {gid, static_cast<dindex>(

core/include/detray/builders/material_map_factory.hpp

@@ -109,7 +109,7 @@ class material_map_factory final : public factory_decorator<detector_t> {
                       std::vector<std::vector<scalar_type>> &&axis_spans,
                       std::vector<index_type> &&indices) {
 
-        DETRAY_DEBUG_HOST("Add map data");
+        DETRAY_VERBOSE_HOST("Received material map data");
 
         auto [sf_index, mat, thickness] = std::move(mat_data).get_data();
 
@@ -150,7 +150,7 @@ class material_map_factory final : public factory_decorator<detector_t> {
         std::map<dindex, darray<std::size_t, N>> &n_bins,
         std::map<dindex, darray<std::vector<scalar_type>, N>> &axis_spans) {
 
-        DETRAY_DEBUG_HOST("Add material maps...");
+        DETRAY_VERBOSE_HOST("Add material maps...");
 
         using link_t = typename detector_t::surface_type::material_link;
 
@@ -162,14 +162,16 @@ class material_map_factory final : public factory_decorator<detector_t> {
 
         assert(surfaces.size() >= n_materials);
 
-        DETRAY_DEBUG_HOST("-> Have " << m_materials.size()
-                                     << " configured material maps to assign");
+        DETRAY_VERBOSE_HOST("-> Have "
+                            << m_materials.size()
+                            << " configured material map(s) to assign");
 
         // Add the material only to those surfaces that the data links against
         for (auto &[i, materials] : m_materials) {
             const auto sf_idx{static_cast<dindex>(i)};
 
-            DETRAY_DEBUG_HOST("-> sf=" << surfaces.at(sf_idx));
+            DETRAY_DEBUG_HOST("--> #" << sf_idx
+                                      << " sf = " << surfaces.at(sf_idx));
 
             // Copy the number of bins to the builder
             assert(m_n_bins.at(sf_idx).size() == N);
@@ -204,9 +206,11 @@ class material_map_factory final : public factory_decorator<detector_t> {
             }
 
             auto map_id{m_links.at(sf_idx).first};
-            DETRAY_DEBUG_HOST("-> material map factory sets map_id=" << map_id);
+            DETRAY_DEBUG_HOST("--> Set surface material id: " << map_id);
             surfaces[sf_idx].material() = link_t{map_id, dindex_invalid};
         }
+
+        DETRAY_DEBUG_HOST("Finished");
     }
 
     private:

core/include/detray/builders/surface_factory.hpp

@@ -160,7 +160,7 @@ class surface_factory : public surface_factory_interface<detector_t> {
         // In case the surfaces container is prefilled with other surfaces
         const auto surfaces_offset{static_cast<dindex>(surfaces.size())};
 
-        DETRAY_VERBOSE_HOST("-> Have " << size() << " surfaces");
+        DETRAY_VERBOSE_HOST("-> Adding " << size() << " surfaces");
 
         // Nothing to construct
         if (size() == 0u) {

core/include/detray/builders/volume_builder.hpp

@@ -275,7 +275,8 @@ class volume_builder : public volume_builder_interface<detector_t> {
 
             // Add only the portals to the brute force method
             DETRAY_VERBOSE_HOST(
-                "-> Register portals/passives with brute force accelerator");
+                "-> Register only portals/passives with brute force "
+                "accelerator");
 
             det._accelerators.template push_back<default_acc_id>(
                 std::move(portals));

core/include/detray/propagator/propagator.hpp

@@ -227,7 +227,9 @@ struct propagator {
         for (unsigned int i = 0; i % 2 == 0 || propagation.is_alive(); ++i) {
 
             if (i % 2 == 0) {
-                DETRAY_DEBUG_HOST("Propagation step: " << i / 2);
+                DETRAY_VERBOSE_HOST_DEVICE("Propagation step: %d", i / 2);
+                DETRAY_VERBOSE_HOST_DEVICE("Path length: %f mm",
+                                           stepping.path_length());
 
                 // Run all registered actors/aborters
                 run_actors(actor_state_refs, propagation);
@@ -280,11 +282,11 @@ struct propagator {
         // Pass on the whether the propagation was successful
         DETRAY_VERBOSE_HOST("Finished propagation for track:\n" << track);
         if (is_complete(propagation)) {
-            DETRAY_VERBOSE_HOST("Satus: COMPLETE");
+            DETRAY_VERBOSE_HOST_DEVICE("Satus: COMPLETE");
         } else if (is_paused(propagation)) {
-            DETRAY_VERBOSE_HOST("Satus: PAUSED");
+            DETRAY_VERBOSE_HOST_DEVICE("Satus: PAUSED");
         } else {
-            DETRAY_VERBOSE_HOST("Satus: ABORT");
+            DETRAY_VERBOSE_HOST_DEVICE("Satus: ABORT");
         }
 
         return is_complete(propagation) || is_paused(propagation);

core/include/detray/utils/grid/detail/axis.hpp

@@ -189,7 +189,7 @@ struct single_axis {
         os << "bounds: " << ax.bounds() << std::endl;
         os << "binning: " << ax.binning() << std::endl;
         os << "n-bins: " << ax.nbins() << std::endl;
-        os << "min: " << ax.min() << ", max: " << ax.max();
+        os << "span: [" << ax.min() << ", " << ax.max() << "]";
 
         return os;
     }