[WiP] Make sure to always only have valid collection names

k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h

@@ -23,12 +23,14 @@
 
 #include "k4EDM4hep2LcioConv/k4EDM4hep2LcioConv.h"
 
+#include "k4FWCore/ICollectionFromObjectSvc.h"
+
 #include <Gaudi/Property.h>
 #include <GaudiKernel/AlgTool.h>
 
 #include <map>
 #include <string>
-#include <tuple>
+#include <unordered_map>
 #include <vector>
 
 class PodioDataSvc;
@@ -71,10 +73,12 @@ class EDM4hep2LcioTool : public AlgTool, virtual public IEDMConverter {
   ServiceHandle<IDataProviderSvc> m_eventDataSvc;
   // Metadata service from k4FWCore that is used together with IOSvc
   SmartIF<IMetadataSvc> m_metadataSvc;
+  /// Service to retrive collection names from objects
+  SmartIF<ICollectionFromObjectSvc> m_collFromObjSvc;
+
   /// A (caching) "map" of original to new collection names that will be populated
   /// during the first conversion
-  std::vector<std::tuple<std::string, std::string>> m_collsToConvert{};
-  std::map<uint32_t, std::string> m_idToName;
+  std::unordered_map<std::string, std::string> m_collsToConvert{};
 
   void convertTracks(TrackMap& tracks_vec, const std::string& e4h_coll_name, const std::string& lcio_coll_name,
                      lcio::LCEventImpl* lcio_event);
@@ -109,7 +113,8 @@ class EDM4hep2LcioTool : public AlgTool, virtual public IEDMConverter {
   void convertReconstructedParticles(RecoParticleMap& recoparticles_vec, const std::string& e4h_coll_name,
                                      const std::string& lcio_coll_name, lcio::LCEventImpl* lcio_event);
 
-  void convertParticleIDs(ParticleIDMap& pidMap, const std::string& e4h_coll_name, int32_t algoId);
+  void convertParticleIDs(ParticleIDMap& pidMap, std::vector<EDM4hep2LCIOConv::ParticleIDConvData>& pidCollections,
+                          lcio::LCEventImpl* lcio_event, const podio::Frame& edmEvent);
 
   void convertMCParticles(MCParticleMap& mc_particles_vec, const std::string& e4h_coll_name,
                           const std::string& lcio_coll_name, lcio::LCEventImpl* lcio_event);
@@ -123,6 +128,13 @@ class EDM4hep2LcioTool : public AlgTool, virtual public IEDMConverter {
 
   const podio::Frame& getEDM4hepEvent() const;
 
+  /// Try to attach the ParticleID related metadata (algorithm name, parameter
+  /// names) to the corresponding LCIO *ReconstructedParticle* collection and
+  /// return the value of the algorithmType that has been used (in case of
+  /// success).
+  std::optional<int32_t> attacheParticleIDMetaInfo(const EDM4hep2LCIOConv::ParticleIDConvData& pidCollMeta,
+                                                   lcio::LCEventImpl* lcio_event, const podio::Frame& edmEvent) const;
+
   /// Get an EDM4hep collection by name, consulting either the podio based data
   /// svc or the IOSvc
   podio::CollectionBase* getEDM4hepCollection(const std::string& name) const;

k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp

@@ -34,7 +34,12 @@
 #include "GaudiKernel/IDataManagerSvc.h"
 #include "GaudiKernel/IDataProviderSvc.h"
 
+#include <IMPL/LCEventImpl.h>
+#include <fmt/format.h>
+#include <fmt/ranges.h>
+
 #include <functional>
+#include <k4EDM4hep2LcioConv/k4EDM4hep2LcioConv.h>
 #include <memory>
 
 DECLARE_COMPONENT(EDM4hep2LcioTool);
@@ -79,6 +84,12 @@ StatusCode EDM4hep2LcioTool::initialize() {
     }
   }
 
+  m_collFromObjSvc = service("CollectionFromObjectSvc", false);
+  if (!m_collFromObjSvc) {
+    error() << "Could not retrieve CollectionFromObjectSvc" << endmsg;
+    return StatusCode::FAILURE;
+  }
+
   return AlgTool::initialize();
 }
 
@@ -117,10 +128,17 @@ void EDM4hep2LcioTool::convertTrackerHits(TrackerHitMap& trackerhits_vec, const
   lcio_event->addCollection(conv_trackerhits.release(), lcio_coll_name);
 }
 
-void EDM4hep2LcioTool::convertParticleIDs(ParticleIDMap& pidMap, const std::string& e4h_coll_name, int32_t algoId) {
-  k4FWCore::DataHandle<edm4hep::ParticleIDCollection> pidHandle{e4h_coll_name, Gaudi::DataHandle::Reader, this};
+void EDM4hep2LcioTool::convertParticleIDs(ParticleIDMap& pidMap,
+                                          std::vector<EDM4hep2LCIOConv::ParticleIDConvData>& pidCollections,
+                                          lcio::LCEventImpl* lcio_event, const podio::Frame& edmEvent) {
 
-  EDM4hep2LCIOConv::convertParticleIDs(pidHandle.get(), pidMap, algoId);
+  EDM4hep2LCIOConv::sortParticleIDs(pidCollections);
+  for (const auto& pidCollMeta : pidCollections) {
+    DataHandle<edm4hep::ParticleIDCollection> pidHandle{pidCollMeta.name, Gaudi::DataHandle::Reader, this};
+
+    auto algoId = attacheParticleIDMetaInfo(pidCollMeta, lcio_event, edmEvent);
+    EDM4hep2LCIOConv::convertParticleIDs(pidHandle.get(), pidMap, algoId.value_or(-1));
+  }
 }
 
 void EDM4hep2LcioTool::convertTrackerHitPlanes(TrackerHitPlaneMap& trackerhits_vec, const std::string& e4h_coll_name,
@@ -381,11 +399,9 @@ void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::s
             << "SimCalorimeterHit collection to be converted in order to be able to attach to them" << endmsg;
   } else {
     warning() << "Error trying to convert requested " << fulltype << " with name " << e4h_coll_name << endmsg;
-    warning() << "List of supported types: "
-              << "Track, TrackerHit3D, TrackerHitPlane, SimTrackerHit, "
-              << "Cluster, CalorimeterHit, RawCalorimeterHit, "
-              << "SimCalorimeterHit, Vertex, ReconstructedParticle, "
-              << "MCParticle." << endmsg;
+    warning() << "List of supported types: Track, TrackerHit3D, TrackerHitPlane, SimTrackerHit, Cluster, "
+              << "CalorimeterHit, RawCalorimeterHit, SimCalorimeterHit, Vertex, ReconstructedParticle, MCParticle."
+              << endmsg;
   }
 }
 
@@ -424,6 +440,72 @@ const podio::Frame& EDM4hep2LcioTool::getEDM4hepEvent() const {
   return tmp->getData();
 }
 
+std::optional<int32_t>
+EDM4hep2LcioTool::attacheParticleIDMetaInfo(const EDM4hep2LCIOConv::ParticleIDConvData& pidCollMeta,
+                                            lcio::LCEventImpl* lcio_event, const podio::Frame& edmEvent) const {
+
+  // We first have to figure out how the corresponding reco collection is
+  // named in EDM4hep. With that we can map it to the correct LCIO name and
+  // then attach the corresponding meta information via the PIDHandler
+  const auto& [pidCollName, pidColl, pidMetaInfo] = pidCollMeta;
+  debug() << fmt::format(
+                 "Attaching PID meta information for ParticleID collection {}, PID meta information available? {}",
+                 pidCollName, pidMetaInfo.has_value())
+          << endmsg;
+  if (msgLevel(MSG::DEBUG) && pidMetaInfo.has_value()) {
+    debug() << fmt::format("PID meta information: algoName: {}, algoType: {}, paramNames: {}", pidMetaInfo->algoName,
+                           pidMetaInfo->algoType(), pidMetaInfo->paramNames)
+            << endmsg;
+  }
+  if (!pidMetaInfo.has_value()) {
+    info() << "Could not collect PID meta information for ParticleID collection " << pidCollName << endmsg;
+  }
+  const auto recoCollName = [&]() {
+    auto name = m_collFromObjSvc->getCollectionNameFor((*pidColl)[0].getParticle());
+    if (!name.has_value()) {
+      // If we can't get a name from the TES we try again via the PodioDataSvc
+      if (m_podioDataSvc) {
+        name = edmEvent.getName((*pidColl)[0].getParticle());
+      }
+    }
+    return name;
+  }();
+  if (!recoCollName.has_value()) {
+    warning() << "Could not determine the name of the (LCIO) ReconstructedParticle collection to attach ParticleID "
+                 "metadata for (EDM4hep) ParticleID collection "
+              << pidCollName << endmsg;
+  }
+
+  std::optional<int32_t> algoId{std::nullopt};
+  if (recoCollName.has_value()) {
+    debug() << "Corresponding ReconstructedParticle (EDM4hep) collection is " << recoCollName.value() << endmsg;
+    if (const auto it = m_collsToConvert.find(recoCollName.value()); it != m_collsToConvert.end()) {
+      const auto lcioRecoName = it->second;
+      debug() << "Corresponding ReconstructedParticle (LCIO) collection is " << lcioRecoName << endmsg;
+      UTIL::PIDHandler pidHandler(lcio_event->getCollection(lcioRecoName));
+      if (pidMetaInfo.has_value()) {
+        try {
+          algoId = pidHandler.getAlgorithmID(pidMetaInfo->algoName);
+          debug() << "PID Algorithm already present with id " << algoId.value() << endmsg;
+          debug() << fmt::format("params: {}", pidHandler.getParameterNames(algoId.value())) << endmsg;
+          debug() << fmt::format("our params: {}", pidMetaInfo->paramNames) << endmsg;
+        } catch (const UTIL::UnknownAlgorithm&) {
+          algoId = pidHandler.addAlgorithm(pidMetaInfo->algoName, pidMetaInfo->paramNames);
+          debug() << "Determined algoId via LCIO PIDHandler: " << algoId.value() << endmsg;
+        }
+      }
+    } else {
+      warning() << "Could not find a name mapping for ReconstructedParticle collection " << recoCollName.value()
+                << " when trying to attach ParticleID meta information" << endmsg;
+    }
+  } else if (pidMetaInfo.has_value()) {
+    // We can still set the value we collected along the way. NOTE: It will
+    // almost certainly not be consistent in roundtrip conversions.
+    algoId = pidMetaInfo->algoType();
+  }
+  return algoId;
+}
+
 // Parse property parameters and convert the indicated collections.
 // Use the collection names in the parameters to read and write them
 StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) {
@@ -436,7 +518,7 @@ StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) {
     auto collNameMapping = m_collNames.value();
 
     // We *always* want to convert the EventHeader
-    m_collsToConvert.emplace_back(edm4hep::labels::EventHeader, "<directly into LCEvent>");
+    m_collsToConvert.emplace(edm4hep::labels::EventHeader, "<directly into LCEvent>");
 
     if (m_convertAll) {
       info() << "Converting all collections from EDM4hep to LCIO" << endmsg;
@@ -450,15 +532,13 @@ StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) {
       // Always check the contents of the TES because algorithms that do not use
       // the PodioDataSvc (e.g. all Functional ones) go to the TES directly and
       // the PodioDataSvc Frame doesn't now about them.
-      std::optional<std::map<uint32_t, std::string>> idToNameOpt(std::move(m_idToName));
-      for (const auto& name : getAvailableCollectionsFromStore(this, idToNameOpt)) {
+      for (const auto& name : getAvailableCollectionsFromStore(this)) {
         const auto& [_, inserted] = collNameMapping.emplace(name, name);
         debug() << fmt::format("Adding '{}' from TES to conversion? {}", name, inserted) << endmsg;
       }
-      m_idToName = std::move(idToNameOpt.value());
 
       for (auto&& [origName, newName] : collNameMapping) {
-        m_collsToConvert.emplace_back(std::move(origName), std::move(newName));
+        m_collsToConvert.emplace(std::move(origName), std::move(newName));
       }
     }
   }
@@ -485,32 +565,7 @@ StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) {
   }
   debug() << "Event: " << lcio_event->getEventNumber() << " Run: " << lcio_event->getRunNumber() << endmsg;
 
-  EDM4hep2LCIOConv::sortParticleIDs(pidCollections);
-
-  for (const auto& pidCollMeta : pidCollections) {
-    auto algoId = attachParticleIDMetaData(lcio_event, edmEvent, pidCollMeta);
-    if (!algoId.has_value()) {
-      // Now go over the collections that have been produced in a functional algorithm (if any)
-      bool found = false;
-      if (!m_podioDataSvc) {
-        const auto id = (*pidCollMeta.coll)[0].getParticle().id().collectionID;
-        if (auto it = m_idToName.find(id); it != m_idToName.end()) {
-          auto name = it->second;
-          if (pidCollMeta.metadata.has_value()) {
-            UTIL::PIDHandler pidHandler(lcio_event->getCollection(name));
-            algoId =
-                pidHandler.addAlgorithm(pidCollMeta.metadata.value().algoName, pidCollMeta.metadata.value().paramNames);
-            found = true;
-          }
-        }
-      }
-      if (!found) {
-        warning() << "Could not determine algorithm type for ParticleID collection " << pidCollMeta.name
-                  << " for setting consistent metadata" << endmsg;
-      }
-    }
-    convertParticleIDs(collection_pairs.particleIDs, pidCollMeta.name, algoId.value_or(-1));
-  }
+  convertParticleIDs(collection_pairs.particleIDs, pidCollections, lcio_event, edmEvent);
 
   EDM4hep2LCIOConv::attachDqdxInfo(collection_pairs.tracks, dQdxCollections);
 

k4MarlinWrapper/src/components/Lcio2EDM4hep.cpp

@@ -74,8 +74,7 @@ bool Lcio2EDM4hepTool::collectionExist(const std::string& collection_name) {
   if (m_podioDataSvc) {
     collections = m_podioDataSvc->getEventFrame().getAvailableCollections();
   } else {
-    std::optional<std::map<uint32_t, std::string>> dummy = std::nullopt;
-    collections = getAvailableCollectionsFromStore(this, dummy, true);
+    collections = getAvailableCollectionsFromStore(this, true);
   }
   if (std::find(collections.begin(), collections.end(), collection_name) != collections.end()) {
     debug() << "Collection named " << collection_name << " already registered, skipping conversion." << endmsg;
@@ -92,11 +91,11 @@ void Lcio2EDM4hepTool::registerCollection(
     return;
   }
 
-  auto wrapper = new DataWrapper<podio::CollectionBase>();
-  wrapper->setData(e4hColl.release());
-
   // No need to check for pre-existing collections, since we only ever end up
   // here if that is not the case
+  // NOTE: This also takes care of assigning a collectionID
+  auto wrapper = new DataWrapper<podio::CollectionBase>();
+  wrapper->setData(e4hColl.release());
   auto sc = m_eventDataSvc->registerObject("/Event", "/" + std::string(name), wrapper);
   if (sc == StatusCode::FAILURE) {
     error() << "Could not register collection " << name << endmsg;
@@ -149,14 +148,17 @@ struct ObjectMappings {
 } // namespace
 
 StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) {
+  debug() << "Converting from EDM4hep to LCIO" << endmsg;
   // Convert event parameters
   if (m_podioDataSvc) {
+    debug() << "Converting event parameters directly into the Frame of the PodioDataSvc" << endmsg;
     LCIO2EDM4hepConv::convertObjectParameters(the_event, m_podioDataSvc->m_eventframe);
   } else {
     DataObject* p;
     StatusCode code = m_eventDataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p);
     if (code.isSuccess()) {
       auto* frameWrapper = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
+      debug() << "Converting event parameters into the Frame stored in the TES" << endmsg;
       LCIO2EDM4hepConv::convertObjectParameters(the_event, frameWrapper->getData());
     } else {
       warning() << "Could not retrieve the event frame; event parameters will not be converted. This is a known "
@@ -167,16 +169,19 @@ StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) {
 
   // Convert Event Header outside the collections loop
   if (!collectionExist(edm4hep::labels::EventHeader)) {
+    debug() << "Converting the EventHeader" << endmsg;
     registerCollection(edm4hep::labels::EventHeader, LCIO2EDM4hepConv::createEventHeader(the_event));
   }
 
   // Start off with the pre-defined collection name mappings
   auto collsToConvert{m_collNames.value()};
   if (m_convertAll) {
+    info() << "Converting all collections from LCIO to EDM4hep" << endmsg;
     const auto* collections = the_event->getCollectionNames();
     for (const auto& collName : *collections) {
       // And simply add the rest, exploiting the fact that emplace will not
       // replace existing entries with the same key
+      debug() << "Adding '" << collName << "' to be converted from the LCIO Event" << endmsg;
       collsToConvert.emplace(collName, collName);
     }
   }
@@ -196,7 +201,7 @@ StatusCode Lcio2EDM4hepTool::convertCollections(lcio::LCEventImpl* the_event) {
   for (const auto& [lcioName, edm4hepName] : collsToConvert) {
     try {
       auto* lcio_coll = the_event->getCollection(lcioName);
-      debug() << "Converting collection " << lcioName << " (storing it as " << edm4hepName << "). ";
+      debug() << "Converting collection " << lcioName << " (storing it as " << edm4hepName << "). " << endmsg;
       if (collectionExist(edm4hepName)) {
         debug() << "Collection already exists, skipping." << endmsg;
         continue; // No need to convert again

k4MarlinWrapper/src/components/StoreUtils.cpp

@@ -28,6 +28,9 @@
 
 #include "k4FWCore/FunctionalUtils.h"
 
+#include <fmt/format.h>
+#include <fmt/ranges.h>
+
 #include <stdexcept>
 #include <string>
 #include <vector>
@@ -36,9 +39,7 @@
 // store that can be found in Writer.cpp in k4FWCore with some modifications
 // that are specific to the usage of this function in the converters, like
 // returning also a map from collection ID to collection name
-std::vector<std::string> getAvailableCollectionsFromStore(const AlgTool* thisClass,
-                                                          std::optional<std::map<uint32_t, std::string>>& idToName,
-                                                          bool returnFrameCollections) {
+std::vector<std::string> getAvailableCollectionsFromStore(const AlgTool* thisClass, bool returnFrameCollections) {
   std::vector<std::string> collectionNames;
 
   SmartIF<IDataManagerSvc> mgr;
@@ -59,6 +60,7 @@ std::vector<std::string> getAvailableCollectionsFromStore(const AlgTool* thisCla
     throw std::runtime_error("Failed to retrieve object leaves");
   }
   for (const auto& pReg : leaves) {
+    thisClass->verbose() << fmt::format("Now processing {} while obtaining collections", pReg->name()) << endmsg;
     if (pReg->name() == k4FWCore::frameLocation) {
       if (!returnFrameCollections)
         continue;
@@ -69,6 +71,9 @@ std::vector<std::string> getAvailableCollectionsFromStore(const AlgTool* thisCla
       for (const auto& name : wrapper->getData().getAvailableCollections()) {
         collectionNames.push_back(name);
       }
+      thisClass->verbose() << fmt::format("Retrieved the following collection names from Frame in TES: {}",
+                                          collectionNames)
+                           << endmsg;
     }
     DataObject* p;
     sc = thisClass->evtSvc()->retrieveObject("/Event" + pReg->name(), p);
@@ -88,14 +93,9 @@ std::vector<std::string> getAvailableCollectionsFromStore(const AlgTool* thisCla
       }
     }
     // Remove the leading /
-    collectionNames.push_back(pReg->name().substr(1, pReg->name().size() - 1));
-    if (idToName) {
-      if (functionalWrapper) {
-        idToName->emplace(functionalWrapper->getData()->getID(), pReg->name());
-      } else {
-        idToName->emplace(algorithmWrapper->collectionBase()->getID(), pReg->name());
-      }
-    }
+    auto name = pReg->name().substr(1, pReg->name().size() - 1);
+    thisClass->verbose() << "Adding '" << name << "' as collection name obtained from TES" << endmsg;
+    collectionNames.push_back(name);
   }
   return collectionNames;
 }

k4MarlinWrapper/src/components/StoreUtils.h

@@ -27,7 +27,6 @@
 // here with some additions to make it useful for converting both from EDM4hep
 // to LCIO and vice versa
 std::vector<std::string> getAvailableCollectionsFromStore(const AlgTool* thisClass,
-                                                          std::optional<std::map<uint32_t, std::string>>& idToName,
                                                           bool returnFrameCollections = false);
 
 k4MarlinWrapper::GlobalConvertedObjectsMap& getGlobalObjectMap(AlgTool* thisTool);