diff --git a/k4FWCore/components/CollectionFromObjectSvc.cpp b/k4FWCore/components/CollectionFromObjectSvc.cpp new file mode 100644 index 00000000..cd373e79 --- /dev/null +++ b/k4FWCore/components/CollectionFromObjectSvc.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "CollectionFromObjectSvc.h" + +#include "k4FWCore/FunctionalUtils.h" + +#include +#include + +#include +#include + +template <> +struct fmt::formatter : ostream_formatter {}; + +StatusCode CollectionFromObjectSvc::initialize() { + auto sc = Service::initialize(); + if (sc.isFailure()) { + error() << "Unable to initialize base class Serivce." << endmsg; + return sc; + } + m_dataSvc = service("EventDataSvc"); + if (!m_dataSvc) { + error() << "Unable to locate the EventDataSvc" << endmsg; + return StatusCode::FAILURE; + } + + return StatusCode::SUCCESS; +} + +const std::optional CollectionFromObjectSvc::getCollectionNameFor(const podio::ObjectID id) const { + debug() << "Trying to retrieve collection name for object " << id << endmsg; + const auto& idTable = k4FWCore::details::getTESCollectionIDTable(m_dataSvc, this); + auto name = idTable.name(id.collectionID); + if (!name.has_value()) { + error() << "Could not get a collection name for object " << id << endmsg; + return std::nullopt; + } + return name.value(); +} + +const podio::CollectionBase* CollectionFromObjectSvc::getCollectionFor(const podio::ObjectID id) const { + debug() << "Trying to retrieve collection for object " << id << endmsg; + const auto& idTable = k4FWCore::details::getTESCollectionIDTable(m_dataSvc, this); + auto name = idTable.name(id.collectionID); + if (!name.has_value()) { + error() << "Could not get a collection name for object " << id << endmsg; + return nullptr; + } + + DataObject* p{nullptr}; + if (m_dataSvc->retrieveObject("/Event/" + name.value(), p).isFailure()) { + error() << "Could not get the collection '" << name.value() << "' from the Event store" << endmsg; + return nullptr; + } + const auto collWrapper = dynamic_cast>*>(p); + if (!collWrapper) { + error() << "Could not cast data object to the necessary type for collection " << name.value() << endmsg; + return nullptr; + } + + return collWrapper->getData().get(); +} + +DECLARE_COMPONENT(CollectionFromObjectSvc) diff --git a/k4FWCore/components/CollectionFromObjectSvc.h b/k4FWCore/components/CollectionFromObjectSvc.h new file mode 100644 index 00000000..46b72dd9 --- /dev/null +++ b/k4FWCore/components/CollectionFromObjectSvc.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FWCORE_COLLECTIONFROMOBJECTSVC_H +#define FWCORE_COLLECTIONFROMOBJECTSVC_H + +#include +#include + +#include +#include + +#include "k4FWCore/ICollectionFromObjectSvc.h" + +class CollectionFromObjectSvc : public extends { + using extends::extends; + +public: + StatusCode initialize() override; + +protected: + const podio::CollectionBase* getCollectionFor(const podio::ObjectID id) const override; + const std::optional getCollectionNameFor(const podio::ObjectID id) const override; + +private: + SmartIF m_dataSvc; +}; + +#endif diff --git a/k4FWCore/components/Reader.cpp b/k4FWCore/components/Reader.cpp index c69de4c2..14ba6243 100644 --- a/k4FWCore/components/Reader.cpp +++ b/k4FWCore/components/Reader.cpp @@ -25,6 +25,7 @@ #include "GaudiKernel/StatusCode.h" #include "podio/CollectionBase.h" +#include "podio/CollectionIDTable.h" #include "podio/Frame.h" #include "IIOSvc.h" @@ -136,12 +137,19 @@ class Reader final : public CollectionPusher { auto eds = eventSvc().as(); auto frame = std::move(std::get(val)); + auto idTable = frame.getCollectionIDTableForWrite(); auto tmp = new AnyDataWrapper(std::move(frame)); if (eds->registerObject("/Event" + k4FWCore::frameLocation, tmp).isFailure()) { error() << "Failed to register Frame object" << endmsg; } + // We also store the collection id table to make sure we can detect collisions + auto idTableWrapper = new AnyDataWrapper(std::move(idTable)); + if (eds->registerObject("/Event" + k4FWCore::idTableLocation, idTableWrapper).isFailure()) { + error() << "Failed to register CollectionIDTable object" << endmsg; + } + return std::make_tuple(std::get<0>(val), std::get<1>(val)); } }; diff --git a/k4FWCore/include/k4FWCore/FunctionalUtils.h b/k4FWCore/include/k4FWCore/FunctionalUtils.h index f8b38c50..a45c1670 100644 --- a/k4FWCore/include/k4FWCore/FunctionalUtils.h +++ b/k4FWCore/include/k4FWCore/FunctionalUtils.h @@ -29,6 +29,7 @@ #include #include "podio/CollectionBase.h" +#include #include "k4FWCore/DataWrapper.h" @@ -43,6 +44,7 @@ namespace k4FWCore { static const std::string frameLocation = "/_Frame"; +static const std::string idTableLocation = "/_CollectionIDTable"; namespace details { @@ -208,6 +210,49 @@ namespace details { } } + podio::CollectionIDTable& getTESCollectionIDTable(IDataProviderSvc* eds, auto thisClass) { + DataObject* p; + auto sc = eds->retrieveObject("/Event" + k4FWCore::idTableLocation, p); + if (!sc.isSuccess()) { + // We are not reading from file so we create one on the fly + thisClass->debug() << "Could not retrieve CollectionIDTable for assigning a collection id" << endmsg; + auto idTableWrapper = new AnyDataWrapper(podio::CollectionIDTable{}); + if (eds->registerObject("/Event" + k4FWCore::idTableLocation, idTableWrapper).isFailure()) { + thisClass->error() << "Failed to place an empty CollectionIDTable into the TES" << endmsg; + } else { + return idTableWrapper->getData(); + } + } else { + auto idTableWrapper = dynamic_cast*>(p); + if (!idTableWrapper) { + thisClass->error() << "Object at /Event" << k4FWCore::idTableLocation + << " is not the expected CollectionIDTable" << endmsg; + } + return idTableWrapper->getData(); + } + + throw std::runtime_error("Could neither retrieve an existing nor place an empty CollectionIDTable in the TES"); + } + + podio::CollectionIDTable& getTESCollectionIDTable(auto thisClass) { + auto eds = thisClass->eventSvc().template as(); + return getTESCollectionIDTable(eds, thisClass); + } + + void putCollectionSetID(std::unique_ptr coll, + const DataObjectWriteHandle>& handle, auto thisClass) { + auto& idTable = getTESCollectionIDTable(thisClass); + if (idTable.present(handle.objKey())) { + thisClass->error() << "Collection with name " << handle.objKey() << " is already stored in the TES" << endmsg; + } + const auto id = idTable.add(handle.objKey()); + coll->setID(id); + thisClass->verbose() << fmt::format("Assigning collection id {:0>8x} to collection '{}'", coll->getID(), + handle.objKey()) + << endmsg; + Gaudi::Functional::details::put(handle, std::move(coll)); + } + template void putVectorOutputs(std::tuple&& handles, const auto& m_outputs, auto thisClass) { if constexpr (Index < sizeof...(Handles)) { @@ -221,12 +266,12 @@ namespace details { throw GaudiException(thisClass->name(), msg, StatusCode::FAILURE); } for (auto& val : std::get(handles)) { - Gaudi::Functional::details::put(std::get(m_outputs)[i], convertToUniquePtr(std::move(val))); + putCollectionSetID(convertToUniquePtr(std::move(val)), std::get(m_outputs)[i], thisClass); i++; } } else { - Gaudi::Functional::details::put(std::get(m_outputs)[0], - convertToUniquePtr(std::move(std::get(handles)))); + putCollectionSetID(convertToUniquePtr(std::move(std::get(handles))), std::get(m_outputs)[0], + thisClass); } // Recursive call for the next index diff --git a/k4FWCore/include/k4FWCore/ICollectionFromObjectSvc.h b/k4FWCore/include/k4FWCore/ICollectionFromObjectSvc.h new file mode 100644 index 00000000..562faa19 --- /dev/null +++ b/k4FWCore/include/k4FWCore/ICollectionFromObjectSvc.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef FWCORE_ICOLLECTIONFROMOBJECTSVC_H +#define FWCORE_ICOLLECTIONFROMOBJECTSVC_H + +#include + +#include +#include +#include + +#include +#include + +class ICollectionFromObjectSvc : virtual public IInterface { +public: + DeclareInterfaceID(ICollectionFromObjectSvc, 1, 0); + + template + const typename O::collection_type* getCollectionFor(const O& object) const { + return dynamic_cast(getCollectionFor(object.id())); + } + + // TODO: return string_view? Some form of DataHandle? + template + const std::optional getCollectionNameFor(const O& object) const { + return getCollectionNameFor(object.id()); + } + +protected: + virtual const podio::CollectionBase* getCollectionFor(const podio::ObjectID) const = 0; + virtual const std::optional getCollectionNameFor(const podio::ObjectID) const = 0; +}; + +#endif diff --git a/k4FWCore/include/k4FWCore/Transformer.h b/k4FWCore/include/k4FWCore/Transformer.h index 64056b99..5b47f0ee 100644 --- a/k4FWCore/include/k4FWCore/Transformer.h +++ b/k4FWCore/include/k4FWCore/Transformer.h @@ -116,9 +116,9 @@ namespace details { std::tuple tmp = filter_evtcontext_tt::apply(*this, ctx, this->m_inputs); putVectorOutputs<0, Out>(std::move(tmp), m_outputs, this); } else { - Gaudi::Functional::details::put( - std::get<0>(this->m_outputs)[0], - convertToUniquePtr(std::move(filter_evtcontext_tt::apply(*this, ctx, this->m_inputs)))); + putCollectionSetID( + convertToUniquePtr(std::move(filter_evtcontext_tt::apply(*this, ctx, this->m_inputs))), + std::get<0>(this->m_outputs)[0], this); } return Gaudi::Functional::FilterDecision::PASSED; } catch (GaudiException& e) { diff --git a/python/k4FWCore/ApplicationMgr.py b/python/k4FWCore/ApplicationMgr.py index e91460dd..a82472cb 100644 --- a/python/k4FWCore/ApplicationMgr.py +++ b/python/k4FWCore/ApplicationMgr.py @@ -20,7 +20,14 @@ from Gaudi import Configuration from Configurables import ApplicationMgr as AppMgr -from Configurables import Reader, Writer, IOSvc, Gaudi__Sequencer, EventLoopMgr +from Configurables import ( + Reader, + Writer, + IOSvc, + Gaudi__Sequencer, + EventLoopMgr, + CollectionFromObjectSvc, +) from k4FWCore.utils import get_logger logger = get_logger() @@ -118,6 +125,9 @@ def fix_properties(self): if "MetadataSvc" in self._mgr.allConfigurables: self._mgr.ExtSvc.append(self._mgr.allConfigurables["MetadataSvc"]) + # Always attach the CollectionFromObjectSvc + self._mgr.ExtSvc.append(CollectionFromObjectSvc("CollectionFromObjectSvc")) + if "IOSvc" not in self._mgr.allConfigurables: return if not isinstance(self._mgr.allConfigurables["IOSvc"], IOSvc): diff --git a/test/k4FWCoreTest/CMakeLists.txt b/test/k4FWCoreTest/CMakeLists.txt index d9898b50..6f13454f 100644 --- a/test/k4FWCoreTest/CMakeLists.txt +++ b/test/k4FWCoreTest/CMakeLists.txt @@ -202,6 +202,7 @@ add_test_with_env(ReadLimitedInputsAllEventsIOSvc options/ExampleIOSvcLimitInput add_test_with_env(ParticleIDMetadataFramework options/ExampleParticleIDMetadata.py) +add_test_with_env(CollectionFromObjectRetrieval options/TestCollectionFromObjectRetrieval.py) # The following is done to make the tests work without installing the files in # the installation directory. The k4FWCore in the build directory is populated by diff --git a/test/k4FWCoreTest/options/TestCollectionFromObjectRetrieval.py b/test/k4FWCoreTest/options/TestCollectionFromObjectRetrieval.py new file mode 100644 index 00000000..afd5fd70 --- /dev/null +++ b/test/k4FWCoreTest/options/TestCollectionFromObjectRetrieval.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2014-2024 Key4hep-Project. +# +# This file is part of Key4hep. +# See https://key4hep.github.io/key4hep-doc/ for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is a simple test checking that the collection IDs that are assigned for +# the Functional algorithms are the same as we get when adding to a frame. +# Effectively, it checks whether we hash the right thing when we assign a +# collection ID. We need this check because for Functional algorithms the +# insertion into a Frame (where the usual assignment happens) happens just +# before writing. + +from Gaudi.Configuration import VERBOSE +from Configurables import ( + TestCollectionFromObjectRetrieval, + ExampleFunctionalProducerMultiple, + CollectionFromObjectSvc, +) + +from k4FWCore import ApplicationMgr +from Configurables import EventDataSvc + +producer = ExampleFunctionalProducerMultiple( + "Producer", OutputCollectionParticles1=["UnconventionalCollName"], OutputLevel=VERBOSE +) + + +collid_checker = TestCollectionFromObjectRetrieval( + "CollectionIDChecker", InputCollection=["UnconventionalCollName"], OutputLevel=VERBOSE +) + +ApplicationMgr( + TopAlg=[producer, collid_checker], + EvtSel="NONE", + EvtMax=1, + ExtSvc=[EventDataSvc(), CollectionFromObjectSvc("CollectionFromObjSvc")], +) diff --git a/test/k4FWCoreTest/src/components/TestCollectionFromObjectRetrieval.cpp b/test/k4FWCoreTest/src/components/TestCollectionFromObjectRetrieval.cpp new file mode 100644 index 00000000..31238b23 --- /dev/null +++ b/test/k4FWCoreTest/src/components/TestCollectionFromObjectRetrieval.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2014-2024 Key4hep-Project. + * + * This file is part of Key4hep. + * See https://key4hep.github.io/key4hep-doc/ for further info. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include + +struct TestCollectionFromObjectRetrieval final : k4FWCore::Consumer { + TestCollectionFromObjectRetrieval(const std::string& name, ISvcLocator* svcLoc) + : Consumer(name, svcLoc, {KeyValues("InputCollection", {"MCParticles"})}) {} + + StatusCode initialize() final { + m_collFromObjSvc = service("CollectionFromObjSvc", false); + if (!m_collFromObjSvc) { + return StatusCode::FAILURE; + } + return StatusCode::SUCCESS; + } + + void operator()(const edm4hep::MCParticleCollection& inputColl) const final { + const auto mc = inputColl[0]; + debug() << "Retrieving collection for object with id " << mc.id() << endmsg; + const auto* collFromObj = m_collFromObjSvc->getCollectionFor(mc); + const auto checkMC = (*collFromObj)[0]; + if (mc != checkMC) { + throw std::runtime_error("Could not get the expected collection from the object"); + } + + const auto name = m_collFromObjSvc->getCollectionNameFor(mc); + if (name.value() != inputLocations(0)[0]) { + throw std::runtime_error("Collection name retrieved via object of collection is not as expected"); + } + + auto newMCParticle = edm4hep::MutableMCParticle{}; + const auto invalidName = m_collFromObjSvc->getCollectionNameFor(newMCParticle); + if (invalidName.has_value()) { + throw std::runtime_error("Retrieved a name for an object that is not known to the TES"); + } + + const auto* invalidColl = m_collFromObjSvc->getCollectionFor(newMCParticle); + if (invalidColl) { + throw std::runtime_error("Could get a collection for an object that is not in a collection"); + } + + auto newCollection = edm4hep::MCParticleCollection(); + newCollection->push_back(newMCParticle); + const auto* stillInvalidColl = m_collFromObjSvc->getCollectionFor(newMCParticle); + if (stillInvalidColl) { + throw std::runtime_error("Could get a collection for an object that is not in a collection"); + } + } + +private: + SmartIF m_collFromObjSvc; +}; + +DECLARE_COMPONENT(TestCollectionFromObjectRetrieval)