Skip to content

[WIP] Add MPI to candidates example #1593

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bmhan12 wants to merge 3 commits into
base: develop
Choose a base branch
from
Open

[WIP] Add MPI to candidates example #1593

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b57cf8b
Account for int64 conduit values; remove implicit perfect square check
bmhan12 Jun 20, 2025
beb01b8
Add +mpi dependency to quest_candidates_examples; can run with MPI, c…
bmhan12 Jun 24, 2025
1151496
Merge branch 'develop' into feature/han12/candidates_support
bmhan12 Oct 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/axom/quest/examples/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,12 +88,12 @@ axom_add_executable(
)

# BVH silo example ------------------------------------------------------------
if (CONDUIT_FOUND AND UMPIRE_FOUND)
if (AXOM_ENABLE_MPI AND CONDUIT_FOUND AND UMPIRE_FOUND)
axom_add_executable(
NAME quest_candidates_example_ex
SOURCES quest_candidates_example.cpp
OUTPUT_DIR ${EXAMPLE_OUTPUT_DIRECTORY}
DEPENDS_ON ${quest_example_depends} conduit::conduit
DEPENDS_ON ${quest_example_depends} conduit::conduit conduit::conduit_mpi
FOLDER axom/quest/examples
)

Expand Down
129 changes: 100 additions & 29 deletions src/axom/quest/examples/quest_candidates_example.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,19 @@
#error This example requires axom to be configured with Umpire support
#endif

#ifdef AXOM_USE_MPI
#include "mpi.h"
#else
#error This example requires axom to be configured with MPI support
#endif

#ifdef AXOM_USE_CONDUIT
#include "conduit_relay.hpp"
#include "conduit_blueprint.hpp"
#include "conduit_blueprint_mpi.hpp"
#include "conduit_relay_mpi_io_blueprint.hpp"
#else
#error This example requires axom to be configured with Conduit support
#error This example requires axom to be configured with Conduit + MPI support
#endif

#include "axom/mint.hpp"
Expand All @@ -47,6 +55,10 @@ using UMesh = axom::mint::UnstructuredMesh<axom::mint::SINGLE_SHAPE>;
using IndexPair = std::pair<axom::IndexType, axom::IndexType>;
using RuntimePolicy = axom::runtime_policy::Policy;

// MPI globals, set in main().
int myRank = -1;
int numRanks = -1;

//-----------------------------------------------------------------------------
/// Basic RAII utility class for initializing and finalizing slic logger
//-----------------------------------------------------------------------------
Expand All @@ -55,21 +67,21 @@ struct BasicLogger
BasicLogger()
{
namespace slic = axom::slic;

// Initialize the SLIC logger
slic::initialize();
slic::setLoggingMsgLevel(slic::message::Debug);
slic::setLoggingMsgLevel(slic::message::Info);

// Customize logging levels and formatting
const std::string slicFormatStr = "[<LEVEL>] <MESSAGE> \n";
slic::LogStream* logStream;

slic::addStreamToMsgLevel(new slic::GenericOutputStream(&std::cerr), slic::message::Error);
slic::addStreamToMsgLevel(new slic::GenericOutputStream(&std::cerr, slicFormatStr),
slic::message::Warning);
std::string fmt = "[<RANK>][<LEVEL>]: <MESSAGE>\n";
#ifdef AXOM_USE_LUMBERJACK
const int RLIMIT = 8;
logStream = new slic::LumberjackStream(&std::cout, MPI_COMM_WORLD, RLIMIT, fmt);
#else
logStream = new slic::SynchronizedStream(&std::cout, MPI_COMM_WORLD, fmt);
#endif // AXOM_USE_MPI

auto* compactStream = new slic::GenericOutputStream(&std::cout, slicFormatStr);
slic::addStreamToMsgLevel(compactStream, slic::message::Info);
slic::addStreamToMsgLevel(compactStream, slic::message::Debug);
slic::addStreamToAllMsgLevels(logStream);
}

~BasicLogger() { axom::slic::finalize(); }
Expand Down Expand Up @@ -244,8 +256,24 @@ HexMesh loadBlueprintHexMesh(const std::string& mesh_path, bool verboseOutput =

// Load Blueprint mesh into Conduit node
conduit::Node n_load;

// Read mesh as single domain first; if partitioned, then reset and reload with MPI
conduit::relay::io::blueprint::read_mesh(mesh_path, n_load);

if(conduit::blueprint::mesh::number_of_domains(n_load) > 1)
{
n_load.reset();
conduit::relay::mpi::io::blueprint::read_mesh(mesh_path, n_load, MPI_COMM_WORLD);
}

// Requirement that each rank has 1 domain
if(conduit::blueprint::mesh::number_of_domains(n_load) != 1)
{
SLIC_ERROR(axom::fmt::format("Rank {} has {} domains. Must have 1 domain per rank!\n",
myRank,
conduit::blueprint::mesh::number_of_domains(n_load)));
}

// Check if Blueprint mesh conforms
conduit::Node n_info;
if(conduit::blueprint::verify("mesh", n_load, n_info) == false)
Expand All @@ -268,16 +296,6 @@ HexMesh loadBlueprintHexMesh(const std::string& mesh_path, bool verboseOutput =
int connectivity_size =
(n_load[0]["topologies/topo/elements/connectivity"]).dtype().number_of_elements();

// Sanity check for number of cells
int cell_calc_from_nodes = std::round(std::pow(std::pow(num_nodes, 1.0 / 3.0) - 1, 3));
int cell_calc_from_connectivity = connectivity_size / HEX_OFFSET;
if(cell_calc_from_nodes != cell_calc_from_connectivity)
{
SLIC_ERROR("Number of cells is not expected!\n"
<< "First calculation is " << cell_calc_from_nodes << " and second calculation is "
<< cell_calc_from_connectivity);
}

// extract hexes into an axom::Array
auto x_vals_h = axom::ArrayView<double>(n_load[0]["coordsets/coords/values/x"].value(), num_nodes);
auto y_vals_h = axom::ArrayView<double>(n_load[0]["coordsets/coords/values/y"].value(), num_nodes);
Expand All @@ -297,9 +315,32 @@ HexMesh loadBlueprintHexMesh(const std::string& mesh_path, bool verboseOutput =
auto z_vals_view = on_device ? z_vals_d.view() : z_vals_h;

// Move connectivity information onto device
auto connectivity_h =
axom::ArrayView<int>(n_load[0]["topologies/topo/elements/connectivity"].value(),
connectivity_size);

int* conn_data = nullptr;
axom::Array<int> temp_conn_data;
int conn_id = n_load[0]["topologies/topo/elements/connectivity"].dtype().id();
if(conn_id == conduit::DataType::INT32_ID)
{
conn_data = n_load[0]["topologies/topo/elements/connectivity"].as_int32_ptr();
}
else if(conn_id == conduit::DataType::INT64_ID)
{
temp_conn_data.resize(connectivity_size);
auto conn_int64_data = n_load[0]["topologies/topo/elements/connectivity"].as_int64_ptr();
for(int i = 0; i < connectivity_size; ++i)
{
temp_conn_data[i] = static_cast<int>(conn_int64_data[i]);
}
conn_data = temp_conn_data.data();
}
else
{
SLIC_ERROR("Node connectivity data type "
<< n_load[0]["topologies/topo/elements/connectivity"].dtype().name()
<< " is unsupported!");
}

auto connectivity_h = axom::ArrayView<int>(conn_data, connectivity_size);

axom::Array<int> connectivity_d =
on_device ? axom::Array<int>(connectivity_h, kernel_allocator) : axom::Array<int>();
Expand Down Expand Up @@ -377,8 +418,10 @@ HexMesh loadBlueprintHexMesh(const std::string& mesh_path, bool verboseOutput =
}

// Write out to vtk for test viewing
SLIC_INFO("Writing out Blueprint mesh to test.vtk for debugging...");
axom::mint::write_vtk(mesh, "test.vtk");
std::string vtk_file_name = "test_" + std::to_string(myRank) + ".vtk";
SLIC_INFO(axom::fmt::format("Writing out Blueprint mesh to {} for debugging...", vtk_file_name));

axom::mint::write_vtk(mesh, vtk_file_name);

delete mesh;
mesh = nullptr;
Expand Down Expand Up @@ -645,6 +688,10 @@ std::vector<IndexPair> findCandidatesImplicit(const HexMesh& insertMesh,

int main(int argc, char** argv)
{
axom::utilities::raii::MPIWrapper mpi_raii_wrapper(argc, argv);
myRank = mpi_raii_wrapper.my_rank();
numRanks = mpi_raii_wrapper.num_ranks();

#if defined(AXOM_RUNTIME_POLICY_USE_OPENMP)
using omp_exec = axom::OMP_EXEC;
#endif
Expand Down Expand Up @@ -672,7 +719,18 @@ int main(int argc, char** argv)
}
catch(const axom::CLI::ParseError& e)
{
return app.exit(e);
int retval = -1;
if(myRank == 0)
{
retval = app.exit(e);
}

#ifdef AXOM_USE_MPI
MPI_Bcast(&retval, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Finalize();
#endif

exit(retval);
}
}

Expand Down Expand Up @@ -802,6 +860,7 @@ int main(int argc, char** argv)

// print first few pairs
const int numCandidates = candidatePairs.size();

if(numCandidates > 0 && params.isVerbose())
{
constexpr int MAX_PRINT = 20;
Expand All @@ -818,15 +877,27 @@ int main(int argc, char** argv)
}

// Write out candidate pairs
SLIC_INFO("Writing out candidate pairs...");
std::ofstream outf("candidates.txt");
std::string candidates_file_name = "candidates_" + std::to_string(myRank) + ".txt";
SLIC_INFO(axom::fmt::format("Writing out candidate pairs to {}...", candidates_file_name));
std::ofstream outf(candidates_file_name);

outf << candidatePairs.size() << " candidate pairs:" << std::endl;
for(unsigned long i = 0; i < candidatePairs.size(); ++i)
{
outf << candidatePairs[i].first << " " << candidatePairs[i].second << std::endl;
}
}

// Print total number of pairs across all ranks
int totalNumCandidates = 0;
MPI_Reduce(&numCandidates, &totalNumCandidates, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
if(myRank == 0)
{
SLIC_INFO(axom::fmt::format(axom::utilities::locale(),
"Mesh had {:L} a total of candidates pairs across all ranks",
totalNumCandidates));
}

AXOM_ANNOTATE_END("find candidates");

return 0;
Expand Down