Skip to content

Commit d72f966

Browse files
cwpearsoncwpearson
committed
comm mode rework cleanup
1 parent de3b85d commit d72f966

9 files changed

Lines changed: 27 additions & 44 deletions

‎perf_tests/test_2dhalo.cpp‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -82,14 +82,12 @@ void benchmark_2dhalo(benchmark::State &state) {
8282
const int ry = rank / rs;
8383

8484
if (rank < rs * rs) {
85-
auto mode = KokkosComm::DefaultCommMode();
8685
auto space = Kokkos::DefaultExecutionSpace();
8786
// grid of elements, each with 3 properties, and a radius-1 halo
8887
grid_type grid("", nx + 2, ny + 2, nprops);
8988
while (state.KeepRunning()) {
90-
do_iteration(state, MPI_COMM_WORLD,
91-
send_recv<KokkosComm::DefaultCommMode, Kokkos::DefaultExecutionSpace, grid_type>, mode, space, nx,
92-
ny, rx, ry, rs, grid);
89+
do_iteration(state, MPI_COMM_WORLD, send_recv<Kokkos::DefaultExecutionSpace, grid_type>, space, nx, ny, rx, ry,
90+
rs, grid);
9391
}
9492
} else {
9593
while (state.KeepRunning()) {

‎perf_tests/test_osu_latency.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@ void benchmark_osu_latency_Kokkos_Comm_mpi_sendrecv(benchmark::State &state) {
6363
state.SkipWithError("benchmark_osu_latency_KokkosComm needs exactly 2 ranks");
6464
}
6565

66-
auto mode = KokkosComm::DefaultCommMode();
6766
auto space = Kokkos::DefaultExecutionSpace();
6867
using view_type = Kokkos::View<char *>;
6968
view_type a("A", state.range(0));

‎perf_tests/test_sendrecv.cpp‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,14 @@
1818

1919
#include "KokkosComm.hpp"
2020

21-
template <KokkosComm::CommunicationMode Mode, typename Space, typename View>
22-
void send_recv(benchmark::State &, MPI_Comm comm, const Mode &mode, const Space &space, int rank, const View &v) {
21+
template <KokkosComm::mpi::CommunicationMode Mode, typename Space, typename View>
22+
void send_recv(benchmark::State &, MPI_Comm comm, const Space &space, int rank, const View &v) {
2323
if (0 == rank) {
24-
KokkosComm::mpi::send(space, v, 1, 0, comm);
24+
KokkosComm::mpi::send(space, v, 1, 0, comm, Mode{});
2525
KokkosComm::mpi::recv(space, v, 1, 0, comm);
2626
} else if (1 == rank) {
2727
KokkosComm::mpi::recv(space, v, 0, 0, comm);
28-
KokkosComm::mpi::send(space, v, 0, 0, comm);
28+
KokkosComm::mpi::send(space, v, 0, 0, comm, Mode{});
2929
}
3030
}
3131

@@ -39,15 +39,13 @@ void benchmark_sendrecv(benchmark::State &state) {
3939

4040
using Scalar = double;
4141

42-
auto mode = KokkosComm::DefaultCommMode();
42+
using Mode = KokkosComm::mpi::DefaultCommMode;
4343
auto space = Kokkos::DefaultExecutionSpace();
4444
using view_type = Kokkos::View<Scalar *>;
4545
view_type a("", 1000000);
4646

4747
while (state.KeepRunning()) {
48-
do_iteration(state, MPI_COMM_WORLD,
49-
send_recv<KokkosComm::DefaultCommMode, Kokkos::DefaultExecutionSpace, view_type>, mode, space, rank,
50-
a);
48+
do_iteration(state, MPI_COMM_WORLD, send_recv<Mode, Kokkos::DefaultExecutionSpace, view_type>, space, rank, a);
5149
}
5250

5351
state.SetBytesProcessed(sizeof(Scalar) * state.iterations() * a.size() * 2);

‎src/KokkosComm_collective.hpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@
2525

2626
namespace KokkosComm {
2727

28-
template <KokkosExecutionSpace ExecSpace = Kokkos::DefaultExecutionSpace, CommunicationSpace CommSpace = DefaultCommunicationSpace>
28+
template <KokkosExecutionSpace ExecSpace = Kokkos::DefaultExecutionSpace,
29+
CommunicationSpace CommSpace = DefaultCommunicationSpace>
2930
void barrier(Handle<ExecSpace, CommSpace> &&h) {
3031
Impl::Barrier<ExecSpace, CommSpace>{std::forward<Handle<ExecSpace, CommSpace>>(h)};
3132
}

‎src/KokkosComm_fwd.hpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ using FallbackCommunicationSpace = Mpi;
3333
template <CommunicationSpace CommSpace = DefaultCommunicationSpace>
3434
class Req;
3535

36-
template <KokkosExecutionSpace ExecSpace = Kokkos::DefaultExecutionSpace, CommunicationSpace CommSpace = DefaultCommunicationSpace>
36+
template <KokkosExecutionSpace ExecSpace = Kokkos::DefaultExecutionSpace,
37+
CommunicationSpace CommSpace = DefaultCommunicationSpace>
3738
class Handle;
3839

3940
namespace Impl {
@@ -44,7 +45,8 @@ struct Recv;
4445
template <KokkosView SendView, KokkosExecutionSpace ExecSpace = Kokkos::DefaultExecutionSpace,
4546
CommunicationSpace CommSpace = DefaultCommunicationSpace>
4647
struct Send;
47-
template <KokkosExecutionSpace ExecSpace = Kokkos::DefaultExecutionSpace, CommunicationSpace CommSpace = DefaultCommunicationSpace>
48+
template <KokkosExecutionSpace ExecSpace = Kokkos::DefaultExecutionSpace,
49+
CommunicationSpace CommSpace = DefaultCommunicationSpace>
4850
struct Barrier;
4951

5052
} // namespace Impl

‎src/mpi/KokkosComm_mpi_commmode.hpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@
1616

1717
#pragma once
1818

19+
#include <type_traits>
20+
1921
// See section 3.4 of the MPI standard for a complete specification.
2022

21-
namespace KokkosComm::mpi{
23+
namespace KokkosComm::mpi {
2224
// Standard mode: MPI implementation decides whether outgoing messages will
2325
// be buffered. Send operations can be started whether or not a matching
2426
// receive has been started. They may complete before a matching receive is
@@ -44,7 +46,6 @@ using DefaultCommMode = CommModeSynchronous;
4446
using DefaultCommMode = CommModeStandard;
4547
#endif
4648

47-
4849
template <typename T>
4950
struct is_communication_mode : std::false_type {};
5051

@@ -63,5 +64,4 @@ inline constexpr bool is_communication_mode_v = is_communication_mode<T>::value;
6364
template <typename T>
6465
concept CommunicationMode = is_communication_mode_v<T>;
6566

66-
6767
} // namespace KokkosComm::mpi

‎src/mpi/KokkosComm_mpi_send.hpp‎

Lines changed: 8 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -24,19 +24,21 @@
2424

2525
namespace KokkosComm::mpi {
2626

27-
template <CommunicationMode SendMode, KokkosView SendView>
28-
void send(const SendMode &, const SendView &sv, int dest, int tag, MPI_Comm comm) {
27+
template <KokkosView SendView, CommunicationMode SendMode>
28+
void send(const SendView &sv, int dest, int tag, MPI_Comm comm, SendMode) {
2929
Kokkos::Tools::pushRegion("KokkosComm::Impl::send");
3030
using KCT = typename KokkosComm::Traits<SendView>;
3131

3232
auto mpi_send_fn = [](void *mpi_view, int mpi_count, MPI_Datatype mpi_datatype, int mpi_dest, int mpi_tag,
3333
MPI_Comm mpi_comm) {
34-
if constexpr (std::is_same_v<SendMode, StandardCommMode>) {
34+
if constexpr (std::is_same_v<SendMode, CommModeStandard>) {
3535
MPI_Send(mpi_view, mpi_count, mpi_datatype, mpi_dest, mpi_tag, mpi_comm);
36-
} else if constexpr (std::is_same_v<SendMode, ReadyCommMode>) {
36+
} else if constexpr (std::is_same_v<SendMode, CommModeReady>) {
3737
MPI_Rsend(mpi_view, mpi_count, mpi_datatype, mpi_dest, mpi_tag, mpi_comm);
38-
} else if constexpr (std::is_same_v<SendMode, SynchronousCommMode>) {
38+
} else if constexpr (std::is_same_v<SendMode, CommModeSynchronous>) {
3939
MPI_Ssend(mpi_view, mpi_count, mpi_datatype, mpi_dest, mpi_tag, mpi_comm);
40+
} else {
41+
static_assert(std::is_void_v<SendMode>, "unexpected communication mode");
4042
}
4143
};
4244

@@ -50,33 +52,22 @@ void send(const SendMode &, const SendView &sv, int dest, int tag, MPI_Comm comm
5052
Kokkos::Tools::popRegion();
5153
}
5254

53-
<<<<<<< HEAD
54-
template <KokkosView SendView>
55-
void send(const SendView &sv, int dest, int tag, MPI_Comm comm) {
56-
send(KokkosComm::DefaultCommMode(), sv, dest, tag, comm);
57-
}
58-
59-
template <CommunicationMode SendMode, KokkosExecutionSpace ExecSpace, KokkosView SendView>
60-
void send(const SendMode &, const ExecSpace &space, const SendView &sv, int dest, int tag, MPI_Comm comm) {
61-
=======
6255
template <KokkosExecutionSpace ExecSpace, KokkosView SendView, CommunicationMode SendMode>
6356
void send(const ExecSpace &space, const SendView &sv, int dest, int tag, MPI_Comm comm, SendMode) {
64-
>>>>>>> 58bcb64 (Communication mode rework)
6557
Kokkos::Tools::pushRegion("KokkosComm::Impl::send");
6658

6759
using Packer = typename KokkosComm::PackTraits<SendView>::packer_type;
6860

6961
auto mpi_send_fn = [](void *mpi_view, int mpi_count, MPI_Datatype mpi_datatype, int mpi_dest, int mpi_tag,
7062
MPI_Comm mpi_comm) {
71-
7263
if constexpr (std::is_same_v<SendMode, CommModeStandard>) {
7364
MPI_Send(mpi_view, mpi_count, mpi_datatype, mpi_dest, mpi_tag, mpi_comm);
7465
} else if constexpr (std::is_same_v<SendMode, CommModeReady>) {
7566
MPI_Rsend(mpi_view, mpi_count, mpi_datatype, mpi_dest, mpi_tag, mpi_comm);
7667
} else if constexpr (std::is_same_v<SendMode, CommModeSynchronous>) {
7768
MPI_Ssend(mpi_view, mpi_count, mpi_datatype, mpi_dest, mpi_tag, mpi_comm);
7869
} else {
79-
static_assert(std::is_void_v<SendMode>, "Unexpected Communication Mode");
70+
static_assert(std::is_void_v<SendMode>, "unexpected communication mode");
8071
}
8172
};
8273

‎unit_tests/mpi/test_isendrecv.cpp‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -97,9 +97,7 @@ TYPED_TEST(IsendRecv, 1D_contig_standard) {
9797
isend_comm_mode_1d_contig<CommModeStandard, typename TestFixture::Scalar>();
9898
}
9999

100-
TYPED_TEST(IsendRecv, 1D_contig_ready) {
101-
isend_comm_mode_1d_contig<CommModeReady, typename TestFixture::Scalar>();
102-
}
100+
TYPED_TEST(IsendRecv, 1D_contig_ready) { isend_comm_mode_1d_contig<CommModeReady, typename TestFixture::Scalar>(); }
103101

104102
TYPED_TEST(IsendRecv, 1D_contig_synchronous) {
105103
isend_comm_mode_1d_contig<CommModeSynchronous, typename TestFixture::Scalar>();

‎unit_tests/mpi/test_sendrecv.cpp‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,6 @@ class MpiSendRecv : public testing::Test {
3232
using ScalarTypes = ::testing::Types<int, int64_t, float, double, Kokkos::complex<float>, Kokkos::complex<double>>;
3333
TYPED_TEST_SUITE(MpiSendRecv, ScalarTypes);
3434

35-
36-
3735
template <CommunicationMode SendMode, typename Scalar>
3836
void send_comm_mode_1d_contig() {
3937
if constexpr (std::is_same_v<SendMode, CommModeReady>) {
@@ -96,9 +94,7 @@ TYPED_TEST(MpiSendRecv, 1D_contig_standard) {
9694
send_comm_mode_1d_contig<CommModeStandard, typename TestFixture::Scalar>();
9795
}
9896

99-
TYPED_TEST(MpiSendRecv, 1D_contig_ready) {
100-
send_comm_mode_1d_contig<CommModeReady, typename TestFixture::Scalar>();
101-
}
97+
TYPED_TEST(MpiSendRecv, 1D_contig_ready) { send_comm_mode_1d_contig<CommModeReady, typename TestFixture::Scalar>(); }
10298

10399
TYPED_TEST(MpiSendRecv, 1D_contig_synchronous) {
104100
send_comm_mode_1d_contig<CommModeSynchronous, typename TestFixture::Scalar>();

0 commit comments

Comments
 (0)