From d2d9051e935aaef01dbdb55cbc4dac8d1d71e27c Mon Sep 17 00:00:00 2001 From: Roland Haas Date: Tue, 1 Dec 2020 13:40:32 -0600 Subject: [PATCH 1/4] correctly captilize PMIx package name --- cmake/{FindPMIx.cmake => FindPMIX.cmake} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cmake/{FindPMIx.cmake => FindPMIX.cmake} (100%) diff --git a/cmake/FindPMIx.cmake b/cmake/FindPMIX.cmake similarity index 100% rename from cmake/FindPMIx.cmake rename to cmake/FindPMIX.cmake From 1e35d4df15e6f11818e39f4107b578edea997908 Mon Sep 17 00:00:00 2001 From: Roland Haas Date: Tue, 1 Dec 2020 13:56:31 -0600 Subject: [PATCH 2/4] scr_env: make work for PMIx --- CMakeLists.txt | 2 +- src/scr_env.c | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e9a71641..91d46a72 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -178,7 +178,7 @@ OPTION(SCR_BGQ "Enable proper BGQ compilation" OFF) ## PMIx IF(${SCR_RESOURCE_MANAGER} STREQUAL "PMIX") FIND_PACKAGE(PMIX REQUIRED) - SET(HAVE_PMIX TRUE) + SET(HAVE_LIBPMIX TRUE) INCLUDE_DIRECTORIES(${PMIX_INCLUDE_DIRS}) LIST(APPEND SCR_EXTERNAL_LIBS ${PMIX_LIBRARIES}) ENDIF(${SCR_RESOURCE_MANAGER} STREQUAL "PMIX") diff --git a/src/scr_env.c b/src/scr_env.c index c387bbd4..11988251 100644 --- a/src/scr_env.c +++ b/src/scr_env.c @@ -53,9 +53,9 @@ #include "cppr.h" #endif /* HAVE_CPPR */ -#ifdef HAVE_PMIX +#ifdef HAVE_LIBPMIX #include "pmix.h" -#endif /* HAVE_PMIX */ +#endif /* HAVE_LIBPMIX */ #ifdef SCR_BGQ #include "firmware/include/personality.h" /* Personality_t */ @@ -126,8 +126,8 @@ char* scr_env_jobid() { char* jobid = NULL; - char* value; #ifdef SCR_RESOURCE_MANAGER_SLURM + char* value; /* read $SLURM_JOBID environment variable for jobid string */ if ((value = getenv("SLURM_JOBID")) != NULL) { jobid = strdup(value); @@ -139,6 +139,7 @@ char* scr_env_jobid() } #endif #ifdef SCR_RESOURCE_MANAGER_APRUN + char* value; /* read $PBS_JOBID environment variable for jobid string */ if ((value = getenv("PBS_JOBID")) != NULL) { jobid = strdup(value); @@ -178,6 +179,7 @@ char* scr_env_jobid() PMIX_PDATA_FREE(pmix_query_data, 1); #endif #ifdef SCR_RESOURCE_MANAGER_LSF + char* value; /* read $PBS_JOBID environment variable for jobid string */ if ((value = getenv("LSB_JOBID")) != NULL) { jobid = strdup(value); @@ -263,12 +265,12 @@ int scr_env_init(void) #ifdef SCR_RESOURCE_MANAGER_PMIX /* init pmix */ + pmix_proc_t scr_pmix_proc; int retval = PMIx_Init(&scr_pmix_proc, NULL, 0); if (retval != PMIX_SUCCESS) { - scr_err("PMIx_Init failed: rc=%d @ %s:%d", + scr_abort(-1, "PMIx_Init failed: rc=%d @ %s:%d", retval, __FILE__, __LINE__ ); - return SCR_FAILURE; } scr_dbg(1, "PMIx_Init succeeded @ %s:%d", __FILE__, __LINE__); #endif /* SCR_MACHINE_TYPE == SCR_PMIX */ From 419a8dc9510cbd40fc78325232dd93b1f6745611 Mon Sep 17 00:00:00 2001 From: Roland Haas Date: Tue, 1 Dec 2020 14:47:52 -0600 Subject: [PATCH 3/4] scr_env: finalize PMIx if we initialized it --- src/scr_env.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/scr_env.c b/src/scr_env.c index 11988251..851f8506 100644 --- a/src/scr_env.c +++ b/src/scr_env.c @@ -291,5 +291,8 @@ int scr_env_init(void) int scr_env_finalize(void) { +#ifdef SCR_RESOURCE_MANAGER_PMIX + PMIx_Finalize(NULL, 0); +#endif return SCR_SUCCESS; } From e9d032449486eb0e3d3348abf3e1d3b341f6a2e3 Mon Sep 17 00:00:00 2001 From: Roland Haas Date: Tue, 1 Dec 2020 14:48:15 -0600 Subject: [PATCH 4/4] scr_env: query PMIx for remaining time if YOGRT is not present --- src/scr_env.c | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/scr_env.c b/src/scr_env.c index 851f8506..1f830613 100644 --- a/src/scr_env.c +++ b/src/scr_env.c @@ -73,6 +73,28 @@ machine-dependent information. */ /* returns the number of seconds remaining in the time allocation */ + +#ifdef SCR_RESOURCE_MANAGER_PMIX +/* callback for PMIX to query remaining walltime */ +static uint32_t seconds_remaining; +static void seconds_remaining_cbfunc(pmix_status_t status, pmix_info_t *info, + size_t ninfo, void *cbdata, + pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + volatile int *waiting = (volatile int*)cbdata; + /* read time_remaining */ + if (info != NULL) { + seconds_remaining = info->value.data.uint32; + } + + if (release_fn != NULL) { + release_fn(release_cbdata); + } + *waiting = 0; +} +#endif + long int scr_env_seconds_remaining() { /* returning a negative number tells the caller this functionality is disabled */ @@ -84,6 +106,26 @@ long int scr_env_seconds_remaining() if (secs < 0) { secs = 0; } + #elif defined(SCR_RESOURCE_MANAGER_PMIX) + /* query time remaining */ + pmix_status_t retval; + pmix_query_t *query; + const int nq = 1; + PMIX_QUERY_CREATE(query, nq); + if (query != NULL) { + PMIX_ARGV_APPEND(retval, query[0].keys, PMIX_TIME_REMAINING); + if (retval == PMIX_SUCCESS) { + volatile int waiting = 1; + retval = PMIx_Query_info_nb(query, nq, seconds_remaining_cbfunc, + (void*)&waiting); + if (retval == PMIX_SUCCESS) { + while (waiting) { + sleep(1); + } + secs = (long int)seconds_remaining; + } + } + } #else char* scr_end_time = getenv("SCR_END_TIME"); if (scr_end_time) {