Merge branch 'master' of github.com:mllg/batchtools

Author: mllg

.aspell/batchtools.rds

@@ -0,0 +1,2 @@
+Rd_files <- vignettes <- R_files <- description <-
+    list(encoding = "UTF-8", language = "en", dictionaries = c("en_stats", "batchtools"))

.aspell/defaults.R

@@ -1,6 +1,6 @@
 Package: batchtools
 Title: Tools for Computation on Batch Systems
-Version: 0.9.9
+Version: 0.9.10-9000
 Authors@R: c(
     person("Michel", "Lang", NULL, "[email protected]",
       role = c("cre", "aut"), comment = c(ORCID = "0000-0001-9754-0393")),
@@ -35,7 +35,7 @@ Imports:
     brew,
     checkmate (>= 1.8.5),
     digest (>= 0.6.9),
-    fs (>= 1.1.0),
+    fs (>= 1.2.0),
     parallel,
     progress (>= 1.1.1),
     R6,
@@ -50,6 +50,7 @@ Suggests:
     doMPI,
     e1071,
     foreach,
+    future,
     future.batchtools,
     knitr,
     parallelMap,

DESCRIPTION

@@ -40,11 +40,11 @@ export(cfHandleUnknownSubmitError)
 export(cfKillJob)
 export(cfReadBrewTemplate)
 export(chunk)
-export(chunkIds)
 export(clearRegistry)
 export(doJobCollection)
 export(estimateRuntimes)
 export(execJob)
+export(findConfFile)
 export(findDone)
 export(findErrors)
 export(findExperiments)
@@ -59,6 +59,7 @@ export(findRunning)
 export(findStarted)
 export(findSubmitted)
 export(findTagged)
+export(findTemplateFile)
 export(flatten)
 export(getDefaultRegistry)
 export(getErrorMessages)

NAMESPACE

@@ -1,3 +1,13 @@
+# batchtools 0.9.11
+
+* Removed deprecated function `chunkIds()`.
+
+# batchtools 0.9.10
+
+* Exported functions `findConfFile()` and `findTemplateFile()`.
+* Dropped support for providing a template file directly as string. A valid file is now always required.
+* Fixed writing to `TMPDIR` instead of the R session's temporary directory.
+
 # batchtools 0.9.9
 
 * RDS files are explicitly stored in version 2 to ensure backward compatibility with R versions prior to 3.5.0.

NEWS.md

@@ -55,15 +55,15 @@
 #'   In the configuration file you can define how \pkg{batchtools} interacts with the system via \code{\link{ClusterFunctions}}.
 #'   Separating the configuration of the underlying host system from the R code allows to easily move computation to another site.
 #'
-#'   The file lookup is implemented in the internal function \code{findConfFile} which returns the first file found of the following candidates:
+#'   The file lookup is implemented in the internal (but exported) function \code{findConfFile} which returns the first file found of the following candidates:
 #'   \enumerate{
 #'    \item{File \dQuote{batchtools.conf.R} in the path specified by the environment variable \dQuote{R_BATCHTOOLS_SEARCH_PATH}.}
 #'    \item{File \dQuote{batchtools.conf.R} in the current working directory.}
 #'    \item{File \dQuote{config.R} in the user configuration directory as reported by \code{rappdirs::user_config_dir("batchtools", expand = FALSE)} (depending on OS, e.g., on linux this usually resolves to \dQuote{~/.config/batchtools/config.R}).}
 #'    \item{\dQuote{.batchtools.conf.R} in the home directory (\dQuote{~}).}
 #'    \item{\dQuote{config.R} in the site config directory as reported by \code{rappdirs::site_config_dir("batchtools")} (depending on OS). This file can be used for admins to set sane defaults for a computation site.}
 #'   }
-#'   Set to \code{character(0)} if you want to suppress reading any configuration file.
+#'   Set to \code{NA} if you want to suppress reading any configuration file.
 #'   If a configuration file is found, it gets sourced inside the environment of the registry after the defaults for all variables are set.
 #'   Therefore you can set and overwrite slots, e.g. \code{default.resources = list(walltime = 3600)} to set default resources or \dQuote{max.concurrent.jobs} to
 #'   limit the number of jobs allowed to run simultaneously on the system.
@@ -128,7 +128,7 @@ makeRegistry = function(file.dir = "registry", work.dir = getwd(), conf.file = f
     assertPathForOutput(file.dir, overwrite = FALSE)
   assertString(work.dir)
   assertDirectoryExists(work.dir, access = "r")
-  assertCharacter(conf.file, any.missing = FALSE, max.len = 1L)
+  assertString(conf.file, na.ok = TRUE)
   assertCharacter(packages, any.missing = FALSE, min.chars = 1L)
   assertCharacter(namespaces, any.missing = FALSE, min.chars = 1L)
   assertCharacter(source, any.missing = FALSE, min.chars = 1L)
@@ -185,8 +185,8 @@ makeRegistry = function(file.dir = "registry", work.dir = getwd(), conf.file = f
   "!DEBUG [makeRegistry]: Creating directories in '`reg$file.dir`'"
 
   fs::dir_create(c(reg$file.dir, reg$work.dir))
-  reg$file.dir = path_real(reg$file.dir)
-  reg$work.dir = path_real(reg$work.dir)
+  reg$file.dir = fs::path_abs(reg$file.dir)
+  reg$work.dir = fs::path_abs(reg$work.dir)
 
   fs::dir_create(fs::path(reg$file.dir, c("jobs", "results", "updates", "logs", "exports", "external")))
   with_dir(reg$work.dir, loadRegistryDependencies(reg))

R/Registry.R

@@ -1,45 +1,8 @@
-#' @title Chunk Jobs for Sequential Execution
-#'
-#' @description
-#' This function is deprecated in favor of the more flexible \code{chunk}, \code{lpt} and \code{binpack}.
-#'
-#' @templateVar ids.default all
-#' @template ids
-#' @param group.by [\code{character(0)}]\cr
-#'   If \code{ids} is a \code{\link{data.frame}} with additional columns
-#'   (in addition to the required column \dQuote{job.id}), then the chunking is performed using
-#'   subgroups defined by the columns set in \code{group.by}.
-#'   See example.
-#' @inheritParams chunk
-#' @template reg
-#' @return [\code{\link[data.table]{data.table}}] with columns \dQuote{job.id} and \dQuote{chunk}.
-#' @seealso \code{\link{chunk}} \code{\link{binpack}} \code{\link{lpt}}
-#' @export
-chunkIds = function(ids = NULL, n.chunks = NULL, chunk.size = NULL, group.by = character(0L), reg = getDefaultRegistry()) {
-  .Deprecated("chunk", package = "batchtools")
-  assertRegistry(reg)
-  assertCharacter(group.by, any.missing = FALSE, min.chars = 1L)
-  ids = convertIds(reg, ids, default = allIds(reg), keep.extra = group.by)
-
-  if (length(group.by) > 0L) {
-    job.id = NULL
-    if (any(group.by %chnin% names(ids)))
-      stop("All columns to group by must be provided in the 'ids' table")
-    ids[, "chunk" := chunk(job.id, n.chunks = n.chunks, chunk.size = chunk.size), by = group.by]
-    ids[, "chunk" := .GRP, by = c(group.by, "chunk")]
-  } else {
-    ids[, "chunk" := chunk(job.id, n.chunks = n.chunks, chunk.size = chunk.size)]
-  }
-
-  return(ids[, c("job.id", "chunk")])
-}
-
-
 #' @title Chunk Jobs for Sequential Execution
 #'
 #' @description
 #' Jobs can be partitioned into \dQuote{chunks} to be executed sequentially on the computational nodes.
-#' Chunks are defined by providing a data frame with columns \dQuote{job.id} and \dQuote{chunk} (integer).
+#' Chunks are defined by providing a data frame with columns \dQuote{job.id} and \dQuote{chunk} (integer)
 #' to \code{\link{submitJobs}}.
 #' All jobs with the same chunk number will be grouped together on one node to form a single
 #' computational job.

R/chunkIds.R

@@ -141,21 +141,15 @@ print.SubmitJobResult = function(x, ...) {
 #' Simply reads your template file and returns it as a character vector.
 #'
 #' @param template [\code{character(1)}]\cr
-#'   Path to template file or single string (containing newlines) which is then passed
-#'   to \code{\link[brew]{brew}}.
+#'   Path to template file which is then passed to \code{\link[brew]{brew}}.
 #' @param comment.string [\code{character(1)}]\cr
 #'   Ignore lines starting with this string.
 #' @return [\code{character}].
 #' @family ClusterFunctionsHelper
 #' @export
 cfReadBrewTemplate = function(template, comment.string = NA_character_) {
-  if (stri_detect_fixed(template, "\n")) {
-    "!DEBUG [cfReadBrewTemplate]: Parsing template from string"
-    lines = stri_trim_both(stri_split_lines(template)[[1L]])
-  } else {
-    "!DEBUG [cfReadBrewTemplate]: Parsing template file '`template`'"
-    lines = stri_trim_both(readLines(template))
-  }
+  "!DEBUG [cfReadBrewTemplate]: Parsing template file '`template`'"
+  lines = stri_trim_both(readLines(template))
 
   lines = lines[!stri_isempty(lines)]
   if (!is.na(comment.string))
@@ -288,42 +282,49 @@ getBatchIds = function(reg, status = "all") {
   tab[batch.id %in% reg$status$batch.id]
 }
 
-findTemplateFile = function(name) {
-  assertString(name, min.chars = 1L)
 
-  if (stri_detect_fixed(name, "\n"))
-    return(name)
+#' @title Find a batchtools Template File
+#'
+#' @description
+#' This functions returns the path to a template file on the file system.
+#' @template template
+#' @return [\code{character}] Path to the file or \code{NA} if no template template file was found.
+#' @keywords internal
+#' @export
+findTemplateFile = function(template) {
+  assertString(template, min.chars = 1L)
 
-  if (stri_endswith_fixed(name, ".tmpl")) {
-    return(assertFileExists(name, access = "r"))
+  if (stri_endswith_fixed(template, ".tmpl")) {
+    assertFileExists(template, access = "r")
+    return(fs::path_abs(template))
   }
 
   x = Sys.getenv("R_BATCHTOOLS_SEARCH_PATH")
   if (nzchar(x)) {
-    x = fs::path(x, sprintf("batchtools.%s.tmpl", name))
-    if (testFileExists(x, access = "r"))
-      return(fs::path_real(x))
+    x = fs::path(x, sprintf("batchtools.%s.tmpl", template))
+    if (fs::file_access(x, "read"))
+      return(fs::path_abs(x))
   }
 
-  x = sprintf("batchtools.%s.tmpl", name)
-  if (testFileExists(x, access = "r"))
-    return(fs::path_real(x))
+  x = sprintf("batchtools.%s.tmpl", template)
+  if (fs::file_access(x, "read"))
+    return(fs::path_abs(x))
 
-  x = fs::path(user_config_dir("batchtools", expand = FALSE), sprintf("%s.tmpl", name))
-  if (testFileExists(x, access = "r"))
+  x = fs::path(user_config_dir("batchtools", expand = FALSE), sprintf("%s.tmpl", template))
+  if (fs::file_access(x, "read"))
     return(x)
 
-  x = fs::path("~", sprintf(".batchtools.%s.tmpl", name))
-  if (testFileExists(x, access = "r"))
-    return(fs::path_real(x))
+  x = fs::path("~", sprintf(".batchtools.%s.tmpl", template))
+  if (fs::file_access(x, "read"))
+    return(fs::path_abs(x))
 
-  x = fs::path(site_config_dir("batchtools"), sprintf("%s.tmpl", name))
-  if (testFileExists(x, access = "r"))
+  x = fs::path(site_config_dir("batchtools"), sprintf("%s.tmpl", template))
+  if (fs::file_access(x, "read"))
     return(x)
 
-  x = system.file("templates", sprintf("%s.tmpl", name), package = "batchtools")
-  if (testFileExists(x, access = "r"))
+  x = system.file("templates", sprintf("%s.tmpl", template), package = "batchtools")
+  if (fs::file_access(x, "read"))
     return(x)
 
-  stopf("Argument 'template' (=\"%s\") must point to a readable template file or contain the template itself as string (containing at least one newline)", name)
+  return(NA_character_)
 }

R/clusterFunctions.R

@@ -18,14 +18,15 @@
 #' @note
 #' Array jobs are currently not supported.
 #'
-#' @templateVar cf.name lsf
 #' @template template
 #' @inheritParams makeClusterFunctions
 #' @return [\code{\link{ClusterFunctions}}].
 #' @family ClusterFunctions
 #' @export
 makeClusterFunctionsLSF = function(template = "lsf", scheduler.latency = 1, fs.latency = 65) { # nocov start
   template = findTemplateFile(template)
+  if (testScalarNA(template))
+    stopf("Argument 'template' (=\"%s\") must point to a readable template file or contain the template itself as string (containing at least one newline)", template)
   template = cfReadBrewTemplate(template)
 
   # When LSB_BJOBS_CONSISTENT_EXIT_CODE = Y, the bjobs command exits with 0 only

R/clusterFunctionsLSF.R

@@ -18,14 +18,15 @@
 #' @note
 #' Array jobs are currently not supported.
 #'
-#' @templateVar cf.name openlava
 #' @template template
 #' @inheritParams makeClusterFunctions
 #' @return [\code{\link{ClusterFunctions}}].
 #' @family ClusterFunctions
 #' @export
 makeClusterFunctionsOpenLava = function(template = "openlava", scheduler.latency = 1, fs.latency = 65) { # nocov start
   template = findTemplateFile(template)
+  if (testScalarNA(template))
+    stopf("Argument 'template' (=\"%s\") must point to a readable template file", template)
   template = cfReadBrewTemplate(template)
 
   # When LSB_BJOBS_CONSISTENT_EXIT_CODE = Y, the bjobs command exits with 0 only