remove dependency on {futile.logger}

.ci/build-docs.sh

@@ -10,7 +10,7 @@ sudo tlmgr --verify-repo=none update --self
 sudo tlmgr --verify-repo=none install inconsolata helvetic rsfs
 
 # install dependencies
-Rscript -e "install.packages(c('assertthat', 'curl', 'data.table', 'futile.logger', 'jsonlite', 'knitr', 'markdown', 'pkgdown', 'purrr', 'roxygen2', 'stringr'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())"
+Rscript -e "install.packages(c('assertthat', 'curl', 'data.table', 'jsonlite', 'knitr', 'markdown', 'pkgdown', 'purrr', 'roxygen2', 'stringr'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())"
 
 cp NEWS.md ./r-pkg/
 cp README.md ./r-pkg/

.ci/setup.sh

@@ -19,5 +19,5 @@ sudo apt-get install \
     tidy \
     qpdf
 
-Rscript -e "install.packages(c('covr', 'curl', 'data.table', 'futile.logger', 'jsonlite', 'knitr', 'lintr', 'markdown', 'purrr', 'stringr', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())"
+Rscript -e "install.packages(c('covr', 'curl', 'data.table', 'jsonlite', 'knitr', 'lintr', 'markdown', 'purrr', 'stringr', 'testthat'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())"
 cp test-data/* r-pkg/inst/testdata/

.github/workflows/build-docs.yaml

@@ -21,6 +21,7 @@ jobs:
       - uses: actions/checkout@v6
         with:
           fetch-depth: 0
+          persist-credentials: false
       - name: set up R
         uses: r-lib/actions/setup-r@v2
         with:

.github/workflows/ci.yml

@@ -39,6 +39,7 @@ jobs:
         uses: actions/checkout@v6
         with:
           fetch-depth: 0
+          persist-credentials: false
       - uses: pre-commit/[email protected]
       - name: set up R
         uses: r-lib/actions/setup-r@v2

.pre-commit-config.yaml

@@ -5,17 +5,17 @@ exclude: |
     )$
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/maxwinterstein/shfmt-py
-    rev: v3.11.0.2
+    rev: v3.12.0.1
     hooks:
       - id: shfmt
         args: ["--indent=4", "--space-redirects", "--write"]
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.10.0.1
+    rev: v0.11.0.1
     hooks:
       - id: shellcheck
         args: ["--exclude=SC2002"]

CONTRIBUTING.md

@@ -127,6 +127,9 @@ Look at the source code of `setup_local.sh` for a list of the valid arguments.
 # Start up Elasticsearch on localhost:9200 and seed it with data
 ./setup_local.sh 8.17.2
 
+# enable the integration tests
+export NOT_CRAN=true
+
 # Run tests
 make test
 

r-pkg/DESCRIPTION

@@ -24,7 +24,6 @@ Depends:
 Imports:
     curl,
     data.table,
-    futile.logger,
     jsonlite,
     purrr,
     stats,
@@ -36,6 +35,6 @@ Suggests:
 License: BSD_3_clause + file LICENSE
 URL: https://github.com/uptake/uptasticsearch
 BugReports: https://github.com/uptake/uptasticsearch/issues
-RoxygenNote: 7.3.2
+RoxygenNote: 7.3.3
 VignetteBuilder: knitr
 Encoding: UTF-8

r-pkg/NAMESPACE

@@ -20,10 +20,6 @@ importFrom(data.table,setcolorder)
 importFrom(data.table,setkeyv)
 importFrom(data.table,setnames)
 importFrom(data.table,uniqueN)
-importFrom(futile.logger,flog.debug)
-importFrom(futile.logger,flog.fatal)
-importFrom(futile.logger,flog.info)
-importFrom(futile.logger,flog.warn)
 importFrom(jsonlite,fromJSON)
 importFrom(parallel,clusterMap)
 importFrom(parallel,detectCores)

r-pkg/R/es_search.R

@@ -8,7 +8,7 @@
 #'                 as it has pulled this many hits. Default is \code{Inf}, meaning that
 #'                 all possible hits will be pulled.
 #' @param size Number of records per page of results.
-#'             See \href{https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#request-body-search-from-size}{Elasticsearch docs} for more.
+#'             See \href{https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-scroll}{Elasticsearch docs} for more.
 #'             Note that this will be reset to 0 if you submit a \code{query_body} with
 #'             an "aggs" request in it. Also see \code{max_hits}.
 #' @param query_body String with a valid Elasticsearch query. Default is an empty query.
@@ -17,7 +17,7 @@
 #'               The scroll context will be refreshed every time you ask Elasticsearch
 #'               for another record, so this parameter should just be the amount of
 #'               time you expect to pass between requests. See the
-#'               \href{https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#request-body-search-scroll}{Elasticsearch scroll/pagination docs}
+#'               \href{https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-scroll}{Elasticsearch scroll/pagination docs}
 #'               for more information.
 #' @param n_cores Number of cores to distribute fetching and processing over.
 #' @param break_on_duplicates Boolean, defaults to TRUE. \code{es_search} uses the size of the
@@ -84,6 +84,7 @@ es_search <- function(es_host
                       , break_on_duplicates = TRUE
                       , ignore_scroll_restriction = FALSE
                       , intermediates_dir = getwd()
+                      , verbose = FALSE
 ) {
 
     # Check if this is an aggs or straight-up search query
@@ -135,6 +136,7 @@ es_search <- function(es_host
             , es_index = es_index
             , trailing_args = "size=0"
             , query_body = query_body
+            , verbose = verbose
         )
 
         return(chomp_aggs(aggs_json = result))
@@ -151,7 +153,8 @@ es_search <- function(es_host
                       , n_cores = n_cores
                       , break_on_duplicates = break_on_duplicates
                       , ignore_scroll_restriction = ignore_scroll_restriction
-                      , intermediates_dir = intermediates_dir))
+                      , intermediates_dir = intermediates_dir
+                      , verbose = verbose))
 }
 
 # nolint start
@@ -162,7 +165,7 @@ es_search <- function(es_host
 # [param] es_host A string identifying an Elasticsearch host. This should be of the form
 #        [transfer_protocol][hostname]:[port]. For example, 'http://myindex.thing.com:9200'.
 # [param] es_index The name of an Elasticsearch index to be queried.
-# [param] size Number of records per page of results. See \href{https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#request-body-search-from-size}{Elasticsearch docs} for more
+# [param] size Number of records per page of results. See \href{https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-scroll}{Elasticsearch docs} for more
 # [param] query_body String with a valid Elasticsearch query to be passed to \code{\link[elastic]{Search}}.
 #                  Default is an empty query.
 # [param] scroll How long should the scroll context be held open? This should be a
@@ -190,6 +193,7 @@ es_search <- function(es_host
 #                                value longer than an hour, set \code{ignore_scroll_restriction}
 #                                to \code{TRUE}.
 # [param] intermediates_dir passed through from es_search. See es_search docs.
+# [param] verbose TRUE to print DEBUG-level logs.
 # [examples]
 # \dontrun{
 #
@@ -224,6 +228,7 @@ es_search <- function(es_host
                      , break_on_duplicates
                      , ignore_scroll_restriction
                      , intermediates_dir
+                     , verbose
 ) {
 
     # Check es_host
@@ -238,7 +243,7 @@ es_search <- function(es_host
                       "\n\nIf you understand the costs and would like to make requests ",
                       "with a longer-lived context, re-run this function with ",
                       "ignore_scroll_restriction = TRUE.\n",
-                      "\nPlease see https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#request-body-search-scroll ",  # nolint[line_length]
+                      "\nPlease see https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-scroll ",  # nolint[line_length]
                       "for more information.")
         .log_fatal(msg)
     }
@@ -284,12 +289,13 @@ es_search <- function(es_host
         , es_index = es_index
         , trailing_args = paste0("size=", size, "&scroll=", scroll)
         , query_body = query_body
+        , verbose = verbose
     )
 
     # Parse to JSON to get total number of documents matching the query
     firstResult <- jsonlite::fromJSON(firstResultJSON, simplifyVector = FALSE)
 
-    major_version <- .get_es_version(es_host)
+    major_version <- .get_es_version(es_host, verbose = verbose)
     if (as.integer(major_version) > 6) {
       hits_to_pull <- min(firstResult[["hits"]][["total"]][["value"]], max_hits)
     } else {
@@ -458,6 +464,7 @@ es_search <- function(es_host
 #          hits_to_pull - Total hits to be pulled (documents matching user's query).
 #                       Or, in the case where max_hits < number of matching docs,
 #                       max_hits.
+#          verbose - TRUE to print DEBUG-level logs.
 #' @importFrom jsonlite fromJSON
 .keep_on_pullin <- function(scroll_id
                             , out_path
@@ -466,11 +473,12 @@ es_search <- function(es_host
                             , scroll
                             , hits_pulled
                             , hits_to_pull
+                            , verbose
 ) {
 
     # Note that the old scrolling strategy was deprecated in Elasticsearch 5.x and
     # officially dropped in Elasticsearch 6.x. Need to grab the correct method here
-    major_version <- .get_es_version(es_host)
+    major_version <- .get_es_version(es_host, verbose = verbose)
     scrolling_request <- switch(
         major_version
         , "1" = .legacy_scroll_request
@@ -487,6 +495,7 @@ es_search <- function(es_host
             es_host = es_host
             , scroll = scroll
             , scroll_id = scroll_id
+            , verbose = verbose
         )
         .stop_for_status(result)
         resultJSON <- .content(result, as = "text")
@@ -531,7 +540,7 @@ es_search <- function(es_host
 # [name] .new_scroll_request
 # [description] Make a scrolling request and return the result
 # [references] https://www.elastic.co/guide/en/elasticsearch/reference/6.7/search-request-scroll.html
-.new_scroll_request <- function(es_host, scroll, scroll_id) {
+.new_scroll_request <- function(es_host, scroll, scroll_id, verbose) {
 
     # Set up scroll_url
     scroll_url <- paste0(es_host, "/_search/scroll")  # nolint[absolute_path,non_portable_path]
@@ -541,14 +550,15 @@ es_search <- function(es_host
         verb = "POST"
         , url = scroll_url
         , body = sprintf('{"scroll": "%s", "scroll_id": "%s"}', scroll, scroll_id)
+        , verbose = verbose
     )
     return(result)
 }
 
 # [title] Make a scroll request with the strategy supported by Elasticsearch 1.x and Elasticsearch 2.x
 # [name] .legacy_scroll_request
 # [description] Make a scrolling request and return the result
-.legacy_scroll_request <- function(es_host, scroll, scroll_id) {
+.legacy_scroll_request <- function(es_host, scroll, scroll_id, verbose) {
 
     # Set up scroll_url
     scroll_url <- paste0(es_host, "/_search/scroll?scroll=", scroll)
@@ -558,6 +568,7 @@ es_search <- function(es_host
         verb = "POST"
         , url = scroll_url
         , body = scroll_id
+        , verbose = verbose
     )
     return(result)
 }
@@ -594,7 +605,7 @@ es_search <- function(es_host
     portPattern <- ":[0-9]+$"
     if (! grepl(portPattern, es_host) == 1) {
         msg <- paste0("No port found in es_host! es_host should be a string of the"
-                      , "form [transfer_protocol][hostname]:[port]). for "
+                      , " form [transfer_protocol][hostname]:[port]. For "
                       , "example: 'http://myindex.mysite.com:9200'")
         .log_fatal(msg)
     }
@@ -625,14 +636,15 @@ es_search <- function(es_host
 #               version of Elasticsearch.
 # [param] es_host A string identifying an Elasticsearch host. This should be of the form
 #         [transfer_protocol][hostname]:[port]. For example, 'http://myindex.thing.com:9200'.
-.get_es_version <- function(es_host) {
+.get_es_version <- function(es_host, verbose) {
 
     # Hit the cluster root to get metadata
     .log_info("Checking Elasticsearch version...")
     result <- .request(
         verb = "GET"
         , url = es_host
         , body = NULL
+        , verbose = verbose
     )
     .stop_for_status(result)
 
@@ -669,6 +681,7 @@ es_search <- function(es_host
 #         For example, to limit the size of the returned results, you might pass
 #         "size=0". This can be a single string or a character vector of params, e.g.
 #         \code{c('size=0', 'scroll=5m')}
+# [param] verbose TRUE to print DEBUG-level logs.
 # [param] query_body A JSON string with valid Elasticsearch DSL
 # [examples]
 # \dontrun{
@@ -701,6 +714,7 @@ es_search <- function(es_host
                           , es_index
                           , trailing_args = NULL
                           , query_body
+                          , verbose
 ) {
 
     # Input checking
@@ -717,6 +731,7 @@ es_search <- function(es_host
         verb = "POST"
         , url = reqURL
         , body = query_body
+        , verbose = verbose
     )
     .stop_for_status(result)
     result <- .content(result, as = "text")

r-pkg/R/get_fields.R

@@ -52,6 +52,7 @@
 #' }
 get_fields <- function(es_host
                        , es_indices = "_all"
+                       , verbose = FALSE
 ) {
 
     # Input checking
@@ -76,6 +77,7 @@ get_fields <- function(es_host
 
     major_version <- .get_es_version(
         es_host = es_host
+        , verbose = verbose
     )
 
     # The use of "_all" to indicate "all indices" was removed in Elasticsearch 7.
@@ -91,6 +93,7 @@ get_fields <- function(es_host
             verb = "GET"
             , url = sprintf("%s/_cat/indices?format=json", es_url)
             , body = NULL
+            , verbose = verbose
         )
         indexDT <- data.table::as.data.table(
             jsonlite::fromJSON(
@@ -112,6 +115,7 @@ get_fields <- function(es_host
         verb = "GET"
         , url = es_url
         , body = NULL
+        , verbose = verbose
     )
     .stop_for_status(result)
     resultContent <- .content(result, as = "parsed")
@@ -142,7 +146,7 @@ get_fields <- function(es_host
     }
 
     ##################### get aliases for index names #########################
-    rawAliasDT <- .get_aliases(es_host = es_host)
+    rawAliasDT <- .get_aliases(es_host = es_host, verbose = verbose)
     if (!is.null(rawAliasDT)) {
 
         .log_info("Replacing index names with aliases")
@@ -221,7 +225,7 @@ get_fields <- function(es_host
 # [es_host] A string identifying an Elasticsearch host.
 #' @importFrom data.table as.data.table
 #' @importFrom jsonlite fromJSON
-.get_aliases <- function(es_host) {
+.get_aliases <- function(es_host, verbose) {
 
     # construct the url to the alias endpoint
     url <- paste0(es_host, "/_cat/aliases")  # nolint[absolute_path, non_portable_path]
@@ -231,6 +235,7 @@ get_fields <- function(es_host
         verb = "GET"
         , url = url
         , body = NULL
+        , verbose = verbose
     )
     .stop_for_status(result)
     resultContent <- .content(result, as = "text")