v2.5.1 (#331)

* v2.4.0 (#327)

* feat(query): support live inserts into parted tables

Add immutable live-tail growth with explicit partition keys, shared FILE-domain symbol handling, atomic symbol rebinding, adversarial coverage, documentation, and a runnable rollover example.

* fix(core): restore total-core -c semantics

* fix(parse) Fix nonstring if not defined

---------

Co-authored-by: Karim <k.nassar@lynxtrading.com>

Author: singaraiona

docs/docs/namespaces/sys.md

@@ -31,7 +31,7 @@ Signature: `(.sys.args)`. Returns the process's command-line arguments as a dict
 |---|---|---|---|
 | `file` | str | `-f` / positional | Script path; empty if none. |
 | `port` | i64 | `-p` | IPC listen port; `0` if unset. |
-| `cores` | i64 | `-c` | Worker-pool size; `0` = auto. |
+| `cores` | i64 | `-c` | Total execution cores, including the main thread; `0` = auto. |
 | `timeit` | bool | `-t` | Profiler enabled at startup. |
 | `querylog` | bool | `-Q` | Query-statistics logging enabled at startup. |
 | `interactive` | bool | `-i` | Force the REPL after a script. |

src/app/main.c

@@ -81,7 +81,7 @@ int main(int argc, char** argv) {
     int interactive = 0;
     const char* file = NULL;
     uint16_t port = 0;
-    int n_cores = -1;      /* -1 = leave pool at lazy ncpu-1 default */
+    int n_cores = -1;      /* total execution cores; -1 = leave pool lazy */
     int timeit_init = 0;   /* -t N: enable profiler at startup */
     int qlog_init = 0;     /* -Q N: enable query-statistics logging at startup */
     const char* auth_pw = NULL;
@@ -92,7 +92,7 @@ int main(int argc, char** argv) {
     /* Parse args. Supported flags:
      *   -f FILE          run script file
      *   -p PORT          IPC listen port
-     *   -c N             worker-pool size (0 = auto: ncpu - 1)
+     *   -c N             total execution cores, main included (0 = auto)
      *   -t N             enable timeit at startup (N != 0 turns on)
      *   -Q N             enable query-statistics logging (N != 0 turns on)
      *   -i               interactive
@@ -152,7 +152,7 @@ int main(int argc, char** argv) {
                 " [-u PW | -U PW] [-l BASE | -L BASE] [-m SIZE] [file.rfl] [-- app args]\n"
                 "  -f, --file FILE     run script file (or pass as a positional arg)\n"
                 "  -p, --port PORT     listen for IPC clients on PORT\n"
-                "  -c, --cores N       worker-pool size (0 = auto: ncpu - 1, default)\n"
+                "  -c, --cores N       total execution cores, main included (0 = auto)\n"
                 "  -t, --timeit N      enable profiler at startup (N != 0)\n"
                 "  -i, --interactive   start the REPL even after running a file\n"
                 "  -u PW               set plain auth password\n"
@@ -191,9 +191,9 @@ int main(int argc, char** argv) {
      * (file load, REPL eval, builtins).  If -c wasn't given, leave the
      * pool to its lazy default on first use. */
     if (n_cores >= 0) {
-        ray_err_t err = ray_pool_init((uint32_t)n_cores);
+        ray_err_t err = ray_pool_init_total((uint32_t)n_cores);
         if (err != RAY_OK)
-            fprintf(stderr, "warning: ray_pool_init(%d) failed (%d)\n",
+            fprintf(stderr, "warning: ray_pool_init_total(%d) failed (%d)\n",
                     n_cores, (int)err);
     }
 

src/core/pool.c

@@ -120,7 +120,8 @@ static void worker_loop(void* arg) {
  * ray_pool_create
  * -------------------------------------------------------------------------- */
 
-ray_err_t ray_pool_create(ray_pool_t* pool, uint32_t n_workers) {
+static ray_err_t ray_pool_create_impl(ray_pool_t* pool, uint32_t n_workers,
+                                      bool auto_size) {
     /* conc-L7: memset zeroes all fields including the `cancelled` atomic,
      * which resets any cancellation state from a prior pool instance. */
     memset(pool, 0, sizeof(*pool));
@@ -133,14 +134,14 @@ ray_err_t ray_pool_create(ray_pool_t* pool, uint32_t n_workers) {
     atomic_init(&pool->pending, 0);
     atomic_init(&pool->cancelled, 0);
 
-    if (n_workers == 0) {
+    if (auto_size) {
         /* Auto-size to ncpu-1. The RAYFORCE_CORES env var overrides this
          * default worker count — the test harness sets it (see the Makefile
          * `test` target) so neither the in-process runtime nor the server
          * children it spawns via .sys.exec each create ncpu-1 threads on a
-         * many-core box. An explicit -c (which passes n_workers > 0 through
-         * ray_pool_init) bypasses this entirely, and a non-test run with the
-         * var unset keeps the historical ncpu-1 default. */
+         * many-core box. An explicit positive -c uses ray_pool_init_total()
+         * with exact sizing and bypasses this entirely; a non-test run with
+         * the var unset keeps the historical ncpu-1 default. */
         const char* env = getenv("RAYFORCE_CORES");
         if (env && *env) {
             long v = strtol(env, NULL, 10);
@@ -221,6 +222,10 @@ ray_err_t ray_pool_create(ray_pool_t* pool, uint32_t n_workers) {
     return RAY_OK;
 }
 
+ray_err_t ray_pool_create(ray_pool_t* pool, uint32_t n_workers) {
+    return ray_pool_create_impl(pool, n_workers, n_workers == 0);
+}
+
 /* --------------------------------------------------------------------------
  * ray_pool_free
  * -------------------------------------------------------------------------- */
@@ -506,11 +511,11 @@ ray_pool_t* ray_pool_get(void) {
  * Public API wrappers (declared in rayforce.h)
  * -------------------------------------------------------------------------- */
 
-/* conc-L4: If ray_pool_init() is called when the pool is already initialized
- * (state==2), the n_workers parameter is silently ignored and the existing
- * pool configuration is preserved. This is by design — the pool is a
- * singleton and reconfiguration requires ray_pool_destroy() + ray_pool_init(). */
-ray_err_t ray_pool_init(uint32_t n_workers) {
+/* conc-L4: If an initializer is called when the pool is already initialized
+ * (state==2), its requested size is silently ignored and the existing pool
+ * configuration is preserved. This is by design — the pool is a singleton
+ * and reconfiguration requires ray_pool_destroy() followed by initialization. */
+static ray_err_t ray_pool_init_impl(uint32_t n_workers, bool auto_size) {
     uint32_t expected = 0;
     if (!atomic_compare_exchange_strong_explicit(&g_pool_init_state, &expected, 1,
                                                  memory_order_acq_rel,
@@ -527,7 +532,7 @@ ray_err_t ray_pool_init(uint32_t n_workers) {
         }
         return RAY_OK;  /* already initialized or completed during our spin */
     }
-    ray_err_t err = ray_pool_create(&g_pool, n_workers);
+    ray_err_t err = ray_pool_create_impl(&g_pool, n_workers, auto_size);
     if (err == RAY_OK) {
         atomic_store_explicit(&g_pool_init_state, 2, memory_order_release);
     } else {
@@ -536,6 +541,16 @@ ray_err_t ray_pool_init(uint32_t n_workers) {
     return err;
 }
 
+ray_err_t ray_pool_init(uint32_t n_workers) {
+    return ray_pool_init_impl(n_workers, n_workers == 0);
+}
+
+ray_err_t ray_pool_init_total(uint32_t total_workers) {
+    if (total_workers == 0)
+        return ray_pool_init_impl(0, true);
+    return ray_pool_init_impl(total_workers - 1, false);
+}
+
 void ray_pool_destroy(void) {
     uint32_t expected = 2;
     if (!atomic_compare_exchange_strong_explicit(&g_pool_init_state, &expected, 3,

src/core/pool.h

@@ -107,6 +107,9 @@ ray_pool_t* ray_pool_get(void);
 
 /* Public pool init/destroy (moved from rayforce.h) */
 ray_err_t ray_pool_init(uint32_t n_workers);
+/* Initialize by total execution participants, including the main thread.
+ * Pass 0 to auto-detect. Used by the CLI, where -c is a total-core count. */
+ray_err_t ray_pool_init_total(uint32_t total_workers);
 void     ray_pool_destroy(void);
 
 #endif /* RAY_POOL_H */

src/lang/parse.c

@@ -53,7 +53,13 @@
 #define PA_MINUS   14
 #define PA_SEMI    15   /* ; comment */
 
-static const char _PA[128] =
+#if defined(__has_attribute) && __has_attribute(nonstring)
+#define RAY_NONSTRING __attribute__((nonstring))
+#else
+#define RAY_NONSTRING
+#endif
+
+static const char _PA[128] RAY_NONSTRING =
 /*  NUL                              \t \n                         */
     "\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x0c\x0c\x00\x00\x0c\x00\x00"
 /*                                                                  */

test/test_pool.c

@@ -661,6 +661,36 @@ static test_result_t test_pool_total_workers(void) {
     PASS();
 }
 
+/* --------------------------------------------------------------------------
+ * Test: total-worker initialization used by the CLI.
+ *
+ * The pool API normally accepts a background-worker count and reserves worker
+ * 0 for the calling thread. The CLI's -c contract is different: it counts all
+ * participants. In particular, total=1 must create an exact zero-background
+ * pool rather than taking ray_pool_init(0)'s auto-size path.
+ * -------------------------------------------------------------------------- */
+
+static test_result_t test_pool_init_total(void) {
+    ray_pool_destroy();
+    TEST_ASSERT_EQ_I(ray_pool_init_total(1), RAY_OK);
+    ray_pool_t* pool = ray_pool_get();
+    TEST_ASSERT_NOT_NULL(pool);
+    TEST_ASSERT_EQ_U(pool->n_workers, 0u);
+    TEST_ASSERT_EQ_U(ray_pool_total_workers(pool), 1u);
+
+    ray_pool_destroy();
+    TEST_ASSERT_EQ_I(ray_pool_init_total(4), RAY_OK);
+    pool = ray_pool_get();
+    TEST_ASSERT_NOT_NULL(pool);
+    TEST_ASSERT_EQ_U(pool->n_workers, 3u);
+    TEST_ASSERT_EQ_U(ray_pool_total_workers(pool), 4u);
+
+    /* Restore the lazy/default configuration for later tests. */
+    ray_pool_destroy();
+    TEST_ASSERT_EQ_I(ray_pool_init(0), RAY_OK);
+    PASS();
+}
+
 /* --------------------------------------------------------------------------
  * Test: ray_pool_free(NULL) is a no-op (covers the early-return guard).
  * -------------------------------------------------------------------------- */
@@ -1217,6 +1247,7 @@ const test_entry_t pool_entries[] = {
     { "pool/dispatch_cancelled",    test_dispatch_cancelled,    NULL, NULL },
     { "pool/zero_workers",          test_pool_zero_workers,     NULL, NULL },
     { "pool/total_workers",         test_pool_total_workers,    NULL, NULL },
+    { "pool/init_total",            test_pool_init_total,       NULL, NULL },
     { "pool/free_null",             test_pool_free_null,        NULL, NULL },
     { "pool/init_idempotent",       test_pool_init_idempotent,  NULL, NULL },
     { "pool/destroy_reinit",        test_pool_destroy_and_reinit, NULL, NULL },
@@ -1235,4 +1266,3 @@ const test_entry_t pool_entries[] = {
     { NULL, NULL, NULL, NULL },
 };
 
-

test/test_repl.c

@@ -2905,9 +2905,52 @@ static test_result_t test_repl_pty_ctrl_c_during_lazy_materialize(void) {
     PASS();
 }
 
+#if defined(__linux__)
+/* Run the real launcher in piped REPL mode and count its live task threads.
+ * /proc/self/task observes the exact process-wide total, so this catches both
+ * the ordinary off-by-one and the -c 1 collision with the pool's auto mode. */
+static int run_cli_task_count(unsigned cores, long* out_count) {
+    char command[256];
+    snprintf(command, sizeof(command),
+             "printf '(count (.fs.list \"/proc/self/task\"))\\n'"
+             " | ./rayforce -c %u -i", cores);
+
+    FILE* pipe = popen(command, "r");
+    if (!pipe) return -1;
+
+    char line[128];
+    bool have_line = fgets(line, sizeof(line), pipe) != NULL;
+    int status = pclose(pipe);
+    if (!have_line || status == -1 || !WIFEXITED(status) ||
+        WEXITSTATUS(status) != 0)
+        return -2;
+
+    char* end = NULL;
+    long count = strtol(line, &end, 10);
+    if (end == line) return -3;
+    *out_count = count;
+    return 0;
+}
+
+static test_result_t test_repl_cli_cores_are_total(void) {
+    long count = 0;
+    int rc = run_cli_task_count(1, &count);
+    TEST_ASSERT_FMT(rc == 0, "-c 1 launcher probe failed: %d", rc);
+    TEST_ASSERT_FMT(count == 1, "-c 1 created %ld total threads", count);
+
+    rc = run_cli_task_count(3, &count);
+    TEST_ASSERT_FMT(rc == 0, "-c 3 launcher probe failed: %d", rc);
+    TEST_ASSERT_FMT(count == 3, "-c 3 created %ld total threads", count);
+    PASS();
+}
+#endif
+
 /* ─── Suite definition ───────────────────────────────────────────── */
 
 const test_entry_t repl_entries[] = {
+#if defined(__linux__)
+    { "repl/cli/cores_are_total",       test_repl_cli_cores_are_total,       NULL,       NULL          },
+#endif
     /* file-batch entrypoint — ray_repl_run_file */
     { "repl/run_file/happy",            test_repl_run_file_happy,            repl_setup, repl_teardown },
     { "repl/run_file/multi_form",       test_repl_run_file_multi_form,       repl_setup, repl_teardown },