feat: lazy-load priority extraction

--lazy-load previously failed at runtime: priorities used absolute
/app paths that never matched tar's payload-relative entries (and
listed the host interpreter as a payload file). Priorities are now
payload-relative, the entrypoint script is always a priority while the
interpreter never is, and --lazy-priority / [build] lazy_priority add
app-relative or payload-relative specs (dirs expand recursively, 2048
cap) via daedalus-core::lazy::expand_priority_paths.

Fixes: non-priority tar entries are drained before skipping (stream
misalignment caused 'Invalid argument (os error 22)'), and namespaced
builds (isolation >= 2) extract synchronously -- spawning any thread
makes the later unshare(CLONE_NEWUSER|CLONE_NEWNS) fail EINVAL and
pivot_root strands a bg thread's cache path. Small hosts (< 4 cores,
< 3 GiB) auto-disable lazy-load without an explicit priority list.

Author: Tednoob17

CHANGELOG.md

@@ -7,6 +7,10 @@ Format based on [Keep a Changelog](https://keepachangelog.com/).
 ## [Unreleased]
 
 ### Added
+- **Lazy-loading priority extraction**: `daedalus build --lazy-load` now fast-starts by extracting only the entrypoint (plus an explicit `--lazy-priority file,dir,...` list or `[build] lazy_priority`) before launching, then fills the rest of the app tree in the background. Priority specs may be app-relative (`main.py`) or payload-relative (`app/main.py`); directories expand recursively (skipping `__pycache__`/`.git`), capped at 2048 files. Dry-run reports `Lazy:` on/off and the priority count. Namespaced builds (isolation ≥ 2) extract synchronously so the user-namespace `unshare` stays single-threaded.
+- `daedalus-core::lazy::expand_priority_paths` — build-time expansion of `--lazy-priority` specs into payload-relative paths with tar-exclusion parity
+- Constrained-host auto-disable: `--lazy-load` without an explicit priority list turns itself off on small hosts (< 4 cores or < 3 GiB RAM, via `daedalus-core::system_info::is_constrained_host`)
+- `daedalus inspect` reports the `lazy_priority` metadata (`Lazy priority:` line)
 - **GPU passthrough for accelerated inference**: `daedalus build --gpu auto|nvidia|rocm|none` (or `[build] gpu` in `.daedalus.toml`) records the compute backend in the binary metadata. When `nvidia` or `rocm` is set, the launcher bind-mounts the GPU device nodes (NVIDIA `/dev/nvidia*` + caps, ROCm `/dev/kfd` + `/dev/dri` render nodes) into the sandbox via a regular-file overlay and pins the matching visibility vars (`CUDA_VISIBLE_DEVICES`, `NVIDIA_VISIBLE_DEVICES`, `HIP_VISIBLE_DEVICES`, `ROCR_VISIBLE_DEVICES`) — Ollama/llama.cpp offload works headless, inside the sandbox.
 - `daedalus-core::gpu` detection probe (`--gpu auto`): `/dev/kfd` → ROCm, `/proc/driver/nvidia/gpus` → NVIDIA; CPU fallback is safe (missing nodes are skipped)
 - `daedalus inspect` shows the embedded `GPU:` backend; dry-run reports the resolved backend
@@ -18,6 +22,8 @@ Format based on [Keep a Changelog](https://keepachangelog.com/).
 - System config detection module `daedalus-core/src/system_info.rs` (`SystemConfig`, `detect()`, `compute_universal_slices()`) for resource-adaptive builds
 
 ### Fixed
+- **`--lazy-load` runtime failure**: priority files used absolute `/app/...` paths that never matched the tar's payload-relative entries (and wrongly listed the host PATH interpreter as a payload file), so every lazy build errored with `priority files not found in payload: /app/python3`. Priorities are now payload-relative and the interpreter is never one.
+- Stub lazy extraction skipped non-priority tar entries without draining them, misaligning the stream and breaking extraction with `Invalid argument (os error 22)` whenever a non-priority entry was followed by more data — entries are now drained before skipping
 - Stub multi-service supervisor on Unix now execs via `execvp` (PATH search) instead of `execve`, so bare interpreter names like `python3` resolve at runtime — multi-service binaries can actually launch their services
 - `find_in_bin_paths`/`is_executable_path` made cross-platform (were Windows-only), so the Unix supervisor shares the same interpreter-resolution fallback as the Windows and single-service paths
 - Removed unused `libc_execve` FFI declaration and `env_to_cstrings` helper

daedalus-cli/src/commands/build/args.rs

@@ -89,6 +89,9 @@ pub(crate) struct BuildPlan {
     /// Identifier of the AI model bundled for offline inference (derived from
     /// `--model` filename). Exposed in the build metadata as `model_id`.
     pub(crate) model_id: Option<String>,
+    /// Payload-relative paths to eager-extract when lazy loading is enabled
+    /// (from `--lazy-priority` or `[build] lazy_priority`).
+    pub(crate) lazy_priority: Vec<String>,
 }
 
 /// Compute backend selection for `--gpu`.
@@ -396,6 +399,10 @@ pub(crate) fn config_fingerprint(args: &BuildArgs, plan: &BuildPlan) -> String {
             .unwrap_or_else(|_| "unreadable".to_string());
         canonical.push(format!("env_file={}:{content}", v.display()));
     }
+    canonical.push(format!("lazy_load={}", args.lazy_load));
+    if !plan.lazy_priority.is_empty() {
+        canonical.push(format!("lazy_priority={:?}", plan.lazy_priority));
+    }
     let mut env: Vec<&String> = args.env.iter().collect();
     env.sort();
     for e in env {
@@ -445,6 +452,23 @@ pub(crate) fn config_fingerprint(args: &BuildArgs, plan: &BuildPlan) -> String {
     hex::encode(hasher.finalize())[..16].to_string()
 }
 
+/// Whether lazy loading is effective for this build.
+///
+/// Background decompression steals CPU from the app on a constrained
+/// host, so `--lazy` without an explicit priority list is auto-
+/// disabled there. An explicit `--lazy-priority` (or `[build]
+/// lazy_priority`) overrides the auto-disable because the operator
+/// asked for it deliberately.
+pub(crate) fn lazy_enabled(lazy_load: bool, lazy_priority: &[String]) -> bool {
+    if !lazy_load {
+        return false;
+    }
+    if !lazy_priority.is_empty() {
+        return true;
+    }
+    !daedalus_core::system_info::is_constrained_host(&daedalus_core::system_info::detect())
+}
+
 #[derive(Args)]
 #[command(after_help = "\
 Examples:
@@ -713,6 +737,12 @@ pub struct BuildArgs {
     #[arg(long)]
     pub lazy_load: bool,
 
+    /// Paths to eager-extract when lazy loading is enabled. App-relative
+    /// (`main.py`) or payload-relative (`app/main.py`); a directory expands
+    /// to every file under it. Comma-separated or repeated.
+    #[arg(long, value_delimiter = ',')]
+    pub lazy_priority: Vec<String>,
+
     /// Bundle an AI model (`.gguf`) into the binary for offline inference.
     ///
     /// Copies the file into the rootfs at `app/models/<name>` and forces the
@@ -747,6 +777,9 @@ pub(crate) struct BuildConfig {
     /// Identifier of the AI model to bundle for offline inference, set in
     /// `.daedalus.toml` as `[build] model_id = "gemma-2b-it-q4"`.
     pub model_id: Option<String>,
+    /// Paths to eager-extract when lazy loading is enabled, set in
+    /// `.daedalus.toml` as `[build] lazy_priority = ["main.py", "models/"]`.
+    pub lazy_priority: Option<Vec<String>>,
 }
 
 #[derive(Default, Deserialize)]
@@ -835,6 +868,7 @@ pub(crate) fn default_build_args() -> BuildArgs {
         token: None,
         universal: false,
         lazy_load: false,
+        lazy_priority: Vec::new(),
         encrypt: None,
         model: None,
     }
@@ -1128,6 +1162,7 @@ mod tests {
             services: Vec::new(),
             entrypoint: Vec::new(),
             model_id: None,
+            lazy_priority: Vec::new(),
         };
         let base = config_fingerprint(&default_build_args(), &plan(false));
 
@@ -1202,4 +1237,27 @@ mod tests {
         let err = apply_service_overrides(&mut services, &["ghost=8080".into()], &[] as &[String]);
         assert!(err.is_err(), "unknown service name must fail");
     }
+
+    #[test]
+    /// lazy_enabled_requires_the_flag - lazy enabled requires the flag.
+    ///
+    /// Description:
+    ///
+    /// Return: nothing
+    fn lazy_enabled_requires_the_flag() {
+        assert!(!lazy_enabled(false, &[]));
+        assert!(!lazy_enabled(false, &["main.py".into()]));
+    }
+
+    #[test]
+    /// lazy_enabled_honors_explicit_priority - lazy enabled honors explicit priority.
+    ///
+    /// Description:
+    /// An operator-supplied priority list keeps lazy loading on even on a
+    /// host too small for background extraction.
+    ///
+    /// Return: nothing
+    fn lazy_enabled_honors_explicit_priority() {
+        assert!(lazy_enabled(true, &["main.py".into()]));
+    }
 }

daedalus-cli/src/commands/build/mod.rs

@@ -47,6 +47,7 @@ pub fn run(args: BuildArgs, verbose: bool) -> Result<()> {
     // below, so the helper can borrow-build the id without tripping the
     // borrow checker on the whole-struct reference.
     let model_id_from_config = config.build.model_id.clone();
+    let lazy_priority_from_config = config.build.lazy_priority.clone();
     let (runtime, model_id) =
         resolve_build_runtime(&app_dir, &args, model_id_from_config, verbose)?;
     let runtime_name = runtime.name().to_string();
@@ -98,6 +99,7 @@ pub fn run(args: BuildArgs, verbose: bool) -> Result<()> {
         services,
         entrypoint: entrypoint_args,
         model_id,
+        lazy_priority: resolve_lazy_priority(&args, lazy_priority_from_config),
     };
 
     if args.dry_run {
@@ -143,13 +145,22 @@ pub fn run(args: BuildArgs, verbose: bool) -> Result<()> {
     Ok(())
 }
 
+/// CLI flags win over `[build] lazy_priority` from config.
+fn resolve_lazy_priority(args: &BuildArgs, from_config: Option<Vec<String>>) -> Vec<String> {
+    if args.lazy_priority.is_empty() {
+        from_config.unwrap_or_default()
+    } else {
+        args.lazy_priority.clone()
+    }
+}
+
 /// Resolve the build runtime and model id.
 ///
 /// `--model` implies an offline Gemma bundle: embedding weights for local
-/// inference pins the runtime to Gemma regardless of the source layout, so the
-/// stub serves the model from the bundled `.gguf`. The model id is derived from
-/// the `--model` filename (stable across rebuilds of the same weights) and falls
-/// back to `[build] model_id` from `.daedalus.toml`.
+/// inference pins the runtime to Gemma regardless of the source layout, so
+/// the stub serves the model from the bundled `.gguf`. The model id is derived
+/// from the `--model` filename (stable across rebuilds of the same weights)
+/// and falls back to `[build] model_id` from `.daedalus.toml`.
 fn resolve_build_runtime(
     app_dir: &Path,
     args: &BuildArgs,
@@ -283,6 +294,23 @@ fn expand_path(path: &str) -> std::path::PathBuf {
 /// Description:
 ///
 /// Return: nothing
+/// Reference detection + install lines for the dry-run summary.
+fn pkgmgr_lines(plan: &BuildPlan) -> Vec<String> {
+    let mgrs = daedalus_core::pkgmgr::detect_all_pkgmgrs(&plan.app_dir, &plan.runtime_name);
+    if mgrs.is_empty() {
+        return vec!["Pkg mgr:   (none)".into()];
+    }
+    let mut lines = Vec::new();
+    for mgr in mgrs {
+        lines.push(format!("Pkg mgr:   {}", mgr.name()));
+        if !plan.no_install {
+            lines.push(format!("Install:   {}", mgr.install_cmd().join(" ")));
+        }
+    }
+    lines
+}
+
+/// Print what a build would do without touching the app directory.
 fn print_dry_run(args: &BuildArgs, plan: &BuildPlan, target: Option<&str>, output: &Path) {
     eprintln!("Dry run — would build:");
     eprintln!("  App:       {}", plan.app_dir.display());
@@ -331,6 +359,11 @@ fn print_dry_run(args: &BuildArgs, plan: &BuildPlan, target: Option<&str>, outpu
     if let Some(ref e) = plan.env_file {
         eprintln!("  Env file:  {}", e.display());
     }
+    let lazy_on = args::lazy_enabled(args.lazy_load, &plan.lazy_priority);
+    eprintln!("  Lazy:    {}", if lazy_on { "on" } else { "off" });
+    if lazy_on {
+        eprintln!("  Lazy priority: {} files", plan.lazy_priority.len());
+    }
     if plan.no_install {
         eprintln!("  No install: yes");
     }
@@ -367,18 +400,8 @@ fn print_dry_run(args: &BuildArgs, plan: &BuildPlan, target: Option<&str>, outpu
         eprintln!("  Services:  {}", describe_services(plan));
     }
 
-    // Detect package managers
-    let all_mgrs = daedalus_core::pkgmgr::detect_all_pkgmgrs(&plan.app_dir, &plan.runtime_name);
-    if all_mgrs.is_empty() {
-        eprintln!("  Pkg mgr:   (none)");
-    } else {
-        for mgr in &all_mgrs {
-            eprintln!("  Pkg mgr:   {}", mgr.name());
-            if !plan.no_install {
-                let cmd = mgr.install_cmd();
-                eprintln!("  Install:   {}", cmd.join(" "));
-            }
-        }
+    for line in pkgmgr_lines(plan) {
+        eprintln!("  {line}");
     }
 
     // Estimate sizes

daedalus-cli/src/commands/build/pipeline.rs

@@ -4,6 +4,7 @@ use daedalus_core::detect;
 use daedalus_core::embed::{self, RuntimeProfile};
 use daedalus_core::encrypt;
 use daedalus_core::layer::{Capability, RuntimeLayer, SerializableLayer};
+use daedalus_core::lazy;
 use daedalus_core::metadata::{BunFeatures, EmbeddedInterpreter};
 use daedalus_core::paths::cache_dir;
 use daedalus_core::pkgmgr;
@@ -285,6 +286,18 @@ pub(crate) fn build_single_target(
         None
     };
 
+    let lazy_on = super::args::lazy_enabled(args.lazy_load, &plan.lazy_priority);
+    let lazy_priority = if lazy_on {
+        if plan.lazy_priority.is_empty() {
+            Vec::new()
+        } else {
+            lazy::expand_priority_paths(&plan.lazy_priority, &rootfs)
+                .with_context(|| "invalid --lazy-priority; give paths relative to the app dir")?
+        }
+    } else {
+        Vec::new()
+    };
+
     let meta = daedalus_core::assembly::build_meta_json(
         &app_name,
         runtime_name,
@@ -311,7 +324,8 @@ pub(crate) fn build_single_target(
             post_hooks,
             layers: Some(layers),
             entrypoint_layer: Some(runtime_name.clone()),
-            lazy_load: args.lazy_load,
+            lazy_load: lazy_on,
+            lazy_priority,
             mcp_tools,
             model_id: plan.model_id.clone(),
             services: plan

daedalus-cli/tests/cli_tests.rs

@@ -266,6 +266,53 @@ fn test_build_dry_run_gpu_default_is_cpu() {
         .stderr(predicate::str::contains("GPU:       none (CPU)"));
 }
 
+#[test]
+fn test_build_help_lists_lazy_priority_flag() {
+    daedalus()
+        .args(["build", "--help"])
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("--lazy-priority"));
+}
+
+#[test]
+fn test_build_dry_run_shows_lazy_priority() {
+    let dir = tempdir().unwrap();
+    std::fs::create_dir_all(dir.path().join("app")).unwrap();
+    std::fs::write(dir.path().join("app/package.json"), "{\"name\":\"app\"}").unwrap();
+    daedalus()
+        .args([
+            "build",
+            dir.path().join("app").to_str().unwrap(),
+            "--lazy-load",
+            "--lazy-priority",
+            "main.py,config.json",
+            "--dry-run",
+            "--no-install",
+        ])
+        .assert()
+        .success()
+        .stderr(predicate::str::contains("Lazy:    on"))
+        .stderr(predicate::str::contains("Lazy priority: 2"));
+}
+
+#[test]
+fn test_build_dry_run_lazy_off_by_default() {
+    let dir = tempdir().unwrap();
+    std::fs::create_dir_all(dir.path().join("app")).unwrap();
+    std::fs::write(dir.path().join("app/package.json"), "{\"name\":\"app\"}").unwrap();
+    daedalus()
+        .args([
+            "build",
+            dir.path().join("app").to_str().unwrap(),
+            "--dry-run",
+            "--no-install",
+        ])
+        .assert()
+        .success()
+        .stderr(predicate::str::contains("Lazy:    off"));
+}
+
 #[test]
 fn test_build_dry_run_rejects_unknown_service_port() {
     let dir = tempdir().unwrap();