Cap canonical-form expansion to fix #1069 OOM/hang in big_o

Composed-path overheads that traverse quadratic-overhead reductions
(e.g. QuadraticAssignment) form nested (sum)^2*(sum)^2 expressions.
canonical_form() expanded these into sum-of-monomials, which is
exponential in nesting depth, blowing up to multi-GB and OOM/hanging
`pred path --all`. On CI this SIGTERM-killed the Test job (exit 143)
via the integration test test_path_all_max_paths_truncates.

Add a hard cap (MAX_CANONICAL_TERMS) on the intermediate term count,
checked before the Cartesian product is materialized. On overflow,
expansion is abandoned with CanonicalizationError::Unsupported; the
existing big_o_of fallback then prints the compact, un-expanded
expression instead of hanging. Paths are still enumerated and shown
— only the Big-O rendering of pathological paths degrades to
un-simplified form. Stopgap until the symbolic system is reworked to
derive Big-O structurally without full expansion.

Regression tests:
- canonical: nested-power blowup returns Unsupported; moderate power
  still expands normally (cap does not perturb legitimate exprs)
- big_o: pathological nesting errors instead of hanging

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: isPANN

src/canonical.rs

@@ -7,6 +7,22 @@ use std::collections::BTreeMap;
 
 use crate::expr::{CanonicalizationError, Expr};
 
+/// Hard cap on the number of additive terms produced while expanding an
+/// expression into canonical sum-of-monomials form.
+///
+/// Expanding a nested `(sum)^2 * (sum)^2` structure is exponential in nesting
+/// depth: composed-path overheads that traverse quadratic-overhead reductions
+/// (e.g. `QuadraticAssignment`) blow up to multi-GB of monomials and OOM/hang.
+/// When the intermediate term count would exceed this cap we abandon expansion
+/// and report the expression as `Unsupported`; callers (e.g. `big_o_of`) fall
+/// back to printing the compact, un-expanded expression. See issue #1069.
+///
+/// Legitimate overhead expressions stay far below this bound (the worst
+/// non-pathological case is a few hundred terms), so this never affects normal
+/// output — it only stops pathological blowups. This is a stopgap guard; the
+/// symbolic system is slated for a larger rework.
+const MAX_CANONICAL_TERMS: usize = 50_000;
+
 /// An opaque non-polynomial factor (exp, log, fractional-power base).
 ///
 /// Stored by its canonical string representation for deterministic ordering.
@@ -184,6 +200,19 @@ impl CanonicalSum {
         CanonicalSum { terms }
     }
 
+    /// Multiply with a guard against pathological expansion (see
+    /// [`MAX_CANONICAL_TERMS`]). The Cartesian product size is checked *before*
+    /// it is materialized, so this never allocates the blown-up vector.
+    fn try_mul(&self, other: &CanonicalSum) -> Result<CanonicalSum, CanonicalizationError> {
+        let product = self.terms.len().saturating_mul(other.terms.len());
+        if product > MAX_CANONICAL_TERMS {
+            return Err(CanonicalizationError::Unsupported(format!(
+                "expression too large to canonicalize ({product} terms exceeds cap of {MAX_CANONICAL_TERMS})"
+            )));
+        }
+        Ok(self.mul(other))
+    }
+
     /// Merge terms with the same signature and drop zero-coefficient terms.
     /// Sort the result deterministically.
     fn simplify(self) -> Self {
@@ -238,7 +267,7 @@ fn expr_to_canonical(expr: &Expr) -> Result<CanonicalSum, CanonicalizationError>
         Expr::Mul(a, b) => {
             let ca = expr_to_canonical(a)?;
             let cb = expr_to_canonical(b)?;
-            Ok(ca.mul(&cb))
+            ca.try_mul(&cb)
         }
         Expr::Pow(base, exp) => canonicalize_pow(base, exp),
         Expr::Exp(arg) => {
@@ -300,7 +329,7 @@ fn canonicalize_pow(base: &Expr, exp: &Expr) -> Result<CanonicalSum, Canonicaliz
                 }
                 let mut result = base_sum.clone();
                 for _ in 1..n {
-                    result = result.mul(&base_sum);
+                    result = result.try_mul(&base_sum)?;
                 }
                 Ok(result)
             } else {

src/models/mod.rs

@@ -15,8 +15,8 @@ pub use algebraic::{
     ConsecutiveOnesMatrixAugmentation, ConsecutiveOnesSubmatrix, EquilibriumPoint,
     FeasibleBasisExtension, MinimumMatrixCover, MinimumMatrixDomination, MinimumWeightDecoding,
     MinimumWeightSolutionToLinearEquations, QuadraticAssignment, QuadraticCongruences,
-    QuadraticDiophantineEquations, SimultaneousIncongruences,
-    SparseMatrixCompression, BMF, ILP, QUBO,
+    QuadraticDiophantineEquations, SimultaneousIncongruences, SparseMatrixCompression, BMF, ILP,
+    QUBO,
 };
 pub use decision::Decision;
 pub use formula::{

src/unit_tests/big_o.rs

@@ -217,3 +217,13 @@ fn test_big_o_multivar_exp_dominates_poly() {
         "expected n*m to survive (different var set), got: {s}"
     );
 }
+
+#[test]
+fn test_big_o_pathological_nesting_errors_instead_of_hanging() {
+    // Regression for issue #1069: a deeply-nested power that expands
+    // exponentially must return an error promptly (so callers like `big_o_of`
+    // fall back to the un-expanded expression) rather than OOM/hang.
+    let sum = Expr::Var("a") + Expr::Var("b") + Expr::Var("c") + Expr::Var("d");
+    let e = Expr::pow(Expr::pow(sum, Expr::Const(4.0)), Expr::Const(4.0));
+    assert!(big_o_normal_form(&e).is_err());
+}

src/unit_tests/canonical.rs

@@ -137,3 +137,29 @@ fn test_canonical_sqrt_as_power() {
     let b = canonical_form(&Expr::pow(Expr::Var("n"), Expr::Const(0.5))).unwrap();
     assert_eq!(a.to_string(), b.to_string());
 }
+
+#[test]
+fn test_canonical_nested_power_blowup_is_capped() {
+    // Regression for issue #1069: a "square of a square of a sum" structure —
+    // the shape composed-path overheads take when they traverse
+    // quadratic-overhead reductions — expands exponentially. Before the cap
+    // this OOM'd / hung indefinitely; now it must fail fast with Unsupported
+    // rather than try to materialize the blown-up monomial expansion.
+    let sum = Expr::Var("a") + Expr::Var("b") + Expr::Var("c") + Expr::Var("d");
+    // ((a+b+c+d)^4)^4 expands to >50_000 intermediate terms.
+    let e = Expr::pow(Expr::pow(sum, Expr::Const(4.0)), Expr::Const(4.0));
+    let err = canonical_form(&e).unwrap_err();
+    assert!(matches!(err, CanonicalizationError::Unsupported(_)));
+}
+
+#[test]
+fn test_canonical_moderate_power_still_expands() {
+    // The cap must not perturb legitimate, modestly-sized expressions:
+    // (a+b)^3 stays well under the cap and expands normally.
+    let e = Expr::pow(Expr::Var("a") + Expr::Var("b"), Expr::Const(3.0));
+    let c = canonical_form(&e).unwrap();
+    // a^3 + 3 a^2 b + 3 a b^2 + b^3 — compare against the same expansion
+    // written out flat (both go through canonical_form for identical ordering).
+    let expected = canonical_form(&Expr::parse("a^3 + 3*a^2*b + 3*a*b^2 + b^3")).unwrap();
+    assert_eq!(c.to_string(), expected.to_string());
+}