quantize: Pre-clamp scalefactor deltas to fix encoder/decoder mismatch (#100)

Problem:
The post-loop delta clamping in BlocQuant() modified sf[] after qlevel()
had already quantized the spectral data. When a negative delta exceeded
-SF_DELTA, the decoder reconstructed using the clamped (larger) sf value
while the encoder used the smaller (natural) gain. This gain mismatch
caused post-echo artifacts in quiet bands following loud bands.

Solution:
Relocate the delta clamp inside qlevel(), prior to the quantization step,
ensuring the encoder and decoder always operate on identical gain values.

To support this, gain calculations and Huffman overflow limits are
extracted into a single gain_with_overflow_clamp() helper. Variable
names are updated (sf_rel, sf_abs) to explicitly distinguish between
relative bitstream values and absolute energy levels, ensuring the
pre-loaded IS stereo biases from AACstereo() are safely preserved.

Author: nschimme

libfaac/quantize.c

@@ -18,6 +18,7 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 ****************************************************************************/
 
+#include <limits.h>
 #include <math.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -59,6 +60,9 @@ static QuantizeFunc qfunc = quantize_scalar;
 static faac_real sfstep;
 static faac_real max_quant_limit;
 
+/* Sentinel: delta chain has no previous band yet (first active regular band). */
+#define SF_CHAIN_UNSET INT_MIN
+
 void QuantizeInit(void)
 {
 #if defined(HAVE_SSE2)
@@ -76,6 +80,21 @@ void QuantizeInit(void)
      * Pre-calculated to avoid redundant runtime power functions. */
     max_quant_limit = FAAC_POW((faac_real)MAX_HUFF_ESC_VAL + 1.0 - MAGIC_NUMBER, 4.0/3.0);
 }
+
+/* Compute gain from integer sfac, clamping against Huffman overflow.
+ * Updates *sfac if clamping was applied. Returns the usable gain. */
+static faac_real gain_with_overflow_clamp(int *sfac, faac_real band_peak)
+{
+    faac_real gain = FAAC_POW(10, *sfac / sfstep);
+    if (band_peak > 0.0 && gain * band_peak > max_quant_limit)
+    {
+        gain = max_quant_limit / band_peak;
+        *sfac = (int)FAAC_FLOOR(FAAC_LOG10(gain) * sfstep);
+        gain = FAAC_POW(10, *sfac / sfstep);
+    }
+    return gain;
+}
+
 #define NOISEFLOOR 0.4
 
 // band sound masking
@@ -179,7 +198,8 @@ static void qlevel(CoderInfo * __restrict coderInfo,
                    const faac_real * __restrict bandenrg,
                    const faac_real * __restrict bandmaxe,
                    int gnum,
-                   int pnslevel
+                   int pnslevel,
+                   int *p_last_abs  /* previous active band's absolute stored scalefactor */
                   )
 {
     int sb;
@@ -190,6 +210,7 @@ static void qlevel(CoderInfo * __restrict coderInfo,
     {
       faac_real sfacfix;
       int sfac;
+      int sf_rel;   /* relative scalefactor index: SF_OFFSET - sfac */
       faac_real rmsx;
       faac_real etot;
       int xitab[8 * MAXSHORTBAND];
@@ -226,22 +247,50 @@ static void qlevel(CoderInfo * __restrict coderInfo,
       }
 
       sfac = FAAC_LRINT(FAAC_LOG10(bandqual[sb] / rmsx) * sfstep);
+      sf_rel = SF_OFFSET - sfac;
+
+      /* sf_bias: IS intensity stereo energy offset pre-loaded into sf[] by
+       * AACstereo() before qlevel() runs; zero for regular bands.
+       * sf_abs = sf_bias + sf_rel is the value written to the bitstream.
+       * Delta chain comparisons must use sf_abs, not sf_rel alone. */
+      int sf_bias = coderInfo->sf[coderInfo->bandcnt];
+      int sf_abs  = sf_bias + sf_rel;
 
-      if ((SF_OFFSET - sfac) < SF_MIN)
+      if (sf_rel < SF_MIN)
+      {
           sfacfix = 0.0;
+      }
       else
       {
-          sfacfix = FAAC_POW(10, sfac / sfstep);
-
-          /* Bitstream saturation check: if gain * peak exceeds the Huffman limit,
-           * clamp gain and re-sync the integer scalefactor to prevent overflow. */
-          if (sfacfix * bandmaxe[sb] > max_quant_limit)
+          /* Compute gain and clamp against Huffman overflow. */
+          sfacfix = gain_with_overflow_clamp(&sfac, bandmaxe[sb]);
+          sf_rel = SF_OFFSET - sfac;
+          sf_abs = sf_bias + sf_rel;
+
+          /* Pre-clamp: enforce delta limits against the previous band's stored
+           * value so encoder and decoder use the same gain (sf_abs). */
+          if (*p_last_abs != SF_CHAIN_UNSET)
           {
-              sfacfix = max_quant_limit / bandmaxe[sb];
-              sfac = (int)FAAC_FLOOR(FAAC_LOG10(sfacfix) * sfstep);
-              /* Re-derive gain from the floored scalefactor to ensure bit-exact
-               * sync with the decoder's inverse quantizer. */
-              sfacfix = FAAC_POW(10, sfac / sfstep);
+              int diff         = sf_abs - *p_last_abs;
+              int clamped_diff = clamp_sf_diff(diff);
+              if (clamped_diff != diff)
+              {
+                  sf_abs = *p_last_abs + clamped_diff;
+                  sf_rel = sf_abs - sf_bias;
+                  sfac   = SF_OFFSET - sf_rel;
+                  if (clamped_diff > 0)
+                  {
+                      /* Upward clamp raised gain; re-check Huffman overflow. */
+                      sfacfix = gain_with_overflow_clamp(&sfac, bandmaxe[sb]);
+                      sf_rel = SF_OFFSET - sfac;
+                      sf_abs = sf_bias + sf_rel;
+                  }
+                  else
+                  {
+                      /* Downward clamp lowered gain; overflow impossible. */
+                      sfacfix = FAAC_POW(10, sfac / sfstep);
+                  }
+              }
           }
       }
 
@@ -261,7 +310,11 @@ static void qlevel(CoderInfo * __restrict coderInfo,
           }
       }
       huffbook(coderInfo, xitab, gsize * end);
-      coderInfo->sf[coderInfo->bandcnt++] += SF_OFFSET - sfac;
+      /* Track sf_abs (full bitstream value) for the next band's delta check.
+       * HCB_ZERO bands don't participate in the regular-band delta chain. */
+      if (coderInfo->book[coderInfo->bandcnt] != HCB_ZERO)
+          *p_last_abs = sf_abs;
+      coderInfo->sf[coderInfo->bandcnt++] += sf_rel;
     }
 }
 
@@ -278,63 +331,52 @@ int BlocQuant(CoderInfo * __restrict coder, faac_real * __restrict xr, AACQuantC
     coder->bandcnt = 0;
     coder->datacnt = 0;
 
+    int lastsf = SF_CHAIN_UNSET;  /* no previous band yet; first active band skips delta clamp */
+
+    gxr = xr;
+    for (cnt = 0; cnt < coder->groups.n; cnt++)
     {
-        int lastis;
-        int lastsf;
+        bmask(coder, gxr, bandlvl, bandenrg, bandmaxe, cnt,
+              (faac_real)aacquantCfg->quality/DEFQUAL);
+        qlevel(coder, gxr, bandlvl, bandenrg, bandmaxe, cnt,
+               aacquantCfg->pnslevel, &lastsf);
+        gxr += coder->groups.len[cnt] * BLOCK_LEN_SHORT;
+    }
 
-        gxr = xr;
-        for (cnt = 0; cnt < coder->groups.n; cnt++)
+    coder->global_gain = 0;
+    for (cnt = 0; cnt < coder->bandcnt; cnt++)
+    {
+        int book = coder->book[cnt];
+        if (!book)
+            continue;
+        if ((book != HCB_INTENSITY) && (book != HCB_INTENSITY2))
         {
-            bmask(coder, gxr, bandlvl, bandenrg, bandmaxe, cnt,
-                  (faac_real)aacquantCfg->quality/DEFQUAL);
-            qlevel(coder, gxr, bandlvl, bandenrg, bandmaxe, cnt, aacquantCfg->pnslevel);
-            gxr += coder->groups.len[cnt] * BLOCK_LEN_SHORT;
+            coder->global_gain = coder->sf[cnt];
+            break;
         }
+    }
 
-        coder->global_gain = 0;
-        for (cnt = 0; cnt < coder->bandcnt; cnt++)
+    int lastis  = 0;
+    int lastpns = coder->global_gain - SF_PNS_OFFSET;
+    for (cnt = 0; cnt < coder->bandcnt; cnt++)
+    {
+        int book = coder->book[cnt];
+        if ((book == HCB_INTENSITY) || (book == HCB_INTENSITY2))
         {
-            int book = coder->book[cnt];
-            if (!book)
-                continue;
-            if ((book != HCB_INTENSITY) && (book != HCB_INTENSITY2))
-            {
-                coder->global_gain = coder->sf[cnt];
-                break;
-            }
+            int diff = coder->sf[cnt] - lastis;
+            diff = clamp_sf_diff(diff);
+            lastis += diff;
+            coder->sf[cnt] = lastis;
         }
-
-        lastsf = coder->global_gain;
-        lastis = 0;
-        int lastpns = coder->global_gain - SF_PNS_OFFSET;
-        for (cnt = 0; cnt < coder->bandcnt; cnt++)
+        else if (book == HCB_PNS)
         {
-            int book = coder->book[cnt];
-            if ((book == HCB_INTENSITY) || (book == HCB_INTENSITY2))
-            {
-                int diff = coder->sf[cnt] - lastis;
-                diff = clamp_sf_diff(diff);
-                lastis += diff;
-                coder->sf[cnt] = lastis;
-            }
-            else if (book == HCB_PNS)
-            {
-                int diff = coder->sf[cnt] - lastpns;
-                diff = clamp_sf_diff(diff);
-                lastpns += diff;
-                coder->sf[cnt] = lastpns;
-            }
-            else if ((book != HCB_ZERO) && (book != HCB_NONE))
-            {
-                int diff = coder->sf[cnt] - lastsf;
-                diff = clamp_sf_diff(diff);
-                lastsf += diff;
-                coder->sf[cnt] = lastsf;
-            }
+            int diff = coder->sf[cnt] - lastpns;
+            diff = clamp_sf_diff(diff);
+            lastpns += diff;
+            coder->sf[cnt] = lastpns;
         }
-        return 1;
     }
-    return 0;
+    return 1;
 }
 
 void CalcBW(unsigned *bw, int rate, SR_INFO *sr, AACQuantCfg *aacquantCfg)