Skip to content

Commit f76d916

Browse files
enarjordclaude
andcommitted
Require strict contiguity for source-dir candle data
Source-dir data bypasses CandlestickManager gap filling, so any non-contiguous candles would crash the downstream strict continuity assertion. Replace the tolerance-based gap check with a strict contiguity check (all intervals must be exactly 60_000ms); fall back to CandlestickManager otherwise. Adds test for small-gap fallback and CHANGELOG "Fixed" entry. Credit: contiguity fix identified by Codex review of PR 559. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent f2b0617 commit f76d916

3 files changed

Lines changed: 75 additions & 5 deletions

File tree

CHANGELOG.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ All notable user-facing changes will be documented in this file.
99
- **Total wallet exposure plot** - Backtests now output `total_wallet_exposure.png` showing long TWE (positive, blue) and short TWE (negative, red) over time.
1010
- **External OHLCV source dir** - New `backtest.ohlcv_source_dir` config option to load 1m candle data from a pre-populated directory tree before falling back to exchange archives. Supports both `.npy` and `.npz` file formats.
1111

12+
### Fixed
13+
- **OHLCV source-dir fallback behavior** - Non-contiguous source-dir candle data now falls back to CandlestickManager instead of propagating gappy series into downstream strict continuity checks.
14+
1215
## v7.8.1 - 2026-02-07
1316

1417
### Fixed

src/hlcv_preparation.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -482,10 +482,10 @@ def _try_load_ohlcvs_from_source_dir(
482482
ts = df["timestamp"].astype(np.int64, copy=False).values
483483
if ts.size > 1:
484484
intervals = np.diff(ts)
485-
greatest_gap_ms = int(intervals.max(initial=60_000))
486-
if greatest_gap_ms > int(self.gap_tolerance_ohlcvs_minutes * 60_000):
485+
if not np.all(intervals == 60_000):
486+
greatest_gap_ms = int(intervals.max(initial=60_000))
487487
logging.warning(
488-
"[%s] source dir gaps detected for %s; greatest gap %.1f minutes. Falling back.",
488+
"[%s] source dir non-contiguous data for %s; greatest gap %.1f minutes. Falling back.",
489489
self.exchange,
490490
coin,
491491
greatest_gap_ms / 60_000.0,

tests/test_hlcv_preparation.py

Lines changed: 69 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -830,6 +830,72 @@ async def test_source_dir_fallback_missing_file(self, tmp_path, mock_exchange):
830830
assert not df.empty
831831
assert len(df) == 1441 # end_ts is inclusive (00:00:00 on day 15 to 00:00:00 on day 16)
832832

833+
@pytest.mark.asyncio
834+
async def test_source_dir_fallback_non_contiguous_small_gap(self, tmp_path, mock_exchange):
835+
"""Test fallback when source dir data has a small non-contiguous gap."""
836+
from ohlcv_utils import dump_ohlcv_data
837+
from utils import date_to_ts
838+
839+
source_dir = tmp_path / "ohlcv_source"
840+
exchange_dir = source_dir / "binance" / "1m" / "BTC"
841+
exchange_dir.mkdir(parents=True)
842+
843+
day = "2024-01-15"
844+
day_ts = int(date_to_ts(day))
845+
846+
# Remove 5 candles to create a 6-minute gap (< default 120-minute tolerance).
847+
timestamps_full = np.arange(day_ts, day_ts + 24 * 60 * 60 * 1000, 60_000)
848+
timestamps = np.concatenate([timestamps_full[:100], timestamps_full[105:]])
849+
850+
data = np.column_stack([
851+
timestamps,
852+
np.full(len(timestamps), 50000.0),
853+
np.full(len(timestamps), 50100.0),
854+
np.full(len(timestamps), 49900.0),
855+
np.full(len(timestamps), 50050.0),
856+
np.full(len(timestamps), 100.0),
857+
])
858+
dump_ohlcv_data(data, str(exchange_dir / f"{day}.npy"))
859+
860+
om = HLCVManager(
861+
"binanceusdm",
862+
start_date=day,
863+
end_date="2024-01-16",
864+
cc=mock_exchange,
865+
ohlcv_source_dir=str(source_dir),
866+
gap_tolerance_ohlcvs_minutes=120.0,
867+
)
868+
869+
om.markets = {
870+
"BTC/USDT:USDT": {
871+
"symbol": "BTC/USDT:USDT",
872+
"base": "BTC",
873+
"quote": "USDT",
874+
"maker": 0.0002,
875+
"taker": 0.0004,
876+
"contractSize": 1.0,
877+
"limits": {"cost": {"min": 5.0}, "amount": {"min": 0.001}},
878+
"precision": {"price": 0.01, "amount": 0.001},
879+
}
880+
}
881+
882+
with patch.object(CandlestickManager, 'get_candles') as mock_get_candles:
883+
full_timestamps = np.arange(day_ts, day_ts + 24 * 60 * 60 * 1000, 60_000, dtype=np.int64)
884+
mock_candles = np.zeros(len(full_timestamps), dtype=CANDLE_DTYPE)
885+
mock_candles['ts'] = full_timestamps
886+
mock_candles['o'] = 50000.0
887+
mock_candles['h'] = 50100.0
888+
mock_candles['l'] = 49900.0
889+
mock_candles['c'] = 50050.0
890+
mock_candles['bv'] = 100.0
891+
mock_get_candles.return_value = mock_candles
892+
893+
df = await om.get_ohlcvs("BTC")
894+
895+
mock_get_candles.assert_called_once()
896+
assert not df.empty
897+
assert len(df) == 1441 # end_ts is inclusive (00:00:00 on day 15 to 00:00:00 on day 16)
898+
833899
@pytest.mark.asyncio
834900
async def test_source_dir_fallback_excessive_gaps(self, tmp_path, mock_exchange):
835901
"""Test fallback when gaps exceed tolerance."""
@@ -1011,18 +1077,19 @@ async def test_prepare_hlcvs_combined_basic_structure(self, tmp_path, sample_con
10111077
- Fetch from exchange
10121078
- Cache loading
10131079
1014-
✅ OHLCV Source Dir (5 tests):
1080+
✅ OHLCV Source Dir (6 tests):
10151081
- Load from .npy files
10161082
- Load from .npz files
10171083
- Fallback on missing files
1084+
- Fallback on non-contiguous small gaps
10181085
- Fallback on excessive gaps
10191086
- Fallback on corrupt/malformed .npz
10201087
10211088
✅ Integration (2 tests):
10221089
- prepare_hlcvs structure
10231090
- prepare_hlcvs_combined structure
10241091
1025-
Total: 22 tests covering critical functionality
1092+
Total: 23 tests covering critical functionality
10261093
10271094
Note: Full integration tests for prepare_hlcvs and prepare_hlcvs_combined
10281095
would require extensive mocking of CandlestickManager, async operations,

0 commit comments

Comments
 (0)