From c72122b084dae3f99cec82c43d548caa47021852 Mon Sep 17 00:00:00 2001 From: woohyun kim Date: Thu, 2 Apr 2026 09:42:10 +0900 Subject: [PATCH] fix: implement paginated getCandlesByRange for historical backtest data Fixes the backtest failure when using past date ranges. The previous fetchHistoricalCandles() only fetched the latest 200 candles and filtered by date, which excluded any historical window older than ~33 days (4h) or ~8 days (1h). Changes: - Add getCandlesByRange(symbol, interval, startTime, endTime) to IExchangeRest interface - BinanceRest: forward-paginate via startTime/endTime params (1000/page) - BybitRest: forward-paginate via start/end params (1000/page) - UpbitRest: backward-paginate via `to` param (200/page) - backtests.service.ts: replace simplified filter with getCandlesByRange Closes #75 Co-Authored-By: Paperclip --- .../src/backtests/backtests.service.ts | 41 +++------ .../src/binance/binance.rest.ts | 72 ++++++++++++++++ .../exchange-adapters/src/bybit/bybit.rest.ts | 74 ++++++++++++++++ .../src/interfaces/exchange-rest.ts | 6 ++ .../exchange-adapters/src/upbit/upbit.rest.ts | 84 +++++++++++++++++++ 5 files changed, 247 insertions(+), 30 deletions(-) diff --git a/apps/worker-service/src/backtests/backtests.service.ts b/apps/worker-service/src/backtests/backtests.service.ts index 6fa7873..5fe5ef8 100644 --- a/apps/worker-service/src/backtests/backtests.service.ts +++ b/apps/worker-service/src/backtests/backtests.service.ts @@ -212,8 +212,7 @@ export class BacktestsService implements OnModuleInit, OnModuleDestroy { } /** - * Fetch historical candles by paginating through exchange API. - * 200 candles per request, walking backwards from endDate. + * Fetch historical candles for the given date range using paginated exchange API calls. */ private async fetchHistoricalCandles( exchange: ExchangeId, @@ -233,36 +232,18 @@ export class BacktestsService implements OnModuleInit, OnModuleDestroy { if (!adapterFactory) throw new Error(`Unsupported exchange: ${exchange}`); const adapter = adapterFactory(); - const allCandles: Candle[] = []; - const startMs = startDate.getTime(); - const endMs = endDate.getTime(); - - // Fetch in pages of 200 candles - // Most exchanges return candles in reverse chronological order, - // but our adapter reverses them to chronological - let fetchedCandles = await adapter.getCandles(symbol, interval, 200); - - // Filter to date range - for (const c of fetchedCandles) { - if (c.timestamp >= startMs && c.timestamp <= endMs) { - allCandles.push(c); - } - } - - // If we need more historical data, paginate - // This is a simplified approach — for a production system we'd - // use exchange-specific pagination parameters (e.g., `endTime`) - if (allCandles.length > 0) { - // Sort chronologically - allCandles.sort((a, b) => a.timestamp - b.timestamp); - } - - // Cache for reuse - if (allCandles.length > 0) { - await this.redis.set(cacheKey, JSON.stringify(allCandles), 'EX', CANDLE_CACHE_TTL); + const candles = await adapter.getCandlesByRange( + symbol, + interval, + startDate.getTime(), + endDate.getTime(), + ); + + if (candles.length > 0) { + await this.redis.set(cacheKey, JSON.stringify(candles), 'EX', CANDLE_CACHE_TTL); } - return allCandles; + return candles; } /** diff --git a/packages/exchange-adapters/src/binance/binance.rest.ts b/packages/exchange-adapters/src/binance/binance.rest.ts index b8cd487..9932458 100644 --- a/packages/exchange-adapters/src/binance/binance.rest.ts +++ b/packages/exchange-adapters/src/binance/binance.rest.ts @@ -11,6 +11,18 @@ import { IExchangeRest } from '../interfaces/exchange-rest'; const BASE_URL = 'https://api.binance.com'; +function parseIntervalMs(interval: string): number { + const INTERVAL_MS: Record = { + '1m': 60_000, + '5m': 300_000, + '15m': 900_000, + '1h': 3_600_000, + '4h': 14_400_000, + '1d': 86_400_000, + }; + return INTERVAL_MS[interval] ?? 60_000; +} + interface BinanceOrderResponse { orderId: number; symbol: string; @@ -168,6 +180,66 @@ export class BinanceRest implements IExchangeRest { })); } + async getCandlesByRange( + symbol: string, + interval: string, + startTime: number, + endTime: number, + ): Promise { + const PAGE_LIMIT = 1000; + const intervalMs = parseIntervalMs(interval); + const allCandles: Candle[] = []; + let pageStart = startTime; + const MAX_PAGES = 100; + + for (let page = 0; page < MAX_PAGES; page++) { + const res = await fetch( + `${BASE_URL}/api/v3/klines?symbol=${symbol}&interval=${interval}&limit=${PAGE_LIMIT}&startTime=${pageStart}&endTime=${endTime}`, + ); + if (!res.ok) { + const body = await res.text(); + throw new Error(`Binance API error ${res.status}: ${body}`); + } + const data = (await res.json()) as Array< + [ + number, + string, + string, + string, + string, + string, + number, + string, + number, + string, + string, + string, + ] + >; + if (data.length === 0) break; + + for (const k of data) { + allCandles.push({ + exchange: this.exchangeId, + symbol, + interval, + open: k[1], + high: k[2], + low: k[3], + close: k[4], + volume: k[5], + timestamp: k[0], + }); + } + + if (data.length < PAGE_LIMIT) break; + pageStart = data[data.length - 1][0] + intervalMs; + if (pageStart > endTime) break; + } + + return allCandles; + } + private mapOrderResult(o: BinanceOrderResponse): OrderResult { return { exchange: this.exchangeId, diff --git a/packages/exchange-adapters/src/bybit/bybit.rest.ts b/packages/exchange-adapters/src/bybit/bybit.rest.ts index 0a55e97..36760e3 100644 --- a/packages/exchange-adapters/src/bybit/bybit.rest.ts +++ b/packages/exchange-adapters/src/bybit/bybit.rest.ts @@ -12,6 +12,18 @@ import { IExchangeRest } from '../interfaces/exchange-rest'; const BASE_URL = 'https://api.bybit.com'; const RECV_WINDOW = '5000'; +function parseIntervalMs(interval: string): number { + const INTERVAL_MS: Record = { + '1m': 60_000, + '5m': 300_000, + '15m': 900_000, + '1h': 3_600_000, + '4h': 14_400_000, + '1d': 86_400_000, + }; + return INTERVAL_MS[interval] ?? 60_000; +} + export class BybitRest implements IExchangeRest { readonly exchangeId = 'bybit' as const; @@ -214,6 +226,68 @@ export class BybitRest implements IExchangeRest { })); } + async getCandlesByRange( + symbol: string, + interval: string, + startTime: number, + endTime: number, + ): Promise { + const INTERVAL_MAP: Record = { + '1m': '1', + '5m': '5', + '15m': '15', + '1h': '60', + '4h': '240', + '1d': 'D', + }; + const bybitInterval = INTERVAL_MAP[interval] || '1'; + const intervalMs = parseIntervalMs(interval); + const PAGE_LIMIT = 1000; + const allCandles: Candle[] = []; + let pageStart = startTime; + const MAX_PAGES = 100; + + for (let page = 0; page < MAX_PAGES; page++) { + const res = await fetch( + `${BASE_URL}/v5/market/kline?category=spot&symbol=${symbol}&interval=${bybitInterval}&limit=${PAGE_LIMIT}&start=${pageStart}&end=${endTime}`, + ); + if (!res.ok) { + const body = await res.text(); + throw new Error(`Bybit API error ${res.status}: ${body}`); + } + const data = (await res.json()) as { + retCode: number; + retMsg: string; + result: { list: Array<[string, string, string, string, string, string, string]> }; + }; + if (data.retCode !== 0) { + throw new Error(`Bybit API error: ${data.retMsg}`); + } + const list = data.result?.list ?? []; + if (list.length === 0) break; + + // Bybit returns newest-first; reverse to chronological + const page_candles = list.reverse().map((k) => ({ + exchange: this.exchangeId, + symbol, + interval, + open: k[1], + high: k[2], + low: k[3], + close: k[4], + volume: k[5], + timestamp: Number(k[0]), + })); + allCandles.push(...page_candles); + + if (list.length < PAGE_LIMIT) break; + pageStart = page_candles[page_candles.length - 1].timestamp + intervalMs; + if (pageStart > endTime) break; + } + + return allCandles; + } + private mapOrder(o: BybitOrder): OrderResult { return { exchange: this.exchangeId, diff --git a/packages/exchange-adapters/src/interfaces/exchange-rest.ts b/packages/exchange-adapters/src/interfaces/exchange-rest.ts index e619e9e..df0462a 100644 --- a/packages/exchange-adapters/src/interfaces/exchange-rest.ts +++ b/packages/exchange-adapters/src/interfaces/exchange-rest.ts @@ -25,4 +25,10 @@ export interface IExchangeRest { ): Promise; getMarkets(): Promise; getCandles(symbol: string, interval: string, limit?: number): Promise; + getCandlesByRange( + symbol: string, + interval: string, + startTime: number, + endTime: number, + ): Promise; } diff --git a/packages/exchange-adapters/src/upbit/upbit.rest.ts b/packages/exchange-adapters/src/upbit/upbit.rest.ts index 5131daf..677d3bf 100644 --- a/packages/exchange-adapters/src/upbit/upbit.rest.ts +++ b/packages/exchange-adapters/src/upbit/upbit.rest.ts @@ -13,6 +13,18 @@ import { IExchangeRest } from '../interfaces/exchange-rest'; const BASE_URL = 'https://api.upbit.com'; +function parseIntervalMs(interval: string): number { + const INTERVAL_MS: Record = { + '1m': 60_000, + '5m': 300_000, + '15m': 900_000, + '1h': 3_600_000, + '4h': 14_400_000, + '1d': 86_400_000, + }; + return INTERVAL_MS[interval] ?? 60_000; +} + interface UpbitTrade { market: string; uuid: string; @@ -177,6 +189,78 @@ export class UpbitRest implements IExchangeRest { })); } + async getCandlesByRange( + symbol: string, + interval: string, + startTime: number, + endTime: number, + ): Promise { + const INTERVAL_MAP: Record = { + '1m': 'minutes/1', + '5m': 'minutes/5', + '15m': 'minutes/15', + '1h': 'minutes/60', + '4h': 'minutes/240', + '1d': 'days', + }; + const path = INTERVAL_MAP[interval] || 'minutes/1'; + const intervalMs = parseIntervalMs(interval); + const PAGE_LIMIT = 200; + const allCandles: Candle[] = []; + // Upbit paginates backward via `to` (exclusive end, ISO 8601) + let pageEnd = endTime; + const MAX_PAGES = 100; + + for (let page = 0; page < MAX_PAGES; page++) { + const toParam = new Date(pageEnd + 1).toISOString(); + const res = await fetch( + `${BASE_URL}/v1/candles/${path}?market=${symbol}&count=${PAGE_LIMIT}&to=${toParam}`, + ); + if (!res.ok) { + const body = await res.text(); + throw new Error(`Upbit API error ${res.status}: ${body}`); + } + const data = (await res.json()) as Array<{ + candle_date_time_utc: string; + opening_price: number; + high_price: number; + low_price: number; + trade_price: number; + candle_acc_trade_volume: number; + timestamp: number; + }>; + if (data.length === 0) break; + + // data is newest-first; filter to range and prepend to result + const inRange: Candle[] = []; + for (const k of data) { + if (k.timestamp < startTime) continue; + if (k.timestamp > endTime) continue; + inRange.push({ + exchange: this.exchangeId, + symbol, + interval, + open: String(k.opening_price), + high: String(k.high_price), + low: String(k.low_price), + close: String(k.trade_price), + volume: String(k.candle_acc_trade_volume), + timestamp: k.timestamp, + }); + } + // prepend so final array is chronological + allCandles.unshift(...inRange.reverse()); + + const oldestInPage = data[data.length - 1].timestamp; + if (oldestInPage <= startTime) break; + if (data.length < PAGE_LIMIT) break; + pageEnd = oldestInPage - intervalMs; + if (pageEnd < startTime) break; + } + + return allCandles; + } + private mapOrderResponse(o: UpbitOrderResponse): OrderResult { const execVol = parseFloat(o.executed_volume);