FEATURE: csvsource backtest (algo-boyz)

.gitignore

@@ -50,6 +50,7 @@ testoutput
 
 *.swp
 /pkg/backtest/assets.go
+/data/backtest
 
 coverage.txt
 coverage_dum.txt

doc/topics/back-testing.md

@@ -1,9 +1,13 @@
 ## Back-testing
 
-*Before you start back-testing, you need to setup [MySQL](../../README.md#configure-mysql-database) or [SQLite3
+Currently bbgo supports two ways to run backtests:
+
+  1: Through csv data source (supported right now are binance, bybit and OkEx)
+
+  2: Alternatively run backtests through [MySQL](../../README.md#configure-mysql-database) or [SQLite3
 ](../../README.md#configure-sqlite3-database). Using MySQL is highly recommended.*
 
-First, you need to add the back-testing config to your `bbgo.yaml`:
+Let's start by adding the back-testing section to your config eg: `bbgo.yaml`:
 
 ```yaml
 backtest:
@@ -41,8 +45,11 @@ Note on date formats, the following date formats are supported:
 * RFC822, which looks like `02 Jan 06 15:04 MST`
 * You can also use `2021-11-26T15:04:56`
 
-And then, you can sync remote exchange k-lines (candle bars) data for back-testing:
-
+And then, you can sync remote exchange k-lines (candle bars) data for back-testing through csv data source:
+```sh
+bbgo backtest -v --csv --verify --config config/grid.yaml
+```
+or use the sql data source like so:
 ```sh
 bbgo backtest -v --sync --config config/grid.yaml
 ```
@@ -67,6 +74,11 @@ Run back-test:
 ```sh
 bbgo backtest --base-asset-baseline --config config/grid.yaml
 ```
+or through csv data source
+
+```sh
+bbgo backtest -v --csv --base-asset-baseline --config config/grid.yaml --output data/backtest
+```
 
 If you're developing a strategy, you might want to start with a command like this:
 

pkg/backtest/exchange.go

@@ -55,7 +55,7 @@ var ErrEmptyOrderType = errors.New("order type can not be empty string")
 type Exchange struct {
 	sourceName     types.ExchangeName
 	publicExchange types.Exchange
-	srv            *service.BacktestService
+	srv            service.BackTestable
 	currentTime    time.Time
 
 	account *types.Account
@@ -78,7 +78,7 @@ type Exchange struct {
 }
 
 func NewExchange(
-	sourceName types.ExchangeName, sourceExchange types.Exchange, srv *service.BacktestService, config *bbgo.Backtest,
+	sourceName types.ExchangeName, sourceExchange types.Exchange, srv service.BackTestable, config *bbgo.Backtest,
 ) (*Exchange, error) {
 	ex := sourceExchange
 
@@ -381,6 +381,7 @@ func (e *Exchange) SubscribeMarketData(
 			loadedIntervals[sub.Options.Interval] = struct{}{}
 
 		default:
+			// todo support stream back test with csv tick source
 			// Since Environment is not yet been injected at this point, no hard error
 			log.Errorf("stream channel %s is not supported in backtest", sub.Channel)
 		}

pkg/backtest/matching.go

@@ -75,7 +75,7 @@ type SimplePriceMatching struct {
 	balanceUpdateCallbacks []func(balances types.BalanceMap)
 }
 
-func (m *SimplePriceMatching) Prepare(srv *service.BacktestService, exchange *Exchange, startTime time.Time) error {
+func (m *SimplePriceMatching) Prepare(srv service.BackTestable, exchange *Exchange, startTime time.Time) error {
 	if !m.lastPrice.IsZero() {
 		// no prepare work needed
 		return nil

pkg/bbgo/config.go

@@ -13,6 +13,7 @@ import (
 	"gopkg.in/yaml.v3"
 
 	bbgochart "github.com/c9s/bbgo/pkg/chart/v1"
+	"github.com/c9s/bbgo/pkg/datasource/csvsource"
 	"github.com/c9s/bbgo/pkg/datatype"
 	"github.com/c9s/bbgo/pkg/dynamic"
 	"github.com/c9s/bbgo/pkg/fixedpoint"
@@ -154,7 +155,8 @@ type Backtest struct {
 	Sessions []string                   `json:"sessions" yaml:"sessions"`
 
 	// sync 1 second interval KLines
-	SyncSecKLines bool `json:"syncSecKLines,omitempty" yaml:"syncSecKLines,omitempty"`
+	SyncSecKLines bool                 `json:"syncSecKLines,omitempty" yaml:"syncSecKLines,omitempty"`
+	CsvSource     *csvsource.CsvConfig `json:"csvConfig,omitempty" yaml:"csvConfig,omitempty"`
 }
 
 func (b *Backtest) GetAccount(n string) BacktestAccount {

pkg/bbgo/environment.go

@@ -45,9 +45,9 @@ var defaultSyncBufferPeriod = 30 * time.Minute
 // IsBackTesting is a global variable that indicates the current environment is back-test or not.
 var IsBackTesting = false
 
-var BackTestService *service.BacktestService
+var BackTestService service.BackTestable
 
-func SetBackTesting(s *service.BacktestService) {
+func SetBackTesting(s service.BackTestable) {
 	BackTestService = s
 	IsBackTesting = s != nil
 }
@@ -101,7 +101,7 @@ type Environment struct {
 	TradeService      *service.TradeService
 	ProfitService     *service.ProfitService
 	PositionService   *service.PositionService
-	BacktestService   *service.BacktestService
+	BacktestService   service.BackTestable
 	RewardService     *service.RewardService
 	MarginService     *service.MarginService
 	SyncService       *service.SyncService

pkg/cmd/backtest.go

@@ -32,6 +32,7 @@ import (
 )
 
 func init() {
+	BacktestCmd.Flags().Bool("csv", false, "use csv data source for exchange (if supported)")
 	BacktestCmd.Flags().Bool("sync", false, "sync backtest data")
 	BacktestCmd.Flags().Bool("sync-only", false, "sync backtest data only, do not run backtest")
 	BacktestCmd.Flags().String("sync-from", "", "sync backtest data from the given time, which will override the time range in the backtest config")
@@ -76,6 +77,11 @@ var BacktestCmd = &cobra.Command{
 			return err
 		}
 
+		modeCsv, err := cmd.Flags().GetBool("csv")
+		if err != nil {
+			return err
+		}
+
 		wantSync, err := cmd.Flags().GetBool("sync")
 		if err != nil {
 			return err
@@ -155,15 +161,30 @@ var BacktestCmd = &cobra.Command{
 		log.Infof("starting backtest with startTime %s", startTime.Format(time.RFC3339))
 
 		environ := bbgo.NewEnvironment()
-		if err := bbgo.BootstrapBacktestEnvironment(ctx, environ); err != nil {
-			return err
-		}
 
-		if environ.DatabaseService == nil {
-			return errors.New("database service is not enabled, please check your environment variables DB_DRIVER and DB_DSN")
-		}
+		var backtestService service.BackTestable
+		if modeCsv {
+			if userConfig.Backtest.CsvSource == nil {
+				return fmt.Errorf("user config backtest section needs csvsource config")
+			}
+			backtestService = service.NewBacktestServiceCSV(
+				outputDirectory,
+				userConfig.Backtest.CsvSource.Market,
+				userConfig.Backtest.CsvSource.Granularity,
+			)
+			if err := bbgo.BootstrapEnvironmentLightweight(ctx, environ, userConfig); err != nil {
+				return err
+			}
+		} else {
+			backtestService = service.NewBacktestService(environ.DatabaseService.DB)
+			if err := bbgo.BootstrapBacktestEnvironment(ctx, environ); err != nil {
+				return err
+			}
 
-		backtestService := &service.BacktestService{DB: environ.DatabaseService.DB}
+			if environ.DatabaseService == nil {
+				return errors.New("database service is not enabled, please check your environment variables DB_DRIVER and DB_DSN")
+			}
+		}
 		environ.BacktestService = backtestService
 		bbgo.SetBackTesting(backtestService)
 
@@ -798,7 +819,7 @@ func n(v float64) fixedpoint.Value {
 }
 
 func verify(
-	userConfig *bbgo.Config, backtestService *service.BacktestService,
+	userConfig *bbgo.Config, backtestService service.BackTestable,
 	sourceExchanges map[types.ExchangeName]types.Exchange, startTime, endTime time.Time,
 ) error {
 	for _, sourceExchange := range sourceExchanges {
@@ -842,7 +863,7 @@ func getExchangeIntervals(ex types.Exchange) types.IntervalMap {
 }
 
 func sync(
-	ctx context.Context, userConfig *bbgo.Config, backtestService *service.BacktestService,
+	ctx context.Context, userConfig *bbgo.Config, backtestService service.BackTestable,
 	sourceExchanges map[types.ExchangeName]types.Exchange, syncFrom, syncTo time.Time,
 ) error {
 	for _, symbol := range userConfig.Backtest.Symbols {
@@ -857,10 +878,8 @@ func sync(
 			var intervals = supportIntervals.Slice()
 			intervals.Sort()
 
-			for _, interval := range intervals {
-				if err := backtestService.Sync(ctx, sourceExchange, symbol, interval, syncFrom, syncTo); err != nil {
-					return err
-				}
+			if err := backtestService.Sync(ctx, sourceExchange, symbol, intervals, syncFrom, syncTo); err != nil {
+				return err
 			}
 		}
 	}

pkg/cmd/pnl.go

@@ -135,7 +135,8 @@ var PnLCmd = &cobra.Command{
 
 				// we need the backtest klines for the daily prices
 				backtestService := &service.BacktestService{DB: environ.DatabaseService.DB}
-				if err := backtestService.Sync(ctx, exchange, symbol, types.Interval1d, since, until); err != nil {
+				intervals := []types.Interval{types.Interval1d}
+				if err := backtestService.Sync(ctx, exchange, symbol, intervals, since, until); err != nil {
 					return err
 				}
 			}

pkg/datasource/csvsource/csv_errors.go

@@ -0,0 +1,26 @@
+package csvsource
+
+import "errors"
+
+var (
+	// ErrNotEnoughColumns is returned when the CSV price record does not have enough columns.
+	ErrNotEnoughColumns = errors.New("not enough columns")
+
+	// ErrInvalidTimeFormat is returned when the CSV price record does not have a valid time unix milli format.
+	ErrInvalidIDFormat = errors.New("cannot parse trade id string")
+
+	// ErrInvalidBoolFormat is returned when the CSV isBuyerMaker record does not have a valid bool representation.
+	ErrInvalidBoolFormat = errors.New("cannot parse bool to string")
+
+	// ErrInvalidTimeFormat is returned when the CSV price record does not have a valid time unix milli format.
+	ErrInvalidTimeFormat = errors.New("cannot parse time string")
+
+	// ErrInvalidOrderSideFormat is returned when the CSV side record does not have a valid buy or sell string.
+	ErrInvalidOrderSideFormat = errors.New("cannot parse order side string")
+
+	// ErrInvalidPriceFormat is returned when the CSV price record does not prices in expected format.
+	ErrInvalidPriceFormat = errors.New("OHLC prices must be valid number format")
+
+	// ErrInvalidVolumeFormat is returned when the CSV price record does not have a valid volume format.
+	ErrInvalidVolumeFormat = errors.New("volume must be valid number format")
+)

pkg/datasource/csvsource/csv_kline_decoder.go

@@ -0,0 +1,63 @@
+package csvsource
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/c9s/bbgo/pkg/fixedpoint"
+	"github.com/c9s/bbgo/pkg/types"
+)
+
+// parseCsvKLineRecord parse a CSV record into a KLine.
+func parseCsvKLineRecord(record []string, interval time.Duration) (types.KLine, error) {
+	var (
+		k, empty types.KLine
+		err      error
+	)
+	if len(record) < 5 {
+		return k, ErrNotEnoughColumns
+	}
+	ts, err := strconv.ParseFloat(record[0], 64) // check for e numbers "1.70027E+12"
+	if err != nil {
+		return empty, ErrInvalidTimeFormat
+	}
+	open, err := fixedpoint.NewFromString(record[1])
+	if err != nil {
+		return empty, ErrInvalidPriceFormat
+	}
+	high, err := fixedpoint.NewFromString(record[2])
+	if err != nil {
+		return empty, ErrInvalidPriceFormat
+	}
+	low, err := fixedpoint.NewFromString(record[3])
+	if err != nil {
+		return empty, ErrInvalidPriceFormat
+	}
+	closing, err := fixedpoint.NewFromString(record[4])
+	if err != nil {
+		return empty, ErrInvalidPriceFormat
+	}
+
+	volume := fixedpoint.Zero
+	if len(record) >= 6 {
+		volume, err = fixedpoint.NewFromString(record[5])
+		if err != nil {
+			return empty, ErrInvalidVolumeFormat
+		}
+	}
+
+	// ts is in milliseconds, convert to seconds and nanoseconds
+	tsMs := int64(ts)
+	tsSec := tsMs / 1000
+	tsNsec := (tsMs % 1000) * 1000000
+
+	k.StartTime = types.NewTimeFromUnix(tsSec, tsNsec)
+	k.EndTime = types.NewTimeFromUnix(k.StartTime.Time().Add(interval).Unix(), 0)
+	k.Open = open
+	k.High = high
+	k.Low = low
+	k.Close = closing
+	k.Volume = volume
+
+	return k, nil
+}

pkg/datasource/csvsource/csv_kline_reader.go

@@ -0,0 +1,52 @@
+package csvsource
+
+import (
+	"encoding/csv"
+	"io"
+	"time"
+
+	"github.com/c9s/bbgo/pkg/types"
+)
+
+var _ KLineReader = (*CSVKLineReader)(nil)
+
+// CSVKLineReader is a KLineReader that reads from a CSV file.
+type CSVKLineReader struct {
+	csv *csv.Reader
+}
+
+// NewCSVKLineReader creates a new CSVKLineReader with the default Binance decoder.
+func NewCSVKLineReader(csv *csv.Reader) *CSVKLineReader {
+	return &CSVKLineReader{
+		csv: csv,
+	}
+}
+
+// Read reads the next KLine from the underlying CSV data.
+func (r *CSVKLineReader) Read(interval time.Duration) (types.KLine, error) {
+	var k types.KLine
+
+	rec, err := r.csv.Read()
+	if err != nil {
+		return k, err
+	}
+
+	return parseCsvKLineRecord(rec, interval)
+}
+
+// ReadAll reads all the KLines from the underlying CSV data.
+func (r *CSVKLineReader) ReadAll(interval time.Duration) ([]types.KLine, error) {
+	var ks []types.KLine
+	for {
+		k, err := r.Read(interval)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		ks = append(ks, k)
+	}
+
+	return ks, nil
+}