service: rewrite kline sync check

doc/development/release-process.md

@@ -1,6 +1,12 @@
 # Release Process
 
-## 1. Prepare the release note
+## 1. Run the release test script
+
+```shell
+bash scripts/release-test.sh
+```
+
+## 2. Prepare the release note
 
 You need to prepare the release note for your next release version.
 
@@ -20,7 +26,7 @@ bash utils/changelog.sh > doc/release/v1.20.2.md
 
 Edit your changelog.
 
-## 2. Make the release
+## 3. Make the release
 
 Run the following command to create the release:
 
@@ -35,5 +41,4 @@ The above command wilL:
 - Run git tag to create the tag.
 - Run git push to push the created tag.
 
-
 You can go to <https://github.com/c9s/bbgo/releases/v1.20.2> to modify the changelog

pkg/cmd/backtest.go

@@ -673,22 +673,8 @@ func sync(ctx context.Context, userConfig *bbgo.Config, backtestService *service
 			})
 
 			for _, interval := range intervals {
-				firstKLine, err := backtestService.QueryFirstKLine(sourceExchange.Name(), symbol, interval)
-				if err != nil {
-					return errors.Wrapf(err, "failed to query backtest kline")
-				}
-
-				// if we don't have klines before the start time endpoint, the back-test will fail.
-				// because the last price will be missing.
-				if firstKLine != nil {
-					if err := backtestService.SyncPartial(ctx, sourceExchange, symbol, interval, syncFrom, syncTo); err != nil {
-						return err
-					}
-				} else {
-					log.Debugf("starting a fresh kline data sync...")
-					if err := backtestService.SyncFresh(ctx, sourceExchange, symbol, interval, syncFrom, syncTo); err != nil {
-						return err
-					}
+				if err := backtestService.Sync(ctx, sourceExchange, symbol, interval, syncFrom, syncTo); err != nil {
+					return err
 				}
 			}
 		}

pkg/service/backtest.go

@@ -109,15 +109,12 @@ func (s *BacktestService) Verify(sourceExchange types.Exchange, symbols []string
 }
 
 func (s *BacktestService) SyncFresh(ctx context.Context, exchange types.Exchange, symbol string, interval types.Interval, startTime, endTime time.Time) error {
+	log.Infof("starting fresh sync %s %s %s: %s <=> %s", exchange.Name(), symbol, interval, startTime, endTime)
 	startTime = startTime.Truncate(time.Minute).Add(-2 * time.Second)
 	endTime = endTime.Truncate(time.Minute).Add(2 * time.Second)
 	return s.SyncKLineByInterval(ctx, exchange, symbol, interval, startTime, endTime)
 }
 
-func (s *BacktestService) QueryFirstKLine(ex types.ExchangeName, symbol string, interval types.Interval) (*types.KLine, error) {
-	return s.QueryKLine(ex, symbol, interval, "ASC", 1)
-}
-
 // QueryKLine queries the klines from the database
 func (s *BacktestService) QueryKLine(ex types.ExchangeName, symbol string, interval types.Interval, orderBy string, limit int) (*types.KLine, error) {
 	log.Infof("querying last kline exchange = %s AND symbol = %s AND interval = %s", ex, symbol, interval)
@@ -330,12 +327,28 @@ func (t *TimeRange) String() string {
 	return t.Start.String() + " ~ " + t.End.String()
 }
 
+func (s *BacktestService) Sync(ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, since, until time.Time) error {
+	t1, t2, err := s.QueryExistingDataRange(ctx, ex, symbol, interval, since, until)
+	if err != nil && err != sql.ErrNoRows {
+		return err
+	}
+
+	if err == sql.ErrNoRows || t1 == nil || t2 == nil {
+		// fallback to fresh sync
+		return s.SyncFresh(ctx, ex, symbol, interval, since, until)
+	}
+
+	return s.SyncPartial(ctx, ex, symbol, interval, since, until)
+}
+
 // SyncPartial
 // find the existing data time range (t1, t2)
 // scan if there is a missing part
 // create a time range slice []TimeRange
 // iterate the []TimeRange slice to sync data.
 func (s *BacktestService) SyncPartial(ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, since, until time.Time) error {
+	log.Infof("starting partial sync %s %s %s: %s <=> %s", ex.Name(), symbol, interval, since, until)
+
 	t1, t2, err := s.QueryExistingDataRange(ctx, ex, symbol, interval, since, until)
 	if err != nil && err != sql.ErrNoRows {
 		return err
@@ -352,7 +365,7 @@ func (s *BacktestService) SyncPartial(ctx context.Context, ex types.Exchange, sy
 	}
 
 	if len(timeRanges) > 0 {
-		log.Infof("found missing time ranges: %v", timeRanges)
+		log.Infof("found missing data time ranges: %v", timeRanges)
 	}
 
 	// there are few cases: