Merge pull request #692 from c9s/fix/pnl-cmd

fix: fix pnl command calculation and add warning logs

.github/workflows/python.yml

@@ -3,8 +3,13 @@ name: Python
 on:
   push:
     branches: [ main ]
+    paths:
+    - python
+
   pull_request:
     branches: [ main ]
+    paths:
+    - python
 
 jobs:
 
@@ -13,31 +18,31 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [ 3.8 ]
 
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
+    - name: Checkout code
+      uses: actions/checkout@v2
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
 
-      - name: Install poetry
-        run: pip install poetry==1.1.13
+    - name: Install poetry
+      run: pip install poetry==1.1.13
 
-      - name: Install package
-        run: |
-          cd python
-          poetry install
+    - name: Install package
+      run: |
+        cd python
+        poetry install
 
-      - name: Test
-        run: |
-          cd python
-          poetry run pytest -v -s tests
+    - name: Test
+      run: |
+        cd python
+        poetry run pytest -v -s tests
 
-      - name: Lint
-        run: |
-          cd python
-          poetry run flake8 .
+    - name: Lint
+      run: |
+        cd python
+        poetry run flake8 .

Makefile

@@ -230,7 +230,7 @@ frontend/out/index.html: frontend/node_modules
 	cd frontend && yarn export
 
 pkg/server/assets.go: frontend/out/index.html
-	go run ./util/embed -package server -tag web -output $@ $(FRONTEND_EXPORT_DIR)
+	go run ./utils/embed -package server -tag web -output $@ $(FRONTEND_EXPORT_DIR)
 
 $(BACKTEST_REPORT_APP_DIR)/node_modules:
 	cd $(BACKTEST_REPORT_APP_DIR) && yarn install
@@ -239,7 +239,7 @@ $(BACKTEST_REPORT_APP_DIR)/out/index.html: .FORCE $(BACKTEST_REPORT_APP_DIR)/nod
 	cd $(BACKTEST_REPORT_APP_DIR) && yarn build && yarn export
 
 pkg/backtest/assets.go: $(BACKTEST_REPORT_APP_DIR)/out/index.html
-	go run ./util/embed -package backtest -tag web -output $@ $(BACKTEST_REPORT_EXPORT_DIR)
+	go run ./utils/embed -package backtest -tag web -output $@ $(BACKTEST_REPORT_EXPORT_DIR)
 
 embed: pkg/server/assets.go pkg/backtest/assets.go
 

config/sync.yaml

@@ -15,6 +15,14 @@ sessions:
     exchange: max
     envVarPrefix: max
 
+  kucoin:
+    exchange: kucoin
+    envVarPrefix: kucoin
+
+  okex:
+    exchange: okex
+    envVarPrefix: okex
+
 sync:
   # userDataStream is used to sync the trading data in real-time
   # it uses the websocket connection to insert the trades
@@ -31,6 +39,8 @@ sync:
   - binance
   - binance_margin_dotusdt
   - max
+  - okex
+  - kucoin
 
   # symbols is the list of symbols you want to sync
   # by default, BBGO try to guess your symbols by your existing account balances.
@@ -47,6 +57,6 @@ sync:
   marginAssets:
   - USDT
 
-  # depositHistory: true
-  # rewardHistory: true
+  depositHistory: true
+  rewardHistory: true
   withdrawHistory: true

doc/development/release-process.md

@@ -1,6 +1,12 @@
 # Release Process
 
-## 1. Prepare the release note
+## 1. Run the release test script
+
+```shell
+bash scripts/release-test.sh
+```
+
+## 2. Prepare the release note
 
 You need to prepare the release note for your next release version.
 
@@ -20,7 +26,7 @@ bash utils/changelog.sh > doc/release/v1.20.2.md
 
 Edit your changelog.
 
-## 2. Make the release
+## 3. Make the release
 
 Run the following command to create the release:
 
@@ -35,5 +41,4 @@ The above command wilL:
 - Run git tag to create the tag.
 - Run git push to push the created tag.
 
-
 You can go to <https://github.com/c9s/bbgo/releases/v1.20.2> to modify the changelog

pkg/accounting/pnl/avg_cost.go

@@ -92,12 +92,12 @@ func (c *AverageCostCalculator) Calculate(symbol string, trades []types.Trade, c
 		BuyVolume:  bidVolume,
 		SellVolume: askVolume,
 
-		Stock:            position.GetBase(),
-		Profit:           totalProfit,
-		NetProfit:        totalNetProfit,
-		UnrealizedProfit: unrealizedProfit,
-		AverageCost:      position.AverageCost,
-		FeeInUSD:         totalProfit.Sub(totalNetProfit),
-		CurrencyFees:     currencyFees,
+		BaseAssetPosition: position.GetBase(),
+		Profit:            totalProfit,
+		NetProfit:         totalNetProfit,
+		UnrealizedProfit:  unrealizedProfit,
+		AverageCost:       position.AverageCost,
+		FeeInUSD:          totalProfit.Sub(totalNetProfit),
+		CurrencyFees:      currencyFees,
 	}
 }

pkg/accounting/pnl/report.go

@@ -20,16 +20,16 @@ type AverageCostPnlReport struct {
 	Symbol    string           `json:"symbol"`
 	Market    types.Market     `json:"market"`
 
-	NumTrades        int                         `json:"numTrades"`
-	Profit           fixedpoint.Value            `json:"profit"`
-	NetProfit        fixedpoint.Value            `json:"netProfit"`
-	UnrealizedProfit fixedpoint.Value            `json:"unrealizedProfit"`
-	AverageCost      fixedpoint.Value            `json:"averageCost"`
-	BuyVolume        fixedpoint.Value            `json:"buyVolume,omitempty"`
-	SellVolume       fixedpoint.Value            `json:"sellVolume,omitempty"`
-	FeeInUSD         fixedpoint.Value            `json:"feeInUSD"`
-	Stock            fixedpoint.Value            `json:"stock"`
-	CurrencyFees     map[string]fixedpoint.Value `json:"currencyFees"`
+	NumTrades         int                         `json:"numTrades"`
+	Profit            fixedpoint.Value            `json:"profit"`
+	NetProfit         fixedpoint.Value            `json:"netProfit"`
+	UnrealizedProfit  fixedpoint.Value            `json:"unrealizedProfit"`
+	AverageCost       fixedpoint.Value            `json:"averageCost"`
+	BuyVolume         fixedpoint.Value            `json:"buyVolume,omitempty"`
+	SellVolume        fixedpoint.Value            `json:"sellVolume,omitempty"`
+	FeeInUSD          fixedpoint.Value            `json:"feeInUSD"`
+	BaseAssetPosition fixedpoint.Value            `json:"baseAssetPosition"`
+	CurrencyFees      map[string]fixedpoint.Value `json:"currencyFees"`
 }
 
 func (report *AverageCostPnlReport) JSON() ([]byte, error) {
@@ -39,7 +39,10 @@ func (report *AverageCostPnlReport) JSON() ([]byte, error) {
 func (report AverageCostPnlReport) Print() {
 	color.Green("TRADES SINCE: %v", report.StartTime)
 	color.Green("NUMBER OF TRADES: %d", report.NumTrades)
+
 	color.Green("AVERAGE COST: %s", types.USD.FormatMoney(report.AverageCost))
+	color.Green("BASE ASSET POSITION: %s", report.BaseAssetPosition.String())
+
 	color.Green("TOTAL BUY VOLUME: %v", report.BuyVolume)
 	color.Green("TOTAL SELL VOLUME: %v", report.SellVolume)
 
@@ -83,7 +86,7 @@ func (report AverageCostPnlReport) SlackAttachment() slack.Attachment {
 
 			// FIXME:
 			// {Title: "Fee (USD)", Value: types.USD.FormatMoney(report.FeeInUSD), Short: true},
-			{Title: "Stock", Value: report.Stock.String(), Short: true},
+			{Title: "Base Asset Position", Value: report.BaseAssetPosition.String(), Short: true},
 			{Title: "Number of Trades", Value: strconv.Itoa(report.NumTrades), Short: true},
 		},
 		Footer:     report.StartTime.Format(time.RFC822),

pkg/bbgo/environment.go

@@ -102,11 +102,13 @@ type Environment struct {
 }
 
 func NewEnvironment() *Environment {
+
+	now := time.Now()
 	return &Environment{
 		// default trade scan time
-		syncStartTime: time.Now().AddDate(-1, 0, 0), // defaults to sync from 1 year ago
+		syncStartTime: now.AddDate(-1, 0, 0), // defaults to sync from 1 year ago
 		sessions:      make(map[string]*ExchangeSession),
-		startTime:     time.Now().UTC(),
+		startTime:     now,
 
 		syncStatus: SyncNotStarted,
 		PersistenceServiceFacade: &service.PersistenceServiceFacade{

pkg/cmd/backtest.go

@@ -127,14 +127,6 @@ var BacktestCmd = &cobra.Command{
 			return err
 		}
 
-		if verboseCnt == 2 {
-			log.SetLevel(log.DebugLevel)
-		} else if verboseCnt > 0 {
-			log.SetLevel(log.InfoLevel)
-		} else {
-			// default mode, disable strategy logging and order executor logging
-			log.SetLevel(log.ErrorLevel)
-		}
 
 		if userConfig.Backtest == nil {
 			return errors.New("backtest config is not defined")
@@ -247,6 +239,15 @@ var BacktestCmd = &cobra.Command{
 			}
 		}
 
+		if verboseCnt == 2 {
+			log.SetLevel(log.DebugLevel)
+		} else if verboseCnt > 0 {
+			log.SetLevel(log.InfoLevel)
+		} else {
+			// default mode, disable strategy logging and order executor logging
+			log.SetLevel(log.ErrorLevel)
+		}
+
 		environ.SetStartTime(startTime)
 
 		// exchangeNameStr is the session name.
@@ -673,22 +674,8 @@ func sync(ctx context.Context, userConfig *bbgo.Config, backtestService *service
 			})
 
 			for _, interval := range intervals {
-				firstKLine, err := backtestService.QueryFirstKLine(sourceExchange.Name(), symbol, interval)
-				if err != nil {
-					return errors.Wrapf(err, "failed to query backtest kline")
-				}
-
-				// if we don't have klines before the start time endpoint, the back-test will fail.
-				// because the last price will be missing.
-				if firstKLine != nil {
-					if err := backtestService.SyncPartial(ctx, sourceExchange, symbol, interval, syncFrom, syncTo); err != nil {
-						return err
-					}
-				} else {
-					log.Debugf("starting a fresh kline data sync...")
-					if err := backtestService.SyncFresh(ctx, sourceExchange, symbol, interval, syncFrom, syncTo); err != nil {
-						return err
-					}
+				if err := backtestService.Sync(ctx, sourceExchange, symbol, interval, syncFrom, syncTo); err != nil {
+					return err
 				}
 			}
 		}

pkg/cmd/deposit.go

@@ -3,10 +3,8 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"os"
 	"time"
 
-	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
@@ -29,34 +27,7 @@ var depositsCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		ctx := context.Background()
 
-		configFile, err := cmd.Flags().GetString("config")
-		if err != nil {
-			return err
-		}
-
-		if len(configFile) == 0 {
-			return errors.New("--config option is required")
-		}
-
-		// if config file exists, use the config loaded from the config file.
-		// otherwise, use a empty config object
-		var userConfig *bbgo.Config
-		if _, err := os.Stat(configFile); err == nil {
-			// load successfully
-			userConfig, err = bbgo.Load(configFile, false)
-			if err != nil {
-				return err
-			}
-		} else if os.IsNotExist(err) {
-			// config file doesn't exist
-			userConfig = &bbgo.Config{}
-		} else {
-			// other error
-			return err
-		}
-
 		environ := bbgo.NewEnvironment()
-
 		if err := environ.ConfigureExchangeSessions(userConfig); err != nil {
 			return err
 		}

pkg/cmd/pnl.go

@@ -2,16 +2,15 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
-	"os"
+	"sort"
 	"strings"
 	"time"
 
-	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/c9s/bbgo/pkg/accounting"
 	"github.com/c9s/bbgo/pkg/accounting/pnl"
 	"github.com/c9s/bbgo/pkg/bbgo"
 	"github.com/c9s/bbgo/pkg/service"
@@ -22,35 +21,18 @@ func init() {
 	PnLCmd.Flags().String("session", "", "target exchange")
 	PnLCmd.Flags().String("symbol", "", "trading symbol")
 	PnLCmd.Flags().Bool("include-transfer", false, "convert transfer records into trades")
-	PnLCmd.Flags().Int("limit", 500, "number of trades")
+	PnLCmd.Flags().Int("limit", 0, "number of trades")
 	RootCmd.AddCommand(PnLCmd)
 }
 
 var PnLCmd = &cobra.Command{
 	Use:          "pnl",
-	Short:        "pnl calculator",
+	Short:        "Average Cost Based PnL Calculator",
+	Long:         "This command calculates the average cost-based profit from your total trades",
 	SilenceUsage: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		ctx := context.Background()
 
-		configFile, err := cmd.Flags().GetString("config")
-		if err != nil {
-			return err
-		}
-
-		if len(configFile) == 0 {
-			return errors.New("--config option is required")
-		}
-
-		if _, err := os.Stat(configFile); os.IsNotExist(err) {
-			return err
-		}
-
-		userConfig, err := bbgo.Load(configFile, false)
-		if err != nil {
-			return err
-		}
-
 		sessionName, err := cmd.Flags().GetString("session")
 		if err != nil {
 			return err
@@ -124,11 +106,16 @@ var PnLCmd = &cobra.Command{
 			if err != nil {
 				return err
 			}
-			_ = withdrawals
+
+			sort.Slice(withdrawals, func(i, j int) bool {
+				a := withdrawals[i].ApplyTime.Time()
+				b := withdrawals[j].ApplyTime.Time()
+				return a.Before(b)
+			})
 
 			// we need the backtest klines for the daily prices
 			backtestService := &service.BacktestService{DB: environ.DatabaseService.DB}
-			if err := backtestService.SyncKLineByInterval(ctx, exchange, symbol, types.Interval1d, since, until); err != nil {
+			if err := backtestService.Sync(ctx, exchange, symbol, types.Interval1d, since, until); err != nil {
 				return err
 			}
 		}
@@ -140,9 +127,8 @@ var PnLCmd = &cobra.Command{
 			trades, err = environ.TradeService.QueryForTradingFeeCurrency(exchange.Name(), symbol, tradingFeeCurrency)
 		} else {
 			trades, err = environ.TradeService.Query(service.QueryTradesOptions{
-				Exchange: exchange.Name(),
-				Symbol:   symbol,
-				Limit:    limit,
+				Symbol: symbol,
+				Limit:  limit,
 			})
 		}
 
@@ -150,41 +136,34 @@ var PnLCmd = &cobra.Command{
 			return err
 		}
 
+		if len(trades) == 0 {
+			return errors.New("empty trades, you need to run sync command to sync the trades from the exchange first")
+		}
+
+		trades = types.SortTradesAscending(trades)
+
 		log.Infof("%d trades loaded", len(trades))
 
-		stockManager := &accounting.StockDistribution{
-			Symbol:             symbol,
-			TradingFeeCurrency: tradingFeeCurrency,
-		}
-
-		checkpoints, err := stockManager.AddTrades(trades)
-		if err != nil {
-			return err
-		}
-
-		log.Infof("found checkpoints: %+v", checkpoints)
-		log.Infof("stock: %v", stockManager.Stocks.Quantity())
-
 		tickers, err := exchange.QueryTickers(ctx, symbol)
-
 		if err != nil {
 			return err
 		}
 
 		currentTick, ok := tickers[symbol]
-
 		if !ok {
 			return errors.New("no ticker data for current price")
 		}
 
 		currentPrice := currentTick.Last
-
 		calculator := &pnl.AverageCostCalculator{
 			TradingFeeCurrency: tradingFeeCurrency,
 		}
 
 		report := calculator.Calculate(symbol, trades, currentPrice)
 		report.Print()
+
+		log.Warnf("note that if you're using cross-exchange arbitrage, the PnL won't be accurate")
+		log.Warnf("withdrawal and deposits are not considered in the PnL")
 		return nil
 	},
 }

pkg/service/backtest.go

@@ -109,15 +109,12 @@ func (s *BacktestService) Verify(sourceExchange types.Exchange, symbols []string
 }
 
 func (s *BacktestService) SyncFresh(ctx context.Context, exchange types.Exchange, symbol string, interval types.Interval, startTime, endTime time.Time) error {
+	log.Infof("starting fresh sync %s %s %s: %s <=> %s", exchange.Name(), symbol, interval, startTime, endTime)
 	startTime = startTime.Truncate(time.Minute).Add(-2 * time.Second)
 	endTime = endTime.Truncate(time.Minute).Add(2 * time.Second)
 	return s.SyncKLineByInterval(ctx, exchange, symbol, interval, startTime, endTime)
 }
 
-func (s *BacktestService) QueryFirstKLine(ex types.ExchangeName, symbol string, interval types.Interval) (*types.KLine, error) {
-	return s.QueryKLine(ex, symbol, interval, "ASC", 1)
-}
-
 // QueryKLine queries the klines from the database
 func (s *BacktestService) QueryKLine(ex types.ExchangeName, symbol string, interval types.Interval, orderBy string, limit int) (*types.KLine, error) {
 	log.Infof("querying last kline exchange = %s AND symbol = %s AND interval = %s", ex, symbol, interval)
@@ -330,12 +327,28 @@ func (t *TimeRange) String() string {
 	return t.Start.String() + " ~ " + t.End.String()
 }
 
+func (s *BacktestService) Sync(ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, since, until time.Time) error {
+	t1, t2, err := s.QueryExistingDataRange(ctx, ex, symbol, interval, since, until)
+	if err != nil && err != sql.ErrNoRows {
+		return err
+	}
+
+	if err == sql.ErrNoRows || t1 == nil || t2 == nil {
+		// fallback to fresh sync
+		return s.SyncFresh(ctx, ex, symbol, interval, since, until)
+	}
+
+	return s.SyncPartial(ctx, ex, symbol, interval, since, until)
+}
+
 // SyncPartial
 // find the existing data time range (t1, t2)
 // scan if there is a missing part
 // create a time range slice []TimeRange
 // iterate the []TimeRange slice to sync data.
 func (s *BacktestService) SyncPartial(ctx context.Context, ex types.Exchange, symbol string, interval types.Interval, since, until time.Time) error {
+	log.Infof("starting partial sync %s %s %s: %s <=> %s", ex.Name(), symbol, interval, since, until)
+
 	t1, t2, err := s.QueryExistingDataRange(ctx, ex, symbol, interval, since, until)
 	if err != nil && err != sql.ErrNoRows {
 		return err
@@ -352,7 +365,7 @@ func (s *BacktestService) SyncPartial(ctx context.Context, ex types.Exchange, sy
 	}
 
 	if len(timeRanges) > 0 {
-		log.Infof("found missing time ranges: %v", timeRanges)
+		log.Infof("found missing data time ranges: %v", timeRanges)
 	}
 
 	// there are few cases:

pkg/types/trade.go

@@ -146,7 +146,7 @@ func trimTrailingZero(a float64) string {
 
 // String is for console output
 func (trade Trade) String() string {
-	return fmt.Sprintf("TRADE %s %s %4s %s @ %s amount %s fee %s %s orderID %d %s",
+	return fmt.Sprintf("TRADE %s %s %4s %-4s @ %6s | amount %s | fee %s %s | orderID %d | %s",
 		trade.Exchange.String(),
 		trade.Symbol,
 		trade.Side,

scripts/release-test.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+echo "testing sync..."
+dotenv -f .env.local.mysql -- go run ./cmd/bbgo sync --session binance  --config config/sync.yaml
+dotenv -f .env.local.sqlite -- go run ./cmd/bbgo sync --session binance  --config config/sync.yaml
+
+echo "backtest sync..."
+echo "backtest mysql sync..."
+dotenv -f .env.local.mysql -- go run ./cmd/bbgo backtest --config config/dca.yaml  --sync --sync-only --verify
+
+echo "backtest sqlite sync..."
+dotenv -f .env.local.sqlite -- go run ./cmd/bbgo backtest --config config/dca.yaml  --sync --sync-only --verify