From ea7b501c268c170d2f7e2d757a9ef0bfaccf0984 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 11 Oct 2020 20:08:54 +0800 Subject: [PATCH 1/2] add transfer history command for calculating baseline and show transfer records --- cmd/pnl.go | 10 +- cmd/transfers.go | 172 +++++++++++++++++++++++++++++ pkg/exchange/binance/exchange.go | 26 ++--- pkg/exchange/max/exchange.go | 124 ++++++++++++++++++--- pkg/exchange/max/maxapi/account.go | 90 ++++++++++++++- pkg/types/deposit.go | 20 ++-- pkg/types/exchange.go | 6 + pkg/types/withdraw.go | 22 ++++ 8 files changed, 418 insertions(+), 52 deletions(-) create mode 100644 cmd/transfers.go create mode 100644 pkg/types/withdraw.go diff --git a/cmd/pnl.go b/cmd/pnl.go index 082695354..ae7e120fd 100644 --- a/cmd/pnl.go +++ b/cmd/pnl.go @@ -20,10 +20,10 @@ import ( ) func init() { - PnLCmd.Flags().String("exchange", "", "target exchange") - PnLCmd.Flags().String("symbol", "BTCUSDT", "trading symbol") - PnLCmd.Flags().String("since", "", "pnl since time") - RootCmd.AddCommand(PnLCmd) + pnlCmd.Flags().String("exchange", "", "target exchange") + pnlCmd.Flags().String("symbol", "BTCUSDT", "trading symbol") + pnlCmd.Flags().String("since", "", "pnl since time") + RootCmd.AddCommand(pnlCmd) } func connectMysql() (*sqlx.DB, error) { @@ -50,7 +50,7 @@ func newExchange(n types.ExchangeName) types.Exchange { return nil } -var PnLCmd = &cobra.Command{ +var pnlCmd = &cobra.Command{ Use: "pnl", Short: "pnl calculator", SilenceUsage: true, diff --git a/cmd/transfers.go b/cmd/transfers.go new file mode 100644 index 000000000..b541b1aea --- /dev/null +++ b/cmd/transfers.go @@ -0,0 +1,172 @@ +package cmd + +import ( + "context" + "sort" + "time" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/c9s/bbgo/pkg/types" +) + +func init() { + transferHistoryCmd.Flags().String("exchange", "", "target exchange") + transferHistoryCmd.Flags().String("asset", "BTC", "trading symbol") + transferHistoryCmd.Flags().String("since", "", "since time") + RootCmd.AddCommand(transferHistoryCmd) +} + + + +type TimeRecord struct { + Record interface{} + Time time.Time +} + +type timeSlice []TimeRecord + +func (p timeSlice) Len() int { + return len(p) +} + +func (p timeSlice) Less(i, j int) bool { + return p[i].Time.Before(p[j].Time) +} + +func (p timeSlice) Swap(i, j int) { + p[i], p[j] = p[j], p[i] +} + + + + + +var transferHistoryCmd = &cobra.Command{ + Use: "transfer-history", + Short: "show transfer history", + + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + _ = ctx + + exchangeNameStr, err := cmd.Flags().GetString("exchange") + if err != nil { + return err + } + + exchangeName, err := types.ValidExchangeName(exchangeNameStr) + if err != nil { + return err + } + + asset, err := cmd.Flags().GetString("asset") + if err != nil { + return err + } + + // default + var now = time.Now() + var since = now.AddDate(-1, 0, 0) + var until = now + + sinceStr, err := cmd.Flags().GetString("since") + if err != nil { + return err + } + + if len(sinceStr) > 0 { + loc, err := time.LoadLocation("Asia/Taipei") + if err != nil { + return err + } + + since, err = time.ParseInLocation("2006-01-02", sinceStr, loc) + if err != nil { + return err + } + } + + + exchange := newExchange(exchangeName) + + var records timeSlice + + deposits, err := exchange.QueryDepositHistory(ctx, asset, since, until) + if err != nil { + return err + } + for _, d := range deposits { + records = append(records, TimeRecord{ + Record: d, + Time: d.EffectiveTime(), + }) + } + + withdraws, err := exchange.QueryWithdrawHistory(ctx, asset, since, until) + if err != nil { + return err + } + for _, w := range withdraws { + records = append(records, TimeRecord{ + Record: w, + Time: w.EffectiveTime(), + }) + } + + sort.Sort(records) + + for _, record := range records { + switch record := record.Record.(type) { + + case types.Deposit: + log.Infof("%s: %s <== (deposit) %f [%s]", record.Time, record.Asset, record.Amount, record.Status) + + case types.Withdraw: + log.Infof("%s: %s ==> (withdraw) %f [%s]", record.ApplyTime, record.Asset, record.Amount, record.Status) + + default: + log.Infof("unknown record: %+v", record) + + } + } + + stats := calBaselineStats(asset, deposits, withdraws) + log.Infof("total %s deposit: %f (x %d)", asset, stats.TotalDeposit, stats.NumOfDeposit) + log.Infof("total %s withdraw: %f (x %d)", asset, stats.TotalWithdraw, stats.NumOfWithdraw) + log.Infof("baseline %s balance: %f", asset, stats.BaselineBalance) + return nil + }, +} + +type BaselineStats struct { + Asset string + NumOfDeposit int + NumOfWithdraw int + TotalDeposit float64 + TotalWithdraw float64 + BaselineBalance float64 +} + +func calBaselineStats(asset string, deposits []types.Deposit, withdraws []types.Withdraw) (stats BaselineStats) { + stats.Asset = asset + stats.NumOfDeposit = len(deposits) + stats.NumOfWithdraw = len(withdraws) + + for _, deposit := range deposits { + if deposit.Status == types.DepositSuccess { + stats.TotalDeposit += deposit.Amount + } + } + + for _, withdraw := range withdraws { + if withdraw.Status == "completed" { + stats.TotalWithdraw += withdraw.Amount + } + } + + stats.BaselineBalance = stats.TotalDeposit - stats.TotalWithdraw + return stats +} diff --git a/pkg/exchange/binance/exchange.go b/pkg/exchange/binance/exchange.go index 06110b945..23921cbc8 100644 --- a/pkg/exchange/binance/exchange.go +++ b/pkg/exchange/binance/exchange.go @@ -33,6 +33,10 @@ func New(key, secret string) *Exchange { } } +func (e *Exchange) Name() types.ExchangeName { + return types.ExchangeBinance +} + func (e *Exchange) QueryAveragePrice(ctx context.Context, symbol string) (float64, error) { resp, err := e.Client.NewAveragePriceService().Symbol(symbol).Do(ctx) if err != nil { @@ -46,22 +50,7 @@ func (e *Exchange) NewStream() types.Stream { return NewStream(e.Client) } -type Withdraw struct { - ID string `json:"id"` - Asset string `json:"asset"` - Amount float64 `json:"amount"` - Address string `json:"address"` - AddressTag string `json:"addressTag"` - Status string `json:"status"` - - TransactionID string `json:"txId"` - TransactionFee float64 `json:"transactionFee"` - WithdrawOrderID string `json:"withdrawOrderId"` - ApplyTime time.Time `json:"applyTime"` - Network string `json:"network"` -} - -func (e *Exchange) QueryWithdrawHistory(ctx context.Context, asset string, since, until time.Time) (allWithdraws []Withdraw, err error) { +func (e *Exchange) QueryWithdrawHistory(ctx context.Context, asset string, since, until time.Time) (allWithdraws []types.Withdraw, err error) { startTime := since txIDs := map[string]struct{}{} @@ -80,7 +69,7 @@ func (e *Exchange) QueryWithdrawHistory(ctx context.Context, asset string, since Do(ctx) if err != nil { - return nil, err + return allWithdraws, err } for _, d := range withdraws { @@ -88,7 +77,6 @@ func (e *Exchange) QueryWithdrawHistory(ctx context.Context, asset string, since continue } - // 0(0:pending,6: credited but cannot withdraw, 1:success) status := "" switch d.Status { case 0: @@ -111,7 +99,7 @@ func (e *Exchange) QueryWithdrawHistory(ctx context.Context, asset string, since } txIDs[d.TxID] = struct{}{} - allWithdraws = append(allWithdraws, Withdraw{ + allWithdraws = append(allWithdraws, types.Withdraw{ ApplyTime: time.Unix(0, d.ApplyTime*int64(time.Millisecond)), Asset: d.Asset, Amount: d.Amount, diff --git a/pkg/exchange/max/exchange.go b/pkg/exchange/max/exchange.go index 539cd2d91..927a746f5 100644 --- a/pkg/exchange/max/exchange.go +++ b/pkg/exchange/max/exchange.go @@ -30,6 +30,10 @@ func New(key, secret string) *Exchange { } } +func (e *Exchange) Name() types.ExchangeName { + return types.ExchangeMax +} + func (e *Exchange) NewStream() types.Stream { return NewStream(e.key, e.secret) } @@ -83,26 +87,110 @@ func (e *Exchange) QueryAccount(ctx context.Context) (*types.Account, error) { }, nil } -func (e *Exchange) QueryDepositHistory(ctx context.Context, asset string, since, until time.Time) (allDeposits []types.Deposit, err error) { - deposits, err := e.client.AccountService.NewGetDepositHistoryRequest(). - Currency(asset). - From(since.Unix()). - To(until.Unix()).Do(ctx) +func (e *Exchange) QueryWithdrawHistory(ctx context.Context, asset string, since, until time.Time) (allWithdraws []types.Withdraw, err error) { + startTime := since + txIDs := map[string]struct{}{} - if err != nil { - return nil, err + for startTime.Before(until) { + // startTime ~ endTime must be in 90 days + endTime := startTime.AddDate(0, 0, 60) + if endTime.After(until) { + endTime = until + } + + log.Infof("querying withdraw %s: %s <=> %s", asset, startTime, endTime) + withdraws, err := e.client.AccountService.NewGetWithdrawalHistoryRequest(). + Currency(toLocalCurrency(asset)). + From(startTime.Unix()). + To(endTime.Unix()). + Do(ctx) + + if err != nil { + return allWithdraws, err + } + + for _, d := range withdraws { + if _, ok := txIDs[d.TxID]; ok { + continue + } + + // we can convert this later + status := d.State + switch d.State { + + case "confirmed": + status = "completed" // make it compatible with binance + + case "submitting", "submitted", "accepted", + "rejected", "suspect", "approved", "delisted_processing", + "processing", "retryable", "sent", "canceled", + "failed", "pending", + "kgi_manually_processing", "kgi_manually_confirmed", "kgi_possible_failed", + "sygna_verifying": + + default: + status = d.State + + } + + txIDs[d.TxID] = struct{}{} + allWithdraws = append(allWithdraws, types.Withdraw{ + ApplyTime: time.Unix(d.CreatedAt, 0), + Asset: toGlobalCurrency(d.Currency), + Amount: util.MustParseFloat(d.Amount), + Address: "", + AddressTag: "", + TransactionID: d.TxID, + TransactionFee: util.MustParseFloat(d.Fee), + // WithdrawOrderID: d.WithdrawOrderID, + // Network: d.Network, + Status: status, + }) + } + + startTime = endTime } - for _, d := range deposits { - allDeposits = append(allDeposits, types.Deposit{ - Time: time.Unix(d.CreatedAt, 0), - Amount: util.MustParseFloat(d.Amount), - Asset: d.Currency, - Address: "", // not supported - AddressTag: "", // not supported - TransactionID: d.TxID, - Status: convertDepositState(d.State), - }) + return allWithdraws, nil +} + +func (e *Exchange) QueryDepositHistory(ctx context.Context, asset string, since, until time.Time) (allDeposits []types.Deposit, err error) { + startTime := since + txIDs := map[string]struct{}{} + for startTime.Before(until) { + // startTime ~ endTime must be in 90 days + endTime := startTime.AddDate(0, 0, 60) + if endTime.After(until) { + endTime = until + } + + log.Infof("querying deposit history %s: %s <=> %s", asset, startTime, endTime) + deposits, err := e.client.AccountService.NewGetDepositHistoryRequest(). + Currency(toLocalCurrency(asset)). + From(startTime.Unix()). + To(endTime.Unix()).Do(ctx) + + if err != nil { + return nil, err + } + + for _, d := range deposits { + if _, ok := txIDs[d.TxID]; ok { + continue + } + + allDeposits = append(allDeposits, types.Deposit{ + Time: time.Unix(d.CreatedAt, 0), + Amount: util.MustParseFloat(d.Amount), + Asset: toGlobalCurrency(d.Currency), + Address: "", // not supported + AddressTag: "", // not supported + TransactionID: d.TxID, + Status: convertDepositState(d.State), + }) + } + + startTime = endTime } return allDeposits, err @@ -119,7 +207,7 @@ func convertDepositState(a string) types.DepositStatus { case "rejected": return types.DepositRejected - case "cancelled": + case "canceled": return types.DepositCancelled case "suspect", "refunded": diff --git a/pkg/exchange/max/maxapi/account.go b/pkg/exchange/max/maxapi/account.go index 0c61960bd..7977f3502 100644 --- a/pkg/exchange/max/maxapi/account.go +++ b/pkg/exchange/max/maxapi/account.go @@ -118,7 +118,7 @@ type Deposit struct { Fee string `json:"fee"` TxID string `json:"txid"` State string `json:"state"` - Confirmations int `json:"confirmations"` + Confirmations string `json:"confirmations"` CreatedAt int64 `json:"created_at"` UpdatedAt int64 `json:"updated_at"` } @@ -129,7 +129,7 @@ type GetDepositHistoryRequestParams struct { Currency string `json:"currency"` From int64 `json:"from,omitempty"` // seconds To int64 `json:"to,omitempty"` // seconds - State string `json:"state,omitempty"` // submitting, submitted, rejected, accepted, checking, refunded, cancelled, suspect + State string `json:"state,omitempty"` // submitting, submitted, rejected, accepted, checking, refunded, canceled, suspect Limit int `json:"limit,omitempty"` } @@ -186,3 +186,89 @@ func (s *AccountService) NewGetDepositHistoryRequest() *GetDepositHistoryRequest client: s.client, } } + + + +type Withdraw struct { + UUID string `json:"uuid"` + Currency string `json:"currency"` + CurrencyVersion string `json:"currency_version"` // "eth" + Amount string `json:"amount"` + Fee string `json:"fee"` + TxID string `json:"txid"` + + // State can be "submitting", "submitted", + // "rejected", "accepted", "suspect", "approved", "delisted_processing", + // "processing", "retryable", "sent", "canceled", + // "failed", "pending", "confirmed", + // "kgi_manually_processing", "kgi_manually_confirmed", "kgi_possible_failed", + // "sygna_verifying" + State string `json:"state"` + Confirmations int `json:"confirmations"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` +} + +type GetWithdrawHistoryRequestParams struct { + *PrivateRequestParams + + Currency string `json:"currency"` + From int64 `json:"from,omitempty"` // seconds + To int64 `json:"to,omitempty"` // seconds + State string `json:"state,omitempty"` // submitting, submitted, rejected, accepted, checking, refunded, canceled, suspect + Limit int `json:"limit,omitempty"` +} + +type GetWithdrawHistoryRequest struct { + client *RestClient + params GetWithdrawHistoryRequestParams +} + +func (r *GetWithdrawHistoryRequest) State(state string) *GetWithdrawHistoryRequest { + r.params.State = state + return r +} + +func (r *GetWithdrawHistoryRequest) Currency(currency string) *GetWithdrawHistoryRequest { + r.params.Currency = currency + return r +} + +func (r *GetWithdrawHistoryRequest) Limit(limit int) *GetWithdrawHistoryRequest { + r.params.Limit = limit + return r +} + +func (r *GetWithdrawHistoryRequest) From(from int64) *GetWithdrawHistoryRequest { + r.params.From = from + return r +} + +func (r *GetWithdrawHistoryRequest) To(to int64) *GetWithdrawHistoryRequest { + r.params.To = to + return r +} + +func (r *GetWithdrawHistoryRequest) Do(ctx context.Context) (withdraws []Withdraw, err error) { + req, err := r.client.newAuthenticatedRequest("GET", "v2/withdrawals", &r.params) + if err != nil { + return withdraws, err + } + + response, err := r.client.sendRequest(req) + if err != nil { + return withdraws, err + } + + if err := response.DecodeJSON(&withdraws); err != nil { + return withdraws, err + } + + return withdraws, err +} + +func (s *AccountService) NewGetWithdrawalHistoryRequest() *GetWithdrawHistoryRequest { + return &GetWithdrawHistoryRequest{ + client: s.client, + } +} diff --git a/pkg/types/deposit.go b/pkg/types/deposit.go index 0034bc834..e75117f99 100644 --- a/pkg/types/deposit.go +++ b/pkg/types/deposit.go @@ -13,18 +13,22 @@ const ( DepositSuccess = DepositStatus("success") - DepositCancelled = DepositStatus("cancelled") + DepositCancelled = DepositStatus("canceled") // created but can not withdraw DepositCredited = DepositStatus("credited") ) type Deposit struct { - Time time.Time `json:"time"` - Amount float64 `json:"amount"` - Asset string `json:"asset"` - Address string `json:"address"` - AddressTag string `json:"addressTag"` - TransactionID string `json:"txId"` - Status DepositStatus `json:"status"` + Time time.Time `json:"time"` + Amount float64 `json:"amount"` + Asset string `json:"asset"` + Address string `json:"address"` + AddressTag string `json:"addressTag"` + TransactionID string `json:"txId"` + Status DepositStatus `json:"status"` +} + +func (d Deposit) EffectiveTime() time.Time { + return d.Time } diff --git a/pkg/types/exchange.go b/pkg/types/exchange.go index 49d6b21ed..55cd12b20 100644 --- a/pkg/types/exchange.go +++ b/pkg/types/exchange.go @@ -28,6 +28,8 @@ func ValidExchangeName(a string) (ExchangeName, error) { } type Exchange interface { + Name() ExchangeName + PlatformFeeCurrency() string NewStream() Stream @@ -42,6 +44,10 @@ type Exchange interface { QueryAveragePrice(ctx context.Context, symbol string) (float64, error) + QueryDepositHistory(ctx context.Context, asset string, since, until time.Time) (allDeposits []Deposit, err error) + + QueryWithdrawHistory(ctx context.Context, asset string, since, until time.Time) (allWithdraws []Withdraw, err error) + SubmitOrder(ctx context.Context, order *SubmitOrder) error } diff --git a/pkg/types/withdraw.go b/pkg/types/withdraw.go new file mode 100644 index 000000000..803d585a6 --- /dev/null +++ b/pkg/types/withdraw.go @@ -0,0 +1,22 @@ +package types + +import "time" + +type Withdraw struct { + ID string `json:"id"` + Asset string `json:"asset"` + Amount float64 `json:"amount"` + Address string `json:"address"` + AddressTag string `json:"addressTag"` + Status string `json:"status"` + + TransactionID string `json:"txId"` + TransactionFee float64 `json:"transactionFee"` + WithdrawOrderID string `json:"withdrawOrderId"` + ApplyTime time.Time `json:"applyTime"` + Network string `json:"network"` +} + +func (w Withdraw) EffectiveTime() time.Time { + return w.ApplyTime +} From 3d625fa738d875358b6a728264b4bed005166cc1 Mon Sep 17 00:00:00 2001 From: c9s Date: Sun, 11 Oct 2020 20:11:22 +0800 Subject: [PATCH 2/2] doc: add command usages --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index f69bf3703..fee3c8f35 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,20 @@ dotenv -f .env.local -- bbgo migrate redo (It internally uses `goose` to run these migration files, see [migrations](migrations)) + +To query transfer history: + +```sh +dotenv -f .env.local -- bbgo transfer-history --exchange max --asset USDT --since "2019-01-01" +``` + +To calculate pnl: + +```sh +dotenv -f .env.local -- bbgo pnl --exchange binance --asset BTC --since "2019-01-01" +``` + + ## Examples Please check out the example directory: [examples](examples)