diff --git a/pkg/cmd/trades.go b/pkg/cmd/trades.go new file mode 100644 index 000000000..162b28c00 --- /dev/null +++ b/pkg/cmd/trades.go @@ -0,0 +1,98 @@ +package cmd + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/types" +) + +func init() { + tradesCmd.Flags().String("session", "", "the exchange session name for querying balances") + tradesCmd.Flags().String("symbol", "", "the trading pair, like btcusdt") + RootCmd.AddCommand(tradesCmd) +} + +// go run ./cmd/bbgo tradesCmd --session=ftx --symbol="BTC/USD" +var tradesCmd = &cobra.Command{ + Use: "trades", + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := context.Background() + + configFile, err := cmd.Flags().GetString("config") + if err != nil { + return err + } + + if len(configFile) == 0 { + return errors.New("--config option is required") + } + + // if config file exists, use the config loaded from the config file. + // otherwise, use a empty config object + var userConfig *bbgo.Config + if _, err := os.Stat(configFile); err == nil { + // load successfully + userConfig, err = bbgo.Load(configFile, false) + if err != nil { + return err + } + } else if os.IsNotExist(err) { + // config file doesn't exist + userConfig = &bbgo.Config{} + } else { + // other error + return err + } + + environ := bbgo.NewEnvironment() + + if err := environ.ConfigureExchangeSessions(userConfig); err != nil { + return err + } + + sessionName, err := cmd.Flags().GetString("session") + if err != nil { + return err + } + + session, ok := environ.Session(sessionName) + if !ok { + return fmt.Errorf("session %s not found", sessionName) + } + + symbol, err := cmd.Flags().GetString("symbol") + if err != nil { + return fmt.Errorf("can't get the symbol from flags: %w", err) + } + if symbol == "" { + return fmt.Errorf("symbol is not found") + } + + until := time.Now() + since := until.Add(-3 * 24 * time.Hour) + trades, err := session.Exchange.QueryTrades(ctx, symbol, &types.TradeQueryOptions{ + StartTime: &since, + EndTime: &until, + Limit: 100, + LastTradeID: 0, + }) + if err != nil { + return err + } + + log.Infof("%d trades", len(trades)) + for _, t := range trades { + log.Infof("trade: %+v", t) + } + return nil + }, +} diff --git a/pkg/exchange/ftx/convert.go b/pkg/exchange/ftx/convert.go index d4b46e0c0..72f58bab4 100644 --- a/pkg/exchange/ftx/convert.go +++ b/pkg/exchange/ftx/convert.go @@ -48,8 +48,8 @@ func toGlobalOrder(r order) (types.Order, error) { OrderID: uint64(r.ID), Status: "", ExecutedQuantity: r.FilledSize, - CreationTime: datatype.Time(r.CreatedAt), - UpdateTime: datatype.Time(r.CreatedAt), + CreationTime: datatype.Time(r.CreatedAt.Time), + UpdateTime: datatype.Time(r.CreatedAt.Time), } // `new` (accepted but not processed yet), `open`, or `closed` (filled or cancelled) @@ -83,13 +83,13 @@ func toGlobalDeposit(input depositHistory) (types.Deposit, error) { log.WithError(err).Warnf("assign empty string to the deposit status") } t := input.Time - if input.ConfirmedTime != (time.Time{}) { + if input.ConfirmedTime.Time != (time.Time{}) { t = input.ConfirmedTime } d := types.Deposit{ GID: 0, Exchange: types.ExchangeFTX, - Time: datatype.Time(t), + Time: datatype.Time(t.Time), Amount: input.Size, Asset: toGlobalCurrency(input.Coin), TransactionID: input.TxID, @@ -108,3 +108,24 @@ func toGlobalDepositStatus(input string) (types.DepositStatus, error) { } return "", fmt.Errorf("unsupported status %s", input) } + +func toGlobalTrade(f fill) (types.Trade, error) { + return types.Trade{ + ID: f.TradeId, + GID: 0, + OrderID: f.OrderId, + Exchange: types.ExchangeFTX.String(), + Price: f.Price, + Quantity: f.Size, + QuoteQuantity: f.Price * f.Size, + Symbol: toGlobalSymbol(f.Market), + Side: f.Side, + IsBuyer: f.Side == types.SideTypeBuy, + IsMaker: f.Liquidity == "maker", + Time: datatype.Time(f.Time.Time), + Fee: f.Fee, + FeeCurrency: f.FeeCurrency, + IsMargin: false, + IsIsolated: false, + }, nil +} diff --git a/pkg/exchange/ftx/exchange.go b/pkg/exchange/ftx/exchange.go index ca5cd3265..352bf075f 100644 --- a/pkg/exchange/ftx/exchange.go +++ b/pkg/exchange/ftx/exchange.go @@ -6,6 +6,7 @@ import ( "net/http" "net/url" "sort" + "strings" "time" "github.com/sirupsen/logrus" @@ -145,7 +146,70 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type } func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) ([]types.Trade, error) { - panic("implement me") + var since, until time.Time + if options.StartTime != nil { + since = *options.StartTime + } + if options.EndTime != nil { + until = *options.EndTime + } + if err := verifySinceUntil(since, until); err != nil { + return nil, err + } + if options.Limit == 1 { + // FTX doesn't provide pagination api, so we have to split the since/until time range into small slices, and paginate ourselves. + // If the limit is 1, we always get the same data from FTX. + return nil, fmt.Errorf("limit can't be 1 which can't be used in pagination") + } + limit := options.Limit + if limit == 0 { + limit = 200 + } + + tradeIDs := make(map[int64]struct{}) + + var lastTradeID int64 + var trades []types.Trade + symbol = strings.ToUpper(symbol) + + for since.Before(until) { + // DO not set limit to `1` since you will always get the same response. + resp, err := e.newRest().Fills(ctx, symbol, since, until, limit, true) + if err != nil { + return nil, err + } + if !resp.Success { + return nil, fmt.Errorf("ftx returns failure") + } + + sort.Slice(resp.Result, func(i, j int) bool { + return resp.Result[i].TradeId < resp.Result[j].TradeId + }) + + for _, r := range resp.Result { + if _, ok := tradeIDs[r.TradeId]; ok { + continue + } + if r.TradeId <= lastTradeID || r.Time.Before(since) || r.Time.After(until) || r.Market != symbol { + continue + } + tradeIDs[r.TradeId] = struct{}{} + lastTradeID = r.TradeId + since = r.Time.Time + + t, err := toGlobalTrade(r) + if err != nil { + return nil, err + } + trades = append(trades, t) + } + + if int64(len(resp.Result)) < limit { + return trades, nil + } + } + + return trades, nil } func (e *Exchange) QueryDepositHistory(ctx context.Context, asset string, since, until time.Time) (allDeposits []types.Deposit, err error) { @@ -158,8 +222,11 @@ func (e *Exchange) QueryDepositHistory(ctx context.Context, asset string, since, if err != nil { return nil, err } + if !resp.Success { + return nil, fmt.Errorf("ftx returns failure") + } sort.Slice(resp.Result, func(i, j int) bool { - return resp.Result[i].Time.Before(resp.Result[j].Time) + return resp.Result[i].Time.Before(resp.Result[j].Time.Time) }) for _, r := range resp.Result { d, err := toGlobalDeposit(r) @@ -255,7 +322,7 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, for _, r := range resp.Result { // There may be more than one orders at the same time, so also have to check the ID - if r.CreatedAt.Before(lastOrder.CreatedAt) || r.ID == lastOrder.ID || r.Status != "closed" || r.ID < int64(lastOrderID) { + if r.CreatedAt.Before(lastOrder.CreatedAt.Time) || r.ID == lastOrder.ID || r.Status != "closed" || r.ID < int64(lastOrderID) { continue } lastOrder = r @@ -274,7 +341,7 @@ func (e *Exchange) QueryClosedOrders(ctx context.Context, symbol string, since, func sortByCreatedASC(orders []order) { sort.Slice(orders, func(i, j int) bool { - return orders[i].CreatedAt.Before(orders[j].CreatedAt) + return orders[i].CreatedAt.Before(orders[j].CreatedAt.Time) }) } diff --git a/pkg/exchange/ftx/exchange_test.go b/pkg/exchange/ftx/exchange_test.go index f91b3df1a..ddb5442b1 100644 --- a/pkg/exchange/ftx/exchange_test.go +++ b/pkg/exchange/ftx/exchange_test.go @@ -2,6 +2,8 @@ package ftx import ( "context" + "database/sql" + "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -484,3 +486,137 @@ func TestExchange_QueryDepositHistory(t *testing.T) { assert.NoError(t, err) assert.Len(t, dh, 0) } + +func TestExchange_QueryTrades(t *testing.T) { + t.Run("empty response", func(t *testing.T) { + respJSON := ` +{ + "success": true, + "result": [] +} +` + var f fillsResponse + assert.NoError(t, json.Unmarshal([]byte(respJSON), &f)) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, respJSON) + })) + defer ts.Close() + + ex := NewExchange("", "", "") + serverURL, err := url.Parse(ts.URL) + assert.NoError(t, err) + ex.restEndpoint = serverURL + + ctx := context.Background() + actualConfirmedTime, err := parseDatetime("2021-02-23T09:29:08.534000+00:00") + assert.NoError(t, err) + + since := actualConfirmedTime.Add(-1 * time.Hour) + until := actualConfirmedTime.Add(1 * time.Hour) + + // ignore unavailable market + trades, err := ex.QueryTrades(ctx, "TSLA/USD", &types.TradeQueryOptions{ + StartTime: &since, + EndTime: &until, + Limit: 0, + LastTradeID: 0, + }) + assert.NoError(t, err) + assert.Len(t, trades, 0) + }) + + t.Run("duplicated response", func(t *testing.T) { + respJSON := ` +{ + "success": true, + "result": [{ + "id": 123, + "market": "TSLA/USD", + "future": null, + "baseCurrency": "TSLA", + "quoteCurrency": "USD", + "type": "order", + "side": "sell", + "price": 672.5, + "size": 1.0, + "orderId": 456, + "time": "2021-02-23T09:29:08.534000+00:00", + "tradeId": 789, + "feeRate": -5e-6, + "fee": -0.0033625, + "feeCurrency": "USD", + "liquidity": "maker" +}, { + "id": 123, + "market": "TSLA/USD", + "future": null, + "baseCurrency": "TSLA", + "quoteCurrency": "USD", + "type": "order", + "side": "sell", + "price": 672.5, + "size": 1.0, + "orderId": 456, + "time": "2021-02-23T09:29:08.534000+00:00", + "tradeId": 789, + "feeRate": -5e-6, + "fee": -0.0033625, + "feeCurrency": "USD", + "liquidity": "maker" +}] +} +` + var f fillsResponse + assert.NoError(t, json.Unmarshal([]byte(respJSON), &f)) + i := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if i == 0 { + fmt.Fprintln(w, respJSON) + return + } + fmt.Fprintln(w, `{"success":true, "result":[]}`) + })) + defer ts.Close() + + ex := NewExchange("", "", "") + serverURL, err := url.Parse(ts.URL) + assert.NoError(t, err) + ex.restEndpoint = serverURL + + ctx := context.Background() + actualConfirmedTime, err := parseDatetime("2021-02-23T09:29:08.534000+00:00") + assert.NoError(t, err) + + since := actualConfirmedTime.Add(-1 * time.Hour) + until := actualConfirmedTime.Add(1 * time.Hour) + + // ignore unavailable market + trades, err := ex.QueryTrades(ctx, "TSLA/USD", &types.TradeQueryOptions{ + StartTime: &since, + EndTime: &until, + Limit: 0, + LastTradeID: 0, + }) + assert.NoError(t, err) + assert.Len(t, trades, 1) + assert.Equal(t, types.Trade{ + ID: 789, + OrderID: 456, + Exchange: types.ExchangeFTX.String(), + Price: 672.5, + Quantity: 1.0, + QuoteQuantity: 672.5 * 1.0, + Symbol: "TSLA/USD", + Side: types.SideTypeSell, + IsBuyer: false, + IsMaker: true, + Time: datatype.Time(actualConfirmedTime), + Fee: -0.0033625, + FeeCurrency: "USD", + IsMargin: false, + IsIsolated: false, + StrategyID: sql.NullString{}, + PnL: sql.NullFloat64{}, + }, trades[0]) + }) +} diff --git a/pkg/exchange/ftx/rest.go b/pkg/exchange/ftx/rest.go index 157b2427f..9e5fee8cd 100644 --- a/pkg/exchange/ftx/rest.go +++ b/pkg/exchange/ftx/rest.go @@ -23,6 +23,7 @@ type restRequest struct { *orderRequest *accountRequest *marketRequest + *fillsRequest key, secret string // Optional sub-account name @@ -49,6 +50,7 @@ func newRestRequest(c *http.Client, baseURL *url.URL) *restRequest { p: make(map[string]interface{}), } + r.fillsRequest = &fillsRequest{restRequest: r} r.marketRequest = &marketRequest{restRequest: r} r.accountRequest = &accountRequest{restRequest: r} r.walletRequest = &walletRequest{restRequest: r} diff --git a/pkg/exchange/ftx/rest_fills_request.go b/pkg/exchange/ftx/rest_fills_request.go new file mode 100644 index 000000000..b08fe3e65 --- /dev/null +++ b/pkg/exchange/ftx/rest_fills_request.go @@ -0,0 +1,50 @@ +package ftx + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "time" +) + +type fillsRequest struct { + *restRequest +} + +func (r *fillsRequest) Fills(ctx context.Context, market string, since, until time.Time, limit int64, orderByASC bool) (fillsResponse, error) { + q := make(map[string]string) + if len(market) > 0 { + q["market"] = market + } + if since != (time.Time{}) { + q["start_time"] = strconv.FormatInt(since.Unix(), 10) + } + if until != (time.Time{}) { + q["end_time"] = strconv.FormatInt(until.Unix(), 10) + } + if limit > 0 { + q["limit"] = strconv.FormatInt(limit, 10) + } + // default is descending + if orderByASC { + q["order"] = "asc" + } + resp, err := r. + Method("GET"). + ReferenceURL("api/fills"). + Query(q). + DoAuthenticatedRequest(ctx) + + if err != nil { + return fillsResponse{}, err + } + + var f fillsResponse + if err := json.Unmarshal(resp.Body, &f); err != nil { + fmt.Println("??? => ", resp.Body) + return fillsResponse{}, fmt.Errorf("failed to unmarshal fills response body to json: %w", err) + } + + return f, nil +} diff --git a/pkg/exchange/ftx/rest_responses.go b/pkg/exchange/ftx/rest_responses.go index f929de9ce..b4b1ab863 100644 --- a/pkg/exchange/ftx/rest_responses.go +++ b/pkg/exchange/ftx/rest_responses.go @@ -1,6 +1,37 @@ package ftx -import "time" +import ( + "strings" + "time" + + "github.com/c9s/bbgo/pkg/types" +) + +//ex: 2019-03-05T09:56:55.728933+00:00 +const timeLayout = "2006-01-02T15:04:05.999999Z07:00" + +type datetime struct { + time.Time +} + +func parseDatetime(s string) (time.Time, error) { + return time.Parse(timeLayout, s) +} + +func (d *datetime) UnmarshalJSON(b []byte) error { + // remove double quote from json string + s := strings.Trim(string(b), "\"") + if len(s) == 0 { + d.Time = time.Time{} + return nil + } + t, err := parseDatetime(s) + if err != nil { + return err + } + d.Time = t + return nil +} /* { @@ -176,8 +207,8 @@ type cancelOrderResponse struct { } type order struct { - CreatedAt time.Time `json:"createdAt"` - FilledSize float64 `json:"filledSize"` + CreatedAt datetime `json:"createdAt"` + FilledSize float64 `json:"filledSize"` // Future field is not defined in the response format table but in the response example. Future string `json:"future"` ID int64 `json:"id"` @@ -226,18 +257,18 @@ type depositHistoryResponse struct { } type depositHistory struct { - ID int64 `json:"id"` - Coin string `json:"coin"` - TxID string `json:"txid"` - Address address `json:"address"` - Confirmations int64 `json:"confirmations"` - ConfirmedTime time.Time `json:"confirmedTime"` - Fee float64 `json:"fee"` - SentTime time.Time `json:"sentTime"` - Size float64 `json:"size"` - Status string `json:"status"` - Time time.Time `json:"time"` - Notes string `json:"notes"` + ID int64 `json:"id"` + Coin string `json:"coin"` + TxID string `json:"txid"` + Address address `json:"address"` + Confirmations int64 `json:"confirmations"` + ConfirmedTime datetime `json:"confirmedTime"` + Fee float64 `json:"fee"` + SentTime datetime `json:"sentTime"` + Size float64 `json:"size"` + Status string `json:"status"` + Time datetime `json:"time"` + Notes string `json:"notes"` } /** @@ -254,3 +285,47 @@ type address struct { Method string `json:"method"` Coin string `json:"coin"` } + +type fillsResponse struct { + Success bool `json:"success"` + Result []fill `json:"result"` +} + +/* +{ + "id": 123, + "market": "TSLA/USD", + "future": null, + "baseCurrency": "TSLA", + "quoteCurrency": "USD", + "type": "order", + "side": "sell", + "price": 672.5, + "size": 1.0, + "orderId": 456, + "time": "2021-02-23T09:29:08.534000+00:00", + "tradeId": 789, + "feeRate": -5e-6, + "fee": -0.0033625, + "feeCurrency": "USD", + "liquidity": "maker" +} +*/ +type fill struct { + ID int64 `json:"id"` + Market string `json:"market"` + Future string `json:"future"` + BaseCurrency string `json:"baseCurrency"` + QuoteCurrency string `json:"quoteCurrency"` + Type string `json:"type"` + Side types.SideType `json:"side"` + Price float64 `json:"price"` + Size float64 `json:"size"` + OrderId uint64 `json:"orderId"` + Time datetime `json:"time"` + TradeId int64 `json:"tradeId"` + FeeRate float64 `json:"feeRate"` + Fee float64 `json:"fee"` + FeeCurrency string `json:"feeCurrency"` + Liquidity string `json:"liquidity"` +}