diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index b70133c9c..e5e203c08 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -2,7 +2,7 @@ ## Our Pledge -We as members, contributors, and leaders pledge to make participation in our +We as members, contributors, and leaders pledge to participate in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, @@ -15,7 +15,7 @@ diverse, inclusive, and healthy community. ## Our Standards Examples of behavior that contributes to a positive environment for our -community include: +community includes: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences @@ -33,7 +33,7 @@ Examples of unacceptable behavior include: * Public or private harassment * Publishing others' private information, such as a physical or email address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a +* Other conduct that could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities @@ -50,7 +50,7 @@ decisions when appropriate. ## Scope -This Code of Conduct applies within all community spaces, and also applies when +This Code of Conduct applies within all community spaces and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed @@ -82,12 +82,11 @@ behavior was inappropriate. A public apology may be requested. ### 2. Warning -**Community Impact**: A violation through a single incident or series -of actions. +**Community Impact**: This violation occurs through a single incident or a series of actions. **Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This +those enforcing the Code of Conduct, for a specified period. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. @@ -98,7 +97,7 @@ permanent ban. sustained inappropriate behavior. **Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or +communication with the community for a specified period. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c92962cdc..f25a27df0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,19 +31,19 @@ Install pre-commit to check your changes before you commit: See for more details. -For new large features, such as integrating binance futures contracts, please propose a discussion first before you start working on it. +For new large features, such as integrating Binance futures contracts, please propose a discussion first before you start working on it. For new small features, you could open a pull request directly. For each contributor, you have chance to receive the BBG token through the polygon network. -Each issue has its BBG label, by completing the issue with a pull request, you can get correspond amount of BBG. +Each issue has its BBG label, by completing the issue with a pull request, you can get corresponding amount of BBG. ## Support ### By contributing pull requests -Any pull request is welcome, documentation, format fixing, testing, features. +Any pull request is welcome, documentation, format fixing, testing, and features. ### By registering account with referral ID @@ -52,7 +52,7 @@ You may register your exchange account with my referral ID to support this proje - For MAX Exchange: (default commission rate to your account) - For Binance Exchange: (5% commission back to your account) -### By small amount cryptos +### By small amount of cryptos - BTC address `3J6XQJNWT56amqz9Hz2BEVQ7W4aNmb5kiU` - USDT ERC20 address `0xeBcf7887A5b767DEb2e0C77E46A22c6Adc64E427` diff --git a/README.md b/README.md index b947f5aca..762f0b082 100644 --- a/README.md +++ b/README.md @@ -30,14 +30,14 @@ You can use BBGO's trading unit and back-test unit to implement your own strateg ### Trading Unit Developers 🧑‍💻 -You can use BBGO's underlying common exchange API, currently it supports 4+ major exchanges, so you don't have to repeat +You can use BBGO's underlying common exchange API, currently, it supports 4+ major exchanges, so you don't have to repeat the implementation. ## Features - Exchange abstraction interface. -- Stream integration (user data websocket, market data websocket). -- Real-time orderBook integration through websocket. +- Stream integration (user data web socket, market data web socket). +- Real-time orderBook integration through a web socket. - TWAP order execution support. See [TWAP Order Execution](./doc/topics/twap.md) - PnL calculation. - Slack/Telegram notification. @@ -101,25 +101,25 @@ the implementation. | xnav | this strategy helps you record the current net asset value | tool | no | | xalign | this strategy aligns your balance position automatically | tool | no | | xfunding | a funding rate fee strategy | funding | no | -| autoborrow | this strategy uses margin to borrow assets, to help you keep the minimal balance | tool | no | -| pivotshort | this strategy finds the pivot low and entry the trade when the price breaks the previous low | long/short | | +| autoborrow | this strategy uses margin to borrow assets, to help you keep a minimal balance | tool | no | +| pivotshort | this strategy finds the pivot low and enters the trade when the price breaks the previous low | long/short | | | schedule | this strategy buy/sell with a fixed quantity periodically, you can use this as a single DCA, or to refill the fee asset like BNB. | tool | | irr | this strategy opens the position based on the predicated return rate | long/short | | -| bollmaker | this strategy holds a long-term long/short position, places maker orders on both side, uses bollinger band to control the position size | maker | | -| wall | this strategy creates wall (large amount order) on the order book | maker | no | -| scmaker | this market making strategy is desgiend for stable coin markets, like USDC/USDT | maker | | +| bollmaker | this strategy holds a long-term long/short position, places maker orders on both sides, and uses a bollinger band to control the position size | maker | | +| wall | this strategy creates a wall (large amount of order) on the order book | maker | no | +| scmaker | this market making strategy is designed for stable coin markets, like USDC/USDT | maker | | | drift | | long/short | | -| rsicross | this strategy opens a long position when the fast rsi cross over the slow rsi, this is a demo strategy for using the v2 indicator | long/short | | +| rsicross | this strategy opens a long position when the fast rsi crosses over the slow rsi, this is a demo strategy for using the v2 indicator | long/short | | | marketcap | this strategy implements a strategy that rebalances the portfolio based on the market capitalization | rebalance | no | | supertrend | this strategy uses DEMA and Supertrend indicator to open the long/short position | long/short | | -| trendtrader | this strategy opens long/short position based on the trendline breakout | long/short | | +| trendtrader | this strategy opens a long/short position based on the trendline breakout | long/short | | | elliottwave | | long/short | | | ewoDgtrd | | long/short | | | fixedmaker | | maker | | | factoryzoo | | long/short | | | fmaker | | maker | | | linregmaker | a linear regression based market maker | maker | | -| convert | convert strategy is a tool that helps you convert specific asset to a target asset | tool | no | +| convert | convert strategy is a tool that helps you convert a specific asset to a target asset | tool | no | @@ -177,7 +177,7 @@ bash <(curl -s https://raw.githubusercontent.com/c9s/bbgo/main/scripts/download. Or refer to the [Release Page](https://github.com/c9s/bbgo/releases) and download manually. -Since v2, we've added new float point implementation from dnum to support decimals with higher precision. To download & +Since v2, we've added a new float point implementation from dnum to support decimals with higher precision. To download & setup, please refer to [Dnum Installation](doc/topics/dnum-binary.md) ### One-click Linode StackScript @@ -250,7 +250,7 @@ To start bbgo with the frontend dashboard: bbgo run --enable-webserver ``` -If you want to switch to other dotenv file, you can add an `--dotenv` option or `--config`: +If you want to switch to another dotenv file, you can add an `--dotenv` option or `--config`: ```sh bbgo sync --dotenv .env.dev --config config/grid.yaml --session binance @@ -292,7 +292,7 @@ You could also add the script to crontab so that the system time could get synch ### Testnet (Paper Trading) -Currently only supports binance testnet. To run bbgo in testnet, apply new API keys +Currently only supports Binance testnet. To run bbgo in testnet, apply new API keys from [Binance Test Network](https://testnet.binance.vision), and set the following env before you start bbgo: ```bash @@ -319,7 +319,7 @@ You can only use one database driver MySQL or SQLite to store your trading data. #### Configure MySQL Database -To use MySQL database for data syncing, first you need to install your mysql server: +To use MySQL database for data syncing, first, you need to install your MySQL server: ```sh # For Ubuntu Linux @@ -406,7 +406,7 @@ Check out the strategy directory [strategy](pkg/strategy) for all built-in strat - `drift` - drift strategy. - `grid2` - the second-generation grid strategy. -To run these built-in strategies, just modify the config file to make the configuration suitable for you, for example if +To run these built-in strategies, just modify the config file to make the configuration suitable for you, for example, if you want to run `buyandhold` strategy: @@ -427,7 +427,7 @@ See [Developing Strategy](./doc/topics/developing-strategy.md) ## Write your own private strategy -Create your go package, and initialize the repository with `go mod` and add bbgo as a dependency: +Create your go package, initialize the repository with `go mod`, and add bbgo as a dependency: ```sh go mod init @@ -488,7 +488,7 @@ See also: ## Command Usages -### Submitting Orders to a specific exchagne session +### Submitting Orders to a specific exchange session ```shell bbgo submit-order --session=okex --symbol=OKBUSDT --side=buy --price=10.0 --quantity=1 @@ -524,7 +524,7 @@ bbgo userdatastream --session binance In order to minimize the strategy code, bbgo supports dynamic dependency injection. -Before executing your strategy, bbgo injects the components into your strategy object if it found the embedded field +Before executing your strategy, bbgo injects the components into your strategy object if it finds the embedded field that is using bbgo component. for example: ```go @@ -550,7 +550,7 @@ following types could be injected automatically: 2. Allocate and initialize exchange sessions. 3. Add exchange sessions to the environment (the data layer). 4. Use the given environment to initialize the trader object (the logic layer). -5. The trader initializes the environment and start the exchange connections. +5. The trader initializes the environment and starts the exchange connections. 6. Call strategy.Run() method sequentially. ## Exchange API Examples @@ -567,7 +567,7 @@ maxRest := maxapi.NewRestClient(maxapi.ProductionAPIURL) maxRest.Auth(key, secret) ``` -Creating user data stream to get the orderbook (depth): +Creating user data stream to get the order book (depth): ```go stream := max.NewStream(key, secret) @@ -591,7 +591,7 @@ streambook.BindStream(stream) 1. Click the "Fork" button from the GitHub repository. 2. Clone your forked repository into `$GOPATH/github.com/c9s/bbgo`. -3. Change directory into `$GOPATH/github.com/c9s/bbgo`. +3. Change the directory to `$GOPATH/github.com/c9s/bbgo`. 4. Create a branch and start your development. 5. Test your changes. 6. Push your changes to your fork. @@ -616,13 +616,13 @@ make embed && go run -tags web ./cmd/bbgo-lorca ### What's Position? - Base Currency & Quote Currency -- How to calculate average cost? +- How to calculate the average cost? ### Looking For A New Strategy? -You can write an article about BBGO in any topic, in 750-1500 words for exchange, and I can implement the strategy for -you (depends on the complexity and efforts). If you're interested in, DM me in telegram or -twitter , we can discuss. +You can write an article about BBGO on any topic, in 750-1500 words for exchange, and I can implement the strategy for +you (depending on the complexity and effort). If you're interested in, DM me in telegram or +twitter , and we can discuss. ### Adding New Crypto Exchange support? diff --git a/config/rebalance.yaml b/config/rebalance.yaml index 14a784c93..bdcd5f6f5 100644 --- a/config/rebalance.yaml +++ b/config/rebalance.yaml @@ -12,9 +12,9 @@ backtest: startTime: "2022-01-01" endTime: "2022-10-01" symbols: - - BTCUSDT - - ETHUSDT - - MAXUSDT + - BTCUSDT + - ETHUSDT + - MAXUSDT account: max: makerFeeRate: 0.075% @@ -28,7 +28,7 @@ backtest: exchangeStrategies: - on: max rebalance: - interval: 1d + cronExpression: "@every 1s" quoteCurrency: USDT targetWeights: BTC: 50% @@ -37,5 +37,5 @@ exchangeStrategies: threshold: 1% maxAmount: 1_000 # max amount to buy or sell per order orderType: LIMIT_MAKER # LIMIT, LIMIT_MAKER or MARKET - dryRun: false + dryRun: true onStart: true diff --git a/config/wall.yaml b/config/wall.yaml index 628088281..2f6104d3c 100644 --- a/config/wall.yaml +++ b/config/wall.yaml @@ -10,6 +10,14 @@ sessions: exchange: max envVarPrefix: MAX + +logging: + trade: true + order: true + # fields: + # env: prod + + exchangeStrategies: - on: max @@ -33,6 +41,6 @@ exchangeStrategies: byLayer: linear: domain: [ 1, 3 ] - range: [ 10.0, 30.0 ] + range: [ 10000.0, 30000.0 ] diff --git a/config/xalign.yaml b/config/xalign.yaml index 51956f74f..6f07ba4b8 100644 --- a/config/xalign.yaml +++ b/config/xalign.yaml @@ -27,7 +27,6 @@ persistence: db: 0 crossExchangeStrategies: - - xalign: interval: 1m sessions: @@ -41,4 +40,10 @@ crossExchangeStrategies: sell: [USDT] expectedBalances: BTC: 0.0440 - + useTakerOrder: false + dryRun: true + balanceToleranceRange: 10% + maxAmounts: + USDT: 100 + USDC: 100 + TWD: 3000 diff --git a/doc/development/adding-new-exchange.md b/doc/development/adding-new-exchange.md index 6f0ae8075..1eb45a2ba 100644 --- a/doc/development/adding-new-exchange.md +++ b/doc/development/adding-new-exchange.md @@ -58,7 +58,7 @@ Stream - [ ] Public trade message parser (optional) - [ ] Ticker message parser (optional) - [ ] ping/pong handling. (you can reuse the existing types.StandardStream) -- [ ] heart-beat hanlding or keep-alive handling. (already included in types.StandardStream) +- [ ] heart-beat handling or keep-alive handling. (already included in types.StandardStream) - [ ] handling reconnect. (already included in types.StandardStream) Database diff --git a/doc/development/release-process.md b/doc/development/release-process.md index 89efb1b35..eaf33d8f8 100644 --- a/doc/development/release-process.md +++ b/doc/development/release-process.md @@ -40,7 +40,7 @@ Run the following command to create the release: make version VERSION=v1.20.2 ``` -The above command wilL: +The above command will: - Update and compile the migration scripts into go files. - Bump the version name in the go code. diff --git a/doc/development/series.md b/doc/development/series.md index 0f23756fc..ec9597377 100644 --- a/doc/development/series.md +++ b/doc/development/series.md @@ -25,7 +25,7 @@ type BoolSeries interface { } ``` -Series were used almost everywhere in indicators to return the calculated numeric results, but the use of BoolSeries is quite limited. At this moment, we only use BoolSeries to check if some condition is fullfilled at some timepoint. For example, in `CrossOver` and `CrossUnder` functions if `Last()` returns true, then there might be a cross event happend on the curves at the moment. +Series were used almost everywhere in indicators to return the calculated numeric results, but the use of BoolSeries is quite limited. At this moment, we only use BoolSeries to check if some condition is fulfilled at some timepoint. For example, in `CrossOver` and `CrossUnder` functions if `Last()` returns true, then there might be a cross event happened on the curves at the moment. #### Expected Implementation @@ -44,7 +44,7 @@ and if any of the method in the interface not been implemented, this would gener #### Extended Series -Instead of simple Series interface, we have `types.SeriesExtend` interface that enriches the functionality of `types.Series`. An indicator struct could simply be extended to `types.SeriesExtend` type by embedding anonymous struct `types.SeriesBase`, and instanced by `types.NewSeries()` function. The `types.SeriesExtend` interface binds commonly used functions, such as `Add`, `Reverse`, `Shfit`, `Covariance` and `Entropy`, to the original `types.Series` object. Please check [pkg/types/seriesbase_imp.go](../../pkg/types/seriesbase_imp.go) for the extendable functions. +Instead of simple Series interface, we have `types.SeriesExtend` interface that enriches the functionality of `types.Series`. An indicator struct could simply be extended to `types.SeriesExtend` type by embedding anonymous struct `types.SeriesBase`, and instanced by `types.NewSeries()` function. The `types.SeriesExtend` interface binds commonly used functions, such as `Add`, `Reverse`, `Shift`, `Covariance` and `Entropy`, to the original `types.Series` object. Please check [pkg/types/seriesbase_imp.go](../../pkg/types/seriesbase_imp.go) for the extendable functions. Example: diff --git a/go.mod b/go.mod index 4ebe211fa..91eca5f8b 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Masterminds/squirrel v1.5.3 github.com/adshao/go-binance/v2 v2.4.2 github.com/c-bata/goptuna v0.8.1 - github.com/c9s/requestgen v1.3.4 + github.com/c9s/requestgen v1.3.5 github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b github.com/cenkalti/backoff/v4 v4.2.0 github.com/cheggaaa/pb/v3 v3.0.8 diff --git a/go.sum b/go.sum index 448c0a214..710eba662 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,8 @@ github.com/c-bata/goptuna v0.8.1 h1:25+n1MLv0yvCsD56xv4nqIus3oLHL9GuPAZDLIqmX1U= github.com/c-bata/goptuna v0.8.1/go.mod h1:knmS8+Iyq5PPy1YUeIEq0pMFR4Y6x7z/CySc9HlZTCY= github.com/c9s/requestgen v1.3.4 h1:kK2rIO3OAt9JoY5gT0OSkSpq0dy/+JeuI22FwSKpUrY= github.com/c9s/requestgen v1.3.4/go.mod h1:wp4saiPdh0zLF5AkopGCqPQfy9Q5xvRh+TQBOA1l1r4= +github.com/c9s/requestgen v1.3.5 h1:iGYAP0rWQW3JOo+Z3S0SoenSt581IQ9mupJxRFCrCJs= +github.com/c9s/requestgen v1.3.5/go.mod h1:QwkZudcv84kJ8g9+E0RDTj+13btFXbTvv2aI+zbuLbc= github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b h1:wT8c03PHLv7+nZUIGqxAzRvIfYHNxMCNVWwvdGkOXTs= github.com/c9s/rockhopper v1.2.2-0.20220617053729-ffdc87df194b/go.mod h1:EKObf66Cp7erWxym2de+07qNN5T1N9PXxHdh97N44EQ= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= diff --git a/pkg/backtest/exchange.go b/pkg/backtest/exchange.go index 851c4df14..28a6b49a4 100644 --- a/pkg/backtest/exchange.go +++ b/pkg/backtest/exchange.go @@ -382,6 +382,14 @@ func (e *Exchange) SubscribeMarketData( } log.Infof("querying klines from database with exchange: %v symbols: %v and intervals: %v for back-testing", e.Name(), symbols, intervals) + if len(symbols) == 0 { + log.Warnf("empty symbols, will not query kline data from the database") + + c := make(chan types.KLine) + close(c) + return c, nil + } + klineC, errC := e.srv.QueryKLinesCh(startTime, endTime, e, symbols, intervals) go func() { if err := <-errC; err != nil { diff --git a/pkg/cmd/backtest.go b/pkg/cmd/backtest.go index f02b1aae0..2695c3fc9 100644 --- a/pkg/cmd/backtest.go +++ b/pkg/cmd/backtest.go @@ -527,8 +527,15 @@ var BacktestCmd = &cobra.Command{ for _, session := range environ.Sessions() { for symbol, trades := range session.Trades { - tradeStats := sessionTradeStats[session.Name][symbol] + if len(trades.Trades) == 0 { + log.Warnf("session has no %s trades", symbol) + continue + } + tradeState := sessionTradeStats[session.Name][symbol] + profitFactor := tradeState.ProfitFactor + winningRatio := tradeState.WinningRatio + intervalProfits := tradeState.IntervalProfits[types.Interval1d] symbolReport, err := createSymbolReport(userConfig, session, symbol, trades.Copy(), tradeStats) if err != nil { return err @@ -719,7 +726,11 @@ func createSymbolReport(userConfig *bbgo.Config, session *bbgo.ExchangeSession, func n(v float64) fixedpoint.Value { return fixedpoint.NewFromFloat(v) } -func verify(userConfig *bbgo.Config, backtestService *service.BacktestService, sourceExchanges map[types.ExchangeName]types.Exchange, startTime, endTime time.Time) error { + +func verify( + userConfig *bbgo.Config, backtestService *service.BacktestService, + sourceExchanges map[types.ExchangeName]types.Exchange, startTime, endTime time.Time, +) error { for _, sourceExchange := range sourceExchanges { err := backtestService.Verify(sourceExchange, userConfig.Backtest.Symbols, startTime, endTime) if err != nil { @@ -759,7 +770,10 @@ func getExchangeIntervals(ex types.Exchange) types.IntervalMap { return types.SupportedIntervals } -func sync(ctx context.Context, userConfig *bbgo.Config, backtestService *service.BacktestService, sourceExchanges map[types.ExchangeName]types.Exchange, syncFrom, syncTo time.Time) error { +func sync( + ctx context.Context, userConfig *bbgo.Config, backtestService *service.BacktestService, + sourceExchanges map[types.ExchangeName]types.Exchange, syncFrom, syncTo time.Time, +) error { for _, symbol := range userConfig.Backtest.Symbols { for _, sourceExchange := range sourceExchanges { var supportIntervals = getExchangeIntervals(sourceExchange) diff --git a/pkg/exchange/batch/closedorders.go b/pkg/exchange/batch/closedorders.go index 51d12f5ff..77e37690b 100644 --- a/pkg/exchange/batch/closedorders.go +++ b/pkg/exchange/batch/closedorders.go @@ -12,7 +12,7 @@ type ClosedOrderBatchQuery struct { types.ExchangeTradeHistoryService } -func (q *ClosedOrderBatchQuery) Query(ctx context.Context, symbol string, startTime, endTime time.Time, lastOrderID uint64) (c chan types.Order, errC chan error) { +func (q *ClosedOrderBatchQuery) Query(ctx context.Context, symbol string, startTime, endTime time.Time, lastOrderID uint64, opts ...Option) (c chan types.Order, errC chan error) { query := &AsyncTimeRangedBatchQuery{ Type: types.Order{}, Q: func(startTime, endTime time.Time) (interface{}, error) { @@ -32,6 +32,10 @@ func (q *ClosedOrderBatchQuery) Query(ctx context.Context, symbol string, startT JumpIfEmpty: 30 * 24 * time.Hour, } + for _, opt := range opts { + opt(query) + } + c = make(chan types.Order, 100) errC = query.Query(ctx, c, startTime, endTime) return c, errC diff --git a/pkg/exchange/batch/option.go b/pkg/exchange/batch/option.go new file mode 100644 index 000000000..67f18e608 --- /dev/null +++ b/pkg/exchange/batch/option.go @@ -0,0 +1,12 @@ +package batch + +import "time" + +type Option func(query *AsyncTimeRangedBatchQuery) + +// JumpIfEmpty jump the startTime + duration when the result is empty +func JumpIfEmpty(duration time.Duration) Option { + return func(query *AsyncTimeRangedBatchQuery) { + query.JumpIfEmpty = duration + } +} diff --git a/pkg/exchange/batch/trade.go b/pkg/exchange/batch/trade.go index 1c91da777..4fce26b65 100644 --- a/pkg/exchange/batch/trade.go +++ b/pkg/exchange/batch/trade.go @@ -17,7 +17,7 @@ type TradeBatchQuery struct { types.ExchangeTradeHistoryService } -func (e TradeBatchQuery) Query(ctx context.Context, symbol string, options *types.TradeQueryOptions) (c chan types.Trade, errC chan error) { +func (e TradeBatchQuery) Query(ctx context.Context, symbol string, options *types.TradeQueryOptions, opts ...Option) (c chan types.Trade, errC chan error) { if options.EndTime == nil { now := time.Now() options.EndTime = &now @@ -45,6 +45,10 @@ func (e TradeBatchQuery) Query(ctx context.Context, symbol string, options *type JumpIfEmpty: 24 * time.Hour, } + for _, opt := range opts { + opt(query) + } + c = make(chan types.Trade, 100) errC = query.Query(ctx, c, startTime, endTime) return c, errC diff --git a/pkg/exchange/bitget/bitgetapi/get_symbols_request.go b/pkg/exchange/bitget/bitgetapi/get_symbols_request.go index c3c4e64a9..48a202a2e 100644 --- a/pkg/exchange/bitget/bitgetapi/get_symbols_request.go +++ b/pkg/exchange/bitget/bitgetapi/get_symbols_request.go @@ -9,6 +9,17 @@ import ( "github.com/c9s/bbgo/pkg/fixedpoint" ) +type SymbolStatus string + +const ( + // SymbolOffline represent market is suspended, users cannot trade. + SymbolOffline SymbolStatus = "offline" + // SymbolGray represents market is online, but user trading is not available. + SymbolGray SymbolStatus = "gray" + // SymbolOnline trading begins, users can trade. + SymbolOnline SymbolStatus = "online" +) + type Symbol struct { Symbol string `json:"symbol"` SymbolName string `json:"symbolName"` @@ -18,10 +29,10 @@ type Symbol struct { MaxTradeAmount fixedpoint.Value `json:"maxTradeAmount"` TakerFeeRate fixedpoint.Value `json:"takerFeeRate"` MakerFeeRate fixedpoint.Value `json:"makerFeeRate"` - PriceScale int `json:"priceScale"` - QuantityScale int `json:"quantityScale"` + PriceScale fixedpoint.Value `json:"priceScale"` + QuantityScale fixedpoint.Value `json:"quantityScale"` MinTradeUSDT fixedpoint.Value `json:"minTradeUSDT"` - Status string `json:"status"` + Status SymbolStatus `json:"status"` BuyLimitPriceRatio fixedpoint.Value `json:"buyLimitPriceRatio"` SellLimitPriceRatio fixedpoint.Value `json:"sellLimitPriceRatio"` } diff --git a/pkg/exchange/bitget/convert.go b/pkg/exchange/bitget/convert.go index 0836b4dd0..1339089fd 100644 --- a/pkg/exchange/bitget/convert.go +++ b/pkg/exchange/bitget/convert.go @@ -1,6 +1,7 @@ package bitget import ( + "math" "strings" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" @@ -23,3 +24,38 @@ func toGlobalBalance(asset bitgetapi.AccountAsset) types.Balance { MaxWithdrawAmount: fixedpoint.Zero, } } + +func toGlobalMarket(s bitgetapi.Symbol) types.Market { + if s.Status != bitgetapi.SymbolOnline { + log.Warnf("The symbol %s is not online", s.Symbol) + } + return types.Market{ + Symbol: s.SymbolName, + LocalSymbol: s.Symbol, + PricePrecision: s.PriceScale.Int(), + VolumePrecision: s.QuantityScale.Int(), + QuoteCurrency: s.QuoteCoin, + BaseCurrency: s.BaseCoin, + MinNotional: s.MinTradeUSDT, + MinAmount: s.MinTradeUSDT, + MinQuantity: s.MinTradeAmount, + MaxQuantity: s.MaxTradeAmount, + StepSize: fixedpoint.NewFromFloat(1.0 / math.Pow10(s.QuantityScale.Int())), + TickSize: fixedpoint.NewFromFloat(1.0 / math.Pow10(s.PriceScale.Int())), + MinPrice: fixedpoint.Zero, + MaxPrice: fixedpoint.Zero, + } +} + +func toGlobalTicker(ticker bitgetapi.Ticker) types.Ticker { + return types.Ticker{ + Time: ticker.Ts.Time(), + Volume: ticker.BaseVol, + Last: ticker.Close, + Open: ticker.OpenUtc0, + High: ticker.High24H, + Low: ticker.Low24H, + Buy: ticker.BuyOne, + Sell: ticker.SellOne, + } +} diff --git a/pkg/exchange/bitget/convert_test.go b/pkg/exchange/bitget/convert_test.go new file mode 100644 index 000000000..770e4e5b8 --- /dev/null +++ b/pkg/exchange/bitget/convert_test.go @@ -0,0 +1,145 @@ +package bitget + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func Test_toGlobalBalance(t *testing.T) { + // sample: + // { + // "coinId":"10012", + // "coinName":"usdt", + // "available":"0", + // "frozen":"0", + // "lock":"0", + // "uTime":"1622697148" + // } + asset := bitgetapi.AccountAsset{ + CoinId: 2, + CoinName: "USDT", + Available: fixedpoint.NewFromFloat(1.2), + Frozen: fixedpoint.NewFromFloat(0.5), + Lock: fixedpoint.NewFromFloat(0.5), + UTime: types.NewMillisecondTimestampFromInt(1622697148), + } + + assert.Equal(t, types.Balance{ + Currency: "USDT", + Available: fixedpoint.NewFromFloat(1.2), + Locked: fixedpoint.NewFromFloat(1), // frozen + lock + Borrowed: fixedpoint.Zero, + Interest: fixedpoint.Zero, + NetAsset: fixedpoint.Zero, + MaxWithdrawAmount: fixedpoint.Zero, + }, toGlobalBalance(asset)) +} + +func Test_toGlobalMarket(t *testing.T) { + // sample: + //{ + // "symbol":"BTCUSDT_SPBL", + // "symbolName":"BTCUSDT", + // "baseCoin":"BTC", + // "quoteCoin":"USDT", + // "minTradeAmount":"0.0001", + // "maxTradeAmount":"10000", + // "takerFeeRate":"0.001", + // "makerFeeRate":"0.001", + // "priceScale":"4", + // "quantityScale":"8", + // "minTradeUSDT":"5", + // "status":"online", + // "buyLimitPriceRatio": "0.05", + // "sellLimitPriceRatio": "0.05" + // } + inst := bitgetapi.Symbol{ + Symbol: "BTCUSDT_SPBL", + SymbolName: "BTCUSDT", + BaseCoin: "BTC", + QuoteCoin: "USDT", + MinTradeAmount: fixedpoint.NewFromFloat(0.0001), + MaxTradeAmount: fixedpoint.NewFromFloat(10000), + TakerFeeRate: fixedpoint.NewFromFloat(0.001), + MakerFeeRate: fixedpoint.NewFromFloat(0.001), + PriceScale: fixedpoint.NewFromFloat(4), + QuantityScale: fixedpoint.NewFromFloat(8), + MinTradeUSDT: fixedpoint.NewFromFloat(5), + Status: bitgetapi.SymbolOnline, + BuyLimitPriceRatio: fixedpoint.NewFromFloat(0.05), + SellLimitPriceRatio: fixedpoint.NewFromFloat(0.05), + } + + exp := types.Market{ + Symbol: inst.SymbolName, + LocalSymbol: inst.Symbol, + PricePrecision: 4, + VolumePrecision: 8, + QuoteCurrency: inst.QuoteCoin, + BaseCurrency: inst.BaseCoin, + MinNotional: inst.MinTradeUSDT, + MinAmount: inst.MinTradeUSDT, + MinQuantity: inst.MinTradeAmount, + MaxQuantity: inst.MaxTradeAmount, + StepSize: fixedpoint.NewFromFloat(0.00000001), + MinPrice: fixedpoint.Zero, + MaxPrice: fixedpoint.Zero, + TickSize: fixedpoint.NewFromFloat(0.0001), + } + + assert.Equal(t, toGlobalMarket(inst), exp) +} + +func Test_toGlobalTicker(t *testing.T) { + // sample: + // { + // "symbol": "BTCUSDT", + // "high24h": "24175.65", + // "low24h": "23677.75", + // "close": "24014.11", + // "quoteVol": "177689342.3025", + // "baseVol": "7421.5009", + // "usdtVol": "177689342.302407", + // "ts": "1660704288118", + // "buyOne": "24013.94", + // "sellOne": "24014.06", + // "bidSz": "0.0663", + // "askSz": "0.0119", + // "openUtc0": "23856.72", + // "changeUtc":"0.00301", + // "change":"0.00069" + // } + ticker := bitgetapi.Ticker{ + Symbol: "BTCUSDT", + High24H: fixedpoint.NewFromFloat(24175.65), + Low24H: fixedpoint.NewFromFloat(23677.75), + Close: fixedpoint.NewFromFloat(24014.11), + QuoteVol: fixedpoint.NewFromFloat(177689342.3025), + BaseVol: fixedpoint.NewFromFloat(7421.5009), + UsdtVol: fixedpoint.NewFromFloat(177689342.302407), + Ts: types.NewMillisecondTimestampFromInt(1660704288118), + BuyOne: fixedpoint.NewFromFloat(24013.94), + SellOne: fixedpoint.NewFromFloat(24014.06), + BidSz: fixedpoint.NewFromFloat(0.0663), + AskSz: fixedpoint.NewFromFloat(0.0119), + OpenUtc0: fixedpoint.NewFromFloat(23856.72), + ChangeUtc: fixedpoint.NewFromFloat(0.00301), + Change: fixedpoint.NewFromFloat(0.00069), + } + + assert.Equal(t, types.Ticker{ + Time: types.NewMillisecondTimestampFromInt(1660704288118).Time(), + Volume: fixedpoint.NewFromFloat(7421.5009), + Last: fixedpoint.NewFromFloat(24014.11), + Open: fixedpoint.NewFromFloat(23856.72), + High: fixedpoint.NewFromFloat(24175.65), + Low: fixedpoint.NewFromFloat(23677.75), + Buy: fixedpoint.NewFromFloat(24013.94), + Sell: fixedpoint.NewFromFloat(24014.06), + }, toGlobalTicker(ticker)) +} diff --git a/pkg/exchange/bitget/exchange.go b/pkg/exchange/bitget/exchange.go index 0d17c1b00..700bd2ea7 100644 --- a/pkg/exchange/bitget/exchange.go +++ b/pkg/exchange/bitget/exchange.go @@ -2,12 +2,13 @@ package bitget import ( "context" - "math" + "fmt" + "time" "github.com/sirupsen/logrus" + "golang.org/x/time/rate" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" - "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" ) @@ -19,6 +20,17 @@ var log = logrus.WithFields(logrus.Fields{ "exchange": ID, }) +var ( + // queryMarketRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-symbols + queryMarketRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) + // queryAccountRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-account-assets + queryAccountRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + // queryTickerRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-single-ticker + queryTickerRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) + // queryTickersRateLimiter has its own rate limit. https://bitgetlimited.github.io/apidoc/en/spot/#get-all-tickers + queryTickersRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 5) +) + type Exchange struct { key, secret, passphrase string @@ -54,7 +66,10 @@ func (e *Exchange) NewStream() types.Stream { } func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { - // TODO implement me + if err := queryMarketRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("markets rate limiter wait error: %w", err) + } + req := e.client.NewGetSymbolsRequest() symbols, err := req.Do(ctx) if err != nil { @@ -64,50 +79,57 @@ func (e *Exchange) QueryMarkets(ctx context.Context) (types.MarketMap, error) { markets := types.MarketMap{} for _, s := range symbols { symbol := toGlobalSymbol(s.SymbolName) - markets[symbol] = types.Market{ - Symbol: s.SymbolName, - LocalSymbol: s.Symbol, - PricePrecision: s.PriceScale, - VolumePrecision: s.QuantityScale, - QuoteCurrency: s.QuoteCoin, - BaseCurrency: s.BaseCoin, - MinNotional: s.MinTradeUSDT, - MinAmount: s.MinTradeUSDT, - MinQuantity: s.MinTradeAmount, - MaxQuantity: s.MaxTradeAmount, - StepSize: fixedpoint.NewFromFloat(math.Pow10(-s.QuantityScale)), - TickSize: fixedpoint.NewFromFloat(math.Pow10(-s.PriceScale)), - MinPrice: fixedpoint.Zero, - MaxPrice: fixedpoint.Zero, - } + markets[symbol] = toGlobalMarket(s) } return markets, nil } func (e *Exchange) QueryTicker(ctx context.Context, symbol string) (*types.Ticker, error) { - req := e.client.NewGetTickerRequest() - req.Symbol(symbol) - ticker, err := req.Do(ctx) - if err != nil { - return nil, err + if err := queryTickerRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("ticker rate limiter wait error: %w", err) } - return &types.Ticker{ - Time: ticker.Ts.Time(), - Volume: ticker.BaseVol, - Last: ticker.Close, - Open: ticker.OpenUtc0, - High: ticker.High24H, - Low: ticker.Low24H, - Buy: ticker.BuyOne, - Sell: ticker.SellOne, - }, nil + req := e.client.NewGetTickerRequest() + req.Symbol(symbol) + resp, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query ticker: %w", err) + } + + ticker := toGlobalTicker(*resp) + return &ticker, nil } -func (e *Exchange) QueryTickers(ctx context.Context, symbol ...string) (map[string]types.Ticker, error) { - // TODO implement me - panic("implement me") +func (e *Exchange) QueryTickers(ctx context.Context, symbols ...string) (map[string]types.Ticker, error) { + tickers := map[string]types.Ticker{} + if len(symbols) > 0 { + for _, s := range symbols { + t, err := e.QueryTicker(ctx, s) + if err != nil { + return nil, err + } + + tickers[s] = *t + } + + return tickers, nil + } + + if err := queryTickersRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("tickers rate limiter wait error: %w", err) + } + + resp, err := e.client.NewGetAllTickersRequest().Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query tickers: %w", err) + } + + for _, s := range resp { + tickers[s.Symbol] = toGlobalTicker(s) + } + + return tickers, nil } func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval types.Interval, options types.KLineQueryOptions) ([]types.KLine, error) { @@ -116,26 +138,34 @@ func (e *Exchange) QueryKLines(ctx context.Context, symbol string, interval type } func (e *Exchange) QueryAccount(ctx context.Context) (*types.Account, error) { - req := e.client.NewGetAccountAssetsRequest() - resp, err := req.Do(ctx) + bals, err := e.QueryAccountBalances(ctx) if err != nil { return nil, err } - bals := types.BalanceMap{} - for _, asset := range resp { - b := toGlobalBalance(asset) - bals[asset.CoinName] = b - } - account := types.NewAccount() account.UpdateBalances(bals) return account, nil } func (e *Exchange) QueryAccountBalances(ctx context.Context) (types.BalanceMap, error) { - // TODO implement me - panic("implement me") + if err := queryAccountRateLimiter.Wait(ctx); err != nil { + return nil, fmt.Errorf("account rate limiter wait error: %w", err) + } + + req := e.client.NewGetAccountAssetsRequest() + resp, err := req.Do(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query account assets: %w", err) + } + + bals := types.BalanceMap{} + for _, asset := range resp { + b := toGlobalBalance(asset) + bals[asset.CoinName] = b + } + + return bals, nil } func (e *Exchange) SubmitOrder(ctx context.Context, order types.SubmitOrder) (createdOrder *types.Order, err error) { diff --git a/pkg/exchange/bitget/stream.go b/pkg/exchange/bitget/stream.go index eadf25b48..039c65127 100644 --- a/pkg/exchange/bitget/stream.go +++ b/pkg/exchange/bitget/stream.go @@ -1,34 +1,48 @@ package bitget import ( + "bytes" "context" "encoding/json" "fmt" + "github.com/gorilla/websocket" + "strings" "github.com/c9s/bbgo/pkg/exchange/bitget/bitgetapi" "github.com/c9s/bbgo/pkg/types" ) +var ( + pingBytes = []byte("ping") + pongBytes = []byte("pong") +) + //go:generate callbackgen -type Stream type Stream struct { types.StandardStream bookEventCallbacks []func(o BookEvent) marketTradeEventCallbacks []func(o MarketTradeEvent) + KLineEventCallbacks []func(o KLineEvent) + + lastCandle map[string]types.KLine } func NewStream() *Stream { stream := &Stream{ StandardStream: types.NewStandardStream(), + lastCandle: map[string]types.KLine{}, } stream.SetEndpointCreator(stream.createEndpoint) stream.SetParser(parseWebSocketEvent) stream.SetDispatcher(stream.dispatchEvent) + stream.SetHeartBeat(stream.ping) stream.OnConnect(stream.handlerConnect) stream.OnBookEvent(stream.handleBookEvent) stream.OnMarketTradeEvent(stream.handleMaretTradeEvent) + stream.OnKLineEvent(stream.handleKLineEvent) return stream } @@ -92,6 +106,15 @@ func (s *Stream) dispatchEvent(event interface{}) { case *MarketTradeEvent: s.EmitMarketTradeEvent(*e) + + case *KLineEvent: + s.EmitKLineEvent(*e) + + case []byte: + // We only handle the 'pong' case. Others are unexpected. + if !bytes.Equal(e, pongBytes) { + log.Errorf("invalid event: %q", e) + } } } @@ -116,6 +139,16 @@ func (s *Stream) handleBookEvent(o BookEvent) { } } +// ping implements the bitget text message of WebSocket PingPong. +func (s *Stream) ping(conn *websocket.Conn) error { + err := conn.WriteMessage(websocket.TextMessage, pingBytes) + if err != nil { + log.WithError(err).Error("ping error", err) + return nil + } + return nil +} + func convertSubscription(sub types.Subscription) (WsArg, error) { arg := WsArg{ // support spot only @@ -140,12 +173,33 @@ func convertSubscription(sub types.Subscription) (WsArg, error) { case types.MarketTradeChannel: arg.Channel = ChannelTrade return arg, nil + + case types.KLineChannel: + interval, found := toLocalInterval[sub.Options.Interval] + if !found { + return WsArg{}, fmt.Errorf("interval %s not supported on KLine subscription", sub.Options.Interval) + } + + arg.Channel = ChannelType(interval) + return arg, nil } return arg, fmt.Errorf("unsupported stream channel: %s", sub.Channel) } func parseWebSocketEvent(in []byte) (interface{}, error) { + switch { + case bytes.Equal(in, pongBytes): + // Return the original raw data may seem redundant because we can validate the string and return nil, + // but we cannot return nil to a lower level handler. This can cause confusion in the next handler, such as + // the dispatch handler. Therefore, I return the original raw data. + return in, nil + default: + return parseEvent(in) + } +} + +func parseEvent(in []byte) (interface{}, error) { var event WsEvent err := json.Unmarshal(in, &event) @@ -157,7 +211,8 @@ func parseWebSocketEvent(in []byte) (interface{}, error) { return &event, nil } - switch event.Arg.Channel { + ch := event.Arg.Channel + switch ch { case ChannelOrderBook, ChannelOrderBook5, ChannelOrderBook15: var book BookEvent err = json.Unmarshal(event.Data, &book.Events) @@ -179,9 +234,26 @@ func parseWebSocketEvent(in []byte) (interface{}, error) { trade.actionType = event.Action trade.instId = event.Arg.InstId return &trade, nil - } - return nil, fmt.Errorf("unhandled websocket event: %+v", string(in)) + default: + + // handle the `KLine` case here to avoid complicating the code structure. + if strings.HasPrefix(string(ch), "candle") { + var kline KLineEvent + err = json.Unmarshal(event.Data, &kline.Events) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal data into KLineEvent, Arg: %+v Data: %s, err: %w", event.Arg, string(event.Data), err) + } + + kline.actionType = event.Action + kline.channel = ch + kline.instId = event.Arg.InstId + return &kline, nil + } + // return an error for any other case + + return nil, fmt.Errorf("unhandled websocket event: %+v", string(in)) + } } func (s *Stream) handleMaretTradeEvent(m MarketTradeEvent) { @@ -199,3 +271,28 @@ func (s *Stream) handleMaretTradeEvent(m MarketTradeEvent) { s.EmitMarketTrade(globalTrade) } } + +func (s *Stream) handleKLineEvent(k KLineEvent) { + if k.actionType == ActionTypeSnapshot { + // we don't support snapshot event + return + } + + interval, found := toGlobalInterval[string(k.channel)] + if !found { + log.Errorf("unexpected interval %s on KLine subscription", k.channel) + return + } + + for _, kline := range k.Events { + last, ok := s.lastCandle[k.CacheKey()] + if ok && kline.StartTime.Time().After(last.StartTime.Time()) { + last.Closed = true + s.EmitKLineClosed(last) + } + + kLine := kline.ToGlobal(interval, k.instId) + s.EmitKLine(kLine) + s.lastCandle[k.CacheKey()] = kLine + } +} diff --git a/pkg/exchange/bitget/stream_callbacks.go b/pkg/exchange/bitget/stream_callbacks.go index 01da4388f..82ef7beae 100644 --- a/pkg/exchange/bitget/stream_callbacks.go +++ b/pkg/exchange/bitget/stream_callbacks.go @@ -23,3 +23,13 @@ func (s *Stream) EmitMarketTradeEvent(o MarketTradeEvent) { cb(o) } } + +func (s *Stream) OnKLineEvent(cb func(o KLineEvent)) { + s.KLineEventCallbacks = append(s.KLineEventCallbacks, cb) +} + +func (s *Stream) EmitKLineEvent(o KLineEvent) { + for _, cb := range s.KLineEventCallbacks { + cb(o) + } +} diff --git a/pkg/exchange/bitget/stream_test.go b/pkg/exchange/bitget/stream_test.go index c25fcd942..b33e6afa2 100644 --- a/pkg/exchange/bitget/stream_test.go +++ b/pkg/exchange/bitget/stream_test.go @@ -106,6 +106,22 @@ func TestStream(t *testing.T) { <-c }) + t.Run("kline test", func(t *testing.T) { + s.Subscribe(types.KLineChannel, "BTCUSDT", types.SubscribeOptions{Interval: types.Interval1w}) + s.SetPublicOnly() + err := s.Connect(context.Background()) + assert.NoError(t, err) + + s.OnKLine(func(kline types.KLine) { + t.Log("got update", kline) + }) + s.OnKLineClosed(func(kline types.KLine) { + t.Log("got closed update", kline) + }) + c := make(chan struct{}) + <-c + }) + } func TestStream_parseWebSocketEvent(t *testing.T) { @@ -453,6 +469,174 @@ func Test_parseWebSocketEvent_MarketTrade(t *testing.T) { }) } +func Test_parseWebSocketEvent_KLine(t *testing.T) { + t.Run("KLine event", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","34361.49","34458.98","34355.53","34416.41","99.6631"] + ], + "ts":1697697791670 + }` + + eventFn := func(in string, actionType ActionType) { + res, err := parseWebSocketEvent([]byte(in)) + assert.NoError(t, err) + kline, ok := res.(*KLineEvent) + assert.True(t, ok) + assert.Equal(t, KLineEvent{ + channel: "candle5m", + Events: KLineSlice{ + { + StartTime: types.NewMillisecondTimestampFromInt(1698744600000), + OpenPrice: fixedpoint.NewFromFloat(34361.49), + HighestPrice: fixedpoint.NewFromFloat(34458.98), + LowestPrice: fixedpoint.NewFromFloat(34355.53), + ClosePrice: fixedpoint.NewFromFloat(34416.41), + Volume: fixedpoint.NewFromFloat(99.6631), + }, + }, + actionType: actionType, + instId: "BTCUSDT", + }, *kline) + } + + t.Run("snapshot type", func(t *testing.T) { + snapshotInput := fmt.Sprintf(input, ActionTypeSnapshot) + eventFn(snapshotInput, ActionTypeSnapshot) + }) + + t.Run("update type", func(t *testing.T) { + snapshotInput := fmt.Sprintf(input, ActionTypeUpdate) + eventFn(snapshotInput, ActionTypeUpdate) + }) + }) + + t.Run("Unexpected length of kline", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","34361.45","34458.98","34355.53","34416.41","99.6631", "123456"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "unexpected kline length") + }) + + t.Run("Unexpected timestamp", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["timestamp","34361.49","34458.98","34355.53","34416.41","99.6631"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "timestamp") + }) + + t.Run("Unexpected open price", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","1p","34458.98","34355.53","34416.41","99.6631"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "open price") + }) + + t.Run("Unexpected highest price", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","34361.45","3p","34355.53","34416.41","99.6631"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "highest price") + }) + + t.Run("Unexpected lowest price", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","34361.45","34458.98","1p","34416.41","99.6631"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "lowest price") + }) + + t.Run("Unexpected close price", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","34361.45","34458.98","34355.53","1c","99.6631"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "close price") + }) + + t.Run("Unexpected volume", func(t *testing.T) { + input := `{ + "action":"%s", + "arg":{ + "instType":"sp", + "channel":"candle5m", + "instId":"BTCUSDT" + }, + "data":[ + ["1698744600000","34361.45","34458.98","34355.53","34416.41", "1v"] + ], + "ts":1697697791670 + }` + _, err := parseWebSocketEvent([]byte(input)) + assert.ErrorContains(t, err, "volume") + }) +} + func Test_convertSubscription(t *testing.T) { t.Run("BookChannel.ChannelOrderBook5", func(t *testing.T) { res, err := convertSubscription(types.Subscription{ @@ -512,4 +696,21 @@ func Test_convertSubscription(t *testing.T) { InstId: "BTCUSDT", }, res) }) + t.Run("CandleChannel", func(t *testing.T) { + for gInterval, localInterval := range toLocalInterval { + res, err := convertSubscription(types.Subscription{ + Symbol: "BTCUSDT", + Channel: types.KLineChannel, + Options: types.SubscribeOptions{ + Interval: gInterval, + }, + }) + assert.NoError(t, err) + assert.Equal(t, WsArg{ + InstType: instSp, + Channel: ChannelType(localInterval), + InstId: "BTCUSDT", + }, res) + } + }) } diff --git a/pkg/exchange/bitget/types.go b/pkg/exchange/bitget/types.go index df75f5bf2..a1107cad6 100644 --- a/pkg/exchange/bitget/types.go +++ b/pkg/exchange/bitget/types.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "time" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" @@ -260,3 +261,134 @@ type MarketTradeEvent struct { actionType ActionType instId string } + +var ( + toLocalInterval = map[types.Interval]string{ + types.Interval1m: "candle1m", + types.Interval5m: "candle5m", + types.Interval15m: "candle15m", + types.Interval30m: "candle30m", + types.Interval1h: "candle1H", + types.Interval4h: "candle4H", + types.Interval12h: "candle12H", + types.Interval1d: "candle1D", + types.Interval1w: "candle1W", + } + + toGlobalInterval = map[string]types.Interval{ + "candle1m": types.Interval1m, + "candle5m": types.Interval5m, + "candle15m": types.Interval15m, + "candle30m": types.Interval30m, + "candle1H": types.Interval1h, + "candle4H": types.Interval4h, + "candle12H": types.Interval12h, + "candle1D": types.Interval1d, + "candle1W": types.Interval1w, + } +) + +type KLine struct { + StartTime types.MillisecondTimestamp + OpenPrice fixedpoint.Value + HighestPrice fixedpoint.Value + LowestPrice fixedpoint.Value + ClosePrice fixedpoint.Value + Volume fixedpoint.Value +} + +func (k KLine) ToGlobal(interval types.Interval, symbol string) types.KLine { + startTime := k.StartTime.Time() + + return types.KLine{ + Exchange: types.ExchangeBitget, + Symbol: symbol, + StartTime: types.Time(startTime), + EndTime: types.Time(startTime.Add(interval.Duration() - time.Millisecond)), + Interval: interval, + Open: k.OpenPrice, + Close: k.ClosePrice, + High: k.HighestPrice, + Low: k.LowestPrice, + Volume: k.Volume, + QuoteVolume: fixedpoint.Zero, // not supported + TakerBuyBaseAssetVolume: fixedpoint.Zero, // not supported + TakerBuyQuoteAssetVolume: fixedpoint.Zero, // not supported + LastTradeID: 0, // not supported + NumberOfTrades: 0, // not supported + Closed: false, + } +} + +type KLineSlice []KLine + +func (m *KLineSlice) UnmarshalJSON(b []byte) error { + if m == nil { + return errors.New("nil pointer of kline slice") + } + s, err := parseKLineSliceJSON(b) + if err != nil { + return err + } + + *m = s + return nil +} + +// parseKLineSliceJSON tries to parse a 2 dimensional string array into a KLineSlice +// +// [ +// +// ["1597026383085", "8533.02", "8553.74", "8527.17", "8548.26", "45247"] +// ] +func parseKLineSliceJSON(in []byte) (slice KLineSlice, err error) { + var rawKLines [][]json.RawMessage + + err = json.Unmarshal(in, &rawKLines) + if err != nil { + return slice, err + } + + for _, raw := range rawKLines { + if len(raw) != 6 { + return nil, fmt.Errorf("unexpected kline length: %d, data: %q", len(raw), raw) + } + var kline KLine + if err = json.Unmarshal(raw[0], &kline.StartTime); err != nil { + return nil, fmt.Errorf("failed to unmarshal into timestamp: %q", raw[0]) + } + if err = json.Unmarshal(raw[1], &kline.OpenPrice); err != nil { + return nil, fmt.Errorf("failed to unmarshal into open price: %q", raw[1]) + } + if err = json.Unmarshal(raw[2], &kline.HighestPrice); err != nil { + return nil, fmt.Errorf("failed to unmarshal into highest price: %q", raw[2]) + } + if err = json.Unmarshal(raw[3], &kline.LowestPrice); err != nil { + return nil, fmt.Errorf("failed to unmarshal into lowest price: %q", raw[3]) + } + if err = json.Unmarshal(raw[4], &kline.ClosePrice); err != nil { + return nil, fmt.Errorf("failed to unmarshal into close price: %q", raw[4]) + } + if err = json.Unmarshal(raw[5], &kline.Volume); err != nil { + return nil, fmt.Errorf("failed to unmarshal into volume: %q", raw[5]) + } + + slice = append(slice, kline) + } + + return slice, nil +} + +type KLineEvent struct { + Events KLineSlice + + // internal use + actionType ActionType + channel ChannelType + instId string +} + +func (k KLineEvent) CacheKey() string { + // e.q: candle5m.BTCUSDT + return fmt.Sprintf("%s.%s", k.channel, k.instId) +} diff --git a/pkg/exchange/bitget/types_test.go b/pkg/exchange/bitget/types_test.go new file mode 100644 index 000000000..185056620 --- /dev/null +++ b/pkg/exchange/bitget/types_test.go @@ -0,0 +1,43 @@ +package bitget + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func TestKLine_ToGlobal(t *testing.T) { + startTime := int64(1698744600000) + interval := types.Interval1m + k := KLine{ + StartTime: types.NewMillisecondTimestampFromInt(startTime), + OpenPrice: fixedpoint.NewFromFloat(34361.49), + HighestPrice: fixedpoint.NewFromFloat(34458.98), + LowestPrice: fixedpoint.NewFromFloat(34355.53), + ClosePrice: fixedpoint.NewFromFloat(34416.41), + Volume: fixedpoint.NewFromFloat(99.6631), + } + + assert.Equal(t, types.KLine{ + Exchange: types.ExchangeBitget, + Symbol: "BTCUSDT", + StartTime: types.Time(types.NewMillisecondTimestampFromInt(startTime).Time()), + EndTime: types.Time(types.NewMillisecondTimestampFromInt(startTime).Time().Add(interval.Duration() - time.Millisecond)), + Interval: interval, + Open: fixedpoint.NewFromFloat(34361.49), + Close: fixedpoint.NewFromFloat(34416.41), + High: fixedpoint.NewFromFloat(34458.98), + Low: fixedpoint.NewFromFloat(34355.53), + Volume: fixedpoint.NewFromFloat(99.6631), + QuoteVolume: fixedpoint.Zero, + TakerBuyBaseAssetVolume: fixedpoint.Zero, + TakerBuyQuoteAssetVolume: fixedpoint.Zero, + LastTradeID: 0, + NumberOfTrades: 0, + Closed: false, + }, k.ToGlobal(interval, "BTCUSDT")) +} diff --git a/pkg/exchange/bybit/convert.go b/pkg/exchange/bybit/convert.go index b13032cd7..d89f194fe 100644 --- a/pkg/exchange/bybit/convert.go +++ b/pkg/exchange/bybit/convert.go @@ -2,7 +2,6 @@ package bybit import ( "fmt" - "math" "strconv" "time" @@ -16,8 +15,8 @@ func toGlobalMarket(m bybitapi.Instrument) types.Market { return types.Market{ Symbol: m.Symbol, LocalSymbol: m.Symbol, - PricePrecision: int(math.Log10(m.LotSizeFilter.QuotePrecision.Float64())), - VolumePrecision: int(math.Log10(m.LotSizeFilter.BasePrecision.Float64())), + PricePrecision: m.LotSizeFilter.QuotePrecision.NumFractionalDigits(), + VolumePrecision: m.LotSizeFilter.BasePrecision.NumFractionalDigits(), QuoteCurrency: m.QuoteCoin, BaseCurrency: m.BaseCoin, MinNotional: m.LotSizeFilter.MinOrderAmt, diff --git a/pkg/exchange/bybit/convert_test.go b/pkg/exchange/bybit/convert_test.go index 786234d8d..a5ddb08bc 100644 --- a/pkg/exchange/bybit/convert_test.go +++ b/pkg/exchange/bybit/convert_test.go @@ -2,7 +2,6 @@ package bybit import ( "fmt" - "math" "strconv" "testing" "time" @@ -67,8 +66,8 @@ func TestToGlobalMarket(t *testing.T) { exp := types.Market{ Symbol: inst.Symbol, LocalSymbol: inst.Symbol, - PricePrecision: int(math.Log10(inst.LotSizeFilter.QuotePrecision.Float64())), - VolumePrecision: int(math.Log10(inst.LotSizeFilter.BasePrecision.Float64())), + PricePrecision: 8, + VolumePrecision: 6, QuoteCurrency: inst.QuoteCoin, BaseCurrency: inst.BaseCoin, MinNotional: inst.LotSizeFilter.MinOrderAmt, diff --git a/pkg/exchange/bybit/exchange.go b/pkg/exchange/bybit/exchange.go index e1c3545c5..7cec992c6 100644 --- a/pkg/exchange/bybit/exchange.go +++ b/pkg/exchange/bybit/exchange.go @@ -26,15 +26,15 @@ const ( ) // https://bybit-exchange.github.io/docs/zh-TW/v5/rate-limit -// sharedRateLimiter indicates that the API belongs to the public API. -// -// The default order limiter apply 5 requests per second and a 5 initial bucket -// this includes QueryMarkets, QueryTicker, QueryAccountBalances, GetFeeRates +// GET/POST method (shared): 120 requests per second for 5 consecutive seconds var ( - sharedRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) - tradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) - orderRateLimiter = rate.NewLimiter(rate.Every(100*time.Millisecond), 10) - closedOrderQueryLimiter = rate.NewLimiter(rate.Every(time.Second), 1) + // sharedRateLimiter indicates that the API belongs to the public API. + // The default order limiter apply 5 requests per second and a 5 initial bucket + // this includes QueryMarkets, QueryTicker, QueryAccountBalances, GetFeeRates + sharedRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + queryOrderTradeRateLimiter = rate.NewLimiter(rate.Every(time.Second/5), 5) + orderRateLimiter = rate.NewLimiter(rate.Every(time.Second/10), 10) + closedOrderQueryLimiter = rate.NewLimiter(rate.Every(time.Second), 1) log = logrus.WithFields(logrus.Fields{ "exchange": "bybit", @@ -159,7 +159,7 @@ func (e *Exchange) QueryOpenOrders(ctx context.Context, symbol string) (orders [ req = req.Cursor(cursor) } - if err = tradeRateLimiter.Wait(ctx); err != nil { + if err = queryOrderTradeRateLimiter.Wait(ctx); err != nil { return nil, fmt.Errorf("place order rate limiter wait error: %w", err) } res, err := req.Do(ctx) @@ -232,7 +232,7 @@ func (e *Exchange) QueryOrderTrades(ctx context.Context, q types.OrderQuery) (tr req.Symbol(q.Symbol) } - if err := tradeRateLimiter.Wait(ctx); err != nil { + if err := queryOrderTradeRateLimiter.Wait(ctx); err != nil { return nil, fmt.Errorf("trade rate limiter wait error: %w", err) } response, err := req.Do(ctx) @@ -463,7 +463,7 @@ func (e *Exchange) QueryTrades(ctx context.Context, symbol string, options *type } req.Limit(limit) - if err := tradeRateLimiter.Wait(ctx); err != nil { + if err := queryOrderTradeRateLimiter.Wait(ctx); err != nil { return nil, fmt.Errorf("trade rate limiter wait error: %w", err) } response, err := req.Do(ctx) diff --git a/pkg/exchange/bybit/stream.go b/pkg/exchange/bybit/stream.go index c6b42cb9b..eb4137ed3 100644 --- a/pkg/exchange/bybit/stream.go +++ b/pkg/exchange/bybit/stream.go @@ -15,10 +15,6 @@ import ( ) const ( - // Bybit: To avoid network or program issues, we recommend that you send the ping heartbeat packet every 20 seconds - // to maintain the WebSocket connection. - pingInterval = 20 * time.Second - // spotArgsLimit can input up to 10 args for each subscription request sent to one connection. spotArgsLimit = 10 ) @@ -244,40 +240,18 @@ func (s *Stream) parseWebSocketEvent(in []byte) (interface{}, error) { } // ping implements the Bybit text message of WebSocket PingPong. -func (s *Stream) ping(ctx context.Context, conn *websocket.Conn, cancelFunc context.CancelFunc) { - defer func() { - log.Debug("[bybit] ping worker stopped") - cancelFunc() - }() - - var pingTicker = time.NewTicker(pingInterval) - defer pingTicker.Stop() - - for { - select { - - case <-ctx.Done(): - return - - case <-s.CloseC: - return - - case <-pingTicker.C: - // it's just for maintaining the liveliness of the connection, so comment out ReqId. - err := conn.WriteJSON(struct { - //ReqId string `json:"req_id"` - Op WsOpType `json:"op"` - }{ - //ReqId: uuid.NewString(), - Op: WsOpTypePing, - }) - if err != nil { - log.WithError(err).Error("ping error", err) - s.Reconnect() - return - } - } +func (s *Stream) ping(conn *websocket.Conn) error { + err := conn.WriteJSON(struct { + Op WsOpType `json:"op"` + }{ + Op: WsOpTypePing, + }) + if err != nil { + log.WithError(err).Error("ping error") + return err } + + return nil } func (s *Stream) handlerConnect() { diff --git a/pkg/exchange/kucoin/convert.go b/pkg/exchange/kucoin/convert.go index d86f84db9..e83ade5d3 100644 --- a/pkg/exchange/kucoin/convert.go +++ b/pkg/exchange/kucoin/convert.go @@ -3,7 +3,6 @@ package kucoin import ( "fmt" "hash/fnv" - "math" "strings" "time" @@ -39,8 +38,8 @@ func toGlobalMarket(m kucoinapi.Symbol) types.Market { return types.Market{ Symbol: symbol, LocalSymbol: m.Symbol, - PricePrecision: int(math.Log10(m.PriceIncrement.Float64())), // convert 0.0001 to 4 - VolumePrecision: int(math.Log10(m.BaseIncrement.Float64())), + PricePrecision: m.PriceIncrement.NumFractionalDigits(), // convert 0.0001 to 4 + VolumePrecision: m.BaseIncrement.NumFractionalDigits(), QuoteCurrency: m.QuoteCurrency, BaseCurrency: m.BaseCurrency, MinNotional: m.QuoteMinSize, diff --git a/pkg/exchange/retry/order.go b/pkg/exchange/retry/order.go index 15b59c6af..df6ef6b59 100644 --- a/pkg/exchange/retry/order.go +++ b/pkg/exchange/retry/order.go @@ -5,10 +5,9 @@ import ( "errors" "strconv" - backoff2 "github.com/cenkalti/backoff/v4" + "github.com/cenkalti/backoff/v4" "github.com/c9s/bbgo/pkg/types" - "github.com/c9s/bbgo/pkg/util/backoff" ) type advancedOrderCancelService interface { @@ -18,7 +17,7 @@ type advancedOrderCancelService interface { } func QueryOrderUntilFilled(ctx context.Context, queryOrderService types.ExchangeOrderQueryService, symbol string, orderId uint64) (o *types.Order, err error) { - err = backoff.RetryGeneral(ctx, func() (err2 error) { + var op = func() (err2 error) { o, err2 = queryOrderService.QueryOrder(ctx, types.OrderQuery{ Symbol: symbol, OrderID: strconv.FormatUint(orderId, 10), @@ -33,20 +32,30 @@ func QueryOrderUntilFilled(ctx context.Context, queryOrderService types.Exchange } return err2 - }) + } + err = GeneralBackoff(ctx, op) return o, err } -func GeneralBackoff(ctx context.Context, op backoff2.Operation) (err error) { - err = backoff2.Retry(op, backoff2.WithContext( - backoff2.WithMaxRetries( - backoff2.NewExponentialBackOff(), +func GeneralBackoff(ctx context.Context, op backoff.Operation) (err error) { + err = backoff.Retry(op, backoff.WithContext( + backoff.WithMaxRetries( + backoff.NewExponentialBackOff(), 101), ctx)) return err } +func GeneralLiteBackoff(ctx context.Context, op backoff.Operation) (err error) { + err = backoff.Retry(op, backoff.WithContext( + backoff.WithMaxRetries( + backoff.NewExponentialBackOff(), + 5), + ctx)) + return err +} + func QueryOpenOrdersUntilSuccessful(ctx context.Context, ex types.Exchange, symbol string) (openOrders []types.Order, err error) { var op = func() (err2 error) { openOrders, err2 = ex.QueryOpenOrders(ctx, symbol) @@ -57,6 +66,16 @@ func QueryOpenOrdersUntilSuccessful(ctx context.Context, ex types.Exchange, symb return openOrders, err } +func QueryOpenOrdersUntilSuccessfulLite(ctx context.Context, ex types.Exchange, symbol string) (openOrders []types.Order, err error) { + var op = func() (err2 error) { + openOrders, err2 = ex.QueryOpenOrders(ctx, symbol) + return err2 + } + + err = GeneralLiteBackoff(ctx, op) + return openOrders, err +} + func QueryOrderUntilSuccessful(ctx context.Context, query types.ExchangeOrderQueryService, opts types.OrderQuery) (order *types.Order, err error) { var op = func() (err2 error) { order, err2 = query.QueryOrder(ctx, opts) diff --git a/pkg/indicator/rma.go b/pkg/indicator/rma.go index a06ad18af..69a7f4fbc 100644 --- a/pkg/indicator/rma.go +++ b/pkg/indicator/rma.go @@ -67,11 +67,6 @@ func (inc *RMA) Update(x float64) { } inc.counter++ - if inc.counter < inc.Window { - inc.Values.Push(0) - return - } - inc.Values.Push(inc.tmp) if len(inc.Values) > MaxNumOfRMA { inc.Values = inc.Values[MaxNumOfRMATruncateSize-1:] diff --git a/pkg/indicator/rma_test.go b/pkg/indicator/rma_test.go new file mode 100644 index 000000000..6e14bcdb1 --- /dev/null +++ b/pkg/indicator/rma_test.go @@ -0,0 +1,72 @@ +package indicator + +import ( + "encoding/json" + "testing" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/stretchr/testify/assert" +) + +/* +python + +import pandas as pd +import pandas_ta as ta + +data = [40105.78, 39935.23, 40183.97, 40182.03, 40212.26, 40149.99, 40378.0, 40618.37, 40401.03, 39990.39, 40179.13, 40097.23, 40014.72, 39667.85, 39303.1, 39519.99, 39693.79, 39827.96, 40074.94, 40059.84] + +close = pd.Series(data) +result = ta.rma(close, length=14) +print(result) +*/ +func Test_RMA(t *testing.T) { + var bytes = []byte(`[40105.78, 39935.23, 40183.97, 40182.03, 40212.26, 40149.99, 40378.0, 40618.37, 40401.03, 39990.39, 40179.13, 40097.23, 40014.72, 39667.85, 39303.1, 39519.99, 39693.79, 39827.96, 40074.94, 40059.84]`) + var values []fixedpoint.Value + err := json.Unmarshal(bytes, &values) + assert.NoError(t, err) + + var kLines []types.KLine + for _, p := range values { + kLines = append(kLines, types.KLine{High: p, Low: p, Close: p}) + } + + tests := []struct { + name string + window int + want []float64 + }{ + { + name: "test_binance_btcusdt_1h", + window: 14, + want: []float64{ + 40129.841000, + 40041.830291, + 39988.157743, + 39958.803719, + 39946.115094, + 39958.296741, + 39967.681562, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rma := RMA{ + IntervalWindow: types.IntervalWindow{Window: tt.window}, + Adjust: true, + } + rma.CalculateAndUpdate(kLines) + + if assert.Equal(t, len(tt.want), len(rma.Values)-tt.window+1) { + for i, v := range tt.want { + j := tt.window - 1 + i + got := rma.Values[j] + assert.InDelta(t, v, got, 0.01, "Expected rma.slice[%d] to be %v, but got %v", j, v, got) + } + } + }) + } +} diff --git a/pkg/indicator/v2/rma.go b/pkg/indicator/v2/rma.go index 1aa08ccdc..b943cf6f8 100644 --- a/pkg/indicator/v2/rma.go +++ b/pkg/indicator/v2/rma.go @@ -34,28 +34,20 @@ func RMA2(source types.Float64Source, window int, adjust bool) *RMAStream { func (s *RMAStream) Calculate(x float64) float64 { lambda := 1 / float64(s.window) - tmp := 0.0 if s.counter == 0 { s.sum = 1 - tmp = x + s.previous = x } else { if s.Adjust { s.sum = s.sum*(1-lambda) + 1 - tmp = s.previous + (x-s.previous)/s.sum + s.previous = s.previous + (x-s.previous)/s.sum } else { - tmp = s.previous*(1-lambda) + x*lambda + s.previous = s.previous*(1-lambda) + x*lambda } } s.counter++ - if s.counter < s.window { - // we can use x, but we need to use 0. to make the same behavior as the result from python pandas_ta - s.Slice.Push(0) - } - - s.Slice.Push(tmp) - s.previous = tmp - return tmp + return s.previous } func (s *RMAStream) Truncate() { diff --git a/pkg/indicator/v2/rma_test.go b/pkg/indicator/v2/rma_test.go new file mode 100644 index 000000000..b06605f75 --- /dev/null +++ b/pkg/indicator/v2/rma_test.go @@ -0,0 +1,65 @@ +package indicatorv2 + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +/* +python + +import pandas as pd +import pandas_ta as ta + +data = [40105.78, 39935.23, 40183.97, 40182.03, 40212.26, 40149.99, 40378.0, 40618.37, 40401.03, 39990.39, 40179.13, 40097.23, 40014.72, 39667.85, 39303.1, 39519.99, 39693.79, 39827.96, 40074.94, 40059.84] + +close = pd.Series(data) +result = ta.rma(close, length=14) +print(result) +*/ +func Test_RMA2(t *testing.T) { + var bytes = []byte(`[40105.78, 39935.23, 40183.97, 40182.03, 40212.26, 40149.99, 40378.0, 40618.37, 40401.03, 39990.39, 40179.13, 40097.23, 40014.72, 39667.85, 39303.1, 39519.99, 39693.79, 39827.96, 40074.94, 40059.84]`) + var values []float64 + err := json.Unmarshal(bytes, &values) + assert.NoError(t, err) + + prices := ClosePrices(nil) + for _, v := range values { + prices.Push(v) + } + + tests := []struct { + name string + window int + want []float64 + }{ + { + name: "test_binance_btcusdt_1h", + window: 14, + want: []float64{ + 40129.841000, + 40041.830291, + 39988.157743, + 39958.803719, + 39946.115094, + 39958.296741, + 39967.681562, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rma := RMA2(prices, tt.window, true) + if assert.Equal(t, len(tt.want), len(rma.Slice)-tt.window+1) { + for i, v := range tt.want { + j := tt.window - 1 + i + got := rma.Slice[j] + assert.InDelta(t, v, got, 0.01, "Expected rma.slice[%d] to be %v, but got %v", j, v, got) + } + } + }) + } +} diff --git a/pkg/service/trade.go b/pkg/service/trade.go index ae8379fa0..9020290a7 100644 --- a/pkg/service/trade.go +++ b/pkg/service/trade.go @@ -23,7 +23,12 @@ type QueryTradesOptions struct { Sessions []string Symbol string LastGID int64 - Since *time.Time + + // inclusive + Since *time.Time + + // exclusive + Until *time.Time // ASC or DESC Ordering string @@ -272,11 +277,19 @@ func (s *TradeService) Query(options QueryTradesOptions) ([]types.Trade, error) sel := sq.Select("*"). From("trades") + if options.LastGID != 0 { + sel = sel.Where(sq.Gt{"gid": options.LastGID}) + } if options.Since != nil { sel = sel.Where(sq.GtOrEq{"traded_at": options.Since}) } + if options.Until != nil { + sel = sel.Where(sq.Lt{"traded_at": options.Until}) + } - sel = sel.Where(sq.Eq{"symbol": options.Symbol}) + if options.Symbol != "" { + sel = sel.Where(sq.Eq{"symbol": options.Symbol}) + } if options.Exchange != "" { sel = sel.Where(sq.Eq{"exchange": options.Exchange}) @@ -412,4 +425,3 @@ func SelectLastTrades(ex types.ExchangeName, symbol string, isMargin, isFutures, OrderBy("traded_at DESC"). Limit(limit) } - diff --git a/pkg/strategy/grid2/active_order_recover.go b/pkg/strategy/grid2/active_order_recover.go index e93e722f0..cfdaeac80 100644 --- a/pkg/strategy/grid2/active_order_recover.go +++ b/pkg/strategy/grid2/active_order_recover.go @@ -2,7 +2,6 @@ package grid2 import ( "context" - "strconv" "time" "github.com/c9s/bbgo/pkg/bbgo" @@ -23,21 +22,21 @@ type SyncActiveOrdersOpts struct { exchange types.Exchange } -func (s *Strategy) initializeRecoverCh() bool { +func (s *Strategy) initializeRecoverC() bool { s.mu.Lock() defer s.mu.Unlock() isInitialize := false - if s.activeOrdersRecoverC == nil { + if s.recoverC == nil { s.logger.Info("initializing recover channel") - s.activeOrdersRecoverC = make(chan struct{}, 1) + s.recoverC = make(chan struct{}, 1) } else { s.logger.Info("recover channel is already initialized, trigger active orders recover") isInitialize = true select { - case s.activeOrdersRecoverC <- struct{}{}: + case s.recoverC <- struct{}{}: s.logger.Info("trigger active orders recover") default: s.logger.Info("activeOrdersRecoverC is full") @@ -49,7 +48,7 @@ func (s *Strategy) initializeRecoverCh() bool { func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { // every time we activeOrdersRecoverC receive signal, do active orders recover - if isInitialize := s.initializeRecoverCh(); isInitialize { + if isInitialize := s.initializeRecoverC(); isInitialize { return } @@ -78,7 +77,7 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { log.WithError(err).Errorf("unable to sync active orders") } - case <-s.activeOrdersRecoverC: + case <-s.recoverC: if err := syncActiveOrders(ctx, opts); err != nil { log.WithError(err).Errorf("unable to sync active orders") } @@ -90,9 +89,10 @@ func (s *Strategy) recoverActiveOrdersPeriodically(ctx context.Context) { func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { opts.logger.Infof("[ActiveOrderRecover] syncActiveOrders") - notAddNonExistingOpenOrdersAfter := time.Now().Add(-5 * time.Minute) + // only sync orders which is updated over 3 min, because we may receive from websocket and handle it twice + syncBefore := time.Now().Add(-3 * time.Minute) - openOrders, err := retry.QueryOpenOrdersUntilSuccessful(ctx, opts.exchange, opts.activeOrderBook.Symbol) + openOrders, err := retry.QueryOpenOrdersUntilSuccessfulLite(ctx, opts.exchange, opts.activeOrderBook.Symbol) if err != nil { opts.logger.WithError(err).Error("[ActiveOrderRecover] failed to query open orders, skip this time") return errors.Wrapf(err, "[ActiveOrderRecover] failed to query open orders, skip this time") @@ -117,6 +117,10 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { delete(openOrdersMap, activeOrder.OrderID) } else { opts.logger.Infof("found active order #%d is not in the open orders, updating...", activeOrder.OrderID) + if activeOrder.UpdateTime.After(syncBefore) { + opts.logger.Infof("active order #%d is updated in 3 min, skip updating...", activeOrder.OrderID) + continue + } // sleep 100ms to avoid DDOS time.Sleep(100 * time.Millisecond) @@ -131,8 +135,10 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { // update open orders not in active orders for _, openOrder := range openOrdersMap { - // we don't add open orders into active orderbook if updated in 5 min - if openOrder.UpdateTime.After(notAddNonExistingOpenOrdersAfter) { + opts.logger.Infof("found open order #%d is not in active orderbook, updating...", openOrder.OrderID) + // we don't add open orders into active orderbook if updated in 3 min, because we may receive message from websocket and add it twice. + if openOrder.UpdateTime.After(syncBefore) { + opts.logger.Infof("open order #%d is updated in 3 min, skip updating...", openOrder.OrderID) continue } @@ -142,18 +148,3 @@ func syncActiveOrders(ctx context.Context, opts SyncActiveOrdersOpts) error { return errs } - -func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64) error { - updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, orderQueryService, types.OrderQuery{ - Symbol: activeOrderBook.Symbol, - OrderID: strconv.FormatUint(orderID, 10), - }) - - if err != nil { - return err - } - - activeOrderBook.Update(*updatedOrder) - - return nil -} diff --git a/pkg/strategy/grid2/grid_recover.go b/pkg/strategy/grid2/grid_recover.go new file mode 100644 index 000000000..945f2c038 --- /dev/null +++ b/pkg/strategy/grid2/grid_recover.go @@ -0,0 +1,362 @@ +package grid2 + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/pkg/errors" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/exchange/retry" + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" +) + +func (s *Strategy) recoverByScanningTrades(ctx context.Context, session *bbgo.ExchangeSession) error { + defer func() { + s.updateGridNumOfOrdersMetricsWithLock() + }() + + historyService, implemented := session.Exchange.(types.ExchangeTradeHistoryService) + // if the exchange doesn't support ExchangeTradeHistoryService, do not run recover + if !implemented { + s.logger.Warn("ExchangeTradeHistoryService is not implemented, can not recover grid") + return nil + } + + openOrders, err := session.Exchange.QueryOpenOrders(ctx, s.Symbol) + if err != nil { + return errors.Wrapf(err, "unable to query open orders when recovering") + } + + s.logger.Infof("found %d open orders left on the %s order book", len(openOrders), s.Symbol) + + if s.GridProfitStats.InitialOrderID != 0 { + s.logger.Info("InitialOrderID is already there, need to recover") + } else if len(openOrders) != 0 { + s.logger.Info("even though InitialOrderID is 0, there are open orders so need to recover") + } else { + s.logger.Info("InitialOrderID is 0 and there is no open orders, query trades to check it") + // initial order id may be new strategy or lost data in redis, so we need to check trades + open orders + // if there are open orders or trades, we need to recover + trades, err := historyService.QueryTrades(ctx, s.Symbol, &types.TradeQueryOptions{ + // from 1, because some API will ignore 0 last trade id + LastTradeID: 1, + // if there is any trades, we need to recover. + Limit: 1, + }) + + if err != nil { + return errors.Wrapf(err, "unable to query trades when recovering") + } + + if len(trades) == 0 { + s.logger.Info("0 trades found, it's a new strategy so no need to recover") + return nil + } + } + + s.logger.Infof("start recovering") + filledOrders, err := s.getFilledOrdersByScanningTrades(ctx, historyService, s.orderQueryService, openOrders) + if err != nil { + return errors.Wrap(err, "grid recover error") + } + s.debugOrders("emit filled orders", filledOrders) + + // add open orders into avtive maker orders + s.addOrdersToActiveOrderBook(openOrders) + + // emit the filled orders + activeOrderBook := s.orderExecutor.ActiveMakerOrders() + for _, filledOrder := range filledOrders { + activeOrderBook.EmitFilled(filledOrder) + } + + // emit ready after recover + s.EmitGridReady() + + // debug and send metrics + // wait for the reverse order to be placed + time.Sleep(2 * time.Second) + debugGrid(s.logger, s.grid, s.orderExecutor.ActiveMakerOrders()) + + defer bbgo.Sync(ctx, s) + + if s.EnableProfitFixer { + until := time.Now() + since := until.Add(-7 * 24 * time.Hour) + if s.FixProfitSince != nil { + since = s.FixProfitSince.Time() + } + + fixer := newProfitFixer(s.grid, s.Symbol, historyService) + fixer.SetLogger(s.logger) + + // set initial order ID = 0 instead of s.GridProfitStats.InitialOrderID because the order ID could be incorrect + if err := fixer.Fix(ctx, since, until, 0, s.GridProfitStats); err != nil { + return err + } + + s.logger.Infof("fixed profitStats: %#v", s.GridProfitStats) + + s.EmitGridProfit(s.GridProfitStats, nil) + } + + return nil +} + +func (s *Strategy) getFilledOrdersByScanningTrades(ctx context.Context, queryTradesService types.ExchangeTradeHistoryService, queryOrderService types.ExchangeOrderQueryService, openOrdersOnGrid []types.Order) ([]types.Order, error) { + // set grid + grid := s.newGrid() + s.setGrid(grid) + + expectedNumOfOrders := s.GridNum - 1 + numGridOpenOrders := int64(len(openOrdersOnGrid)) + s.debugLog("open orders nums: %d, expected nums: %d", numGridOpenOrders, expectedNumOfOrders) + if expectedNumOfOrders == numGridOpenOrders { + // no need to recover, only need to add open orders back to active order book + return nil, nil + } else if expectedNumOfOrders < numGridOpenOrders { + return nil, fmt.Errorf("amount of grid's open orders should not > amount of expected grid's orders") + } + + // 1. build twin-order map + twinOrdersOpen, err := s.buildTwinOrderMap(grid.Pins, openOrdersOnGrid) + if err != nil { + return nil, errors.Wrapf(err, "failed to build pin order map with open orders") + } + + // 2. build the filled twin-order map by querying trades + expectedFilledNum := int(expectedNumOfOrders - numGridOpenOrders) + twinOrdersFilled, err := s.buildFilledTwinOrderMapFromTrades(ctx, queryTradesService, queryOrderService, twinOrdersOpen, expectedFilledNum) + if err != nil { + return nil, errors.Wrapf(err, "failed to build filled pin order map") + } + + // 3. get the filled orders from twin-order map + filledOrders := twinOrdersFilled.AscendingOrders() + + // 4. verify the grid + if err := s.verifyFilledTwinGrid(s.grid.Pins, twinOrdersOpen, filledOrders); err != nil { + return nil, errors.Wrapf(err, "verify grid with error") + } + + return filledOrders, nil +} + +func (s *Strategy) verifyFilledTwinGrid(pins []Pin, twinOrders TwinOrderMap, filledOrders []types.Order) error { + s.debugLog("verifying filled grid - pins: %+v", pins) + s.debugOrders("verifying filled grid - filled orders", filledOrders) + s.debugLog("verifying filled grid - open twin orders:\n%s", twinOrders.String()) + + if err := s.addOrdersIntoTwinOrderMap(twinOrders, filledOrders); err != nil { + return errors.Wrapf(err, "verifying filled grid error when add orders into twin order map") + } + + s.debugLog("verifying filled grid - filled twin orders:\n%+v", twinOrders.String()) + + for i, pin := range pins { + // we use twinOrderMap to make sure there are no duplicated order at one grid, and we use the sell price as key so we skip the pins[0] which is only for buy price + if i == 0 { + continue + } + + twin, exist := twinOrders[fixedpoint.Value(pin)] + if !exist { + return fmt.Errorf("there is no order at price (%+v)", pin) + } + + if !twin.Exist() { + return fmt.Errorf("all the price need a twin") + } + + if !twin.IsValid() { + return fmt.Errorf("all the twins need to be valid") + } + } + + return nil +} + +// buildTwinOrderMap build the pin-order map with grid and open orders. +// The keys of this map contains all required pins of this grid. +// If the Order of the pin is empty types.Order (OrderID == 0), it means there is no open orders at this pin. +func (s *Strategy) buildTwinOrderMap(pins []Pin, openOrders []types.Order) (TwinOrderMap, error) { + twinOrderMap := make(TwinOrderMap) + + for i, pin := range pins { + // twin order map only use sell price as key, so skip 0 + if i == 0 { + continue + } + + twinOrderMap[fixedpoint.Value(pin)] = TwinOrder{} + } + + for _, openOrder := range openOrders { + twinKey, err := findTwinOrderMapKey(s.grid, openOrder) + if err != nil { + return nil, errors.Wrapf(err, "failed to build twin order map") + } + + twinOrder, exist := twinOrderMap[twinKey] + if !exist { + return nil, fmt.Errorf("the price of the openOrder (id: %d) is not in pins", openOrder.OrderID) + } + + if twinOrder.Exist() { + return nil, fmt.Errorf("there are multiple order in a twin") + } + + twinOrder.SetOrder(openOrder) + twinOrderMap[twinKey] = twinOrder + } + + return twinOrderMap, nil +} + +// buildFilledTwinOrderMapFromTrades will query the trades from last 24 hour and use them to build a pin order map +// It will skip the orders on pins at which open orders are already +func (s *Strategy) buildFilledTwinOrderMapFromTrades(ctx context.Context, queryTradesService types.ExchangeTradeHistoryService, queryOrderService types.ExchangeOrderQueryService, twinOrdersOpen TwinOrderMap, expectedFillNum int) (TwinOrderMap, error) { + twinOrdersFilled := make(TwinOrderMap) + + // existedOrders is used to avoid re-query the same orders + existedOrders := twinOrdersOpen.SyncOrderMap() + + // get the filled orders when bbgo is down in order from trades + until := time.Now() + // the first query only query the last 1 hour, because mostly shutdown and recovery happens within 1 hour + since := until.Add(-1 * time.Hour) + // hard limit for recover + recoverSinceLimit := time.Date(2023, time.March, 10, 0, 0, 0, 0, time.UTC) + + if s.RecoverGridWithin != 0 && until.Add(-1*s.RecoverGridWithin).After(recoverSinceLimit) { + recoverSinceLimit = until.Add(-1 * s.RecoverGridWithin) + } + + for { + if err := s.queryTradesToUpdateTwinOrdersMap(ctx, queryTradesService, queryOrderService, twinOrdersOpen, twinOrdersFilled, existedOrders, since, until); err != nil { + return nil, errors.Wrapf(err, "failed to query trades to update twin orders map") + } + + until = since + since = until.Add(-6 * time.Hour) + + if len(twinOrdersFilled) >= expectedFillNum { + s.logger.Infof("stop querying trades because twin orders filled (%d) >= expected filled nums (%d)", len(twinOrdersFilled), expectedFillNum) + break + } + + if s.GridProfitStats != nil && s.GridProfitStats.Since != nil && until.Before(*s.GridProfitStats.Since) { + s.logger.Infof("stop querying trades because the time range is out of the strategy's since (%s)", *s.GridProfitStats.Since) + break + } + + if until.Before(recoverSinceLimit) { + s.logger.Infof("stop querying trades because the time range is out of the limit (%s)", recoverSinceLimit) + break + } + } + + return twinOrdersFilled, nil +} + +func (s *Strategy) queryTradesToUpdateTwinOrdersMap(ctx context.Context, queryTradesService types.ExchangeTradeHistoryService, queryOrderService types.ExchangeOrderQueryService, twinOrdersOpen, twinOrdersFilled TwinOrderMap, existedOrders *types.SyncOrderMap, since, until time.Time) error { + var fromTradeID uint64 = 0 + var limit int64 = 1000 + for { + trades, err := queryTradesService.QueryTrades(ctx, s.Symbol, &types.TradeQueryOptions{ + StartTime: &since, + EndTime: &until, + LastTradeID: fromTradeID, + Limit: limit, + }) + + if err != nil { + return errors.Wrapf(err, "failed to query trades to recover the grid with open orders") + } + + s.debugLog("QueryTrades from %s <-> %s (from: %d) return %d trades", since, until, fromTradeID, len(trades)) + + for _, trade := range trades { + if trade.Time.After(until) { + return nil + } + + s.debugLog(trade.String()) + + if existedOrders.Exists(trade.OrderID) { + // already queries, skip + continue + } + order, err := retry.QueryOrderUntilSuccessful(ctx, queryOrderService, types.OrderQuery{ + Symbol: trade.Symbol, + OrderID: strconv.FormatUint(trade.OrderID, 10), + }) + + if err != nil { + return errors.Wrapf(err, "failed to query order by trade (trade id: %d, order id: %d)", trade.ID, trade.OrderID) + } + + s.debugLog(order.String()) + // avoid query this order again + existedOrders.Add(*order) + // add 1 to avoid duplicate + fromTradeID = trade.ID + 1 + + twinOrderKey, err := findTwinOrderMapKey(s.grid, *order) + if err != nil { + return errors.Wrapf(err, "failed to find grid order map's key when recover") + } + + twinOrderOpen, exist := twinOrdersOpen[twinOrderKey] + if !exist { + return fmt.Errorf("the price of the order with the same GroupID is not in pins") + } + + if twinOrderOpen.Exist() { + continue + } + + if twinOrder, exist := twinOrdersFilled[twinOrderKey]; exist { + to := twinOrder.GetOrder() + if to.UpdateTime.Time().After(order.UpdateTime.Time()) { + s.logger.Infof("twinOrder's update time (%s) should not be after order's update time (%s)", to.UpdateTime, order.UpdateTime) + continue + } + } + + twinOrder := TwinOrder{} + twinOrder.SetOrder(*order) + twinOrdersFilled[twinOrderKey] = twinOrder + } + + // stop condition + if int64(len(trades)) < limit { + return nil + } + } +} + +func (s *Strategy) addOrdersIntoTwinOrderMap(twinOrders TwinOrderMap, orders []types.Order) error { + for _, order := range orders { + k, err := findTwinOrderMapKey(s.grid, order) + if err != nil { + return errors.Wrap(err, "failed to add orders into twin order map") + } + + if v, exist := twinOrders[k]; !exist { + return fmt.Errorf("the price (%+v) is not in pins", k) + } else if v.Exist() { + return fmt.Errorf("there is already a twin order at this price (%+v)", k) + } else { + twin := TwinOrder{} + twin.SetOrder(order) + twinOrders[k] = twin + } + } + + return nil +} diff --git a/pkg/strategy/grid2/grid_recover_test.go b/pkg/strategy/grid2/grid_recover_test.go new file mode 100644 index 000000000..997cac9e5 --- /dev/null +++ b/pkg/strategy/grid2/grid_recover_test.go @@ -0,0 +1,295 @@ +package grid2 + +import ( + "context" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "sort" + "strconv" + "testing" + "time" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/types" + "github.com/stretchr/testify/assert" +) + +type TestData struct { + Market types.Market `json:"market" yaml:"market"` + Strategy Strategy `json:"strategy" yaml:"strategy"` + OpenOrders []types.Order `json:"openOrders" yaml:"openOrders"` + ClosedOrders []types.Order `json:"closedOrders" yaml:"closedOrders"` + Trades []types.Trade `json:"trades" yaml:"trades"` +} + +type TestDataService struct { + Orders map[string]types.Order + Trades []types.Trade +} + +func (t *TestDataService) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) ([]types.Trade, error) { + var i int = 0 + if options.LastTradeID != 0 { + for idx, trade := range t.Trades { + if trade.ID < options.LastTradeID { + continue + } + + i = idx + break + } + } + + var trades []types.Trade + l := len(t.Trades) + for ; i < l && len(trades) < int(options.Limit); i++ { + trades = append(trades, t.Trades[i]) + } + + return trades, nil +} + +func (t *TestDataService) QueryOrder(ctx context.Context, q types.OrderQuery) (*types.Order, error) { + if len(q.OrderID) == 0 { + return nil, fmt.Errorf("order id should not be empty") + } + + order, exist := t.Orders[q.OrderID] + if !exist { + return nil, fmt.Errorf("order not found") + } + + return &order, nil +} + +// dummy method for interface +func (t *TestDataService) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) (orders []types.Order, err error) { + return nil, nil +} + +// dummy method for interface +func (t *TestDataService) QueryOrderTrades(ctx context.Context, q types.OrderQuery) ([]types.Trade, error) { + return nil, nil +} + +func NewStrategy(t *TestData) *Strategy { + s := t.Strategy + s.Debug = true + s.Initialize() + s.Market = t.Market + s.Position = types.NewPositionFromMarket(t.Market) + s.orderExecutor = bbgo.NewGeneralOrderExecutor(&bbgo.ExchangeSession{}, t.Market.Symbol, ID, s.InstanceID(), s.Position) + return &s +} + +func NewTestDataService(t *TestData) *TestDataService { + var orders map[string]types.Order = make(map[string]types.Order) + for _, order := range t.OpenOrders { + orders[strconv.FormatUint(order.OrderID, 10)] = order + } + + for _, order := range t.ClosedOrders { + orders[strconv.FormatUint(order.OrderID, 10)] = order + } + + trades := t.Trades + sort.Slice(t.Trades, func(i, j int) bool { + return trades[i].ID < trades[j].ID + }) + + return &TestDataService{ + Orders: orders, + Trades: trades, + } +} + +func readSpec(fileName string) (*TestData, error) { + content, err := ioutil.ReadFile(fileName) + if err != nil { + return nil, err + } + + market := types.Market{} + if err := json.Unmarshal(content, &market); err != nil { + return nil, err + } + + strategy := Strategy{} + if err := json.Unmarshal(content, &strategy); err != nil { + return nil, err + } + + data := TestData{ + Market: market, + Strategy: strategy, + } + return &data, nil +} + +func readOrdersFromCSV(fileName string) ([]types.Order, error) { + csvFile, err := os.Open(fileName) + if err != nil { + return nil, err + } + defer csvFile.Close() + csvReader := csv.NewReader(csvFile) + + keys, err := csvReader.Read() + if err != nil { + return nil, err + } + + var orders []types.Order + for { + row, err := csvReader.Read() + if err == io.EOF { + break + } + + if err != nil { + return nil, err + } + + if len(row) != len(keys) { + return nil, fmt.Errorf("length of row should be equal to length of keys") + } + + var m map[string]interface{} = make(map[string]interface{}) + for i, key := range keys { + if key == "orderID" { + x, err := strconv.ParseUint(row[i], 10, 64) + if err != nil { + return nil, err + } + m[key] = x + } else { + m[key] = row[i] + } + } + + b, err := json.Marshal(m) + if err != nil { + return nil, err + } + + order := types.Order{} + if err = json.Unmarshal(b, &order); err != nil { + return nil, err + } + + orders = append(orders, order) + } + + return orders, nil +} + +func readTradesFromCSV(fileName string) ([]types.Trade, error) { + csvFile, err := os.Open(fileName) + if err != nil { + return nil, err + } + defer csvFile.Close() + csvReader := csv.NewReader(csvFile) + + keys, err := csvReader.Read() + if err != nil { + return nil, err + } + + var trades []types.Trade + for { + row, err := csvReader.Read() + if err == io.EOF { + break + } + + if err != nil { + return nil, err + } + + if len(row) != len(keys) { + return nil, fmt.Errorf("length of row should be equal to length of keys") + } + + var m map[string]interface{} = make(map[string]interface{}) + for i, key := range keys { + switch key { + case "id", "orderID": + x, err := strconv.ParseUint(row[i], 10, 64) + if err != nil { + return nil, err + } + m[key] = x + default: + m[key] = row[i] + } + } + + b, err := json.Marshal(m) + if err != nil { + return nil, err + } + + trade := types.Trade{} + if err = json.Unmarshal(b, &trade); err != nil { + return nil, err + } + + trades = append(trades, trade) + } + + return trades, nil +} + +func readTestDataFrom(fileDir string) (*TestData, error) { + data, err := readSpec(fmt.Sprintf("%s/spec", fileDir)) + if err != nil { + return nil, err + } + + openOrders, err := readOrdersFromCSV(fmt.Sprintf("%s/open_orders.csv", fileDir)) + if err != nil { + return nil, err + } + + closedOrders, err := readOrdersFromCSV(fmt.Sprintf("%s/closed_orders.csv", fileDir)) + if err != nil { + return nil, err + } + + trades, err := readTradesFromCSV(fmt.Sprintf("%s/trades.csv", fileDir)) + if err != nil { + return nil, err + } + + data.OpenOrders = openOrders + data.ClosedOrders = closedOrders + data.Trades = trades + return data, nil +} + +func TestRecoverByScanningTrades(t *testing.T) { + assert := assert.New(t) + + t.Run("test case 1", func(t *testing.T) { + fileDir := "recovery_testcase/testcase1/" + + data, err := readTestDataFrom(fileDir) + if !assert.NoError(err) { + return + } + + testService := NewTestDataService(data) + strategy := NewStrategy(data) + filledOrders, err := strategy.getFilledOrdersByScanningTrades(context.Background(), testService, testService, data.OpenOrders) + if !assert.NoError(err) { + return + } + + assert.Len(filledOrders, 0) + }) +} diff --git a/pkg/strategy/grid2/profit_stats.go b/pkg/strategy/grid2/profit_stats.go index 5f47effc6..cd8367c23 100644 --- a/pkg/strategy/grid2/profit_stats.go +++ b/pkg/strategy/grid2/profit_stats.go @@ -22,7 +22,6 @@ type GridProfitStats struct { TotalFee map[string]fixedpoint.Value `json:"totalFee,omitempty"` Volume fixedpoint.Value `json:"volume,omitempty"` Market types.Market `json:"market,omitempty"` - ProfitEntries []*GridProfit `json:"profitEntries,omitempty"` Since *time.Time `json:"since,omitempty"` InitialOrderID uint64 `json:"initialOrderID"` } @@ -38,7 +37,6 @@ func newGridProfitStats(market types.Market) *GridProfitStats { TotalFee: make(map[string]fixedpoint.Value), Volume: fixedpoint.Zero, Market: market, - ProfitEntries: nil, } } @@ -69,8 +67,6 @@ func (s *GridProfitStats) AddProfit(profit *GridProfit) { case s.Market.BaseCurrency: s.TotalBaseProfit = s.TotalBaseProfit.Add(profit.Profit) } - - s.ProfitEntries = append(s.ProfitEntries, profit) } func (s *GridProfitStats) SlackAttachment() slack.Attachment { diff --git a/pkg/strategy/grid2/recover.go b/pkg/strategy/grid2/recover.go index 945f2c038..0634b72d4 100644 --- a/pkg/strategy/grid2/recover.go +++ b/pkg/strategy/grid2/recover.go @@ -6,41 +6,63 @@ import ( "strconv" "time" - "github.com/pkg/errors" - "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/exchange/retry" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" + "github.com/pkg/errors" ) -func (s *Strategy) recoverByScanningTrades(ctx context.Context, session *bbgo.ExchangeSession) error { - defer func() { - s.updateGridNumOfOrdersMetricsWithLock() - }() +/* + Background knowledge + 1. active orderbook add orders only when receive new order event or call Add/Update method manually + 2. active orderbook remove orders only when receive filled/cancelled event or call Remove/Update method manually + As a result + 1. at the same twin-order-price, there is order in open orders but not in active orderbook + - not receive new order event + => add order into active orderbook + 2. at the same twin-order-price, there is order in active orderbook but not in open orders + - not receive filled event + => query the filled order and call Update method + 3. at the same twin-order-price, there is no order in open orders and no order in active orderbook + - failed to create the order + => query the last order from trades to emit filled, and it will submit again + - not receive new order event and the order filled before we find it. + => query the untracked order (also is the last order) from trades to emit filled and it will submit the reversed order + 4. at the same twin-order-price, there are different orders in open orders and active orderbook + - should not happen !!! + => log error + 5. at the same twin-order-price, there is the same order in open orders and active orderbook + - normal case + => no need to do anything + After killing pod, active orderbook must be empty. we can think it is the same as not receive new event. + Process + 1. build twin orderbook with pins and open orders. + 2. build twin orderbook with pins and active orders. + 3. compare above twin orderbooks to add open orders into active orderbook and update active orders. + 4. run grid recover to make sure all the twin price has its order. +*/ - historyService, implemented := session.Exchange.(types.ExchangeTradeHistoryService) +func (s *Strategy) recover(ctx context.Context) error { + historyService, implemented := s.session.Exchange.(types.ExchangeTradeHistoryService) // if the exchange doesn't support ExchangeTradeHistoryService, do not run recover if !implemented { s.logger.Warn("ExchangeTradeHistoryService is not implemented, can not recover grid") return nil } - openOrders, err := session.Exchange.QueryOpenOrders(ctx, s.Symbol) + activeOrderBook := s.orderExecutor.ActiveMakerOrders() + activeOrders := activeOrderBook.Orders() + + openOrders, err := retry.QueryOpenOrdersUntilSuccessfulLite(ctx, s.session.Exchange, s.Symbol) if err != nil { - return errors.Wrapf(err, "unable to query open orders when recovering") + return err } - s.logger.Infof("found %d open orders left on the %s order book", len(openOrders), s.Symbol) - - if s.GridProfitStats.InitialOrderID != 0 { - s.logger.Info("InitialOrderID is already there, need to recover") - } else if len(openOrders) != 0 { - s.logger.Info("even though InitialOrderID is 0, there are open orders so need to recover") - } else { - s.logger.Info("InitialOrderID is 0 and there is no open orders, query trades to check it") - // initial order id may be new strategy or lost data in redis, so we need to check trades + open orders - // if there are open orders or trades, we need to recover + // check if it's new strategy or need to recover + if len(activeOrders) == 0 && len(openOrders) == 0 && s.GridProfitStats.InitialOrderID == 0 { + // even though there is no open orders and initial orderID is 0 + // we still need to query trades to make sure if we need to recover or not trades, err := historyService.QueryTrades(ctx, s.Symbol, &types.TradeQueryOptions{ // from 1, because some API will ignore 0 last trade id LastTradeID: 1, @@ -53,181 +75,134 @@ func (s *Strategy) recoverByScanningTrades(ctx context.Context, session *bbgo.Ex } if len(trades) == 0 { - s.logger.Info("0 trades found, it's a new strategy so no need to recover") + s.logger.Info("no open order, no active order, no trade, it's a new strategy so no need to recover") return nil } } - s.logger.Infof("start recovering") - filledOrders, err := s.getFilledOrdersByScanningTrades(ctx, historyService, s.orderQueryService, openOrders) - if err != nil { - return errors.Wrap(err, "grid recover error") - } - s.debugOrders("emit filled orders", filledOrders) + s.logger.Info("start recovering") - // add open orders into avtive maker orders - s.addOrdersToActiveOrderBook(openOrders) - - // emit the filled orders - activeOrderBook := s.orderExecutor.ActiveMakerOrders() - for _, filledOrder := range filledOrders { - activeOrderBook.EmitFilled(filledOrder) + if s.getGrid() == nil { + s.setGrid(s.newGrid()) } - // emit ready after recover - s.EmitGridReady() + s.mu.Lock() + defer s.mu.Unlock() - // debug and send metrics - // wait for the reverse order to be placed - time.Sleep(2 * time.Second) - debugGrid(s.logger, s.grid, s.orderExecutor.ActiveMakerOrders()) + pins := s.getGrid().Pins - defer bbgo.Sync(ctx, s) + activeOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, activeOrders) + openOrdersInTwinOrderBook, err := buildTwinOrderBook(pins, openOrders) - if s.EnableProfitFixer { - until := time.Now() - since := until.Add(-7 * 24 * time.Hour) - if s.FixProfitSince != nil { - since = s.FixProfitSince.Time() + s.logger.Infof("active orders' twin orderbook\n%s", activeOrdersInTwinOrderBook.String()) + s.logger.Infof("open orders in twin orderbook\n%s", openOrdersInTwinOrderBook.String()) + + // remove index 0, because twin orderbook's price is from the second one + pins = pins[1:] + var noTwinOrderPins []fixedpoint.Value + + for _, pin := range pins { + v := fixedpoint.Value(pin) + activeOrder := activeOrdersInTwinOrderBook.GetTwinOrder(v) + openOrder := openOrdersInTwinOrderBook.GetTwinOrder(v) + if activeOrder == nil || openOrder == nil { + return fmt.Errorf("there is no any twin order at this pin, can not recover") } - fixer := newProfitFixer(s.grid, s.Symbol, historyService) - fixer.SetLogger(s.logger) + var activeOrderID uint64 = 0 + if activeOrder.Exist() { + activeOrderID = activeOrder.GetOrder().OrderID + } - // set initial order ID = 0 instead of s.GridProfitStats.InitialOrderID because the order ID could be incorrect - if err := fixer.Fix(ctx, since, until, 0, s.GridProfitStats); err != nil { + var openOrderID uint64 = 0 + if openOrder.Exist() { + openOrderID = openOrder.GetOrder().OrderID + } + + // case 3 + if activeOrderID == 0 && openOrderID == 0 { + noTwinOrderPins = append(noTwinOrderPins, v) + continue + } + + // case 1 + if activeOrderID == 0 { + activeOrderBook.Add(openOrder.GetOrder()) + // also add open orders into active order's twin orderbook, we will use this active orderbook to recover empty price grid + activeOrdersInTwinOrderBook.AddTwinOrder(v, openOrder) + continue + } + + // case 2 + if openOrderID == 0 { + syncActiveOrder(ctx, activeOrderBook, s.orderQueryService, activeOrder.GetOrder().OrderID) + continue + } + + // case 4 + if activeOrderID != openOrderID { + return fmt.Errorf("there are two different orders in the same pin, can not recover") + } + + // case 5 + // do nothing + } + + s.logger.Infof("twin orderbook after adding open orders\n%s", activeOrdersInTwinOrderBook.String()) + + if len(noTwinOrderPins) != 0 { + if err := s.recoverEmptyGridOnTwinOrderBook(ctx, activeOrdersInTwinOrderBook, historyService, s.orderQueryService); err != nil { + s.logger.WithError(err).Error("failed to recover empty grid") return err } - s.logger.Infof("fixed profitStats: %#v", s.GridProfitStats) + s.logger.Infof("twin orderbook after recovering no twin order on grid\n%s", activeOrdersInTwinOrderBook.String()) - s.EmitGridProfit(s.GridProfitStats, nil) + if activeOrdersInTwinOrderBook.EmptyTwinOrderSize() > 0 { + return fmt.Errorf("there is still empty grid in twin orderbook") + } + + for _, pin := range noTwinOrderPins { + twinOrder := activeOrdersInTwinOrderBook.GetTwinOrder(pin) + if twinOrder == nil { + return fmt.Errorf("should not get nil twin order after recovering empty grid, check it") + } + + if !twinOrder.Exist() { + return fmt.Errorf("should not get empty twin order after recovering empty grid, check it") + } + + activeOrderBook.EmitFilled(twinOrder.GetOrder()) + + time.Sleep(100 * time.Millisecond) + } } + // TODO: do not emit ready here, emit ready only once when opening grid or recovering grid after worker stopped + // s.EmitGridReady() + + time.Sleep(2 * time.Second) + debugGrid(s.logger, s.grid, s.orderExecutor.ActiveMakerOrders()) + + bbgo.Sync(ctx, s) + return nil } -func (s *Strategy) getFilledOrdersByScanningTrades(ctx context.Context, queryTradesService types.ExchangeTradeHistoryService, queryOrderService types.ExchangeOrderQueryService, openOrdersOnGrid []types.Order) ([]types.Order, error) { - // set grid - grid := s.newGrid() - s.setGrid(grid) - - expectedNumOfOrders := s.GridNum - 1 - numGridOpenOrders := int64(len(openOrdersOnGrid)) - s.debugLog("open orders nums: %d, expected nums: %d", numGridOpenOrders, expectedNumOfOrders) - if expectedNumOfOrders == numGridOpenOrders { - // no need to recover, only need to add open orders back to active order book - return nil, nil - } else if expectedNumOfOrders < numGridOpenOrders { - return nil, fmt.Errorf("amount of grid's open orders should not > amount of expected grid's orders") +func (s *Strategy) recoverEmptyGridOnTwinOrderBook( + ctx context.Context, + twinOrderBook *TwinOrderBook, + queryTradesService types.ExchangeTradeHistoryService, + queryOrderService types.ExchangeOrderQueryService, +) error { + if twinOrderBook.EmptyTwinOrderSize() == 0 { + s.logger.Info("no empty grid") + return nil } - // 1. build twin-order map - twinOrdersOpen, err := s.buildTwinOrderMap(grid.Pins, openOrdersOnGrid) - if err != nil { - return nil, errors.Wrapf(err, "failed to build pin order map with open orders") - } + existedOrders := twinOrderBook.SyncOrderMap() - // 2. build the filled twin-order map by querying trades - expectedFilledNum := int(expectedNumOfOrders - numGridOpenOrders) - twinOrdersFilled, err := s.buildFilledTwinOrderMapFromTrades(ctx, queryTradesService, queryOrderService, twinOrdersOpen, expectedFilledNum) - if err != nil { - return nil, errors.Wrapf(err, "failed to build filled pin order map") - } - - // 3. get the filled orders from twin-order map - filledOrders := twinOrdersFilled.AscendingOrders() - - // 4. verify the grid - if err := s.verifyFilledTwinGrid(s.grid.Pins, twinOrdersOpen, filledOrders); err != nil { - return nil, errors.Wrapf(err, "verify grid with error") - } - - return filledOrders, nil -} - -func (s *Strategy) verifyFilledTwinGrid(pins []Pin, twinOrders TwinOrderMap, filledOrders []types.Order) error { - s.debugLog("verifying filled grid - pins: %+v", pins) - s.debugOrders("verifying filled grid - filled orders", filledOrders) - s.debugLog("verifying filled grid - open twin orders:\n%s", twinOrders.String()) - - if err := s.addOrdersIntoTwinOrderMap(twinOrders, filledOrders); err != nil { - return errors.Wrapf(err, "verifying filled grid error when add orders into twin order map") - } - - s.debugLog("verifying filled grid - filled twin orders:\n%+v", twinOrders.String()) - - for i, pin := range pins { - // we use twinOrderMap to make sure there are no duplicated order at one grid, and we use the sell price as key so we skip the pins[0] which is only for buy price - if i == 0 { - continue - } - - twin, exist := twinOrders[fixedpoint.Value(pin)] - if !exist { - return fmt.Errorf("there is no order at price (%+v)", pin) - } - - if !twin.Exist() { - return fmt.Errorf("all the price need a twin") - } - - if !twin.IsValid() { - return fmt.Errorf("all the twins need to be valid") - } - } - - return nil -} - -// buildTwinOrderMap build the pin-order map with grid and open orders. -// The keys of this map contains all required pins of this grid. -// If the Order of the pin is empty types.Order (OrderID == 0), it means there is no open orders at this pin. -func (s *Strategy) buildTwinOrderMap(pins []Pin, openOrders []types.Order) (TwinOrderMap, error) { - twinOrderMap := make(TwinOrderMap) - - for i, pin := range pins { - // twin order map only use sell price as key, so skip 0 - if i == 0 { - continue - } - - twinOrderMap[fixedpoint.Value(pin)] = TwinOrder{} - } - - for _, openOrder := range openOrders { - twinKey, err := findTwinOrderMapKey(s.grid, openOrder) - if err != nil { - return nil, errors.Wrapf(err, "failed to build twin order map") - } - - twinOrder, exist := twinOrderMap[twinKey] - if !exist { - return nil, fmt.Errorf("the price of the openOrder (id: %d) is not in pins", openOrder.OrderID) - } - - if twinOrder.Exist() { - return nil, fmt.Errorf("there are multiple order in a twin") - } - - twinOrder.SetOrder(openOrder) - twinOrderMap[twinKey] = twinOrder - } - - return twinOrderMap, nil -} - -// buildFilledTwinOrderMapFromTrades will query the trades from last 24 hour and use them to build a pin order map -// It will skip the orders on pins at which open orders are already -func (s *Strategy) buildFilledTwinOrderMapFromTrades(ctx context.Context, queryTradesService types.ExchangeTradeHistoryService, queryOrderService types.ExchangeOrderQueryService, twinOrdersOpen TwinOrderMap, expectedFillNum int) (TwinOrderMap, error) { - twinOrdersFilled := make(TwinOrderMap) - - // existedOrders is used to avoid re-query the same orders - existedOrders := twinOrdersOpen.SyncOrderMap() - - // get the filled orders when bbgo is down in order from trades until := time.Now() - // the first query only query the last 1 hour, because mostly shutdown and recovery happens within 1 hour since := until.Add(-1 * time.Hour) // hard limit for recover recoverSinceLimit := time.Date(2023, time.March, 10, 0, 0, 0, 0, time.UTC) @@ -237,15 +212,15 @@ func (s *Strategy) buildFilledTwinOrderMapFromTrades(ctx context.Context, queryT } for { - if err := s.queryTradesToUpdateTwinOrdersMap(ctx, queryTradesService, queryOrderService, twinOrdersOpen, twinOrdersFilled, existedOrders, since, until); err != nil { - return nil, errors.Wrapf(err, "failed to query trades to update twin orders map") + if err := queryTradesToUpdateTwinOrderBook(ctx, s.Symbol, twinOrderBook, queryTradesService, queryOrderService, existedOrders, since, until, s.debugLog); err != nil { + return errors.Wrapf(err, "failed to query trades to update twin orderbook") } until = since since = until.Add(-6 * time.Hour) - if len(twinOrdersFilled) >= expectedFillNum { - s.logger.Infof("stop querying trades because twin orders filled (%d) >= expected filled nums (%d)", len(twinOrdersFilled), expectedFillNum) + if twinOrderBook.EmptyTwinOrderSize() == 0 { + s.logger.Infof("stop querying trades because there is no empty twin order on twin orderbook") break } @@ -260,14 +235,54 @@ func (s *Strategy) buildFilledTwinOrderMapFromTrades(ctx context.Context, queryT } } - return twinOrdersFilled, nil + return nil } -func (s *Strategy) queryTradesToUpdateTwinOrdersMap(ctx context.Context, queryTradesService types.ExchangeTradeHistoryService, queryOrderService types.ExchangeOrderQueryService, twinOrdersOpen, twinOrdersFilled TwinOrderMap, existedOrders *types.SyncOrderMap, since, until time.Time) error { +func buildTwinOrderBook(pins []Pin, orders []types.Order) (*TwinOrderBook, error) { + book := newTwinOrderBook(pins) + + for _, order := range orders { + if err := book.AddOrder(order); err != nil { + return nil, err + } + } + + return book, nil +} + +func syncActiveOrder(ctx context.Context, activeOrderBook *bbgo.ActiveOrderBook, orderQueryService types.ExchangeOrderQueryService, orderID uint64) error { + updatedOrder, err := retry.QueryOrderUntilSuccessful(ctx, orderQueryService, types.OrderQuery{ + Symbol: activeOrderBook.Symbol, + OrderID: strconv.FormatUint(orderID, 10), + }) + + if err != nil { + return err + } + + activeOrderBook.Update(*updatedOrder) + + return nil +} + +func queryTradesToUpdateTwinOrderBook( + ctx context.Context, + symbol string, + twinOrderBook *TwinOrderBook, + queryTradesService types.ExchangeTradeHistoryService, + queryOrderService types.ExchangeOrderQueryService, + existedOrders *types.SyncOrderMap, + since, until time.Time, + logger func(format string, args ...interface{}), +) error { + if twinOrderBook == nil { + return fmt.Errorf("twin orderbook should not be nil, please check it") + } + var fromTradeID uint64 = 0 var limit int64 = 1000 for { - trades, err := queryTradesService.QueryTrades(ctx, s.Symbol, &types.TradeQueryOptions{ + trades, err := queryTradesService.QueryTrades(ctx, symbol, &types.TradeQueryOptions{ StartTime: &since, EndTime: &until, LastTradeID: fromTradeID, @@ -275,17 +290,21 @@ func (s *Strategy) queryTradesToUpdateTwinOrdersMap(ctx context.Context, queryTr }) if err != nil { - return errors.Wrapf(err, "failed to query trades to recover the grid with open orders") + return errors.Wrapf(err, "failed to query trades to recover the grid") } - s.debugLog("QueryTrades from %s <-> %s (from: %d) return %d trades", since, until, fromTradeID, len(trades)) + if logger != nil { + logger("QueryTrades from %s <-> %s (from: %d) return %d trades", since, until, fromTradeID, len(trades)) + } for _, trade := range trades { if trade.Time.After(until) { return nil } - s.debugLog(trade.String()) + if logger != nil { + logger(trade.String()) + } if existedOrders.Exists(trade.OrderID) { // already queries, skip @@ -300,37 +319,17 @@ func (s *Strategy) queryTradesToUpdateTwinOrdersMap(ctx context.Context, queryTr return errors.Wrapf(err, "failed to query order by trade (trade id: %d, order id: %d)", trade.ID, trade.OrderID) } - s.debugLog(order.String()) + if logger != nil { + logger(order.String()) + } // avoid query this order again existedOrders.Add(*order) // add 1 to avoid duplicate fromTradeID = trade.ID + 1 - twinOrderKey, err := findTwinOrderMapKey(s.grid, *order) - if err != nil { - return errors.Wrapf(err, "failed to find grid order map's key when recover") + if err := twinOrderBook.AddOrder(*order); err != nil { + return errors.Wrapf(err, "failed to add queried order into twin orderbook") } - - twinOrderOpen, exist := twinOrdersOpen[twinOrderKey] - if !exist { - return fmt.Errorf("the price of the order with the same GroupID is not in pins") - } - - if twinOrderOpen.Exist() { - continue - } - - if twinOrder, exist := twinOrdersFilled[twinOrderKey]; exist { - to := twinOrder.GetOrder() - if to.UpdateTime.Time().After(order.UpdateTime.Time()) { - s.logger.Infof("twinOrder's update time (%s) should not be after order's update time (%s)", to.UpdateTime, order.UpdateTime) - continue - } - } - - twinOrder := TwinOrder{} - twinOrder.SetOrder(*order) - twinOrdersFilled[twinOrderKey] = twinOrder } // stop condition @@ -339,24 +338,3 @@ func (s *Strategy) queryTradesToUpdateTwinOrdersMap(ctx context.Context, queryTr } } } - -func (s *Strategy) addOrdersIntoTwinOrderMap(twinOrders TwinOrderMap, orders []types.Order) error { - for _, order := range orders { - k, err := findTwinOrderMapKey(s.grid, order) - if err != nil { - return errors.Wrap(err, "failed to add orders into twin order map") - } - - if v, exist := twinOrders[k]; !exist { - return fmt.Errorf("the price (%+v) is not in pins", k) - } else if v.Exist() { - return fmt.Errorf("there is already a twin order at this price (%+v)", k) - } else { - twin := TwinOrder{} - twin.SetOrder(order) - twinOrders[k] = twin - } - } - - return nil -} diff --git a/pkg/strategy/grid2/recover_test.go b/pkg/strategy/grid2/recover_test.go index 997cac9e5..bdfd191ee 100644 --- a/pkg/strategy/grid2/recover_test.go +++ b/pkg/strategy/grid2/recover_test.go @@ -2,81 +2,19 @@ package grid2 import ( "context" - "encoding/csv" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "os" - "sort" "strconv" "testing" "time" "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" + "github.com/c9s/bbgo/pkg/types/mocks" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" ) -type TestData struct { - Market types.Market `json:"market" yaml:"market"` - Strategy Strategy `json:"strategy" yaml:"strategy"` - OpenOrders []types.Order `json:"openOrders" yaml:"openOrders"` - ClosedOrders []types.Order `json:"closedOrders" yaml:"closedOrders"` - Trades []types.Trade `json:"trades" yaml:"trades"` -} - -type TestDataService struct { - Orders map[string]types.Order - Trades []types.Trade -} - -func (t *TestDataService) QueryTrades(ctx context.Context, symbol string, options *types.TradeQueryOptions) ([]types.Trade, error) { - var i int = 0 - if options.LastTradeID != 0 { - for idx, trade := range t.Trades { - if trade.ID < options.LastTradeID { - continue - } - - i = idx - break - } - } - - var trades []types.Trade - l := len(t.Trades) - for ; i < l && len(trades) < int(options.Limit); i++ { - trades = append(trades, t.Trades[i]) - } - - return trades, nil -} - -func (t *TestDataService) QueryOrder(ctx context.Context, q types.OrderQuery) (*types.Order, error) { - if len(q.OrderID) == 0 { - return nil, fmt.Errorf("order id should not be empty") - } - - order, exist := t.Orders[q.OrderID] - if !exist { - return nil, fmt.Errorf("order not found") - } - - return &order, nil -} - -// dummy method for interface -func (t *TestDataService) QueryClosedOrders(ctx context.Context, symbol string, since, until time.Time, lastOrderID uint64) (orders []types.Order, err error) { - return nil, nil -} - -// dummy method for interface -func (t *TestDataService) QueryOrderTrades(ctx context.Context, q types.OrderQuery) ([]types.Trade, error) { - return nil, nil -} - -func NewStrategy(t *TestData) *Strategy { +func newStrategy(t *TestData) *Strategy { s := t.Strategy s.Debug = true s.Initialize() @@ -86,210 +24,214 @@ func NewStrategy(t *TestData) *Strategy { return &s } -func NewTestDataService(t *TestData) *TestDataService { - var orders map[string]types.Order = make(map[string]types.Order) - for _, order := range t.OpenOrders { - orders[strconv.FormatUint(order.OrderID, 10)] = order - } - - for _, order := range t.ClosedOrders { - orders[strconv.FormatUint(order.OrderID, 10)] = order - } - - trades := t.Trades - sort.Slice(t.Trades, func(i, j int) bool { - return trades[i].ID < trades[j].ID - }) - - return &TestDataService{ - Orders: orders, - Trades: trades, - } -} - -func readSpec(fileName string) (*TestData, error) { - content, err := ioutil.ReadFile(fileName) - if err != nil { - return nil, err - } - - market := types.Market{} - if err := json.Unmarshal(content, &market); err != nil { - return nil, err - } - - strategy := Strategy{} - if err := json.Unmarshal(content, &strategy); err != nil { - return nil, err - } - - data := TestData{ - Market: market, - Strategy: strategy, - } - return &data, nil -} - -func readOrdersFromCSV(fileName string) ([]types.Order, error) { - csvFile, err := os.Open(fileName) - if err != nil { - return nil, err - } - defer csvFile.Close() - csvReader := csv.NewReader(csvFile) - - keys, err := csvReader.Read() - if err != nil { - return nil, err - } - - var orders []types.Order - for { - row, err := csvReader.Read() - if err == io.EOF { - break - } - - if err != nil { - return nil, err - } - - if len(row) != len(keys) { - return nil, fmt.Errorf("length of row should be equal to length of keys") - } - - var m map[string]interface{} = make(map[string]interface{}) - for i, key := range keys { - if key == "orderID" { - x, err := strconv.ParseUint(row[i], 10, 64) - if err != nil { - return nil, err - } - m[key] = x - } else { - m[key] = row[i] - } - } - - b, err := json.Marshal(m) - if err != nil { - return nil, err - } - - order := types.Order{} - if err = json.Unmarshal(b, &order); err != nil { - return nil, err - } - - orders = append(orders, order) - } - - return orders, nil -} - -func readTradesFromCSV(fileName string) ([]types.Trade, error) { - csvFile, err := os.Open(fileName) - if err != nil { - return nil, err - } - defer csvFile.Close() - csvReader := csv.NewReader(csvFile) - - keys, err := csvReader.Read() - if err != nil { - return nil, err - } - - var trades []types.Trade - for { - row, err := csvReader.Read() - if err == io.EOF { - break - } - - if err != nil { - return nil, err - } - - if len(row) != len(keys) { - return nil, fmt.Errorf("length of row should be equal to length of keys") - } - - var m map[string]interface{} = make(map[string]interface{}) - for i, key := range keys { - switch key { - case "id", "orderID": - x, err := strconv.ParseUint(row[i], 10, 64) - if err != nil { - return nil, err - } - m[key] = x - default: - m[key] = row[i] - } - } - - b, err := json.Marshal(m) - if err != nil { - return nil, err - } - - trade := types.Trade{} - if err = json.Unmarshal(b, &trade); err != nil { - return nil, err - } - - trades = append(trades, trade) - } - - return trades, nil -} - -func readTestDataFrom(fileDir string) (*TestData, error) { - data, err := readSpec(fmt.Sprintf("%s/spec", fileDir)) - if err != nil { - return nil, err - } - - openOrders, err := readOrdersFromCSV(fmt.Sprintf("%s/open_orders.csv", fileDir)) - if err != nil { - return nil, err - } - - closedOrders, err := readOrdersFromCSV(fmt.Sprintf("%s/closed_orders.csv", fileDir)) - if err != nil { - return nil, err - } - - trades, err := readTradesFromCSV(fmt.Sprintf("%s/trades.csv", fileDir)) - if err != nil { - return nil, err - } - - data.OpenOrders = openOrders - data.ClosedOrders = closedOrders - data.Trades = trades - return data, nil -} - -func TestRecoverByScanningTrades(t *testing.T) { +func TestBuildTwinOrderBook(t *testing.T) { assert := assert.New(t) - t.Run("test case 1", func(t *testing.T) { - fileDir := "recovery_testcase/testcase1/" - - data, err := readTestDataFrom(fileDir) + pins := []Pin{ + Pin(fixedpoint.NewFromInt(200)), + Pin(fixedpoint.NewFromInt(300)), + Pin(fixedpoint.NewFromInt(500)), + Pin(fixedpoint.NewFromInt(400)), + Pin(fixedpoint.NewFromInt(100)), + } + t.Run("build twin orderbook with no order", func(t *testing.T) { + b, err := buildTwinOrderBook(pins, nil) if !assert.NoError(err) { return } - testService := NewTestDataService(data) - strategy := NewStrategy(data) - filledOrders, err := strategy.getFilledOrdersByScanningTrades(context.Background(), testService, testService, data.OpenOrders) + assert.Equal(0, b.Size()) + assert.Nil(b.GetTwinOrder(fixedpoint.NewFromInt(100))) + assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(200)).Exist()) + assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(300)).Exist()) + assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(400)).Exist()) + assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(500)).Exist()) + }) + + t.Run("build twin orderbook with some valid orders", func(t *testing.T) { + orders := []types.Order{ + { + OrderID: 1, + SubmitOrder: types.SubmitOrder{ + Side: types.SideTypeBuy, + Price: fixedpoint.NewFromInt(100), + }, + }, + { + OrderID: 5, + SubmitOrder: types.SubmitOrder{ + Side: types.SideTypeSell, + Price: fixedpoint.NewFromInt(500), + }, + }, + } + b, err := buildTwinOrderBook(pins, orders) if !assert.NoError(err) { return } - assert.Len(filledOrders, 0) + assert.Equal(2, b.Size()) + assert.Equal(2, b.EmptyTwinOrderSize()) + assert.Nil(b.GetTwinOrder(fixedpoint.NewFromInt(100))) + assert.True(b.GetTwinOrder(fixedpoint.NewFromInt(200)).Exist()) + assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(300)).Exist()) + assert.False(b.GetTwinOrder(fixedpoint.NewFromInt(400)).Exist()) + assert.True(b.GetTwinOrder(fixedpoint.NewFromInt(500)).Exist()) + }) + + t.Run("build twin orderbook with invalid orders", func(t *testing.T) {}) +} + +func TestSyncActiveOrder(t *testing.T) { + assert := assert.New(t) + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + symbol := "ETHUSDT" + + t.Run("sync filled order in active orderbook, active orderbook should remove this order", func(t *testing.T) { + mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) + activeOrderbook := bbgo.NewActiveOrderBook(symbol) + + order := types.Order{ + OrderID: 1, + Status: types.OrderStatusNew, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + }, + } + activeOrderbook.Add(order) + + updatedOrder := order + updatedOrder.Status = types.OrderStatusFilled + + mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{ + Symbol: symbol, + OrderID: strconv.FormatUint(order.OrderID, 10), + }).Return(&updatedOrder, nil) + + if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) { + return + } + + // verify active orderbook + activeOrders := activeOrderbook.Orders() + assert.Equal(0, len(activeOrders)) + }) + + t.Run("sync partial-filled order in active orderbook, active orderbook should still keep this order", func(t *testing.T) { + mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) + activeOrderbook := bbgo.NewActiveOrderBook(symbol) + + order := types.Order{ + OrderID: 1, + Status: types.OrderStatusNew, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + }, + } + activeOrderbook.Add(order) + + updatedOrder := order + updatedOrder.Status = types.OrderStatusPartiallyFilled + + mockOrderQueryService.EXPECT().QueryOrder(ctx, types.OrderQuery{ + Symbol: symbol, + OrderID: strconv.FormatUint(order.OrderID, 10), + }).Return(&updatedOrder, nil) + + if !assert.NoError(syncActiveOrder(ctx, activeOrderbook, mockOrderQueryService, order.OrderID)) { + return + } + + // verify active orderbook + activeOrders := activeOrderbook.Orders() + assert.Equal(1, len(activeOrders)) + assert.Equal(order.OrderID, activeOrders[0].OrderID) + assert.Equal(updatedOrder.Status, activeOrders[0].Status) + }) +} + +func TestQueryTradesToUpdateTwinOrderBook(t *testing.T) { + assert := assert.New(t) + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + symbol := "ETHUSDT" + pins := []Pin{ + Pin(fixedpoint.NewFromInt(100)), + Pin(fixedpoint.NewFromInt(200)), + Pin(fixedpoint.NewFromInt(300)), + Pin(fixedpoint.NewFromInt(400)), + Pin(fixedpoint.NewFromInt(500)), + } + + t.Run("query trades and update twin orderbook successfully in one page", func(t *testing.T) { + book := newTwinOrderBook(pins) + mockTradeHistoryService := mocks.NewMockExchangeTradeHistoryService(mockCtrl) + mockOrderQueryService := mocks.NewMockExchangeOrderQueryService(mockCtrl) + + trades := []types.Trade{ + { + ID: 1, + OrderID: 1, + Symbol: symbol, + Time: types.Time(time.Now().Add(-2 * time.Hour)), + }, + { + ID: 2, + OrderID: 2, + Symbol: symbol, + Time: types.Time(time.Now().Add(-1 * time.Hour)), + }, + } + orders := []types.Order{ + { + OrderID: 1, + Status: types.OrderStatusNew, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + Side: types.SideTypeBuy, + Price: fixedpoint.NewFromInt(100), + }, + }, + { + OrderID: 2, + Status: types.OrderStatusFilled, + SubmitOrder: types.SubmitOrder{ + Symbol: symbol, + Side: types.SideTypeSell, + Price: fixedpoint.NewFromInt(500), + }, + }, + } + mockTradeHistoryService.EXPECT().QueryTrades(gomock.Any(), gomock.Any(), gomock.Any()).Return(trades, nil).Times(1) + mockOrderQueryService.EXPECT().QueryOrder(gomock.Any(), types.OrderQuery{ + Symbol: symbol, + OrderID: "1", + }).Return(&orders[0], nil) + mockOrderQueryService.EXPECT().QueryOrder(gomock.Any(), types.OrderQuery{ + Symbol: symbol, + OrderID: "2", + }).Return(&orders[1], nil) + + assert.Equal(0, book.Size()) + if !assert.NoError(queryTradesToUpdateTwinOrderBook(ctx, symbol, book, mockTradeHistoryService, mockOrderQueryService, book.SyncOrderMap(), time.Now().Add(-24*time.Hour), time.Now(), nil)) { + return + } + + assert.Equal(2, book.Size()) + assert.True(book.GetTwinOrder(fixedpoint.NewFromInt(200)).Exist()) + assert.Equal(orders[0].OrderID, book.GetTwinOrder(fixedpoint.NewFromInt(200)).GetOrder().OrderID) + assert.True(book.GetTwinOrder(fixedpoint.NewFromInt(500)).Exist()) + assert.Equal(orders[1].OrderID, book.GetTwinOrder(fixedpoint.NewFromInt(500)).GetOrder().OrderID) }) } diff --git a/pkg/strategy/grid2/strategy.go b/pkg/strategy/grid2/strategy.go index 736b1ad67..ce3f77dc2 100644 --- a/pkg/strategy/grid2/strategy.go +++ b/pkg/strategy/grid2/strategy.go @@ -204,7 +204,7 @@ type Strategy struct { tradingCtx, writeCtx context.Context cancelWrite context.CancelFunc - activeOrdersRecoverC chan struct{} + recoverC chan struct{} // this ensures that bbgo.Sync to lock the object sync.Mutex diff --git a/pkg/strategy/grid2/twin_order.go b/pkg/strategy/grid2/twin_order.go index adeeb5263..f3a093b95 100644 --- a/pkg/strategy/grid2/twin_order.go +++ b/pkg/strategy/grid2/twin_order.go @@ -4,6 +4,7 @@ import ( "fmt" "sort" "strings" + "sync" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" @@ -111,3 +112,170 @@ func (m TwinOrderMap) String() string { sb.WriteString("================== END OF PIN ORDER MAP ==================\n") return sb.String() } + +// TwinOrderBook is to verify grid +// For grid trading, there are twin orders between a grid +// e.g. 100, 200, 300, 400, 500 +// BUY 100 and SELL 200 are a twin. +// BUY 200 and SELL 300 are a twin. +// Because they can't be placed on orderbook at the same time. +// We use sell price to be the twin orderbook's key +// New the twin orderbook with pins, and it will sort the pins in asc order. +// There must be a non nil TwinOrder on the every pin (except the first one). +// But the TwinOrder.Exist() may be false. It means there is no twin order on this grid +type TwinOrderBook struct { + // used to protect orderbook update + mu sync.Mutex + + // sort in asc order + pins []fixedpoint.Value + + // pin index, use to find the next or last pin in desc order + pinIdx map[fixedpoint.Value]int + + // orderbook + m map[fixedpoint.Value]*TwinOrder + + // size is the amount on twin orderbook + size int +} + +func newTwinOrderBook(pins []Pin) *TwinOrderBook { + var v []fixedpoint.Value + for _, pin := range pins { + v = append(v, fixedpoint.Value(pin)) + } + + // sort it in asc order + sort.Slice(v, func(i, j int) bool { + return v[j].Compare(v[i]) > 0 + }) + + pinIdx := make(map[fixedpoint.Value]int) + m := make(map[fixedpoint.Value]*TwinOrder) + for i, pin := range v { + // we use sell price for twin orderbook's price, so we skip the first pin as price + if i > 0 { + m[pin] = &TwinOrder{} + } + pinIdx[pin] = i + } + + return &TwinOrderBook{ + pins: v, + pinIdx: pinIdx, + m: m, + size: 0, + } +} + +func (b *TwinOrderBook) String() string { + var sb strings.Builder + + sb.WriteString("================== TWIN ORDERBOOK ==================\n") + for _, pin := range b.pins { + twin := b.m[fixedpoint.Value(pin)] + twinOrder := twin.GetOrder() + sb.WriteString(fmt.Sprintf("-> %8s) %s\n", pin, twinOrder.String())) + } + sb.WriteString("================== END OF TWINORDERBOOK ==================\n") + return sb.String() +} + +func (b *TwinOrderBook) GetTwinOrderPin(order types.Order) (fixedpoint.Value, error) { + idx, exist := b.pinIdx[order.Price] + if !exist { + return fixedpoint.Zero, fmt.Errorf("the order's (%d) price (%s) is not in pins", order.OrderID, order.Price) + } + + if order.Side == types.SideTypeBuy { + // we use sell price as twin orderbook's key, so if the order's side is buy. + // we need to find its next price on grid. + // e.g. + // BUY 100 <- twin -> SELL 200 + // BUY 200 <- twin -> SELL 300 + // BUY 300 <- twin -> SELL 400 + // BUY 400 <- twin -> SELL 500 + // if the order is BUY 100, we need to find its twin order's price to be the twin orderbook's key + // so we plus 1 here and use sorted pins to find the next price (200) + // there must no BUY 500 in the grid, so we need to make sure the idx should always not over the len(pins) + // also, there must no SELL 100 in the grid, so we need to make sure the idx should always not be 0 + idx++ + if idx >= len(b.pins) { + return fixedpoint.Zero, fmt.Errorf("this order's twin order price is not in pins, %+v", order) + } + } else if order.Side == types.SideTypeSell { + if idx == 0 { + return fixedpoint.Zero, fmt.Errorf("this order's twin order price is at zero index, %+v", order) + } + // do nothing + } else { + // should not happen + return fixedpoint.Zero, fmt.Errorf("the order's (%d) side (%s) is not supported", order.OrderID, order.Side) + } + + return b.pins[idx], nil +} + +func (b *TwinOrderBook) AddOrder(order types.Order) error { + b.mu.Lock() + defer b.mu.Unlock() + + pin, err := b.GetTwinOrderPin(order) + if err != nil { + return err + } + + // At all the pins, we already create the empty TwinOrder{} + // As a result,if the exist is false, it means the pin is not in the twin orderbook. + // That's invalid pin, or we have something wrong when new TwinOrderBook + twinOrder, exist := b.m[pin] + if !exist { + // should not happen + return fmt.Errorf("no any empty twin order at pins, should not happen, check it") + } + + // Exist == false means there is no twin order on this pin + if !twinOrder.Exist() { + b.size++ + } + if b.size >= len(b.pins) { + return fmt.Errorf("the maximum size of twin orderbook is len(pins) - 1, need to check it") + } + twinOrder.SetOrder(order) + + return nil +} + +func (b *TwinOrderBook) GetTwinOrder(pin fixedpoint.Value) *TwinOrder { + return b.m[pin] +} + +func (b *TwinOrderBook) AddTwinOrder(pin fixedpoint.Value, order *TwinOrder) { + b.mu.Lock() + defer b.mu.Unlock() + + b.m[pin] = order +} + +// Size is the valid twin order on grid. +func (b *TwinOrderBook) Size() int { + return b.size +} + +// EmptyTwinOrderSize is the amount of grid there is no twin order on it. +func (b *TwinOrderBook) EmptyTwinOrderSize() int { + // for grid, there is only pins - 1 order on the grid, so we need to minus 1. + return len(b.pins) - 1 - b.size +} + +func (b *TwinOrderBook) SyncOrderMap() *types.SyncOrderMap { + orderMap := types.NewSyncOrderMap() + for _, twin := range b.m { + if twin.Exist() { + orderMap.Add(twin.GetOrder()) + } + } + + return orderMap +} diff --git a/pkg/strategy/grid2/twin_order_test.go b/pkg/strategy/grid2/twin_order_test.go new file mode 100644 index 000000000..47395303f --- /dev/null +++ b/pkg/strategy/grid2/twin_order_test.go @@ -0,0 +1,73 @@ +package grid2 + +import ( + "testing" + + "github.com/c9s/bbgo/pkg/fixedpoint" + "github.com/c9s/bbgo/pkg/types" + "github.com/stretchr/testify/assert" +) + +func TestTwinOrderBook(t *testing.T) { + assert := assert.New(t) + pins := []Pin{ + Pin(fixedpoint.NewFromInt(3)), + Pin(fixedpoint.NewFromInt(4)), + Pin(fixedpoint.NewFromInt(1)), + Pin(fixedpoint.NewFromInt(5)), + Pin(fixedpoint.NewFromInt(2)), + } + + book := newTwinOrderBook(pins) + assert.Equal(0, book.Size()) + assert.Equal(4, book.EmptyTwinOrderSize()) + for _, pin := range pins { + twinOrder := book.GetTwinOrder(fixedpoint.Value(pin)) + if fixedpoint.NewFromInt(1) == fixedpoint.Value(pin) { + assert.Nil(twinOrder) + continue + } + + if !assert.NotNil(twinOrder) { + continue + } + + assert.False(twinOrder.Exist()) + } + + orders := []types.Order{ + { + OrderID: 1, + SubmitOrder: types.SubmitOrder{ + Price: fixedpoint.NewFromInt(2), + Side: types.SideTypeBuy, + }, + }, + { + OrderID: 2, + SubmitOrder: types.SubmitOrder{ + Price: fixedpoint.NewFromInt(4), + Side: types.SideTypeSell, + }, + }, + } + + for _, order := range orders { + assert.NoError(book.AddOrder(order)) + } + assert.Equal(2, book.Size()) + assert.Equal(2, book.EmptyTwinOrderSize()) + + for _, order := range orders { + pin, err := book.GetTwinOrderPin(order) + if !assert.NoError(err) { + continue + } + twinOrder := book.GetTwinOrder(pin) + if !assert.True(twinOrder.Exist()) { + continue + } + + assert.Equal(order.OrderID, twinOrder.GetOrder().OrderID) + } +} diff --git a/pkg/strategy/rebalance/multi_market_strategy.go b/pkg/strategy/rebalance/multi_market_strategy.go new file mode 100644 index 000000000..e7d17dd0b --- /dev/null +++ b/pkg/strategy/rebalance/multi_market_strategy.go @@ -0,0 +1,44 @@ +package rebalance + +import ( + "context" + + "github.com/c9s/bbgo/pkg/bbgo" + "github.com/c9s/bbgo/pkg/types" +) + +type MultiMarketStrategy struct { + Environ *bbgo.Environment + Session *bbgo.ExchangeSession + + PositionMap PositionMap `persistence:"positionMap"` + ProfitStatsMap ProfitStatsMap `persistence:"profitStatsMap"` + OrderExecutorMap GeneralOrderExecutorMap + + parent, ctx context.Context + cancel context.CancelFunc +} + +func (s *MultiMarketStrategy) Initialize(ctx context.Context, environ *bbgo.Environment, session *bbgo.ExchangeSession, markets map[string]types.Market, strategyID string) { + s.parent = ctx + s.ctx, s.cancel = context.WithCancel(ctx) + + s.Environ = environ + s.Session = session + + if s.PositionMap == nil { + s.PositionMap = make(PositionMap) + } + s.PositionMap.CreatePositions(markets) + + if s.ProfitStatsMap == nil { + s.ProfitStatsMap = make(ProfitStatsMap) + } + s.ProfitStatsMap.CreateProfitStats(markets) + + s.OrderExecutorMap = NewGeneralOrderExecutorMap(session, s.PositionMap) + s.OrderExecutorMap.BindEnvironment(environ) + s.OrderExecutorMap.BindProfitStats(s.ProfitStatsMap) + s.OrderExecutorMap.Sync(ctx, s) + s.OrderExecutorMap.Bind() +} diff --git a/pkg/strategy/rebalance/position_map.go b/pkg/strategy/rebalance/position_map.go index 772d1726c..73bdda499 100644 --- a/pkg/strategy/rebalance/position_map.go +++ b/pkg/strategy/rebalance/position_map.go @@ -6,17 +6,17 @@ import ( type PositionMap map[string]*types.Position -func (m PositionMap) CreatePositions(markets []types.Market) PositionMap { - for _, market := range markets { - if _, ok := m[market.Symbol]; ok { +func (m PositionMap) CreatePositions(markets map[string]types.Market) PositionMap { + for symbol, market := range markets { + if _, ok := m[symbol]; ok { continue } - log.Infof("creating position for symbol %s", market.Symbol) + log.Infof("creating position for symbol %s", symbol) position := types.NewPositionFromMarket(market) position.Strategy = ID - position.StrategyInstanceID = instanceID(market.Symbol) - m[market.Symbol] = position + position.StrategyInstanceID = instanceID(symbol) + m[symbol] = position } return m } diff --git a/pkg/strategy/rebalance/profit_stats_map.go b/pkg/strategy/rebalance/profit_stats_map.go index a84bf5cc9..29e427a6e 100644 --- a/pkg/strategy/rebalance/profit_stats_map.go +++ b/pkg/strategy/rebalance/profit_stats_map.go @@ -1,17 +1,19 @@ package rebalance -import "github.com/c9s/bbgo/pkg/types" +import ( + "github.com/c9s/bbgo/pkg/types" +) type ProfitStatsMap map[string]*types.ProfitStats -func (m ProfitStatsMap) CreateProfitStats(markets []types.Market) ProfitStatsMap { - for _, market := range markets { - if _, ok := m[market.Symbol]; ok { +func (m ProfitStatsMap) CreateProfitStats(markets map[string]types.Market) ProfitStatsMap { + for symbol, market := range markets { + if _, ok := m[symbol]; ok { continue } - log.Infof("creating profit stats for symbol %s", market.Symbol) - m[market.Symbol] = types.NewProfitStats(market) + log.Infof("creating profit stats for symbol %s", symbol) + m[symbol] = types.NewProfitStats(market) } return m } diff --git a/pkg/strategy/rebalance/strategy.go b/pkg/strategy/rebalance/strategy.go index e15e2507f..22896d24b 100644 --- a/pkg/strategy/rebalance/strategy.go +++ b/pkg/strategy/rebalance/strategy.go @@ -5,6 +5,7 @@ import ( "fmt" "sync" + "github.com/robfig/cron/v3" "github.com/sirupsen/logrus" "github.com/c9s/bbgo/pkg/bbgo" @@ -15,6 +16,7 @@ import ( const ID = "rebalance" var log = logrus.WithField("strategy", ID) +var two = fixedpoint.NewFromFloat(2.0) func init() { bbgo.RegisterStrategy(ID, &Strategy{}) @@ -25,23 +27,24 @@ func instanceID(symbol string) string { } type Strategy struct { + *MultiMarketStrategy + Environment *bbgo.Environment - Interval types.Interval `json:"interval"` - QuoteCurrency string `json:"quoteCurrency"` - TargetWeights types.ValueMap `json:"targetWeights"` - Threshold fixedpoint.Value `json:"threshold"` - MaxAmount fixedpoint.Value `json:"maxAmount"` // max amount to buy or sell per order - OrderType types.OrderType `json:"orderType"` - DryRun bool `json:"dryRun"` - OnStart bool `json:"onStart"` // rebalance on start + CronExpression string `json:"cronExpression"` + QuoteCurrency string `json:"quoteCurrency"` + TargetWeights types.ValueMap `json:"targetWeights"` + Threshold fixedpoint.Value `json:"threshold"` + MaxAmount fixedpoint.Value `json:"maxAmount"` // max amount to buy or sell per order + OrderType types.OrderType `json:"orderType"` + DryRun bool `json:"dryRun"` + OnStart bool `json:"onStart"` // rebalance on start - PositionMap PositionMap `persistence:"positionMap"` - ProfitStatsMap ProfitStatsMap `persistence:"profitStatsMap"` - - session *bbgo.ExchangeSession - orderExecutorMap GeneralOrderExecutorMap - activeOrderBook *bbgo.ActiveOrderBook + session *bbgo.ExchangeSession + symbols []string + markets map[string]types.Market + activeOrderBook *bbgo.ActiveOrderBook + cron *cron.Cron } func (s *Strategy) Defaults() error { @@ -52,6 +55,13 @@ func (s *Strategy) Defaults() error { } func (s *Strategy) Initialize() error { + for currency := range s.TargetWeights { + if currency == s.QuoteCurrency { + continue + } + + s.symbols = append(s.symbols, currency+s.QuoteCurrency) + } return nil } @@ -84,35 +94,22 @@ func (s *Strategy) Validate() error { return nil } -func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) { - for _, symbol := range s.symbols() { - session.Subscribe(types.KLineChannel, symbol, types.SubscribeOptions{Interval: s.Interval}) - } -} +func (s *Strategy) Subscribe(session *bbgo.ExchangeSession) {} func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { s.session = session - markets, err := s.markets() - if err != nil { - return err + s.markets = make(map[string]types.Market) + for _, symbol := range s.symbols { + market, ok := s.session.Market(symbol) + if !ok { + return fmt.Errorf("market %s not found", symbol) + } + s.markets[symbol] = market } - if s.PositionMap == nil { - s.PositionMap = make(PositionMap) - } - s.PositionMap.CreatePositions(markets) - - if s.ProfitStatsMap == nil { - s.ProfitStatsMap = make(ProfitStatsMap) - } - s.ProfitStatsMap.CreateProfitStats(markets) - - s.orderExecutorMap = NewGeneralOrderExecutorMap(session, s.PositionMap) - s.orderExecutorMap.BindEnvironment(s.Environment) - s.orderExecutorMap.BindProfitStats(s.ProfitStatsMap) - s.orderExecutorMap.Bind() - s.orderExecutorMap.Sync(ctx, s) + s.MultiMarketStrategy = &MultiMarketStrategy{} + s.MultiMarketStrategy.Initialize(ctx, s.Environment, session, s.markets, ID) s.activeOrderBook = bbgo.NewActiveOrderBook("") s.activeOrderBook.BindStream(s.session.UserDataStream) @@ -123,16 +120,18 @@ func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo. } }) - s.session.MarketDataStream.OnKLineClosed(func(kline types.KLine) { - s.rebalance(ctx) - }) - // the shutdown handler, you can cancel all orders bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { defer wg.Done() - _ = s.orderExecutorMap.GracefulCancel(ctx) + _ = s.OrderExecutorMap.GracefulCancel(ctx) }) + s.cron = cron.New() + s.cron.AddFunc(s.CronExpression, func() { + s.rebalance(ctx) + }) + s.cron.Start() + return nil } @@ -142,21 +141,24 @@ func (s *Strategy) rebalance(ctx context.Context) { log.WithError(err).Errorf("failed to cancel orders") } - submitOrders, err := s.generateSubmitOrders(ctx) + order, err := s.generateOrder(ctx) if err != nil { - log.WithError(err).Error("failed to generate submit orders") + log.WithError(err).Error("failed to generate order") return } - for _, order := range submitOrders { - log.Infof("generated submit order: %s", order.String()) + + if order == nil { + log.Info("no order generated") + return } + log.Infof("generated order: %s", order.String()) if s.DryRun { log.Infof("dry run, not submitting orders") return } - createdOrders, err := s.orderExecutorMap.SubmitOrders(ctx, submitOrders...) + createdOrders, err := s.OrderExecutorMap.SubmitOrders(ctx, *order) if err != nil { log.WithError(err).Error("failed to submit orders") return @@ -164,7 +166,7 @@ func (s *Strategy) rebalance(ctx context.Context) { s.activeOrderBook.Add(createdOrders...) } -func (s *Strategy) prices(ctx context.Context) (types.ValueMap, error) { +func (s *Strategy) queryMidPrices(ctx context.Context) (types.ValueMap, error) { m := make(types.ValueMap) for currency := range s.TargetWeights { if currency == s.QuoteCurrency { @@ -177,12 +179,12 @@ func (s *Strategy) prices(ctx context.Context) (types.ValueMap, error) { return nil, err } - m[currency] = ticker.Buy.Add(ticker.Sell).Div(fixedpoint.NewFromFloat(2.0)) + m[currency] = ticker.Buy.Add(ticker.Sell).Div(two) } return m, nil } -func (s *Strategy) balances() (types.BalanceMap, error) { +func (s *Strategy) selectBalances() (types.BalanceMap, error) { m := make(types.BalanceMap) balances := s.session.GetAccount().Balances() for currency := range s.TargetWeights { @@ -195,47 +197,37 @@ func (s *Strategy) balances() (types.BalanceMap, error) { return m, nil } -func (s *Strategy) generateSubmitOrders(ctx context.Context) (submitOrders []types.SubmitOrder, err error) { - prices, err := s.prices(ctx) +func (s *Strategy) generateOrder(ctx context.Context) (*types.SubmitOrder, error) { + prices, err := s.queryMidPrices(ctx) if err != nil { return nil, err } - balances, err := s.balances() + + balances, err := s.selectBalances() if err != nil { return nil, err } - marketValues := prices.Mul(balanceToTotal(balances)) - currentWeights := marketValues.Normalize() - for currency, targetWeight := range s.TargetWeights { - if currency == s.QuoteCurrency { - continue - } + values := prices.Mul(toValueMap(balances)) + weights := values.Normalize() - symbol := currency + s.QuoteCurrency - currentWeight := currentWeights[currency] - currentPrice := prices[currency] + for symbol, market := range s.markets { + target := s.TargetWeights[market.BaseCurrency] + weight := weights[market.BaseCurrency] + midPrice := prices[market.BaseCurrency] - log.Infof("%s price: %v, current weight: %v, target weight: %v", - symbol, - currentPrice, - currentWeight, - targetWeight) + log.Infof("%s mid price: %s", symbol, midPrice.String()) + log.Infof("%s weight: %.2f%%, target: %.2f%%", market.BaseCurrency, weight.Float64()*100, target.Float64()*100) // calculate the difference between current weight and target weight // if the difference is less than threshold, then we will not create the order - weightDifference := targetWeight.Sub(currentWeight) - if weightDifference.Abs().Compare(s.Threshold) < 0 { - log.Infof("%s weight distance |%v - %v| = |%v| less than the threshold: %v", - symbol, - currentWeight, - targetWeight, - weightDifference, - s.Threshold) + diff := target.Sub(weight) + if diff.Abs().Compare(s.Threshold) < 0 { + log.Infof("%s weight is close to target, skip", market.BaseCurrency) continue } - quantity := weightDifference.Mul(marketValues.Sum()).Div(currentPrice) + quantity := diff.Mul(values.Sum()).Div(midPrice) side := types.SideTypeBuy if quantity.Sign() < 0 { @@ -243,94 +235,47 @@ func (s *Strategy) generateSubmitOrders(ctx context.Context) (submitOrders []typ quantity = quantity.Abs() } - maxAmount := s.adjustMaxAmountByBalance(side, currency, currentPrice, balances) - if maxAmount.Sign() > 0 { - quantity = bbgo.AdjustQuantityByMaxAmount(quantity, currentPrice, maxAmount) - log.Infof("adjust the quantity %v (%s %s @ %v) by max amount %v", - quantity, + if s.MaxAmount.Float64() > 0 { + quantity = bbgo.AdjustQuantityByMaxAmount(quantity, midPrice, s.MaxAmount) + log.Infof("adjust quantity %s (%s %s @ %s) by max amount %s", + quantity.String(), symbol, side.String(), - currentPrice, - s.MaxAmount) + midPrice.String(), + s.MaxAmount.String()) } - log.Debugf("symbol: %v, quantity: %v", symbol, quantity) + if side == types.SideTypeBuy { + quantity = fixedpoint.Min(quantity, balances[s.QuoteCurrency].Available.Div(midPrice)) + } else if side == types.SideTypeSell { + quantity = fixedpoint.Min(quantity, balances[market.BaseCurrency].Available) + } - order := types.SubmitOrder{ + if market.IsDustQuantity(quantity, midPrice) { + log.Infof("quantity %s (%s %s @ %s) is dust quantity, skip", + quantity.String(), + symbol, + side.String(), + midPrice.String()) + continue + } + + return &types.SubmitOrder{ Symbol: symbol, Side: side, Type: s.OrderType, Quantity: quantity, - Price: currentPrice, - } - - if ok := s.checkMinimalOrderQuantity(order); ok { - submitOrders = append(submitOrders, order) - } + Price: midPrice, + }, nil } - - return submitOrders, err + return nil, nil } -func (s *Strategy) symbols() (symbols []string) { - for currency := range s.TargetWeights { - if currency == s.QuoteCurrency { - continue - } - symbols = append(symbols, currency+s.QuoteCurrency) - } - return symbols -} - -func (s *Strategy) markets() ([]types.Market, error) { - markets := []types.Market{} - for _, symbol := range s.symbols() { - market, ok := s.session.Market(symbol) - if !ok { - return nil, fmt.Errorf("market %s not found", symbol) - } - markets = append(markets, market) - } - return markets, nil -} - -func (s *Strategy) adjustMaxAmountByBalance(side types.SideType, currency string, currentPrice fixedpoint.Value, balances types.BalanceMap) fixedpoint.Value { - var maxAmount fixedpoint.Value - - switch side { - case types.SideTypeBuy: - maxAmount = balances[s.QuoteCurrency].Available - case types.SideTypeSell: - maxAmount = balances[currency].Available.Mul(currentPrice) - default: - log.Errorf("unknown side type: %s", side) - return fixedpoint.Zero - } - - if s.MaxAmount.Sign() > 0 { - maxAmount = fixedpoint.Min(s.MaxAmount, maxAmount) - } - - return maxAmount -} - -func (s *Strategy) checkMinimalOrderQuantity(order types.SubmitOrder) bool { - if order.Quantity.Compare(order.Market.MinQuantity) < 0 { - log.Infof("order quantity is too small: %f < %f", order.Quantity.Float64(), order.Market.MinQuantity.Float64()) - return false - } - - if order.Quantity.Mul(order.Price).Compare(order.Market.MinNotional) < 0 { - log.Infof("order min notional is too small: %f < %f", order.Quantity.Mul(order.Price).Float64(), order.Market.MinNotional.Float64()) - return false - } - return true -} - -func balanceToTotal(balances types.BalanceMap) types.ValueMap { +func toValueMap(balances types.BalanceMap) types.ValueMap { m := make(types.ValueMap) for _, b := range balances { - m[b.Currency] = b.Total() + // m[b.Currency] = b.Net() + m[b.Currency] = b.Available } return m } diff --git a/pkg/strategy/wall/strategy.go b/pkg/strategy/wall/strategy.go index 5cbb4294f..0967b241a 100644 --- a/pkg/strategy/wall/strategy.go +++ b/pkg/strategy/wall/strategy.go @@ -6,12 +6,11 @@ import ( "sync" "time" - "github.com/c9s/bbgo/pkg/core" - "github.com/c9s/bbgo/pkg/util" - "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/c9s/bbgo/pkg/strategy/common" + "github.com/c9s/bbgo/pkg/bbgo" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/types" @@ -31,9 +30,10 @@ func init() { } type Strategy struct { - Environment *bbgo.Environment - StandardIndicatorSet *bbgo.StandardIndicatorSet - Market types.Market + *common.Strategy + + Environment *bbgo.Environment + Market types.Market // Symbol is the market symbol you want to trade Symbol string `json:"symbol"` @@ -60,18 +60,8 @@ type Strategy struct { session *bbgo.ExchangeSession - // persistence fields - Position *types.Position `json:"position,omitempty" persistence:"position"` - ProfitStats *types.ProfitStats `json:"profitStats,omitempty" persistence:"profit_stats"` - activeAdjustmentOrders *bbgo.ActiveOrderBook activeWallOrders *bbgo.ActiveOrderBook - orderStore *core.OrderStore - tradeCollector *core.TradeCollector - - groupID uint32 - - stopC chan struct{} } func (s *Strategy) ID() string { @@ -149,7 +139,6 @@ func (s *Strategy) placeAdjustmentOrders(ctx context.Context, orderExecutor bbgo Price: askPrice, Quantity: quantity, Market: s.Market, - GroupID: s.groupID, }) case types.SideTypeSell: @@ -175,7 +164,6 @@ func (s *Strategy) placeAdjustmentOrders(ctx context.Context, orderExecutor bbgo Price: bidPrice, Quantity: quantity, Market: s.Market, - GroupID: s.groupID, }) } @@ -189,12 +177,13 @@ func (s *Strategy) placeAdjustmentOrders(ctx context.Context, orderExecutor bbgo return err } - s.orderStore.Add(createdOrders...) s.activeAdjustmentOrders.Add(createdOrders...) return nil } func (s *Strategy) placeWallOrders(ctx context.Context, orderExecutor bbgo.OrderExecutor) error { + log.Infof("placing wall orders...") + var submitOrders []types.SubmitOrder var startPrice = s.FixedPrice for i := 0; i < s.NumLayers; i++ { @@ -217,7 +206,6 @@ func (s *Strategy) placeWallOrders(ctx context.Context, orderExecutor bbgo.Order Price: price, Quantity: quantity, Market: s.Market, - GroupID: s.groupID, } submitOrders = append(submitOrders, order) switch s.Side { @@ -240,74 +228,27 @@ func (s *Strategy) placeWallOrders(ctx context.Context, orderExecutor bbgo.Order return err } - s.orderStore.Add(createdOrders...) + log.Infof("wall orders placed: %+v", createdOrders) + s.activeWallOrders.Add(createdOrders...) return err } -func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { +func (s *Strategy) Run(ctx context.Context, _ bbgo.OrderExecutor, session *bbgo.ExchangeSession) error { + s.Strategy = &common.Strategy{} + s.Strategy.Initialize(ctx, s.Environment, session, s.Market, ID, s.InstanceID()) + // initial required information s.session = session - // calculate group id for orders - instanceID := s.InstanceID() - s.groupID = util.FNV32(instanceID) - - // If position is nil, we need to allocate a new position for calculation - if s.Position == nil { - s.Position = types.NewPositionFromMarket(s.Market) - } - - if s.ProfitStats == nil { - s.ProfitStats = types.NewProfitStats(s.Market) - } - - // Always update the position fields - s.Position.Strategy = ID - s.Position.StrategyInstanceID = instanceID - - s.stopC = make(chan struct{}) - s.activeWallOrders = bbgo.NewActiveOrderBook(s.Symbol) s.activeWallOrders.BindStream(session.UserDataStream) s.activeAdjustmentOrders = bbgo.NewActiveOrderBook(s.Symbol) s.activeAdjustmentOrders.BindStream(session.UserDataStream) - s.orderStore = core.NewOrderStore(s.Symbol) - s.orderStore.BindStream(session.UserDataStream) - - s.tradeCollector = core.NewTradeCollector(s.Symbol, s.Position, s.orderStore) - - s.tradeCollector.OnTrade(func(trade types.Trade, profit, netProfit fixedpoint.Value) { - bbgo.Notify(trade) - s.ProfitStats.AddTrade(trade) - - if profit.Compare(fixedpoint.Zero) == 0 { - s.Environment.RecordPosition(s.Position, trade, nil) - } else { - log.Infof("%s generated profit: %v", s.Symbol, profit) - p := s.Position.NewProfit(trade, profit, netProfit) - p.Strategy = ID - p.StrategyInstanceID = instanceID - bbgo.Notify(&p) - - s.ProfitStats.AddProfit(p) - bbgo.Notify(&s.ProfitStats) - - s.Environment.RecordPosition(s.Position, trade, &p) - } - }) - - s.tradeCollector.OnPositionUpdate(func(position *types.Position) { - log.Infof("position changed: %s", s.Position) - bbgo.Notify(s.Position) - }) - - s.tradeCollector.BindStream(session.UserDataStream) - session.UserDataStream.OnStart(func() { - if err := s.placeWallOrders(ctx, orderExecutor); err != nil { + if err := s.placeWallOrders(ctx, s.OrderExecutor); err != nil { log.WithError(err).Errorf("can not place order") } }) @@ -318,9 +259,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se } // check if there is a canceled order had partially filled. - s.tradeCollector.Process() + s.OrderExecutor.TradeCollector().Process() - if err := s.placeAdjustmentOrders(ctx, orderExecutor); err != nil { + if err := s.placeAdjustmentOrders(ctx, s.OrderExecutor); err != nil { log.WithError(err).Errorf("can not place order") } }) @@ -331,9 +272,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se } // check if there is a canceled order had partially filled. - s.tradeCollector.Process() + s.OrderExecutor.TradeCollector().Process() - if err := s.placeWallOrders(ctx, orderExecutor); err != nil { + if err := s.placeWallOrders(ctx, s.OrderExecutor); err != nil { log.WithError(err).Errorf("can not place order") } @@ -342,9 +283,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se } // check if there is a canceled order had partially filled. - s.tradeCollector.Process() + s.OrderExecutor.TradeCollector().Process() - if err := s.placeAdjustmentOrders(ctx, orderExecutor); err != nil { + if err := s.placeAdjustmentOrders(ctx, s.OrderExecutor); err != nil { log.WithError(err).Errorf("can not place order") } }) @@ -365,9 +306,9 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se } // check if there is a canceled order had partially filled. - s.tradeCollector.Process() + s.OrderExecutor.TradeCollector().Process() - if err := s.placeWallOrders(ctx, orderExecutor); err != nil { + if err := s.placeWallOrders(ctx, s.OrderExecutor); err != nil { log.WithError(err).Errorf("can not place order") } } @@ -377,7 +318,6 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se bbgo.OnShutdown(ctx, func(ctx context.Context, wg *sync.WaitGroup) { defer wg.Done() - close(s.stopC) if err := s.activeWallOrders.GracefulCancel(ctx, s.session.Exchange); err != nil { log.WithError(err).Errorf("graceful cancel order error") @@ -387,7 +327,8 @@ func (s *Strategy) Run(ctx context.Context, orderExecutor bbgo.OrderExecutor, se log.WithError(err).Errorf("graceful cancel order error") } - s.tradeCollector.Process() + // check if there is a canceled order had partially filled. + s.OrderExecutor.TradeCollector().Process() }) return nil diff --git a/pkg/strategy/xalign/strategy.go b/pkg/strategy/xalign/strategy.go index 140c1fb50..b6c4f49ad 100644 --- a/pkg/strategy/xalign/strategy.go +++ b/pkg/strategy/xalign/strategy.go @@ -45,6 +45,7 @@ type Strategy struct { DryRun bool `json:"dryRun"` BalanceToleranceRange fixedpoint.Value `json:"balanceToleranceRange"` Duration types.Duration `json:"for"` + MaxAmounts map[string]fixedpoint.Value `json:"maxAmounts"` faultBalanceRecords map[string][]TimeBalance @@ -156,7 +157,7 @@ func (s *Strategy) selectSessionForCurrency(ctx context.Context, sessions map[st switch side { case types.SideTypeBuy: - price := ticker.Sell + var price fixedpoint.Value if taker { price = ticker.Sell } else if spread.Compare(market.TickSize) > 0 { @@ -177,6 +178,12 @@ func (s *Strategy) selectSessionForCurrency(ctx context.Context, sessions map[st continue } + maxAmount, ok := s.MaxAmounts[market.QuoteCurrency] + if ok { + requiredQuoteAmount = bbgo.AdjustQuantityByMaxAmount(requiredQuoteAmount, price, maxAmount) + log.Infof("adjusted quantity %f %s by max amount %f %s", requiredQuoteAmount.Float64(), market.BaseCurrency, maxAmount.Float64(), market.QuoteCurrency) + } + if quantity, ok := market.GreaterThanMinimalOrderQuantity(side, price, requiredQuoteAmount); ok { return session, &types.SubmitOrder{ Symbol: symbol, @@ -190,7 +197,7 @@ func (s *Strategy) selectSessionForCurrency(ctx context.Context, sessions map[st } case types.SideTypeSell: - price := ticker.Buy + var price fixedpoint.Value if taker { price = ticker.Buy } else if spread.Compare(market.TickSize) > 0 { @@ -209,6 +216,12 @@ func (s *Strategy) selectSessionForCurrency(ctx context.Context, sessions map[st continue } + maxAmount, ok := s.MaxAmounts[market.QuoteCurrency] + if ok { + q = bbgo.AdjustQuantityByMaxAmount(q, price, maxAmount) + log.Infof("adjusted quantity %f %s by max amount %f %s", q.Float64(), market.BaseCurrency, maxAmount.Float64(), market.QuoteCurrency) + } + if quantity, ok := market.GreaterThanMinimalOrderQuantity(side, price, q); ok { return session, &types.SubmitOrder{ Symbol: symbol, diff --git a/pkg/types/series_float64.go b/pkg/types/series_float64.go index f12e05d58..8c07b4a13 100644 --- a/pkg/types/series_float64.go +++ b/pkg/types/series_float64.go @@ -46,6 +46,21 @@ func (f *Float64Series) Subscribe(source Float64Source, c func(x float64)) { } } +// AddSubscriber adds the subscriber function and push historical data to the subscriber +func (f *Float64Series) AddSubscriber(fn func(v float64)) { + f.OnUpdate(fn) + + if f.Length() == 0 { + return + } + + // push historical values to the subscriber + for _, vv := range f.Slice { + fn(vv) + } +} + + // Bind binds the source event to the target (Float64Calculator) // A Float64Calculator should be able to calculate the float64 result from a single float64 argument input func (f *Float64Series) Bind(source Float64Source, target Float64Calculator) { diff --git a/pkg/types/stream.go b/pkg/types/stream.go index f65927b61..4ce8c161f 100644 --- a/pkg/types/stream.go +++ b/pkg/types/stream.go @@ -57,8 +57,8 @@ type Parser func(message []byte) (interface{}, error) type Dispatcher func(e interface{}) -// HeartBeat keeps connection alive by sending the heartbeat packet. -type HeartBeat func(ctxConn context.Context, conn *websocket.Conn, cancelConn context.CancelFunc) +// HeartBeat keeps connection alive by sending the ping packet. +type HeartBeat func(conn *websocket.Conn) error type BeforeConnect func(ctx context.Context) error @@ -86,7 +86,7 @@ type StandardStream struct { // sg is used to wait until the previous routines are closed. // only handle routines used internally, avoid including external callback func to prevent issues if they have - // bugs and cannot terminate. e.q. heartBeat + // bugs and cannot terminate. sg SyncGroup // ReconnectC is a signal channel for reconnecting @@ -319,6 +319,14 @@ func (s *StandardStream) ping( return case <-pingTicker.C: + if s.heartBeat != nil { + if err := s.heartBeat(conn); err != nil { + // log errors at the concrete class so that we can identify which exchange encountered an error + s.Reconnect() + return + } + } + if err := conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(writeTimeout)); err != nil { log.WithError(err).Error("ping error", err) s.Reconnect() @@ -432,11 +440,6 @@ func (s *StandardStream) DialAndConnect(ctx context.Context) error { s.ping(connCtx, conn, connCancel, pingInterval) }) s.sg.Run() - - if s.heartBeat != nil { - // not included in wg, as it is an external callback func. - go s.heartBeat(connCtx, conn, connCancel) - } return nil }