From 14fff8dbadc2dcb13c677661691f12cd020e8280 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 24 Aug 2024 11:42:07 +0800 Subject: [PATCH 1/3] xmaker: integrate circuitbreaker --- pkg/risk/circuitbreaker/basic.go | 4 +++- pkg/strategy/xmaker/strategy.go | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/risk/circuitbreaker/basic.go b/pkg/risk/circuitbreaker/basic.go index b58fbe8e6..3a7e06dc3 100644 --- a/pkg/risk/circuitbreaker/basic.go +++ b/pkg/risk/circuitbreaker/basic.go @@ -116,7 +116,7 @@ type BasicCircuitBreaker struct { } func NewBasicCircuitBreaker(strategyID, strategyInstance string) *BasicCircuitBreaker { - return &BasicCircuitBreaker{ + b := &BasicCircuitBreaker{ MaximumConsecutiveLossTimes: 8, MaximumHaltTimes: 3, MaximumHaltTimesExceededPanic: false, @@ -125,6 +125,8 @@ func NewBasicCircuitBreaker(strategyID, strategyInstance string) *BasicCircuitBr strategyInstance: strategyInstance, metricsLabels: prometheus.Labels{"strategy": strategyID, "strategyInstance": strategyInstance}, } + b.updateMetrics() + return b } func (b *BasicCircuitBreaker) getMetricsLabels() prometheus.Labels { diff --git a/pkg/strategy/xmaker/strategy.go b/pkg/strategy/xmaker/strategy.go index f5412fd27..86a71b2ce 100644 --- a/pkg/strategy/xmaker/strategy.go +++ b/pkg/strategy/xmaker/strategy.go @@ -14,6 +14,7 @@ import ( "github.com/c9s/bbgo/pkg/core" "github.com/c9s/bbgo/pkg/fixedpoint" "github.com/c9s/bbgo/pkg/indicator" + "github.com/c9s/bbgo/pkg/risk/circuitbreaker" "github.com/c9s/bbgo/pkg/types" "github.com/c9s/bbgo/pkg/util" ) @@ -98,6 +99,8 @@ type Strategy struct { state *State + CircuitBreaker *circuitbreaker.BasicCircuitBreaker `json:"circuitBreaker"` + // persistence fields Position *types.Position `json:"position,omitempty" persistence:"position"` ProfitStats *ProfitStats `json:"profitStats,omitempty" persistence:"profit_stats"` @@ -630,6 +633,14 @@ func (s *Strategy) tradeRecover(ctx context.Context) { } } +func (s *Strategy) Defaults() error { + if s.CircuitBreaker == nil { + s.CircuitBreaker = circuitbreaker.NewBasicCircuitBreaker(ID, s.InstanceID()) + } + + return nil +} + func (s *Strategy) Validate() error { if s.Quantity.IsZero() || s.QuantityScale == nil { return errors.New("quantity or quantityScale can not be empty") From 5f65e87e8926ed5586580e3e3fce53158a1c3c28 Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 24 Aug 2024 11:43:12 +0800 Subject: [PATCH 2/3] change default HaltDuration to 1h --- pkg/risk/circuitbreaker/basic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/risk/circuitbreaker/basic.go b/pkg/risk/circuitbreaker/basic.go index 3a7e06dc3..03e8183fe 100644 --- a/pkg/risk/circuitbreaker/basic.go +++ b/pkg/risk/circuitbreaker/basic.go @@ -120,7 +120,7 @@ func NewBasicCircuitBreaker(strategyID, strategyInstance string) *BasicCircuitBr MaximumConsecutiveLossTimes: 8, MaximumHaltTimes: 3, MaximumHaltTimesExceededPanic: false, - HaltDuration: types.Duration(30 * time.Minute), + HaltDuration: types.Duration(1 * time.Hour), strategyID: strategyID, strategyInstance: strategyInstance, metricsLabels: prometheus.Labels{"strategy": strategyID, "strategyInstance": strategyInstance}, From 6ef8aa62e574270a02d6ea68dec31ef27962f05c Mon Sep 17 00:00:00 2001 From: c9s Date: Sat, 24 Aug 2024 11:58:09 +0800 Subject: [PATCH 3/3] xmaker: integrate CircuitBreaker --- config/xmaker.yaml | 13 ++++++++++--- pkg/risk/circuitbreaker/basic.go | 8 ++++---- pkg/strategy/xmaker/strategy.go | 22 ++++++++++++++++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/config/xmaker.yaml b/config/xmaker.yaml index e9d10dc9a..dddf31166 100644 --- a/config/xmaker.yaml +++ b/config/xmaker.yaml @@ -59,6 +59,13 @@ crossExchangeStrategies: # 0.1 pip is 0.01, here we use 10, so we will get 18000.00, 18001.00 and # 18002.00 pips: 10 - persistence: - type: redis - + circuitBreaker: + maximumConsecutiveTotalLoss: 36.0 + maximumConsecutiveLossTimes: 10 + maximumLossPerRound: 15.0 + maximumTotalLoss: 80.0 + ignoreConsecutiveDustLoss: true + consecutiveDustLossThreshold: 0.003 + haltDuration: "30m" + maximumHaltTimes: 2 + maximumHaltTimesExceededPanic: true diff --git a/pkg/risk/circuitbreaker/basic.go b/pkg/risk/circuitbreaker/basic.go index 03e8183fe..e18bfef52 100644 --- a/pkg/risk/circuitbreaker/basic.go +++ b/pkg/risk/circuitbreaker/basic.go @@ -223,21 +223,21 @@ func (b *BasicCircuitBreaker) reset() { b.updateMetrics() } -func (b *BasicCircuitBreaker) IsHalted(now time.Time) bool { +func (b *BasicCircuitBreaker) IsHalted(now time.Time) (string, bool) { b.mu.Lock() defer b.mu.Unlock() if !b.halted { - return false + return "", false } // check if it's an expired halt if now.After(b.haltTo) { b.reset() - return false + return "", false } - return true + return b.haltReason, true } func (b *BasicCircuitBreaker) halt(now time.Time, reason string) { diff --git a/pkg/strategy/xmaker/strategy.go b/pkg/strategy/xmaker/strategy.go index 86a71b2ce..6854eb38f 100644 --- a/pkg/strategy/xmaker/strategy.go +++ b/pkg/strategy/xmaker/strategy.go @@ -22,6 +22,9 @@ import ( var defaultMargin = fixedpoint.NewFromFloat(0.003) var Two = fixedpoint.NewFromInt(2) +// circuitBreakerAlertLimiter is for CircuitBreaker alerts +var circuitBreakerAlertLimiter = rate.NewLimiter(rate.Every(3*time.Minute), 2) + const priceUpdateTimeout = 30 * time.Second const ID = "xmaker" @@ -190,6 +193,19 @@ func (s *Strategy) updateQuote(ctx context.Context, orderExecutionRouter bbgo.Or return } + if s.CircuitBreaker != nil { + now := time.Now() + if reason, halted := s.CircuitBreaker.IsHalted(now); halted { + log.Warnf("[arbWorker] strategy is halted, reason: %s", reason) + + if circuitBreakerAlertLimiter.AllowN(now, 1) { + bbgo.Notify("Strategy is halted, reason: %s", reason) + } + + return + } + } + bestBid, bestAsk, hasPrice := s.book.BestBidAndAsk() if !hasPrice { return @@ -573,6 +589,8 @@ func (s *Strategy) Hedge(ctx context.Context, pos fixedpoint.Value) { log.Infof("submitting %s hedge order %s %v", s.Symbol, side.String(), quantity) bbgo.Notify("Submitting %s hedge order %s %v", s.Symbol, side.String(), quantity) + + // TODO: improve order executor orderExecutor := &bbgo.ExchangeOrderExecutor{Session: s.sourceSession} returnOrders, err := orderExecutor.SubmitOrders(ctx, types.SubmitOrder{ Market: s.sourceMarket, @@ -824,6 +842,10 @@ func (s *Strategy) CrossRun( s.ProfitStats.AddProfit(p) s.Environment.RecordPosition(s.Position, trade, &p) + + if s.CircuitBreaker != nil { + s.CircuitBreaker.RecordProfit(profit, trade.Time.Time()) + } } })